feat: actor view panels for the dashboard
Add read-only actor overview and per-actor dossier pages to the dashboard, fed by enriched per-actor runtime snapshots. - `dashboard/swactor/actor_view`: new `ActorPanelView`, a tolerant frame consumer over `runtime.actors`/`runtime.stats` that folds per-actor snapshots and serves `/view/swactor/actor-overview` (roster) and `/view/swactor/actor-dossier` (per-actor detail), each backed by an embedded HTML template (`actor_overview.html`, `actor_dossier.html`) - `dashboard`: register both views in `DashboardHandle` and export `actor_overview_view()`/`actor_dossier_view()` from the swactor module - `swactor` core: enrich `ActorSnapshot` with `actor_type` and `message_type` (populated from `slot.actor.metadata()` in `ActorPool`) and add `ActorAddress::to_full_hex()` for untruncated display - `myelin/orchestration`: publish actor stats to the dashboard via a `runtime.actors` channel producer (`stats_hook_on`) threaded through the distribution stack, and carry the orchestrator actor address into readiness signaling - workspace `Cargo.toml`: add `default-members` for native iteration and a centralized `[workspace.dependencies] tokio` so members share one feature set; `dashboard/Cargo.toml` switches to `tokio.workspace = true` Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
This commit is contained in:
parent
ef9e1c98a3
commit
2bf11e0fdb
35 changed files with 1490 additions and 140 deletions
20
Cargo.toml
20
Cargo.toml
|
|
@ -15,7 +15,27 @@ members = [
|
||||||
"xtask",
|
"xtask",
|
||||||
"tools/vastai",
|
"tools/vastai",
|
||||||
]
|
]
|
||||||
|
default-members = [
|
||||||
|
".",
|
||||||
|
"crates/process",
|
||||||
|
"crates/provisioning",
|
||||||
|
"crates/transport",
|
||||||
|
"crates/distribution",
|
||||||
|
"crates/iroh-driver",
|
||||||
|
"crates/datastream",
|
||||||
|
"crates/data-plane",
|
||||||
|
"crates/dashboard",
|
||||||
|
"apps/myelin",
|
||||||
|
"tools/vastai",
|
||||||
|
]
|
||||||
|
# The language bindings (python, wasm) and xtask are built on demand
|
||||||
|
# (-p / --workspace / cargo-xtask), not during normal native iteration.
|
||||||
exclude = ["crates/bindings/wasm-crypto", "examples"]
|
exclude = ["crates/bindings/wasm-crypto", "examples"]
|
||||||
|
[workspace.dependencies]
|
||||||
|
# Centralized so every member resolves tokio with the SAME feature set.
|
||||||
|
# Without this, members declared tokio with different features, so building
|
||||||
|
# one member vs another (or `run` vs `test`) recompiled tokio each time.
|
||||||
|
tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros", "process", "io-util", "sync", "time", "net", "signal"] }
|
||||||
|
|
||||||
[package]
|
[package]
|
||||||
name = "swactor"
|
name = "swactor"
|
||||||
|
|
|
||||||
|
|
@ -22,7 +22,7 @@ swactor-process = { path = "../../crates/process" }
|
||||||
distribution = { path = "../../crates/distribution" }
|
distribution = { path = "../../crates/distribution" }
|
||||||
iroh-driver = { path = "../../crates/iroh-driver" }
|
iroh-driver = { path = "../../crates/iroh-driver" }
|
||||||
iroh = "0.98"
|
iroh = "0.98"
|
||||||
tokio = { version = "1", features = ["rt-multi-thread", "macros", "process", "io-util", "sync", "time", "net", "signal"] }
|
tokio.workspace = true
|
||||||
swactor-vastai = { path = "../../tools/vastai" }
|
swactor-vastai = { path = "../../tools/vastai" }
|
||||||
parking_lot = "0.12"
|
parking_lot = "0.12"
|
||||||
blake3 = "1"
|
blake3 = "1"
|
||||||
|
|
|
||||||
|
|
@ -56,7 +56,7 @@ OPTIONS:
|
||||||
--relay-mode <mode> Relay mode: default or disabled
|
--relay-mode <mode> Relay mode: default or disabled
|
||||||
--relay-url <url> Custom relay URL passed to myelin-orchestrator
|
--relay-url <url> Custom relay URL passed to myelin-orchestrator
|
||||||
--endpoint-addr-mask <mask> Endpoint address mask: full or relay-only
|
--endpoint-addr-mask <mask> Endpoint address mask: full or relay-only
|
||||||
--cached-model[=<path>] Use discovered or explicit cached GGUF model
|
--cached-model[=<path>] Use discovered or explicit cached GGUF model (default for --process)
|
||||||
--dump-logs[=<path>] Write datastream frame log
|
--dump-logs[=<path>] Write datastream frame log
|
||||||
--run-id <id> Override run id
|
--run-id <id> Override run id
|
||||||
--skip-rebuild Reuse existing Cargo artifacts
|
--skip-rebuild Reuse existing Cargo artifacts
|
||||||
|
|
@ -793,9 +793,15 @@ impl Config {
|
||||||
let gpu_run = args.gpu || env_flag(MYELIN_CHAT_GPU_RUN_ENV, false);
|
let gpu_run = args.gpu || env_flag(MYELIN_CHAT_GPU_RUN_ENV, false);
|
||||||
let endpoint_addr_mask = Self::endpoint_addr_mask(&args, &toml)?;
|
let endpoint_addr_mask = Self::endpoint_addr_mask(&args, &toml)?;
|
||||||
let (relay_mode, relay_url) = Self::relay_settings(&args, &toml, endpoint_addr_mask)?;
|
let (relay_mode, relay_url) = Self::relay_settings(&args, &toml, endpoint_addr_mask)?;
|
||||||
let cached_model = Self::cached_model_source(&args, gpu_run, &provider)
|
let cached_model = match Self::cached_model_source(&args, &provider) {
|
||||||
.map(CachedModelConfig::from_source)
|
None => None,
|
||||||
.transpose()?;
|
Some(CachedModelSource::Path(path)) => Some(CachedModelConfig::from_path(path)?),
|
||||||
|
Some(CachedModelSource::Discover) => match CachedModelConfig::discover() {
|
||||||
|
Ok(config) => Some(config),
|
||||||
|
Err(error) if args.cached_model.is_some() => return Err(error),
|
||||||
|
Err(_) => None,
|
||||||
|
},
|
||||||
|
};
|
||||||
let model = Self::model_config(&provider, &toml, cached_model.as_ref())?;
|
let model = Self::model_config(&provider, &toml, cached_model.as_ref())?;
|
||||||
let datastream_frame_log = Self::datastream_frame_log(&args, &toml);
|
let datastream_frame_log = Self::datastream_frame_log(&args, &toml);
|
||||||
let vastai = if provider == provider_kind::vastai() {
|
let vastai = if provider == provider_kind::vastai() {
|
||||||
|
|
@ -876,16 +882,15 @@ impl Config {
|
||||||
Ok((relay_mode, relay_url))
|
Ok((relay_mode, relay_url))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn cached_model_source(
|
/// Resolve the cached-model source. The process provider defaults to
|
||||||
args: &ParsedArgs,
|
/// best-effort discovery of `.model-cache/` so `cargo myelin-chat` runs a
|
||||||
gpu_run: bool,
|
/// cached GGUF model without explicit flags. Explicit `--cached-model` is
|
||||||
provider: &ProviderKind,
|
/// always honored (and stays strict); the default degrades gracefully to
|
||||||
) -> Option<CachedModelSource> {
|
/// the normal download path when no cached model is present.
|
||||||
|
fn cached_model_source(args: &ParsedArgs, provider: &ProviderKind) -> Option<CachedModelSource> {
|
||||||
match &args.cached_model {
|
match &args.cached_model {
|
||||||
Some(source) => Some(source.clone()),
|
Some(source) => Some(source.clone()),
|
||||||
None if gpu_run && provider == &provider_kind::process() => {
|
None if provider == &provider_kind::process() => Some(CachedModelSource::Discover),
|
||||||
Some(CachedModelSource::Discover)
|
|
||||||
}
|
|
||||||
None => None,
|
None => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -2122,13 +2127,6 @@ struct CachedModelConfig {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CachedModelConfig {
|
impl CachedModelConfig {
|
||||||
fn from_source(source: CachedModelSource) -> Result<Self, String> {
|
|
||||||
match source {
|
|
||||||
CachedModelSource::Discover => Self::discover(),
|
|
||||||
CachedModelSource::Path(path) => Self::from_path(path),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn from_path(path: PathBuf) -> Result<Self, String> {
|
fn from_path(path: PathBuf) -> Result<Self, String> {
|
||||||
let metadata = fs::metadata(&path)
|
let metadata = fs::metadata(&path)
|
||||||
.map_err(|e| format!("stat cached model {}: {e}", path.display()))?;
|
.map_err(|e| format!("stat cached model {}: {e}", path.display()))?;
|
||||||
|
|
@ -2517,6 +2515,34 @@ tag = " alias "
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn config_defaults_use_cached_model_for_process_when_present() {
|
||||||
|
let temp = TempDir::new("cached-model-default");
|
||||||
|
|
||||||
|
with_process_state(&[], Some(temp.path()), || {
|
||||||
|
let cache_dir = temp.path().join(REPO_MODEL_CACHE_DIR);
|
||||||
|
fs::create_dir_all(&cache_dir).expect("create model cache dir");
|
||||||
|
fs::write(
|
||||||
|
cache_dir.join(DEFAULT_PIPELINE_CACHED_MODEL_FILE),
|
||||||
|
Vec::<u8>::new(),
|
||||||
|
)
|
||||||
|
.expect("seed cached model file");
|
||||||
|
|
||||||
|
let defaults = Config::from_args(Vec::<String>::new()).expect("defaults resolve");
|
||||||
|
|
||||||
|
assert_eq!(defaults.provider, provider_kind::process());
|
||||||
|
let cached_model = defaults
|
||||||
|
.cached_model
|
||||||
|
.expect("process provider discovers a cached model by default");
|
||||||
|
assert!(
|
||||||
|
cached_model
|
||||||
|
.host_path
|
||||||
|
.ends_with(DEFAULT_PIPELINE_CACHED_MODEL_FILE),
|
||||||
|
"discovered the seeded cached model"
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn config_max_tokens_drives_orchestrator_args_and_submit_prompt() {
|
fn config_max_tokens_drives_orchestrator_args_and_submit_prompt() {
|
||||||
let temp = TempDir::new("config-max-tokens");
|
let temp = TempDir::new("config-max-tokens");
|
||||||
|
|
|
||||||
|
|
@ -170,6 +170,19 @@ pub(crate) enum NodeAgentMsg {
|
||||||
WorkerCrashed {
|
WorkerCrashed {
|
||||||
reason: Option<String>,
|
reason: Option<String>,
|
||||||
},
|
},
|
||||||
|
StepFailed {
|
||||||
|
step_id: u64,
|
||||||
|
},
|
||||||
|
ObjectFailed {
|
||||||
|
edge_id: u64,
|
||||||
|
object_id: Option<u64>,
|
||||||
|
},
|
||||||
|
OutputFault {
|
||||||
|
edge_id: u64,
|
||||||
|
},
|
||||||
|
EdgeFault {
|
||||||
|
edge_id: u64,
|
||||||
|
},
|
||||||
StopRun {
|
StopRun {
|
||||||
run_id: u64,
|
run_id: u64,
|
||||||
},
|
},
|
||||||
|
|
@ -236,9 +249,6 @@ pub(crate) enum StageCommandWire {
|
||||||
layer_end_exclusive: u32,
|
layer_end_exclusive: u32,
|
||||||
stage_shard_plan: Option<StageShardPlan>,
|
stage_shard_plan: Option<StageShardPlan>,
|
||||||
},
|
},
|
||||||
RewireEdge {
|
|
||||||
edge_id: u64,
|
|
||||||
},
|
|
||||||
ExecuteStep {
|
ExecuteStep {
|
||||||
step_id: u64,
|
step_id: u64,
|
||||||
input_edge_id: u64,
|
input_edge_id: u64,
|
||||||
|
|
@ -531,6 +541,27 @@ impl NodeAgentActor {
|
||||||
self.last_worker_crash = reason;
|
self.last_worker_crash = reason;
|
||||||
self.core.observe(stage::StageEvent::WorkerCrashed)
|
self.core.observe(stage::StageEvent::WorkerCrashed)
|
||||||
}
|
}
|
||||||
|
NodeAgentMsg::StepFailed { step_id } => {
|
||||||
|
self.core.observe(stage::StageEvent::StepFailed {
|
||||||
|
step_id: stage::StepId(step_id),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
NodeAgentMsg::ObjectFailed { edge_id, object_id } => {
|
||||||
|
self.core.observe(stage::StageEvent::ObjectFailed {
|
||||||
|
edge_id: stage::EdgeId(edge_id),
|
||||||
|
object_id: object_id.map(stage::ObjectId),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
NodeAgentMsg::OutputFault { edge_id } => {
|
||||||
|
self.core.observe(stage::StageEvent::OutputFault {
|
||||||
|
edge_id: stage::EdgeId(edge_id),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
NodeAgentMsg::EdgeFault { edge_id } => {
|
||||||
|
self.core.observe(stage::StageEvent::EdgeFault {
|
||||||
|
edge_id: stage::EdgeId(edge_id),
|
||||||
|
})
|
||||||
|
}
|
||||||
NodeAgentMsg::StopRun { run_id } => self.core.observe(stage::StageEvent::StopRun {
|
NodeAgentMsg::StopRun { run_id } => self.core.observe(stage::StageEvent::StopRun {
|
||||||
run_id: stage::RunId(run_id),
|
run_id: stage::RunId(run_id),
|
||||||
}),
|
}),
|
||||||
|
|
@ -692,7 +723,6 @@ impl From<&stage::StageCommand> for StageCommandWire {
|
||||||
layer_end_exclusive: range.end_exclusive,
|
layer_end_exclusive: range.end_exclusive,
|
||||||
stage_shard_plan: shard_plan.clone(),
|
stage_shard_plan: shard_plan.clone(),
|
||||||
},
|
},
|
||||||
stage::StageCommand::RewireEdge { edge_id } => Self::RewireEdge { edge_id: edge_id.0 },
|
|
||||||
stage::StageCommand::ExecuteStep(step) => Self::ExecuteStep {
|
stage::StageCommand::ExecuteStep(step) => Self::ExecuteStep {
|
||||||
step_id: step.step_id.0,
|
step_id: step.step_id.0,
|
||||||
input_edge_id: step.input.edge_id.0,
|
input_edge_id: step.input.edge_id.0,
|
||||||
|
|
|
||||||
|
|
@ -1013,7 +1013,7 @@ impl WorkerEdgeRuntime {
|
||||||
);
|
);
|
||||||
let step_started = Instant::now();
|
let step_started = Instant::now();
|
||||||
let mut pump = || pump_network(driver, stack);
|
let mut pump = || pump_network(driver, stack);
|
||||||
let committed_bytes = worker.execute_step(
|
let committed_bytes = match worker.execute_step(
|
||||||
u64::from(config.stage_index) + 1,
|
u64::from(config.stage_index) + 1,
|
||||||
step_id,
|
step_id,
|
||||||
object_id,
|
object_id,
|
||||||
|
|
@ -1027,7 +1027,15 @@ impl WorkerEdgeRuntime {
|
||||||
config,
|
config,
|
||||||
datastream,
|
datastream,
|
||||||
&mut pump,
|
&mut pump,
|
||||||
)?;
|
) {
|
||||||
|
Ok(bytes) => bytes,
|
||||||
|
Err(e) => {
|
||||||
|
let _ = stack
|
||||||
|
.runtime
|
||||||
|
.send_to(node_actor, NodeAgentMsg::StepFailed { step_id });
|
||||||
|
return Err(e);
|
||||||
|
}
|
||||||
|
};
|
||||||
let helper_execute_ms = duration_ms_u64(step_started.elapsed());
|
let helper_execute_ms = duration_ms_u64(step_started.elapsed());
|
||||||
let egress_read_started = Instant::now();
|
let egress_read_started = Instant::now();
|
||||||
let record = {
|
let record = {
|
||||||
|
|
@ -1035,9 +1043,16 @@ impl WorkerEdgeRuntime {
|
||||||
let lease = arena
|
let lease = arena
|
||||||
.lookup_lease(arena::RingId(output_ring_id))
|
.lookup_lease(arena::RingId(output_ring_id))
|
||||||
.ok_or_else(|| format!("outbound ring {output_ring_id} lease missing"))?;
|
.ok_or_else(|| format!("outbound ring {output_ring_id} lease missing"))?;
|
||||||
arena
|
arena.read_arena(lease.layout.data_offset, committed_bytes)
|
||||||
.read_arena(lease.layout.data_offset, committed_bytes)
|
};
|
||||||
.map_err(|e| format!("read egress ring: {e}"))?
|
let record = match record {
|
||||||
|
Ok(record) => record,
|
||||||
|
Err(e) => {
|
||||||
|
let _ = stack
|
||||||
|
.runtime
|
||||||
|
.send_to(node_actor, NodeAgentMsg::OutputFault { edge_id: outbound.edge_id });
|
||||||
|
return Err(format!("read egress ring: {e}"));
|
||||||
|
}
|
||||||
};
|
};
|
||||||
let egress_read_ms = duration_ms_u64(egress_read_started.elapsed());
|
let egress_read_ms = duration_ms_u64(egress_read_started.elapsed());
|
||||||
let record_bytes = record.len();
|
let record_bytes = record.len();
|
||||||
|
|
@ -1067,7 +1082,12 @@ impl WorkerEdgeRuntime {
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.ok_or_else(|| "outbound edge sender missing".to_owned())?;
|
.ok_or_else(|| "outbound edge sender missing".to_owned())?;
|
||||||
let edge_send_started = Instant::now();
|
let edge_send_started = Instant::now();
|
||||||
sender.send(record)?;
|
if let Err(e) = sender.send(record) {
|
||||||
|
let _ = stack
|
||||||
|
.runtime
|
||||||
|
.send_to(node_actor, NodeAgentMsg::OutputFault { edge_id: outbound.edge_id });
|
||||||
|
return Err(e);
|
||||||
|
}
|
||||||
let edge_send_ms = duration_ms_u64(edge_send_started.elapsed());
|
let edge_send_ms = duration_ms_u64(edge_send_started.elapsed());
|
||||||
node_stage(
|
node_stage(
|
||||||
datastream,
|
datastream,
|
||||||
|
|
@ -1131,8 +1151,17 @@ impl WorkerEdgeRuntime {
|
||||||
buffer.extend_from_slice(&bytes);
|
buffer.extend_from_slice(&bytes);
|
||||||
let buffered_bytes = buffer.len();
|
let buffered_bytes = buffer.len();
|
||||||
let mut records = Vec::new();
|
let mut records = Vec::new();
|
||||||
while let Some(record) = take_complete_ingress_record(buffer, inbound.object_spec)? {
|
loop {
|
||||||
records.push(record);
|
match take_complete_ingress_record(buffer, inbound.object_spec) {
|
||||||
|
Ok(Some(record)) => records.push(record),
|
||||||
|
Ok(None) => break,
|
||||||
|
Err(e) => {
|
||||||
|
let _ = stack
|
||||||
|
.runtime
|
||||||
|
.send_to(node_actor, NodeAgentMsg::ObjectFailed { edge_id, object_id: None });
|
||||||
|
return Err(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
(records, buffered_bytes)
|
(records, buffered_bytes)
|
||||||
};
|
};
|
||||||
|
|
@ -1171,14 +1200,22 @@ impl WorkerEdgeRuntime {
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
let object_load_started = Instant::now();
|
let object_load_started = Instant::now();
|
||||||
let loaded = worker.ring_readable(
|
let loaded = match worker.ring_readable(
|
||||||
ring_id,
|
ring_id,
|
||||||
edge_id,
|
edge_id,
|
||||||
inbound.object_spec,
|
inbound.object_spec,
|
||||||
config,
|
config,
|
||||||
datastream,
|
datastream,
|
||||||
&mut || {},
|
&mut || {},
|
||||||
)?;
|
) {
|
||||||
|
Ok(loaded) => loaded,
|
||||||
|
Err(e) => {
|
||||||
|
let _ = stack
|
||||||
|
.runtime
|
||||||
|
.send_to(node_actor, NodeAgentMsg::ObjectFailed { edge_id, object_id: Some(record.object_id) });
|
||||||
|
return Err(e);
|
||||||
|
}
|
||||||
|
};
|
||||||
let object_load_ms = duration_ms_u64(object_load_started.elapsed());
|
let object_load_ms = duration_ms_u64(object_load_started.elapsed());
|
||||||
let key = ObjectKey {
|
let key = ObjectKey {
|
||||||
edge_id,
|
edge_id,
|
||||||
|
|
@ -1510,11 +1547,9 @@ impl WorkerEdgeRuntime {
|
||||||
.runtime
|
.runtime
|
||||||
.send_to(
|
.send_to(
|
||||||
node_actor,
|
node_actor,
|
||||||
NodeAgentMsg::WorkerCrashed {
|
NodeAgentMsg::EdgeFault { edge_id: edge_id.0 },
|
||||||
reason: Some(format!("edge {} faulted: {reason:?}", edge_id.0)),
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
.map_err(|e| format!("mark worker crashed after edge fault: {e}"))?;
|
.map_err(|e| format!("report edge fault: {e}"))?;
|
||||||
return Err(format!("edge {} faulted: {reason:?}", edge_id.0));
|
return Err(format!("edge {} faulted: {reason:?}", edge_id.0));
|
||||||
}
|
}
|
||||||
edge::EdgeLifecycleEvent::EdgeStopped { .. } => {}
|
edge::EdgeLifecycleEvent::EdgeStopped { .. } => {}
|
||||||
|
|
@ -1691,6 +1726,8 @@ fn run() -> Result<(), String> {
|
||||||
)?;
|
)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let mut datastream = NodeDatastream::new(&config);
|
||||||
|
let worker_stats_hook = datastream.producer.stats_hook();
|
||||||
let stack = DistributionRuntimeStack::new_with_codecs(
|
let stack = DistributionRuntimeStack::new_with_codecs(
|
||||||
driver.node_id(),
|
driver.node_id(),
|
||||||
DistributedNodeConfig::default(),
|
DistributedNodeConfig::default(),
|
||||||
|
|
@ -1698,6 +1735,7 @@ fn run() -> Result<(), String> {
|
||||||
register_myelin_actor_codecs(registry);
|
register_myelin_actor_codecs(registry);
|
||||||
datastream::wire::register_datastream_codec(registry);
|
datastream::wire::register_datastream_codec(registry);
|
||||||
},
|
},
|
||||||
|
Some(worker_stats_hook),
|
||||||
);
|
);
|
||||||
boot(
|
boot(
|
||||||
"distribution_stack",
|
"distribution_stack",
|
||||||
|
|
@ -1750,7 +1788,6 @@ fn run() -> Result<(), String> {
|
||||||
};
|
};
|
||||||
let arena_fd = arena_manager.lock().arena_fd();
|
let arena_fd = arena_manager.lock().arena_fd();
|
||||||
|
|
||||||
let mut datastream = NodeDatastream::new(&config);
|
|
||||||
let node_boot = |ds: &mut NodeDatastream, phase: &str, status: &str, detail: Value| {
|
let node_boot = |ds: &mut NodeDatastream, phase: &str, status: &str, detail: Value| {
|
||||||
emit_node_event(ds, &config, NODE_BOOTSTRAP_CHANNEL, phase, status, detail)
|
emit_node_event(ds, &config, NODE_BOOTSTRAP_CHANNEL, phase, status, detail)
|
||||||
};
|
};
|
||||||
|
|
@ -3560,15 +3597,6 @@ fn handle_stage_command(
|
||||||
);
|
);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
StageCommandWire::RewireEdge { .. } => {
|
|
||||||
node_stage(
|
|
||||||
datastream,
|
|
||||||
"rewire_edge",
|
|
||||||
"skipped",
|
|
||||||
json!({"reason":"not implemented in myelin-worker image path"}),
|
|
||||||
);
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
StageCommandWire::ReleaseInputHandle { handle_id, .. } => {
|
StageCommandWire::ReleaseInputHandle { handle_id, .. } => {
|
||||||
edge_runtime.release_input_handle(handle_id, worker, config, datastream, driver, stack)
|
edge_runtime.release_input_handle(handle_id, worker, config, datastream, driver, stack)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -68,6 +68,13 @@ pub(crate) enum OrchestratorMsg {
|
||||||
stage_index: u32,
|
stage_index: u32,
|
||||||
},
|
},
|
||||||
ObserveTokenEndpointsStopped,
|
ObserveTokenEndpointsStopped,
|
||||||
|
ObserveOperatorStop {
|
||||||
|
run_id: u64,
|
||||||
|
},
|
||||||
|
ObserveMembershipLost {
|
||||||
|
run_id: u64,
|
||||||
|
node_id: u64,
|
||||||
|
},
|
||||||
AdvanceTimeMs(u64),
|
AdvanceTimeMs(u64),
|
||||||
Snapshot {
|
Snapshot {
|
||||||
reply_to: ActorAddress,
|
reply_to: ActorAddress,
|
||||||
|
|
@ -131,7 +138,6 @@ pub(crate) enum RunCommandWire {
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub(crate) enum LifecycleEventWire {
|
pub(crate) enum LifecycleEventWire {
|
||||||
RunRejected { run_id: u64 },
|
|
||||||
RunFaulted { run_id: u64 },
|
RunFaulted { run_id: u64 },
|
||||||
RunCompleted { run_id: u64 },
|
RunCompleted { run_id: u64 },
|
||||||
RunOperatorStopped { run_id: u64 },
|
RunOperatorStopped { run_id: u64 },
|
||||||
|
|
@ -272,6 +278,17 @@ impl OrchestratorActor {
|
||||||
OrchestratorMsg::ObserveTokenEndpointsStopped => {
|
OrchestratorMsg::ObserveTokenEndpointsStopped => {
|
||||||
self.core.observe(core::RunEvent::TokenEndpointsStopped)
|
self.core.observe(core::RunEvent::TokenEndpointsStopped)
|
||||||
}
|
}
|
||||||
|
OrchestratorMsg::ObserveOperatorStop { run_id } => {
|
||||||
|
self.core.observe(core::RunEvent::OperatorStop {
|
||||||
|
run_id: core::RunId(run_id),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
OrchestratorMsg::ObserveMembershipLost { run_id, node_id } => {
|
||||||
|
self.core.observe(core::RunEvent::MembershipLost {
|
||||||
|
run_id: core::RunId(run_id),
|
||||||
|
node_id: core::NodeId(node_id),
|
||||||
|
});
|
||||||
|
}
|
||||||
OrchestratorMsg::AdvanceTimeMs(delta) => self.core.advance_time_ms(delta),
|
OrchestratorMsg::AdvanceTimeMs(delta) => self.core.advance_time_ms(delta),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -449,9 +466,6 @@ impl From<&core::RunCommand> for RunCommandWire {
|
||||||
impl From<&core::LifecycleEvent> for LifecycleEventWire {
|
impl From<&core::LifecycleEvent> for LifecycleEventWire {
|
||||||
fn from(event: &core::LifecycleEvent) -> Self {
|
fn from(event: &core::LifecycleEvent) -> Self {
|
||||||
match event {
|
match event {
|
||||||
core::LifecycleEvent::RunRejected { run_id, .. } => {
|
|
||||||
Self::RunRejected { run_id: run_id.0 }
|
|
||||||
}
|
|
||||||
core::LifecycleEvent::RunFaulted { run_id, .. } => {
|
core::LifecycleEvent::RunFaulted { run_id, .. } => {
|
||||||
Self::RunFaulted { run_id: run_id.0 }
|
Self::RunFaulted { run_id: run_id.0 }
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -19,7 +19,7 @@ use crate::node_actor::{
|
||||||
#[cfg(feature = "dashboard")]
|
#[cfg(feature = "dashboard")]
|
||||||
use crate::observability::dashboard_view::MyelinClusterDashboardView;
|
use crate::observability::dashboard_view::MyelinClusterDashboardView;
|
||||||
use crate::observability::{benchmark, frame_archive::FrameArchive};
|
use crate::observability::{benchmark, frame_archive::FrameArchive};
|
||||||
use crate::orchestration::actor::{OrchestratorActor, OrchestratorReport};
|
use crate::orchestration::actor::{OrchestratorActor, OrchestratorMsg, OrchestratorReport};
|
||||||
use crate::orchestration::config::{DEFAULT_CONFIG_PATH, TomlConfigOverlay};
|
use crate::orchestration::config::{DEFAULT_CONFIG_PATH, TomlConfigOverlay};
|
||||||
const PROVIDER_START_MAX_ATTEMPTS: usize = 4;
|
const PROVIDER_START_MAX_ATTEMPTS: usize = 4;
|
||||||
|
|
||||||
|
|
@ -56,6 +56,7 @@ use datastream::{
|
||||||
};
|
};
|
||||||
use distribution::node::DistributedNodeConfig;
|
use distribution::node::DistributedNodeConfig;
|
||||||
use distribution::telemetry::{MembershipTransition, SwimProbeEvent};
|
use distribution::telemetry::{MembershipTransition, SwimProbeEvent};
|
||||||
|
use distribution::swim::telemetry::ObservedTransition;
|
||||||
use distribution::types::{MemberState, NodeId as DistNodeId};
|
use distribution::types::{MemberState, NodeId as DistNodeId};
|
||||||
use iroh::EndpointAddr;
|
use iroh::EndpointAddr;
|
||||||
use iroh_driver::{
|
use iroh_driver::{
|
||||||
|
|
@ -283,6 +284,8 @@ where
|
||||||
"connectivity_preflight":"ready",
|
"connectivity_preflight":"ready",
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
let actors_channel = orch_datastream.channel_by_name("runtime.actors");
|
||||||
|
let orch_stats_hook = orch_datastream.producer.stats_hook_on(actors_channel);
|
||||||
let stack = DistributionRuntimeStack::new_with_codecs(
|
let stack = DistributionRuntimeStack::new_with_codecs(
|
||||||
driver.node_id(),
|
driver.node_id(),
|
||||||
DistributedNodeConfig::default(),
|
DistributedNodeConfig::default(),
|
||||||
|
|
@ -290,6 +293,7 @@ where
|
||||||
register_myelin_actor_codecs(registry);
|
register_myelin_actor_codecs(registry);
|
||||||
datastream::wire::register_datastream_codec(registry);
|
datastream::wire::register_datastream_codec(registry);
|
||||||
},
|
},
|
||||||
|
Some(orch_stats_hook),
|
||||||
);
|
);
|
||||||
bootstrap(
|
bootstrap(
|
||||||
&mut orch_datastream,
|
&mut orch_datastream,
|
||||||
|
|
@ -455,7 +459,7 @@ where
|
||||||
tx: Mutex::new(obs_tx),
|
tx: Mutex::new(obs_tx),
|
||||||
}));
|
}));
|
||||||
let pipeline_coordinator_endpoint = coordinator_endpoint.clone();
|
let pipeline_coordinator_endpoint = coordinator_endpoint.clone();
|
||||||
let (mut provisioned_nodes, ready) = start_and_provision_workers(
|
let (mut provisioned_nodes, ready, swim_to_node) = start_and_provision_workers(
|
||||||
provisioner,
|
provisioner,
|
||||||
&config,
|
&config,
|
||||||
pipeline_plan.as_ref(),
|
pipeline_plan.as_ref(),
|
||||||
|
|
@ -473,6 +477,7 @@ where
|
||||||
run_id: config.run_id,
|
run_id: config.run_id,
|
||||||
orchestrator_node_id: config.node_id,
|
orchestrator_node_id: config.node_id,
|
||||||
provider: &config.provider,
|
provider: &config.provider,
|
||||||
|
orchestrator_actor,
|
||||||
},
|
},
|
||||||
sink,
|
sink,
|
||||||
coordinator_endpoint,
|
coordinator_endpoint,
|
||||||
|
|
@ -547,6 +552,7 @@ where
|
||||||
run_id: config.run_id,
|
run_id: config.run_id,
|
||||||
orchestrator_node_id: config.node_id,
|
orchestrator_node_id: config.node_id,
|
||||||
provider: &config.provider,
|
provider: &config.provider,
|
||||||
|
orchestrator_actor,
|
||||||
},
|
},
|
||||||
&work_rx,
|
&work_rx,
|
||||||
&prompt_events,
|
&prompt_events,
|
||||||
|
|
@ -558,6 +564,7 @@ where
|
||||||
tokenizer_reply_actor,
|
tokenizer_reply_actor,
|
||||||
pipeline_plan.as_ref(),
|
pipeline_plan.as_ref(),
|
||||||
ready.first_stage.endpoint.clone(),
|
ready.first_stage.endpoint.clone(),
|
||||||
|
&swim_to_node,
|
||||||
);
|
);
|
||||||
if let Err(error) = &result {
|
if let Err(error) = &result {
|
||||||
bootstrap(
|
bootstrap(
|
||||||
|
|
@ -2024,6 +2031,7 @@ struct RuntimeReadyAckLoop<'a> {
|
||||||
run_id: u64,
|
run_id: u64,
|
||||||
orchestrator_node_id: u64,
|
orchestrator_node_id: u64,
|
||||||
provider: &'a ProviderKind,
|
provider: &'a ProviderKind,
|
||||||
|
orchestrator_actor: ActorAddress,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn wait_for_runtime_ready_acks(
|
fn wait_for_runtime_ready_acks(
|
||||||
|
|
@ -2045,6 +2053,7 @@ fn wait_for_runtime_ready_acks(
|
||||||
run_id,
|
run_id,
|
||||||
orchestrator_node_id,
|
orchestrator_node_id,
|
||||||
provider,
|
provider,
|
||||||
|
..
|
||||||
} = ctx;
|
} = ctx;
|
||||||
let bootstrap = |ds: &mut OrchDatastream, phase: &str, status: &str, detail: Value| {
|
let bootstrap = |ds: &mut OrchDatastream, phase: &str, status: &str, detail: Value| {
|
||||||
ds.emit_bootstrap(
|
ds.emit_bootstrap(
|
||||||
|
|
@ -2232,7 +2241,7 @@ fn start_and_provision_workers(
|
||||||
coordinator: EndpointAddr,
|
coordinator: EndpointAddr,
|
||||||
pipeline_coordinator: EndpointAddr,
|
pipeline_coordinator: EndpointAddr,
|
||||||
orchestrator_actor: ActorAddress,
|
orchestrator_actor: ActorAddress,
|
||||||
) -> Result<(ProvisionedClusterGuard, PromptRuntimeReady), String> {
|
) -> Result<(ProvisionedClusterGuard, PromptRuntimeReady, BTreeMap<DistNodeId, u64>), String> {
|
||||||
let RuntimeReadyAckLoop {
|
let RuntimeReadyAckLoop {
|
||||||
driver,
|
driver,
|
||||||
stack,
|
stack,
|
||||||
|
|
@ -2439,6 +2448,7 @@ fn start_and_provision_workers(
|
||||||
run_id: config.run_id,
|
run_id: config.run_id,
|
||||||
orchestrator_node_id: config.node_id,
|
orchestrator_node_id: config.node_id,
|
||||||
provider: &config.provider,
|
provider: &config.provider,
|
||||||
|
orchestrator_actor,
|
||||||
},
|
},
|
||||||
&expected_node_ids,
|
&expected_node_ids,
|
||||||
) {
|
) {
|
||||||
|
|
@ -2468,6 +2478,7 @@ fn start_and_provision_workers(
|
||||||
run_id: config.run_id,
|
run_id: config.run_id,
|
||||||
orchestrator_node_id: config.node_id,
|
orchestrator_node_id: config.node_id,
|
||||||
provider: &config.provider,
|
provider: &config.provider,
|
||||||
|
orchestrator_actor,
|
||||||
}) {
|
}) {
|
||||||
Ok(ready) => ready,
|
Ok(ready) => ready,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
|
|
@ -2512,6 +2523,7 @@ fn start_and_provision_workers(
|
||||||
run_id: config.run_id,
|
run_id: config.run_id,
|
||||||
orchestrator_node_id: config.node_id,
|
orchestrator_node_id: config.node_id,
|
||||||
provider: &config.provider,
|
provider: &config.provider,
|
||||||
|
orchestrator_actor,
|
||||||
},
|
},
|
||||||
&ack_targets,
|
&ack_targets,
|
||||||
&pipeline_coordinator,
|
&pipeline_coordinator,
|
||||||
|
|
@ -2555,6 +2567,7 @@ fn start_and_provision_workers(
|
||||||
run_id: config.run_id,
|
run_id: config.run_id,
|
||||||
orchestrator_node_id: config.node_id,
|
orchestrator_node_id: config.node_id,
|
||||||
provider: &config.provider,
|
provider: &config.provider,
|
||||||
|
orchestrator_actor,
|
||||||
},
|
},
|
||||||
expected_node_ids.len(),
|
expected_node_ids.len(),
|
||||||
pipeline_plan.expect("pipeline mode requires plan"),
|
pipeline_plan.expect("pipeline mode requires plan"),
|
||||||
|
|
@ -2578,6 +2591,7 @@ fn start_and_provision_workers(
|
||||||
run_id: config.run_id,
|
run_id: config.run_id,
|
||||||
orchestrator_node_id: config.node_id,
|
orchestrator_node_id: config.node_id,
|
||||||
provider: &config.provider,
|
provider: &config.provider,
|
||||||
|
orchestrator_actor,
|
||||||
},
|
},
|
||||||
config.stage_index,
|
config.stage_index,
|
||||||
)
|
)
|
||||||
|
|
@ -2640,7 +2654,11 @@ fn start_and_provision_workers(
|
||||||
final_stage: ready,
|
final_stage: ready,
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
Ok((provisioned_nodes, prompt_ready))
|
let swim_to_node = readies
|
||||||
|
.iter()
|
||||||
|
.map(|(node_id, ready)| (ready.swim_node_id, *node_id))
|
||||||
|
.collect::<BTreeMap<DistNodeId, u64>>();
|
||||||
|
Ok((provisioned_nodes, prompt_ready, swim_to_node))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn stage_node_specs(
|
fn stage_node_specs(
|
||||||
|
|
@ -2999,6 +3017,7 @@ fn wait_for_weights_loaded_count(
|
||||||
run_id,
|
run_id,
|
||||||
orchestrator_node_id: node_id,
|
orchestrator_node_id: node_id,
|
||||||
provider,
|
provider,
|
||||||
|
..
|
||||||
} = ctx;
|
} = ctx;
|
||||||
let expected_stages = pipeline_plan
|
let expected_stages = pipeline_plan
|
||||||
.stages
|
.stages
|
||||||
|
|
@ -4126,6 +4145,7 @@ fn wait_for_runtime_ready(ctx: RuntimeReadyAckLoop<'_>) -> Result<RuntimeReady,
|
||||||
run_id,
|
run_id,
|
||||||
orchestrator_node_id: node_id,
|
orchestrator_node_id: node_id,
|
||||||
provider,
|
provider,
|
||||||
|
..
|
||||||
} = ctx;
|
} = ctx;
|
||||||
let mut pending_ready: Option<RuntimeReady> = None;
|
let mut pending_ready: Option<RuntimeReady> = None;
|
||||||
let mut node_swim_started = false;
|
let mut node_swim_started = false;
|
||||||
|
|
@ -4271,6 +4291,7 @@ fn wait_for_weights_loaded(ctx: RuntimeReadyAckLoop<'_>, stage_index: u32) -> Re
|
||||||
run_id,
|
run_id,
|
||||||
orchestrator_node_id: node_id,
|
orchestrator_node_id: node_id,
|
||||||
provider,
|
provider,
|
||||||
|
..
|
||||||
} = ctx;
|
} = ctx;
|
||||||
loop {
|
loop {
|
||||||
pump(driver, stack, frame_tx);
|
pump(driver, stack, frame_tx);
|
||||||
|
|
@ -4913,6 +4934,7 @@ fn serve_prompts(
|
||||||
tokenizer_reply_to: ActorAddress,
|
tokenizer_reply_to: ActorAddress,
|
||||||
pipeline_plan: Option<&run_plan::RunPlan>,
|
pipeline_plan: Option<&run_plan::RunPlan>,
|
||||||
prompt_endpoint: EndpointAddr,
|
prompt_endpoint: EndpointAddr,
|
||||||
|
swim_to_node: &BTreeMap<DistNodeId, u64>,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
let RuntimeReadyAckLoop {
|
let RuntimeReadyAckLoop {
|
||||||
driver,
|
driver,
|
||||||
|
|
@ -4927,6 +4949,7 @@ fn serve_prompts(
|
||||||
run_id,
|
run_id,
|
||||||
orchestrator_node_id: node_id,
|
orchestrator_node_id: node_id,
|
||||||
provider,
|
provider,
|
||||||
|
orchestrator_actor,
|
||||||
..
|
..
|
||||||
} = ctx;
|
} = ctx;
|
||||||
let emit_prompt_evt =
|
let emit_prompt_evt =
|
||||||
|
|
@ -4949,6 +4972,7 @@ fn serve_prompts(
|
||||||
let mut active: Option<ActivePrompt> = None;
|
let mut active: Option<ActivePrompt> = None;
|
||||||
loop {
|
loop {
|
||||||
pump(driver, stack, frame_tx);
|
pump(driver, stack, frame_tx);
|
||||||
|
orch_datastream.flush(dashboard, "orchestrator");
|
||||||
if let Some(pipeline) = pipeline_runtime.as_mut() {
|
if let Some(pipeline) = pipeline_runtime.as_mut() {
|
||||||
pipeline.poll_driver(driver);
|
pipeline.poll_driver(driver);
|
||||||
pipeline.drain_tokenizer_events(
|
pipeline.drain_tokenizer_events(
|
||||||
|
|
@ -4971,6 +4995,21 @@ fn serve_prompts(
|
||||||
)?;
|
)?;
|
||||||
drain_frames(frame_rx, dashboard, orch_datastream);
|
drain_frames(frame_rx, dashboard, orch_datastream);
|
||||||
drain_orch_stdio_capture(orch_stdio_rx, orch_datastream, dashboard, run_id, node_id);
|
drain_orch_stdio_capture(orch_stdio_rx, orch_datastream, dashboard, run_id, node_id);
|
||||||
|
let swim_transitions =
|
||||||
|
emit_swim_transitions(orch_datastream, dashboard, run_id, node_id, stack);
|
||||||
|
for transition in &swim_transitions {
|
||||||
|
if transition.to == MemberState::Dead
|
||||||
|
&& let Some(&lost_node_id) = swim_to_node.get(&transition.peer)
|
||||||
|
{
|
||||||
|
let _ = stack.runtime.send_to(
|
||||||
|
orchestrator_actor,
|
||||||
|
OrchestratorMsg::ObserveMembershipLost {
|
||||||
|
run_id,
|
||||||
|
node_id: lost_node_id,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
if stop_rx.try_recv().is_ok() {
|
if stop_rx.try_recv().is_ok() {
|
||||||
orch_datastream.emit_bootstrap(
|
orch_datastream.emit_bootstrap(
|
||||||
dashboard,
|
dashboard,
|
||||||
|
|
@ -4980,6 +5019,10 @@ fn serve_prompts(
|
||||||
"started",
|
"started",
|
||||||
json!({"source":"stdin"}),
|
json!({"source":"stdin"}),
|
||||||
);
|
);
|
||||||
|
let _ = stack
|
||||||
|
.runtime
|
||||||
|
.send_to(orchestrator_actor, OrchestratorMsg::ObserveOperatorStop { run_id });
|
||||||
|
pump(driver, stack, frame_tx);
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -5337,8 +5380,9 @@ fn emit_swim_transitions(
|
||||||
run_id: u64,
|
run_id: u64,
|
||||||
node_id: u64,
|
node_id: u64,
|
||||||
stack: &DistributionRuntimeStack,
|
stack: &DistributionRuntimeStack,
|
||||||
) {
|
) -> Vec<ObservedTransition> {
|
||||||
for transition in stack.drain_swim_transitions() {
|
let transitions = stack.drain_swim_transitions();
|
||||||
|
for transition in &transitions {
|
||||||
let peer = format!("{:?}", transition.peer);
|
let peer = format!("{:?}", transition.peer);
|
||||||
let from = transition.from.map(|state| format!("{:?}", state));
|
let from = transition.from.map(|state| format!("{:?}", state));
|
||||||
let to = format!("{:?}", transition.to);
|
let to = format!("{:?}", transition.to);
|
||||||
|
|
@ -5366,8 +5410,9 @@ fn emit_swim_transitions(
|
||||||
"member_state":member_state.clone(),
|
"member_state":member_state.clone(),
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
orch_datastream.emit_record(dashboard, &stack.membership_transition(&transition));
|
orch_datastream.emit_record(dashboard, &stack.membership_transition(transition));
|
||||||
}
|
}
|
||||||
|
transitions
|
||||||
}
|
}
|
||||||
|
|
||||||
fn emit_swim_probe_events(
|
fn emit_swim_probe_events(
|
||||||
|
|
@ -5418,18 +5463,24 @@ fn local_tinygrad_worker_env(provider: &str) -> Option<(String, String)> {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn default_local_tinygrad_worker_path() -> Option<PathBuf> {
|
fn default_local_tinygrad_worker_path() -> Option<PathBuf> {
|
||||||
let cwd_candidate = std::env::current_dir()
|
// The tinygrad worker script ships in the node-image build context at
|
||||||
.ok()
|
// `apps/myelin/node-image/tinygrad_worker.py` (see `chat/node_image.rs` and
|
||||||
.map(|cwd| cwd.join("apps").join("myelin-node").join("tinygrad_worker.py"));
|
// the node-image Dockerfile). The process provider runs it directly via
|
||||||
|
// `python3`, so resolve that path from the workspace cwd or this crate's
|
||||||
|
// manifest dir. (Previously looked in `apps/myelin-node/`, a path left stale
|
||||||
|
// by the `mvp-system` -> `myelin` app refactor and never present on disk.)
|
||||||
|
let cwd_candidate = std::env::current_dir().ok().map(|cwd| {
|
||||||
|
cwd.join("apps")
|
||||||
|
.join("myelin")
|
||||||
|
.join("node-image")
|
||||||
|
.join("tinygrad_worker.py")
|
||||||
|
});
|
||||||
if let Some(candidate) = cwd_candidate.filter(|path| path.is_file()) {
|
if let Some(candidate) = cwd_candidate.filter(|path| path.is_file()) {
|
||||||
return Some(candidate.canonicalize().unwrap_or(candidate));
|
return Some(candidate.canonicalize().unwrap_or(candidate));
|
||||||
}
|
}
|
||||||
|
|
||||||
let manifest_candidate = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
let manifest_candidate = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||||
.join("..")
|
.join("node-image")
|
||||||
.join("..")
|
|
||||||
.join("apps")
|
|
||||||
.join("myelin-node")
|
|
||||||
.join("tinygrad_worker.py");
|
.join("tinygrad_worker.py");
|
||||||
manifest_candidate.is_file().then(|| {
|
manifest_candidate.is_file().then(|| {
|
||||||
manifest_candidate
|
manifest_candidate
|
||||||
|
|
|
||||||
|
|
@ -13,6 +13,7 @@ use std::time::{Duration, Instant};
|
||||||
use swactor::actor::{ActorAddress, ActorInterface};
|
use swactor::actor::{ActorAddress, ActorInterface};
|
||||||
use swactor::config::RuntimeConfig;
|
use swactor::config::RuntimeConfig;
|
||||||
use swactor::runtime::{Ctx, Runtime};
|
use swactor::runtime::{Ctx, Runtime};
|
||||||
|
use swactor::stats::StatsHook;
|
||||||
use swactor::std::StdExtension;
|
use swactor::std::StdExtension;
|
||||||
use swactor_transport::{CodecRegistry, CodecRemoteSink, NetworkMessage, TransportRouter};
|
use swactor_transport::{CodecRegistry, CodecRemoteSink, NetworkMessage, TransportRouter};
|
||||||
|
|
||||||
|
|
@ -44,7 +45,6 @@ pub(crate) struct DistributionActorAddrs {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) struct DistributionRuntimeStack {
|
pub(crate) struct DistributionRuntimeStack {
|
||||||
pub node_id: NodeId,
|
|
||||||
pub runtime: Arc<Runtime>,
|
pub runtime: Arc<Runtime>,
|
||||||
pub codec: Arc<CodecRegistry>,
|
pub codec: Arc<CodecRegistry>,
|
||||||
pub outbox: Outbox,
|
pub outbox: Outbox,
|
||||||
|
|
@ -61,6 +61,7 @@ impl DistributionRuntimeStack {
|
||||||
node_id: NodeId,
|
node_id: NodeId,
|
||||||
config: DistributedNodeConfig,
|
config: DistributedNodeConfig,
|
||||||
extend_codecs: impl FnOnce(&mut CodecRegistry),
|
extend_codecs: impl FnOnce(&mut CodecRegistry),
|
||||||
|
stats_hook: Option<Arc<dyn StatsHook>>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
let mut runtime =
|
let mut runtime =
|
||||||
Runtime::new(RuntimeConfig::default()).with_extension(Arc::new(StdExtension::new()));
|
Runtime::new(RuntimeConfig::default()).with_extension(Arc::new(StdExtension::new()));
|
||||||
|
|
@ -72,6 +73,9 @@ impl DistributionRuntimeStack {
|
||||||
Arc::clone(&codec),
|
Arc::clone(&codec),
|
||||||
Arc::clone(&transport_router),
|
Arc::clone(&transport_router),
|
||||||
)));
|
)));
|
||||||
|
if let Some(hook) = stats_hook {
|
||||||
|
runtime.set_stats_hook(hook);
|
||||||
|
}
|
||||||
let runtime = Arc::new(runtime);
|
let runtime = Arc::new(runtime);
|
||||||
|
|
||||||
let outbox: Outbox = Arc::new(Mutex::new(Vec::new()));
|
let outbox: Outbox = Arc::new(Mutex::new(Vec::new()));
|
||||||
|
|
@ -146,7 +150,6 @@ impl DistributionRuntimeStack {
|
||||||
.expect("subscribe MembershipFanout");
|
.expect("subscribe MembershipFanout");
|
||||||
|
|
||||||
Self {
|
Self {
|
||||||
node_id,
|
|
||||||
runtime,
|
runtime,
|
||||||
codec,
|
codec,
|
||||||
outbox,
|
outbox,
|
||||||
|
|
|
||||||
|
|
@ -72,12 +72,6 @@ impl StaticRelayProvider {
|
||||||
parse_relay_url(raw).map(Self::new)
|
parse_relay_url(raw).map(Self::new)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn from_env() -> Result<Option<Self>, String> {
|
|
||||||
selected_relay_url_from_env()
|
|
||||||
.map(|url| Self::from_url_str(&url).map(Some))
|
|
||||||
.unwrap_or(Ok(None))
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn url(&self) -> String {
|
pub(crate) fn url(&self) -> String {
|
||||||
self.url.to_string()
|
self.url.to_string()
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -16,16 +16,6 @@ pub(crate) struct RunPlan {
|
||||||
pub stages: Vec<StageRef>,
|
pub stages: Vec<StageRef>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RunPlan {
|
|
||||||
pub(crate) fn test_linear(run_id: RunId, stages: Vec<StageRef>) -> Self {
|
|
||||||
Self { run_id, stages }
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn stage_nodes(&self) -> Vec<NodeId> {
|
|
||||||
self.stages.iter().map(|stage| stage.node_id).collect()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
pub(crate) struct RunConfig {
|
pub(crate) struct RunConfig {
|
||||||
pub run_id: RunId,
|
pub run_id: RunId,
|
||||||
|
|
@ -125,10 +115,6 @@ pub(crate) enum RunFaultReason {
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
pub(crate) enum LifecycleEvent {
|
pub(crate) enum LifecycleEvent {
|
||||||
RunRejected {
|
|
||||||
run_id: RunId,
|
|
||||||
reason: RunFaultReason,
|
|
||||||
},
|
|
||||||
RunFaulted {
|
RunFaulted {
|
||||||
run_id: RunId,
|
run_id: RunId,
|
||||||
reason: RunFaultReason,
|
reason: RunFaultReason,
|
||||||
|
|
@ -175,8 +161,6 @@ pub(crate) enum RunCommand {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
pub type OrchestratorHarness = OrchestratorRun;
|
|
||||||
|
|
||||||
pub(crate) struct OrchestratorRun {
|
pub(crate) struct OrchestratorRun {
|
||||||
config: RunConfig,
|
config: RunConfig,
|
||||||
plan: Option<RunPlan>,
|
plan: Option<RunPlan>,
|
||||||
|
|
|
||||||
|
|
@ -165,7 +165,6 @@ pub(crate) enum EdgeKind {
|
||||||
pub(crate) enum ObjectKind {
|
pub(crate) enum ObjectKind {
|
||||||
Token,
|
Token,
|
||||||
Activation,
|
Activation,
|
||||||
Weight,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
|
|
||||||
|
|
@ -19,12 +19,6 @@ pub(crate) struct DeviceHandle {
|
||||||
pub id: u64,
|
pub id: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl DeviceHandle {
|
|
||||||
pub(crate) fn new_current(id: u64) -> Self {
|
|
||||||
Self { generation: 1, id }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
pub(crate) struct LayerRange {
|
pub(crate) struct LayerRange {
|
||||||
pub start: u32,
|
pub start: u32,
|
||||||
|
|
@ -50,14 +44,6 @@ impl WeightSource {
|
||||||
tokenizer,
|
tokenizer,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn embedded_gguf(model_id: impl Into<String>, path: impl Into<String>) -> Self {
|
|
||||||
Self::new(
|
|
||||||
model_id,
|
|
||||||
GgufSource::LocalPath(path.into()),
|
|
||||||
TokenizerSource::EmbeddedGguf,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
|
@ -228,9 +214,6 @@ pub(crate) enum StageCommand {
|
||||||
range: LayerRange,
|
range: LayerRange,
|
||||||
shard_plan: Option<StageShardPlan>,
|
shard_plan: Option<StageShardPlan>,
|
||||||
},
|
},
|
||||||
RewireEdge {
|
|
||||||
edge_id: EdgeId,
|
|
||||||
},
|
|
||||||
ExecuteStep(ExecuteStep),
|
ExecuteStep(ExecuteStep),
|
||||||
ReleaseInputHandle {
|
ReleaseInputHandle {
|
||||||
object_id: ObjectId,
|
object_id: ObjectId,
|
||||||
|
|
@ -244,8 +227,6 @@ pub(crate) enum StageCommand {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
pub type StageControllerHarness = StageController;
|
|
||||||
|
|
||||||
pub(crate) struct StageController {
|
pub(crate) struct StageController {
|
||||||
local_node_id: NodeId,
|
local_node_id: NodeId,
|
||||||
provision: Option<ProvisionStage>,
|
provision: Option<ProvisionStage>,
|
||||||
|
|
|
||||||
50
apps/myelin/src/tests/harness.rs
Normal file
50
apps/myelin/src/tests/harness.rs
Normal file
|
|
@ -0,0 +1,50 @@
|
||||||
|
//! Test-only constructors and harness type aliases for the `tests` tree.
|
||||||
|
//!
|
||||||
|
//! These helpers previously lived in the production modules
|
||||||
|
//! (`orchestration/run_fsm`, `staging::control`) but serve only test code.
|
||||||
|
//! Defining them here keeps them compiled under `#[cfg(test)]` only, so
|
||||||
|
//! production builds stay free of the dead-code warnings they would otherwise
|
||||||
|
//! trigger. Inherent methods resolve through their type, so existing call sites
|
||||||
|
//! such as `fsm::RunPlan::test_linear(..)` and
|
||||||
|
//! `stage::WeightSource::embedded_gguf(..)` work unchanged.
|
||||||
|
|
||||||
|
use crate::run_fsm::{NodeId as FsmNodeId, OrchestratorRun, RunId as FsmRunId, RunPlan, StageRef};
|
||||||
|
use crate::run_plan::{GgufSource, TokenizerSource};
|
||||||
|
use crate::staging::{DeviceHandle, StageController, WeightSource};
|
||||||
|
|
||||||
|
impl RunPlan {
|
||||||
|
/// Build a linear pipeline plan (stage 0 -> stage 1 -> ... -> last stage).
|
||||||
|
/// Test-only convenience over the public `RunPlan` fields.
|
||||||
|
pub(crate) fn test_linear(run_id: FsmRunId, stages: Vec<StageRef>) -> Self {
|
||||||
|
Self { run_id, stages }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Node ids in declared stage order.
|
||||||
|
pub(crate) fn stage_nodes(&self) -> Vec<FsmNodeId> {
|
||||||
|
self.stages.iter().map(|stage| stage.node_id).collect()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl DeviceHandle {
|
||||||
|
/// A handle on the worker's current (first) generation.
|
||||||
|
pub(crate) fn new_current(id: u64) -> Self {
|
||||||
|
Self { generation: 1, id }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WeightSource {
|
||||||
|
/// Weights carried by an embedded GGUF file with an embedded tokenizer.
|
||||||
|
pub(crate) fn embedded_gguf(model_id: impl Into<String>, path: impl Into<String>) -> Self {
|
||||||
|
Self::new(
|
||||||
|
model_id,
|
||||||
|
GgufSource::LocalPath(path.into()),
|
||||||
|
TokenizerSource::EmbeddedGguf,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Test alias for the orchestrator run core, retained for readable test prose.
|
||||||
|
pub(crate) type OrchestratorHarness = OrchestratorRun;
|
||||||
|
|
||||||
|
/// Test alias for the stage controller core, retained for readable test prose.
|
||||||
|
pub(crate) type StageControllerHarness = StageController;
|
||||||
|
|
@ -2,6 +2,7 @@ use std::collections::{BTreeMap, BTreeSet};
|
||||||
|
|
||||||
use crate::run_fsm as fsm;
|
use crate::run_fsm as fsm;
|
||||||
use crate::run_plan as plan;
|
use crate::run_plan as plan;
|
||||||
|
use crate::tests::harness::OrchestratorHarness;
|
||||||
use data_plane::edge_actor;
|
use data_plane::edge_actor;
|
||||||
use myelin::observability::lifecycle as obs;
|
use myelin::observability::lifecycle as obs;
|
||||||
use myelin::orchestration::engine_builder as engine;
|
use myelin::orchestration::engine_builder as engine;
|
||||||
|
|
@ -34,7 +35,7 @@ pub struct LocalMockCluster {
|
||||||
engine_events: Vec<engine::EngineEvent>,
|
engine_events: Vec<engine::EngineEvent>,
|
||||||
plan: plan::RunPlan,
|
plan: plan::RunPlan,
|
||||||
nodes: BTreeMap<u32, MockNode>,
|
nodes: BTreeMap<u32, MockNode>,
|
||||||
orchestrator: Option<fsm::OrchestratorHarness>,
|
orchestrator: Option<OrchestratorHarness>,
|
||||||
orchestrator_command_cursor: usize,
|
orchestrator_command_cursor: usize,
|
||||||
orchestrator_event_cursor: usize,
|
orchestrator_event_cursor: usize,
|
||||||
trace: Vec<obs::Event>,
|
trace: Vec<obs::Event>,
|
||||||
|
|
@ -303,7 +304,7 @@ impl LocalMockCluster {
|
||||||
self.orchestrator_command_cursor = 0;
|
self.orchestrator_command_cursor = 0;
|
||||||
self.orchestrator_event_cursor = 0;
|
self.orchestrator_event_cursor = 0;
|
||||||
self.scenario = scenario;
|
self.scenario = scenario;
|
||||||
self.orchestrator = Some(fsm::OrchestratorHarness::new(fsm::RunConfig {
|
self.orchestrator = Some(OrchestratorHarness::new(fsm::RunConfig {
|
||||||
run_id: fsm::RunId(self.run_id.0),
|
run_id: fsm::RunId(self.run_id.0),
|
||||||
max_tokens: u64::from(self.max_tokens),
|
max_tokens: u64::from(self.max_tokens),
|
||||||
prompt: tokenize(prompt),
|
prompt: tokenize(prompt),
|
||||||
|
|
@ -742,7 +743,6 @@ impl LocalMockCluster {
|
||||||
self.push_run(obs::EventKind::RunCompleted, obs::Component::Orchestrator);
|
self.push_run(obs::EventKind::RunCompleted, obs::Component::Orchestrator);
|
||||||
}
|
}
|
||||||
fsm::LifecycleEvent::RunFaulted { .. }
|
fsm::LifecycleEvent::RunFaulted { .. }
|
||||||
| fsm::LifecycleEvent::RunRejected { .. }
|
|
||||||
| fsm::LifecycleEvent::RunOperatorStopped { .. } => {
|
| fsm::LifecycleEvent::RunOperatorStopped { .. } => {
|
||||||
self.push_run(obs::EventKind::RunFaulted, obs::Component::Orchestrator);
|
self.push_run(obs::EventKind::RunFaulted, obs::Component::Orchestrator);
|
||||||
}
|
}
|
||||||
|
|
@ -852,7 +852,7 @@ impl LocalMockCluster {
|
||||||
.count()
|
.count()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn orchestrator_mut(&mut self) -> &mut fsm::OrchestratorHarness {
|
fn orchestrator_mut(&mut self) -> &mut OrchestratorHarness {
|
||||||
self.orchestrator
|
self.orchestrator
|
||||||
.as_mut()
|
.as_mut()
|
||||||
.expect("run_prompt must initialize orchestrator")
|
.expect("run_prompt must initialize orchestrator")
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
use crate::run_plan as plan;
|
use crate::run_plan as plan;
|
||||||
use data_plane::edge_actor;
|
use data_plane::edge_actor;
|
||||||
use myelin::staging as stage;
|
use myelin::staging as stage;
|
||||||
|
use crate::tests::harness::StageControllerHarness;
|
||||||
|
|
||||||
use super::mock_transport::MockObject;
|
use super::mock_transport::MockObject;
|
||||||
use super::mock_worker::MockWorker;
|
use super::mock_worker::MockWorker;
|
||||||
|
|
@ -16,7 +17,7 @@ pub struct MockNode {
|
||||||
inbound_edge: Option<plan::EdgeId>,
|
inbound_edge: Option<plan::EdgeId>,
|
||||||
outbound_edge: Option<plan::EdgeId>,
|
outbound_edge: Option<plan::EdgeId>,
|
||||||
outbound_object_allocator: Option<edge_actor::ObjectIdAllocator>,
|
outbound_object_allocator: Option<edge_actor::ObjectIdAllocator>,
|
||||||
controller: stage::StageControllerHarness,
|
controller: StageControllerHarness,
|
||||||
worker: MockWorker,
|
worker: MockWorker,
|
||||||
event_cursor: usize,
|
event_cursor: usize,
|
||||||
}
|
}
|
||||||
|
|
@ -34,7 +35,7 @@ impl MockNode {
|
||||||
inbound_edge: None,
|
inbound_edge: None,
|
||||||
outbound_edge: None,
|
outbound_edge: None,
|
||||||
outbound_object_allocator: None,
|
outbound_object_allocator: None,
|
||||||
controller: stage::StageControllerHarness::new(stage::NodeId(node_id.0)),
|
controller: StageControllerHarness::new(stage::NodeId(node_id.0)),
|
||||||
worker: MockWorker::new(stage_index, eos_after_sequence),
|
worker: MockWorker::new(stage_index, eos_after_sequence),
|
||||||
event_cursor: 0,
|
event_cursor: 0,
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
mod harness;
|
||||||
mod local_e2e_guarantees;
|
mod local_e2e_guarantees;
|
||||||
mod local_mock;
|
mod local_mock;
|
||||||
mod node_guarantees;
|
mod node_guarantees;
|
||||||
|
|
|
||||||
|
|
@ -502,6 +502,7 @@ mod run_fsm {
|
||||||
//! `specs/BEHAVIOR_GUARANTEES.md`.
|
//! `specs/BEHAVIOR_GUARANTEES.md`.
|
||||||
|
|
||||||
use crate::run_fsm as fsm;
|
use crate::run_fsm as fsm;
|
||||||
|
use crate::tests::harness::OrchestratorHarness;
|
||||||
|
|
||||||
// A three-stage plan proves multi-stage provisioning and readiness without
|
// A three-stage plan proves multi-stage provisioning and readiness without
|
||||||
// making tests depend on any placement heuristic. The plan is already valid;
|
// making tests depend on any placement heuristic. The plan is already valid;
|
||||||
|
|
@ -529,8 +530,8 @@ mod run_fsm {
|
||||||
// The harness is the black-box public boundary for the run FSM. It accepts
|
// The harness is the black-box public boundary for the run FSM. It accepts
|
||||||
// observable events and records emitted commands/events; tests never inspect an
|
// observable events and records emitted commands/events; tests never inspect an
|
||||||
// internal FSM enum or private readiness counter.
|
// internal FSM enum or private readiness counter.
|
||||||
fn new_run() -> fsm::OrchestratorHarness {
|
fn new_run() -> OrchestratorHarness {
|
||||||
fsm::OrchestratorHarness::new(fsm::RunConfig {
|
OrchestratorHarness::new(fsm::RunConfig {
|
||||||
run_id: fsm::RunId(7),
|
run_id: fsm::RunId(7),
|
||||||
max_tokens: 4,
|
max_tokens: 4,
|
||||||
prompt: vec![101, 102, 103],
|
prompt: vec![101, 102, 103],
|
||||||
|
|
@ -680,7 +681,6 @@ mod run_fsm {
|
||||||
});
|
});
|
||||||
assert!(invalid.events().iter().any(|event| {
|
assert!(invalid.events().iter().any(|event| {
|
||||||
matches!(event, fsm::LifecycleEvent::RunFaulted { .. })
|
matches!(event, fsm::LifecycleEvent::RunFaulted { .. })
|
||||||
|| matches!(event, fsm::LifecycleEvent::RunRejected { .. })
|
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,7 @@ mod stage_controller {
|
||||||
//! `specs/BEHAVIOR_GUARANTEES.md`.
|
//! `specs/BEHAVIOR_GUARANTEES.md`.
|
||||||
|
|
||||||
use myelin::staging as stage;
|
use myelin::staging as stage;
|
||||||
|
use crate::tests::harness::StageControllerHarness;
|
||||||
|
|
||||||
// This provision fixture represents a single middle stage. It has one inbound
|
// This provision fixture represents a single middle stage. It has one inbound
|
||||||
// and one outbound edge so tests can prove the controller uses assigned edges
|
// and one outbound edge so tests can prove the controller uses assigned edges
|
||||||
|
|
@ -37,8 +38,8 @@ mod stage_controller {
|
||||||
// The harness exposes only public messages. Tests intentionally do not inspect
|
// The harness exposes only public messages. Tests intentionally do not inspect
|
||||||
// private controller states such as "Preparing" or "Executing"; they infer
|
// private controller states such as "Preparing" or "Executing"; they infer
|
||||||
// controller behavior from emitted commands and lifecycle events.
|
// controller behavior from emitted commands and lifecycle events.
|
||||||
fn new_controller() -> stage::StageControllerHarness {
|
fn new_controller() -> StageControllerHarness {
|
||||||
stage::StageControllerHarness::new(stage::NodeId(11))
|
StageControllerHarness::new(stage::NodeId(11))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Preparation readiness has four independent prerequisites. Listing them as
|
// Preparation readiness has four independent prerequisites. Listing them as
|
||||||
|
|
@ -60,7 +61,7 @@ mod stage_controller {
|
||||||
// This helper provisions and readies a stage through public events. Tests that
|
// This helper provisions and readies a stage through public events. Tests that
|
||||||
// focus on execution use it to avoid duplicating setup while still going through
|
// focus on execution use it to avoid duplicating setup while still going through
|
||||||
// the same observable path as production.
|
// the same observable path as production.
|
||||||
fn ready_stage() -> stage::StageControllerHarness {
|
fn ready_stage() -> StageControllerHarness {
|
||||||
let mut harness = new_controller();
|
let mut harness = new_controller();
|
||||||
harness.observe(stage::StageEvent::ProvisionStage {
|
harness.observe(stage::StageEvent::ProvisionStage {
|
||||||
from: stage::NodeId(99),
|
from: stage::NodeId(99),
|
||||||
|
|
@ -103,15 +104,6 @@ mod stage_controller {
|
||||||
)
|
)
|
||||||
}));
|
}));
|
||||||
|
|
||||||
// The controller must not emit any command that replaces the provisioned
|
|
||||||
// edge ids with a locally chosen edge.
|
|
||||||
assert!(
|
|
||||||
!harness
|
|
||||||
.commands()
|
|
||||||
.iter()
|
|
||||||
.any(|command| { matches!(command, stage::StageCommand::RewireEdge { .. }) })
|
|
||||||
);
|
|
||||||
|
|
||||||
// An unauthorized provision attempt must fault before setup can begin.
|
// An unauthorized provision attempt must fault before setup can begin.
|
||||||
let mut unauthorized = new_controller();
|
let mut unauthorized = new_controller();
|
||||||
unauthorized.observe(stage::StageEvent::ProvisionStage {
|
unauthorized.observe(stage::StageEvent::ProvisionStage {
|
||||||
|
|
|
||||||
|
|
@ -8,4 +8,4 @@ Keep this crate read-only with respect to observed programs.
|
||||||
- It must not send control signals to observed runtimes.
|
- It must not send control signals to observed runtimes.
|
||||||
- It must not require changes outside `crates/dashboard` for dashboard-only work.
|
- It must not require changes outside `crates/dashboard` for dashboard-only work.
|
||||||
|
|
||||||
Main built-in view: `/view/swactor/workers`, backed by `runtime.stats`, `runtime.workers`, and `runtime.actors` frames when present.
|
Main built-in views: `/view/swactor/workers` (worker-centric) and `/view/swactor/actor-overview` + `/view/swactor/actor-dossier` (actor-centric), all backed by `runtime.stats`, `runtime.workers`, and `runtime.actors` frames when present. Actor views are pure frame consumers and tolerant of publisher shape.
|
||||||
|
|
|
||||||
|
|
@ -10,5 +10,5 @@ swactor = { path = "../..", features = ["serde"] }
|
||||||
parking_lot = "0.12"
|
parking_lot = "0.12"
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
tokio = { version = "1", features = ["net", "rt-multi-thread", "sync"] }
|
tokio.workspace = true
|
||||||
tokio-stream = "0.1"
|
tokio-stream = "0.1"
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@
|
||||||
|
|
||||||
Read-only HTML/SSE dashboard over incoming datastream frames.
|
Read-only HTML/SSE dashboard over incoming datastream frames.
|
||||||
|
|
||||||
The crate owns the Axum server, bounded raw frame window, and view registry. Component crates can keep their own view implementations beside their code and register them through `DashboardHandle::register_view`. The built-in swactor worker page is hosted here because worker/actor/message processing is universal to swactor programs.
|
The crate owns the Axum server, bounded raw frame window, and view registry. Component crates can keep their own view implementations beside their code and register them through `DashboardHandle::register_view`. The built-in swactor views are hosted here because worker/actor/message processing is universal to swactor programs: the worker page, and the actor overview (fused roster) plus per-actor dossier.
|
||||||
|
|
||||||
## Routes
|
## Routes
|
||||||
|
|
||||||
|
|
@ -16,5 +16,9 @@ The crate owns the Axum server, bounded raw frame window, and view registry. Com
|
||||||
- `GET /api/view/fleet` — fleet and machine telemetry JSON snapshot
|
- `GET /api/view/fleet` — fleet and machine telemetry JSON snapshot
|
||||||
- `GET /view/swactor/workers` — built-in worker page
|
- `GET /view/swactor/workers` — built-in worker page
|
||||||
- `GET /api/view/swactor/workers` — worker page JSON snapshot
|
- `GET /api/view/swactor/workers` — worker page JSON snapshot
|
||||||
|
- `GET /view/swactor/actor-overview` — built-in actor overview + roster page
|
||||||
|
- `GET /api/view/swactor/actor-overview` — actor overview JSON snapshot
|
||||||
|
- `GET /view/swactor/actor-dossier` — built-in per-actor dossier page
|
||||||
|
- `GET /api/view/swactor/actor-dossier` — actor dossier JSON snapshot
|
||||||
|
|
||||||
All state is derived from observed frames. The dashboard sends no control signals back to producers.
|
All state is derived from observed frames. The dashboard sends no control signals back to producers.
|
||||||
|
|
|
||||||
|
|
@ -97,6 +97,8 @@ impl DashboardHandle {
|
||||||
views.register(Arc::new(live_explorer::LiveDatastreamExplorer::default()));
|
views.register(Arc::new(live_explorer::LiveDatastreamExplorer::default()));
|
||||||
views.register(Arc::new(hardware_view::HardwareDashboardView::default()));
|
views.register(Arc::new(hardware_view::HardwareDashboardView::default()));
|
||||||
views.register(swactor::worker_view());
|
views.register(swactor::worker_view());
|
||||||
|
views.register(swactor::actor_overview_view());
|
||||||
|
views.register(swactor::actor_dossier_view());
|
||||||
let store = Arc::new(DashboardStore::new(
|
let store = Arc::new(DashboardStore::new(
|
||||||
config.raw_frame_history,
|
config.raw_frame_history,
|
||||||
Arc::clone(&views),
|
Arc::clone(&views),
|
||||||
|
|
|
||||||
185
crates/dashboard/src/swactor/actor_dossier.html
Normal file
185
crates/dashboard/src/swactor/actor_dossier.html
Normal file
|
|
@ -0,0 +1,185 @@
|
||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>Actor dossier — swactor</title>
|
||||||
|
<style>
|
||||||
|
:root{
|
||||||
|
--bg:#0f172a; --panel:#1e293b; --panel2:#172033; --border:#334155;
|
||||||
|
--text:#e2e8f0; --muted:#94a3b8; --accent:#60a5fa;
|
||||||
|
--running:#34d399; --poisoned:#f87171;
|
||||||
|
font-family:ui-sans-serif,system-ui,-apple-system,"Segoe UI",sans-serif;
|
||||||
|
}
|
||||||
|
*{box-sizing:border-box}
|
||||||
|
body{margin:0;padding:0;background:var(--bg);color:var(--text);}
|
||||||
|
.mono,.table,th,td,.value,.num{font-variant-numeric:tabular-nums;font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;}
|
||||||
|
header.bar{display:flex;align-items:center;gap:16px;padding:14px 20px;border-bottom:1px solid var(--border);background:var(--panel);position:sticky;top:0;z-index:5;}
|
||||||
|
header.bar h1{font-size:18px;margin:0;font-weight:600;letter-spacing:.01em;}
|
||||||
|
nav.tabs{display:flex;gap:4px;margin-left:auto;}
|
||||||
|
nav.tabs a{padding:6px 12px;border-radius:8px;color:var(--muted);text-decoration:none;font-size:13px;border:1px solid transparent;}
|
||||||
|
nav.tabs a:hover{color:var(--text);background:var(--panel2);}
|
||||||
|
nav.tabs a.cur{color:var(--accent);background:var(--panel2);border-color:var(--border);}
|
||||||
|
select{background:var(--panel2);color:var(--text);border:1px solid var(--border);border-radius:8px;padding:6px 8px;font-size:13px;}
|
||||||
|
.wrap{padding:20px;max-width:1100px;margin:0 auto;}
|
||||||
|
.picker{display:flex;gap:10px;align-items:center;flex-wrap:wrap;margin-bottom:18px;}
|
||||||
|
.picker label{font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.06em;}
|
||||||
|
#pick{min-width:300px;max-width:560px;flex:1;}
|
||||||
|
.back-link{font-size:13px;}
|
||||||
|
.badge{display:inline-block;padding:2px 8px;border-radius:99px;font-size:11px;text-transform:uppercase;letter-spacing:.06em;font-weight:600;}
|
||||||
|
.empty{padding:28px;border:1px dashed #475569;border-radius:12px;color:var(--muted);text-align:center;}
|
||||||
|
.toast{position:fixed;left:50%;bottom:20px;transform:translateX(-50%);background:var(--panel2);border:1px solid var(--border);color:var(--text);padding:10px 16px;border-radius:10px;font-size:13px;max-width:min(90vw,560px);text-align:center;z-index:50;box-shadow:0 8px 24px rgba(0,0,0,.45);}
|
||||||
|
.toast.err{border-color:var(--poisoned);color:var(--poisoned);}
|
||||||
|
.kvs{display:grid;grid-template-columns:max-content 1fr;gap:6px 14px;}
|
||||||
|
.kvs .k{color:var(--muted);font-size:12px;text-transform:uppercase;letter-spacing:.05em;}
|
||||||
|
.kvs .v{font-size:13px;word-break:break-word;}
|
||||||
|
.section{background:var(--panel);border:1px solid var(--border);border-radius:14px;padding:16px 18px;margin-bottom:16px;}
|
||||||
|
h2{font-size:14px;text-transform:uppercase;letter-spacing:.08em;color:var(--muted);margin:0 0 12px;}
|
||||||
|
.mb-dyn{display:flex;gap:24px;align-items:center;flex-wrap:wrap;}
|
||||||
|
.mb-spark{flex:1 1 320px;min-width:0;}
|
||||||
|
.mb-spark .spark{display:block;width:100%;height:120px;}
|
||||||
|
.mb-vals{display:flex;flex-direction:column;gap:16px;min-width:150px;}
|
||||||
|
.mb-val .label{color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.08em;}
|
||||||
|
.mb-val .value{font-size:28px;margin-top:4px;font-weight:600;line-height:1;}
|
||||||
|
.bar-row{display:flex;align-items:center;gap:8px;margin:5px 0;}
|
||||||
|
.bar-row .name{width:48%;color:var(--text);overflow:hidden;text-overflow:ellipsis;white-space:nowrap;}
|
||||||
|
.bar-track{flex:1;height:10px;background:var(--panel2);border-radius:99px;overflow:hidden;}
|
||||||
|
.bar-fill{height:100%;background:var(--accent);border-radius:99px;}
|
||||||
|
.bar-row .num{width:90px;text-align:right;color:var(--muted);}
|
||||||
|
.bar-row.last .bar-fill{background:var(--running);}
|
||||||
|
.bar-row.last{font-weight:600;}
|
||||||
|
a{color:var(--accent);text-decoration:none;}
|
||||||
|
.stale{opacity:.55;}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<header class="bar">
|
||||||
|
<h1>Actor dossier</h1>
|
||||||
|
<select id="runtime"></select>
|
||||||
|
<nav class="tabs">
|
||||||
|
<a href="/view/swactor/actor-overview">Overview</a>
|
||||||
|
<a href="/view/swactor/actor-dossier" class="cur">Dossier</a>
|
||||||
|
<a href="/view/swactor/workers">Workers</a>
|
||||||
|
</nav>
|
||||||
|
</header>
|
||||||
|
<div class="wrap">
|
||||||
|
<div class="picker">
|
||||||
|
<label for="pick">Actor</label>
|
||||||
|
<select id="pick"></select>
|
||||||
|
<a class="back-link" href="/view/swactor/actor-overview">← back to overview</a>
|
||||||
|
</div>
|
||||||
|
<div id="empty" class="toast" hidden>No runtime actor frames yet.</div>
|
||||||
|
<div id="content"></div>
|
||||||
|
</div>
|
||||||
|
<script>
|
||||||
|
const POLL_MS=750;
|
||||||
|
const STATE_COLOR={running:'#34d399',poisoned:'#f87171'};
|
||||||
|
const elRuntime=document.getElementById('runtime');
|
||||||
|
const elPick=document.getElementById('pick');
|
||||||
|
const elEmpty=document.getElementById('empty');
|
||||||
|
const elContent=document.getElementById('content');
|
||||||
|
let runtimes=[],runtimeKey='',selectedAddr='';
|
||||||
|
|
||||||
|
function fmt(n,d=0){if(n==null||n===undefined)return'\u2014';if(typeof n!=='number')return String(n);return n.toLocaleString(undefined,{maximumFractionDigits:d});}
|
||||||
|
function esc(s){return String(s==null?'':s).replace(/[&<>"]/g,function(c){return{'&':'&','<':'<','>':'>','"':'"'}[c];});}
|
||||||
|
function shortAddr(a){a=String(a||'');return a.length<=12?a:a.slice(0,6)+'\u2026'+a.slice(-4);}
|
||||||
|
function urlActor(){try{return new URLSearchParams(location.search).get('actor');}catch(e){return null;}}
|
||||||
|
function stateBadge(a){const s=a.poisoned?'poisoned':'running';const c=STATE_COLOR[s];return '<span class="badge" style="color:'+c+';background:'+c+'22">'+s+'</span>';}
|
||||||
|
function growthColor(g){if(g>1.5)return '#f87171';if(g>0.3)return '#fbbf24';return '#34d399';}
|
||||||
|
function areaChart(data,w,h,color){
|
||||||
|
if(!data||data.length<2)return '<svg class="spark" viewBox="0 0 '+w+' '+h+'" width="100%" height="'+h+'" preserveAspectRatio="none"></svg>';
|
||||||
|
const max=Math.max(1,...data);const n=data.length;
|
||||||
|
let line='',area='0,'+h+' ';
|
||||||
|
for(let i=0;i<n;i++){const x=(i/(n-1)*w).toFixed(1);const y=(h-(data[i]/max)*(h-2)-1).toFixed(1);line+=x+','+y+' ';area+=x+','+y+' ';}
|
||||||
|
area+=' '+w+','+h;
|
||||||
|
return '<svg class="spark" viewBox="0 0 '+w+' '+h+'" width="100%" height="'+h+'" preserveAspectRatio="none">'+
|
||||||
|
'<polygon points="'+area.trim()+'" fill="'+(color||'#60a5fa')+'22"/>'+
|
||||||
|
'<polyline points="'+line.trim()+'" fill="none" stroke="'+(color||'#60a5fa')+'" stroke-width="1.5"/></svg>';
|
||||||
|
}
|
||||||
|
function kv(k,vhtml){return '<div class="k">'+esc(k)+'</div><div class="v">'+vhtml+'</div>';}
|
||||||
|
function typeShort(t){t=String(t||'');const i=t.lastIndexOf('::');return i>=0?t.slice(i+2):t;}
|
||||||
|
function pickRuntime(list){
|
||||||
|
if(!runtimeKey||!list.some(r=>r.stream.key===runtimeKey)){
|
||||||
|
const live=list.find(r=>r.live)||list[0];runtimeKey=live?live.stream.key:'';
|
||||||
|
}
|
||||||
|
return list.find(r=>r.stream.key===runtimeKey)||null;
|
||||||
|
}
|
||||||
|
function currentActors(rt){return rt?(rt.actors||[]):[];}
|
||||||
|
function findActor(rt){
|
||||||
|
const actors=currentActors(rt);
|
||||||
|
if(selectedAddr){const hit=actors.find(a=>a.address===selectedAddr);if(hit)return hit;}
|
||||||
|
return actors[0]||null;
|
||||||
|
}
|
||||||
|
function renderPicker(rt){
|
||||||
|
const actors=currentActors(rt);
|
||||||
|
elPick.innerHTML=actors.map(a=>
|
||||||
|
'<option value="'+esc(a.address)+'">'+esc(shortAddr(a.address))+(a.name?' \u00b7 '+esc(a.name):'')+'</option>'
|
||||||
|
).join('');
|
||||||
|
const cur=findActor(rt);
|
||||||
|
if(cur){selectedAddr=cur.address;elPick.value=selectedAddr;}
|
||||||
|
}
|
||||||
|
function renderRuntimeSelector(){
|
||||||
|
elRuntime.innerHTML=runtimes.map(r=>
|
||||||
|
'<option value="'+esc(r.stream.key)+'">'+esc(r.stream.node)+'#'+r.stream.life+(r.live?'':' (stale)')+'</option>'
|
||||||
|
).join('');
|
||||||
|
if(runtimeKey)elRuntime.value=runtimeKey;
|
||||||
|
}
|
||||||
|
function render(a,rt){
|
||||||
|
if(!a){elContent.innerHTML='<div class="empty">Select an actor. None are present on the current frame.</div>';return;}
|
||||||
|
const lastType=a.last_msg_type||'\u2014';
|
||||||
|
const diet=(a.message_type_counts||[]).slice();
|
||||||
|
diet.sort((x,y)=>y.count-x.count);
|
||||||
|
const dmax=Math.max(1,...diet.map(d=>d.count));
|
||||||
|
const lastName=a.last_msg_type;
|
||||||
|
const dietHtml=diet.length?diet.map(d=>{
|
||||||
|
const cls=d.message_type===lastName?'bar-row last':'bar-row';
|
||||||
|
return '<div class="'+cls+'"><span class="name">'+esc(d.message_type)+'</span>'+
|
||||||
|
'<div class="bar-track"><div class="bar-fill" style="width:'+(d.count/dmax*100).toFixed(1)+'%"></div></div>'+
|
||||||
|
'<span class="num">'+fmt(d.count)+'</span></div>';
|
||||||
|
}).join(''):'<div class="empty">no message-type counts on frame</div>';
|
||||||
|
const hist=(a.history||[]).map(h=>h.mailbox_depth);
|
||||||
|
const g=a.mailbox_growth||0;
|
||||||
|
elContent.innerHTML=
|
||||||
|
'<div class="section"><h2>Identity & lifecycle</h2><div class="kvs">'+
|
||||||
|
kv('address','<span class="mono">'+esc(a.address)+'</span>')+
|
||||||
|
(a.name?kv('name',esc(a.name)):'')+
|
||||||
|
(a.actor_type?kv('actor type','<span class="mono" title="'+esc(a.actor_type)+'">'+esc(typeShort(a.actor_type))+'</span>'):'')+
|
||||||
|
(a.message_type?kv('message type','<span class="mono" title="'+esc(a.message_type)+'">'+esc(typeShort(a.message_type))+'</span>'):'')+
|
||||||
|
kv('state',stateBadge(a))+
|
||||||
|
kv('worker',a.worker_id==null?'\u2014':esc(a.worker_id))+
|
||||||
|
kv('last message',esc(lastType))+
|
||||||
|
kv('last seen',a.last_seen_ms_ago!=null?(a.last_seen_ms_ago/1000).toFixed(1)+'s ago':'\u2014')+
|
||||||
|
'</div></div>'+
|
||||||
|
'<div class="section"><h2>Mailbox dynamics</h2><div class="mb-dyn">'+
|
||||||
|
'<div class="mb-spark">'+areaChart(hist,720,120,growthColor(g))+'</div>'+
|
||||||
|
'<div class="mb-vals">'+
|
||||||
|
'<div class="mb-val"><div class="label">depth</div><div class="value" style="color:'+growthColor(g)+'">'+fmt(a.mailbox_depth)+'</div></div>'+
|
||||||
|
'<div class="mb-val"><div class="label">growth</div><div class="value">'+fmt(g,1)+'/s</div></div>'+
|
||||||
|
'<div class="mb-val"><div class="label">msg/s</div><div class="value">'+fmt(a.msg_per_sec,1)+'</div></div>'+
|
||||||
|
'</div>'+
|
||||||
|
'</div></div>'+
|
||||||
|
'<div class="section"><h2>Message diet</h2>'+dietHtml+'</div>';
|
||||||
|
}
|
||||||
|
async function refresh(){
|
||||||
|
try{
|
||||||
|
const res=await fetch('/api/view/swactor/actor-dossier',{cache:'no-store'});
|
||||||
|
if(!res.ok)throw new Error('HTTP '+res.status);
|
||||||
|
const data=await res.json();
|
||||||
|
runtimes=data.runtimes||[];
|
||||||
|
renderRuntimeSelector();
|
||||||
|
const rt=pickRuntime(runtimes);
|
||||||
|
if(rt&¤tActors(rt).length>0){elEmpty.hidden=true;}else{elEmpty.className='toast';elEmpty.textContent='No runtime actor frames yet.';elEmpty.hidden=false;}
|
||||||
|
renderPicker(rt);
|
||||||
|
render(findActor(rt),rt);
|
||||||
|
}catch(e){
|
||||||
|
elEmpty.className='toast err';elEmpty.hidden=false;elEmpty.textContent='Failed to load actor frames: '+e.message;elContent.innerHTML='';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
elRuntime.addEventListener('change',e=>{runtimeKey=e.target.value;const rt=pickRuntime(runtimes);renderPicker(rt);render(findActor(rt),rt);});
|
||||||
|
elPick.addEventListener('change',e=>{selectedAddr=e.target.value;const rt=pickRuntime(runtimes);render(findActor(rt),rt);});
|
||||||
|
const q=urlActor();if(q)selectedAddr=q;
|
||||||
|
refresh();
|
||||||
|
setInterval(refresh,POLL_MS);
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
312
crates/dashboard/src/swactor/actor_overview.html
Normal file
312
crates/dashboard/src/swactor/actor_overview.html
Normal file
|
|
@ -0,0 +1,312 @@
|
||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>Actor overview — swactor</title>
|
||||||
|
<style>
|
||||||
|
:root{
|
||||||
|
--bg:#0f172a; --panel:#1e293b; --panel2:#172033; --border:#334155;
|
||||||
|
--text:#e2e8f0; --muted:#94a3b8; --accent:#60a5fa;
|
||||||
|
--running:#34d399; --poisoned:#f87171;
|
||||||
|
font-family:ui-sans-serif,system-ui,-apple-system,"Segoe UI",sans-serif;
|
||||||
|
}
|
||||||
|
*{box-sizing:border-box}
|
||||||
|
body{margin:0;padding:0;background:var(--bg);color:var(--text);}
|
||||||
|
.mono,.table,th,td,.value,.num{font-variant-numeric:tabular-nums;font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;}
|
||||||
|
header.bar{display:flex;align-items:center;gap:16px;padding:14px 20px;border-bottom:1px solid var(--border);background:var(--panel);position:sticky;top:0;z-index:5;}
|
||||||
|
header.bar h1{font-size:18px;margin:0;font-weight:600;letter-spacing:.01em;}
|
||||||
|
nav.tabs{display:flex;gap:4px;margin-left:auto;}
|
||||||
|
nav.tabs a{padding:6px 12px;border-radius:8px;color:var(--muted);text-decoration:none;font-size:13px;border:1px solid transparent;}
|
||||||
|
nav.tabs a:hover{color:var(--text);background:var(--panel2);}
|
||||||
|
nav.tabs a.cur{color:var(--accent);background:var(--panel2);border-color:var(--border);}
|
||||||
|
select{background:var(--panel2);color:var(--text);border:1px solid var(--border);border-radius:8px;padding:6px 8px;font-size:13px;}
|
||||||
|
.wrap{padding:20px;max-width:1400px;margin:0 auto;}
|
||||||
|
.grid{display:grid;gap:12px;}
|
||||||
|
.cards{grid-template-columns:repeat(auto-fit,minmax(150px,1fr));margin-bottom:16px;}
|
||||||
|
.card{background:var(--panel);border:1px solid var(--border);border-radius:14px;padding:14px;}
|
||||||
|
.card .label{color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.08em;}
|
||||||
|
.card .value{font-size:26px;margin-top:6px;font-weight:600;}
|
||||||
|
.card .value.bad{color:var(--poisoned);}
|
||||||
|
.badge{display:inline-block;padding:2px 8px;border-radius:99px;font-size:11px;text-transform:uppercase;letter-spacing:.06em;font-weight:600;}
|
||||||
|
.table{width:100%;border-collapse:collapse;background:var(--panel);border:1px solid var(--border);border-radius:12px;overflow:hidden;}
|
||||||
|
.table th,.table td{padding:8px 10px;border-bottom:1px solid #1f2937;text-align:left;white-space:nowrap;}
|
||||||
|
.table th{color:#93c5fd;background:var(--panel2);cursor:pointer;user-select:none;font-size:12px;text-transform:uppercase;letter-spacing:.05em;}
|
||||||
|
.table th:hover{color:#fff;}
|
||||||
|
.table tr{cursor:pointer;}
|
||||||
|
.table tbody tr:hover{background:#1d4ed822;}
|
||||||
|
.table td.muted,.muted{color:var(--muted);}
|
||||||
|
.spark{display:block;}
|
||||||
|
.bar-row{display:flex;align-items:center;gap:8px;margin:5px 0;}
|
||||||
|
.bar-row .name{width:38%;color:var(--text);overflow:hidden;text-overflow:ellipsis;white-space:nowrap;}
|
||||||
|
.bar-track{flex:1;height:10px;background:var(--panel2);border-radius:99px;overflow:hidden;}
|
||||||
|
.bar-fill{height:100%;background:var(--accent);border-radius:99px;}
|
||||||
|
.bar-row .num{width:90px;text-align:right;color:var(--muted);}
|
||||||
|
input{background:var(--panel2);color:var(--text);border:1px solid var(--border);border-radius:8px;padding:6px 8px;font-size:13px;}
|
||||||
|
.filters{display:flex;gap:10px;flex-wrap:wrap;align-items:center;margin-bottom:14px;}
|
||||||
|
.filters label{font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.06em;}
|
||||||
|
h2{font-size:14px;text-transform:uppercase;letter-spacing:.08em;color:var(--muted);margin:22px 0 10px;}
|
||||||
|
.empty{padding:28px;border:1px dashed #475569;border-radius:12px;color:var(--muted);text-align:center;}
|
||||||
|
.toast{position:fixed;left:50%;bottom:20px;transform:translateX(-50%);background:var(--panel2);border:1px solid var(--border);color:var(--text);padding:10px 16px;border-radius:10px;font-size:13px;max-width:min(90vw,560px);text-align:center;z-index:50;box-shadow:0 8px 24px rgba(0,0,0,.45);}
|
||||||
|
.toast.err{border-color:var(--poisoned);color:var(--poisoned);}
|
||||||
|
.stacked{display:flex;height:18px;border-radius:6px;overflow:hidden;border:1px solid var(--border);}
|
||||||
|
.legend{display:flex;gap:14px;flex-wrap:wrap;margin-top:8px;font-size:12px;color:var(--muted);}
|
||||||
|
.legend i{display:inline-block;width:12px;height:12px;margin-right:5px;border-radius:3px;vertical-align:middle;}
|
||||||
|
a{color:var(--accent);text-decoration:none;}
|
||||||
|
.roster-box{max-height:520px;overflow:auto;border:1px solid var(--border);border-radius:12px;}
|
||||||
|
.roster-box .table{border-radius:0;border:none;}
|
||||||
|
.stale{opacity:.55;}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<header class="bar">
|
||||||
|
<h1>Actor overview</h1>
|
||||||
|
<select id="runtime"></select>
|
||||||
|
<nav class="tabs">
|
||||||
|
<a href="/view/swactor/actor-overview" class="cur">Overview</a>
|
||||||
|
<a href="/view/swactor/actor-dossier">Dossier</a>
|
||||||
|
<a href="/view/swactor/workers">Workers</a>
|
||||||
|
</nav>
|
||||||
|
</header>
|
||||||
|
<div class="wrap">
|
||||||
|
<div id="empty" class="toast" hidden>No runtime actor frames yet. A live swactor runtime publishing <code>runtime.actors</code> will populate this page.</div>
|
||||||
|
<div id="cards" class="grid cards"></div>
|
||||||
|
|
||||||
|
<h2>State distribution</h2>
|
||||||
|
<div class="stacked" id="dist"></div>
|
||||||
|
<div class="legend" id="distLegend"></div>
|
||||||
|
|
||||||
|
<h2>Throughput</h2>
|
||||||
|
<div id="throughput"></div>
|
||||||
|
|
||||||
|
<h2>Worker balance</h2>
|
||||||
|
<div id="workers"></div>
|
||||||
|
|
||||||
|
<h2>Hot actors — mailbox growth</h2>
|
||||||
|
<div id="hot"></div>
|
||||||
|
|
||||||
|
<h2>Alerts</h2>
|
||||||
|
<div id="alerts"></div>
|
||||||
|
|
||||||
|
<h2>Roster</h2>
|
||||||
|
<div class="filters">
|
||||||
|
<label for="f-search">search</label>
|
||||||
|
<input id="f-search" type="search" placeholder="name · addr · worker" autocomplete="off" spellcheck="false">
|
||||||
|
<label for="f-state">state</label>
|
||||||
|
<select id="f-state">
|
||||||
|
<option value="all">all states</option>
|
||||||
|
<option value="running">running</option>
|
||||||
|
<option value="poisoned">poisoned</option>
|
||||||
|
</select>
|
||||||
|
<label for="f-worker">worker</label>
|
||||||
|
<select id="f-worker"></select>
|
||||||
|
</div>
|
||||||
|
<div class="roster-box">
|
||||||
|
<table class="table">
|
||||||
|
<thead><tr id="head"></tr></thead>
|
||||||
|
<tbody id="rows"></tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<script>
|
||||||
|
const POLL_MS=750;
|
||||||
|
const elRuntime=document.getElementById('runtime');
|
||||||
|
const elEmpty=document.getElementById('empty');
|
||||||
|
const elCards=document.getElementById('cards');
|
||||||
|
const elDist=document.getElementById('dist');
|
||||||
|
const elDistLegend=document.getElementById('distLegend');
|
||||||
|
const elThroughput=document.getElementById('throughput');
|
||||||
|
const elWorkers=document.getElementById('workers');
|
||||||
|
const elHot=document.getElementById('hot');
|
||||||
|
const elAlerts=document.getElementById('alerts');
|
||||||
|
const elHead=document.getElementById('head');
|
||||||
|
const elRows=document.getElementById('rows');
|
||||||
|
const elSearch=document.getElementById('f-search');
|
||||||
|
const elState=document.getElementById('f-state');
|
||||||
|
const elWorker=document.getElementById('f-worker');
|
||||||
|
|
||||||
|
const STATE_COLOR={running:'#34d399',poisoned:'#f87171'};
|
||||||
|
const COLS=[['actor','Actor'],['type','Type'],['state','State'],['mailbox','Mailbox'],['rate','Msg/s'],
|
||||||
|
['processed','Processed'],['last','Last msg'],['worker','Worker']];
|
||||||
|
const DEFDIR={actor:'asc',type:'asc',state:'asc',mailbox:'desc',rate:'desc',processed:'desc',last:'asc',worker:'asc'};
|
||||||
|
let sortKey='mailbox',sortDir='desc';
|
||||||
|
let fSearch='',fState='all',fWorker='all';
|
||||||
|
let runtimes=[],runtimeKey='';
|
||||||
|
|
||||||
|
function fmt(n,d=0){if(n==null||n===undefined)return'\u2014';if(typeof n!=='number')return String(n);return n.toLocaleString(undefined,{maximumFractionDigits:d});}
|
||||||
|
function ago(sec){sec=Math.max(0,Math.round(sec));if(sec<60)return sec+'s';const m=Math.floor(sec/60);if(m<60)return m+'m';return Math.floor(m/60)+'h';}
|
||||||
|
function esc(s){return String(s==null?'':s).replace(/[&<>"]/g,function(c){return{'&':'&','<':'<','>':'>','"':'"'}[c];});}
|
||||||
|
function shortAddr(a){a=String(a||'');return a.length<=12?a:a.slice(0,6)+'\u2026'+a.slice(-4);}
|
||||||
|
function typeShort(t){t=String(t||'');const i=t.lastIndexOf('::');return i>=0?t.slice(i+2):t;}
|
||||||
|
function stateBadge(a){const s=a.poisoned?'poisoned':'running';const c=STATE_COLOR[s];return '<span class="badge" style="color:'+c+';background:'+c+'22">'+s+'</span>';}
|
||||||
|
function growthColor(g){if(g>1.5)return '#f87171';if(g>0.3)return '#fbbf24';return '#34d399';}
|
||||||
|
function sparkline(data,w,h,color){
|
||||||
|
if(!data||data.length<2)return '<svg class="spark" viewBox="0 0 '+w+' '+h+'" width="'+w+'" height="'+h+'"></svg>';
|
||||||
|
const max=Math.max(1,...data);const n=data.length;
|
||||||
|
let pts='';for(let i=0;i<n;i++)pts+=(i/(n-1)*w).toFixed(1)+','+(h-(data[i]/max)*(h-2)-1).toFixed(1)+' ';
|
||||||
|
return '<svg class="spark" viewBox="0 0 '+w+' '+h+'" width="'+w+'" height="'+h+'" preserveAspectRatio="none"><polyline points="'+pts.trim()+'" fill="none" stroke="'+(color||'#60a5fa')+'" stroke-width="1.5"/></svg>';
|
||||||
|
}
|
||||||
|
function pickRuntime(list){
|
||||||
|
if(!runtimeKey||!list.some(r=>r.stream.key===runtimeKey)){
|
||||||
|
const live=list.find(r=>r.live)||list[0];
|
||||||
|
runtimeKey=live?live.stream.key:'';
|
||||||
|
}
|
||||||
|
return list.find(r=>r.stream.key===runtimeKey)||null;
|
||||||
|
}
|
||||||
|
function renderRuntimeSelector(){
|
||||||
|
elRuntime.innerHTML=runtimes.map(r=>
|
||||||
|
'<option value="'+esc(r.stream.key)+'">'+esc(r.stream.node)+'#'+r.stream.life+(r.live?'':' (stale)')+'</option>'
|
||||||
|
).join('');
|
||||||
|
if(runtimeKey)elRuntime.value=runtimeKey;
|
||||||
|
}
|
||||||
|
function renderCards(rt){
|
||||||
|
const s=rt.summary||{};
|
||||||
|
const uptime=s.uptime_ms!=null?ago(s.uptime_ms/1000):'\u2014';
|
||||||
|
elCards.innerHTML=
|
||||||
|
'<div class="card"><div class="label">Actors</div><div class="value">'+fmt(s.actors)+'</div></div>'+
|
||||||
|
'<div class="card"><div class="label">msg/s</div><div class="value">'+fmt(s.msg_per_sec,1)+'</div></div>'+
|
||||||
|
'<div class="card"><div class="label">Mailbox</div><div class="value">'+fmt(s.mailbox_depth)+'</div></div>'+
|
||||||
|
'<div class="card"><div class="label">Poisoned</div><div class="value'+(s.poisoned>0?' bad':'')+'">'+fmt(s.poisoned)+'</div></div>'+
|
||||||
|
'<div class="card"><div class="label">Uptime</div><div class="value">'+uptime+'</div></div>';
|
||||||
|
}
|
||||||
|
function renderDist(rt){
|
||||||
|
const actors=rt.actors||[];const n=actors.length||1;
|
||||||
|
let running=0,poisoned=0;
|
||||||
|
actors.forEach(a=>{if(a.poisoned)poisoned++;else running++;});
|
||||||
|
let segs='';
|
||||||
|
if(running)segs+='<div style="width:'+(running/n*100).toFixed(2)+'%;background:'+STATE_COLOR.running+';"></div>';
|
||||||
|
if(poisoned)segs+='<div style="width:'+(poisoned/n*100).toFixed(2)+'%;background:'+STATE_COLOR.poisoned+';"></div>';
|
||||||
|
elDist.innerHTML=segs||'<div style="width:100%;background:var(--panel2);"></div>';
|
||||||
|
elDistLegend.innerHTML='<span><i style="background:'+STATE_COLOR.running+'"></i>running: '+running+'</span>'+
|
||||||
|
'<span><i style="background:'+STATE_COLOR.poisoned+'"></i>poisoned: '+poisoned+'</span>';
|
||||||
|
}
|
||||||
|
function renderThroughput(rt){
|
||||||
|
const h=(rt.history||[]).map(x=>x.msg_per_sec);
|
||||||
|
const mb=(rt.history||[]).map(x=>x.mailbox_depth);
|
||||||
|
elThroughput.innerHTML=
|
||||||
|
sparkline(h,760,90,'#60a5fa')+'<div style="height:6px"></div>'+
|
||||||
|
sparkline(mb,760,90,'#fbbf24')+
|
||||||
|
'<div class="legend"><span><i style="background:#60a5fa"></i>msg/s</span><span><i style="background:#fbbf24"></i>mailbox</span></div>';
|
||||||
|
}
|
||||||
|
function renderWorkers(rt){
|
||||||
|
const counts={};
|
||||||
|
(rt.actors||[]).forEach(a=>{const w=a.worker_id==null?'?':a.worker_id;counts[w]=(counts[w]||0)+1;});
|
||||||
|
const keys=Object.keys(counts).sort();
|
||||||
|
const max=Math.max(1,...Object.values(counts));
|
||||||
|
elWorkers.innerHTML=keys.map(w=>{
|
||||||
|
const c=counts[w];
|
||||||
|
return '<div class="bar-row"><span class="name">worker '+esc(w)+'</span>'+
|
||||||
|
'<div class="bar-track"><div class="bar-fill" style="width:'+(c/max*100).toFixed(1)+'%"></div></div>'+
|
||||||
|
'<span class="num">'+c+'</span></div>';
|
||||||
|
}).join('')||'<div class="empty">no worker placement on frame</div>';
|
||||||
|
}
|
||||||
|
function renderHot(rt){
|
||||||
|
const hot=(rt.actors||[]).slice().sort((a,b)=>(b.mailbox_growth||0)-(a.mailbox_growth||0)).slice(0,8);
|
||||||
|
const gmax=Math.max(1,...hot.map(a=>Math.max(0,a.mailbox_growth||0)));
|
||||||
|
elHot.innerHTML=hot.map(a=>{
|
||||||
|
const g=a.mailbox_growth||0;const gc=growthColor(g);
|
||||||
|
return '<div class="bar-row" data-addr="'+esc(a.address)+'">'+
|
||||||
|
'<span class="name">'+shortAddr(a.address)+(a.name?' '+esc(a.name):'')+'</span>'+
|
||||||
|
'<div class="bar-track"><div class="bar-fill" style="width:'+(Math.max(0,g)/gmax*100).toFixed(1)+'%;background:'+gc+'"></div></div>'+
|
||||||
|
'<span class="num">'+fmt(g,1)+'/s</span></div>';
|
||||||
|
}).join('')||'<div class="empty">no actors</div>';
|
||||||
|
}
|
||||||
|
function renderAlerts(rt){
|
||||||
|
const alerted=(rt.actors||[]).filter(a=>a.poisoned||(a.mailbox_growth||0)>1.5);
|
||||||
|
if(!alerted.length){elAlerts.innerHTML='<div class="empty">fleet healthy</div>';return;}
|
||||||
|
alerted.sort((a,b)=>{const ap=a.poisoned?0:1,bp=b.poisoned?0:1;if(ap!==bp)return ap-bp;return (b.mailbox_growth||0)-(a.mailbox_growth||0);});
|
||||||
|
elAlerts.innerHTML='<table class="table"><tbody>'+
|
||||||
|
alerted.map(a=>{
|
||||||
|
const reason=a.poisoned?'poisoned':'saturating';
|
||||||
|
return '<tr data-addr="'+esc(a.address)+'"><td>'+shortAddr(a.address)+(a.name?' '+esc(a.name):'')+' '+stateBadge(a)+'</td>'+
|
||||||
|
'<td class="muted">'+reason+'</td><td style="text-align:right;color:var(--muted);width:1%;">\u2192</td></tr>';
|
||||||
|
}).join('')+'</tbody></table>';
|
||||||
|
}
|
||||||
|
function colValue(a,key){
|
||||||
|
switch(key){
|
||||||
|
case 'actor':return (a.name||a.address).toLowerCase();
|
||||||
|
case 'type':return typeShort(a.actor_type).toLowerCase();
|
||||||
|
case 'state':return a.poisoned?'poisoned':'running';
|
||||||
|
case 'mailbox':return a.mailbox_depth||0;
|
||||||
|
case 'rate':return a.msg_per_sec||0;
|
||||||
|
case 'processed':return a.messages_processed||0;
|
||||||
|
case 'last':return a.last_msg_type||'';
|
||||||
|
case 'worker':return a.worker_id==null?-1:a.worker_id;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
function visible(rt){
|
||||||
|
let list=rt.actors||[];
|
||||||
|
if(fSearch){const q=fSearch.toLowerCase();list=list.filter(a=>((a.name?a.name+' ':'')+(a.actor_type?typeShort(a.actor_type)+' '+a.actor_type+' ':'')+a.address+' '+(a.worker_id==null?'':a.worker_id)).toLowerCase().indexOf(q)>=0);}
|
||||||
|
if(fState!=='all')list=list.filter(a=>(a.poisoned?'poisoned':'running')===fState);
|
||||||
|
if(fWorker!=='all')list=list.filter(a=>String(a.worker_id)===fWorker);
|
||||||
|
const k=sortKey,dir=sortDir==='asc'?1:-1;
|
||||||
|
return list.slice().sort((x,y)=>{const vx=colValue(x,k),vy=colValue(y,k);let r;if(typeof vx==='number'&&typeof vy==='number')r=vx-vy;else r=String(vx).localeCompare(String(vy));return r*dir;});
|
||||||
|
}
|
||||||
|
function renderHead(){
|
||||||
|
elHead.innerHTML=COLS.map(c=>{
|
||||||
|
const key=c[0],label=c[1];
|
||||||
|
const arrow=sortKey===key?(sortDir==='asc'?' \u25B2':' \u25BC'):'';
|
||||||
|
const order=sortKey===key?(sortDir==='asc'?'ascending':'descending'):'none';
|
||||||
|
return '<th data-key="'+key+'" scope="col" aria-sort="'+order+'">'+label+arrow+'</th>';
|
||||||
|
}).join('');
|
||||||
|
}
|
||||||
|
function populateWorkerFilter(rt){
|
||||||
|
const seen={};(rt.actors||[]).forEach(a=>{if(a.worker_id!=null)seen[a.worker_id]=1;});
|
||||||
|
const keys=Object.keys(seen).sort((a,b)=>a-b);
|
||||||
|
elWorker.innerHTML='<option value="all">all workers</option>'+keys.map(w=>'<option value="'+esc(w)+'">'+esc(w)+'</option>').join('');
|
||||||
|
if(fWorker!=='all'&&!keys.includes(fWorker))fWorker='all';
|
||||||
|
elWorker.value=fWorker;
|
||||||
|
}
|
||||||
|
function renderRows(rt){
|
||||||
|
const list=visible(rt);
|
||||||
|
if(!list.length){elRows.innerHTML='<tr><td colspan="'+COLS.length+'"><div class="empty">No actors match the current filters.</div></td></tr>';return;}
|
||||||
|
elRows.innerHTML=list.map(a=>{
|
||||||
|
const hist=(a.history||[]).map(h=>h.mailbox_depth);
|
||||||
|
const gc=growthColor(a.mailbox_growth||0);
|
||||||
|
const mbox='<span style="display:inline-flex;align-items:center;gap:8px">'+
|
||||||
|
'<span style="color:'+gc+';font-weight:600">'+fmt(a.mailbox_depth)+'</span>'+sparkline(hist,60,16,gc)+'</span>';
|
||||||
|
const actorCell=shortAddr(a.address)+(a.name?'<br><span class="muted" style="font-size:11px">'+esc(a.name)+'</span>':'');
|
||||||
|
const typeCell=a.actor_type?'<span class="muted" title="'+esc(a.actor_type)+'">'+esc(typeShort(a.actor_type))+'</span>':'<span class="muted">\u2014</span>';
|
||||||
|
return '<tr data-addr="'+esc(a.address)+'">'+
|
||||||
|
'<td>'+actorCell+'</td><td>'+typeCell+'</td><td>'+stateBadge(a)+'</td><td>'+mbox+'</td>'+
|
||||||
|
'<td>'+fmt(a.msg_per_sec,1)+'</td><td>'+fmt(a.messages_processed)+'</td>'+
|
||||||
|
'<td><span class="muted" style="display:inline-block;max-width:240px;overflow:hidden;text-overflow:ellipsis;vertical-align:bottom">'+esc(a.last_msg_type)+'</span></td>'+
|
||||||
|
'<td>'+(a.worker_id==null?'<span class="muted">\u2014</span>':a.worker_id)+'</td></tr>';
|
||||||
|
}).join('');
|
||||||
|
}
|
||||||
|
function render(rt){
|
||||||
|
if(!rt){[elCards,elDist,elThroughput,elWorkers,elHot,elAlerts,elRows].forEach(e=>e.innerHTML='');return;}
|
||||||
|
elEmpty.hidden=true;
|
||||||
|
renderCards(rt);renderDist(rt);renderThroughput(rt);renderWorkers(rt);renderHot(rt);renderAlerts(rt);
|
||||||
|
populateWorkerFilter(rt);renderHead();renderRows(rt);
|
||||||
|
}
|
||||||
|
async function refresh(){
|
||||||
|
try{
|
||||||
|
const res=await fetch('/api/view/swactor/actor-overview',{cache:'no-store'});
|
||||||
|
if(!res.ok)throw new Error('HTTP '+res.status);
|
||||||
|
const data=await res.json();
|
||||||
|
runtimes=data.runtimes||[];
|
||||||
|
renderRuntimeSelector();
|
||||||
|
const rt=pickRuntime(runtimes);
|
||||||
|
if(rt){elEmpty.hidden=true;}else{elEmpty.className='toast';elEmpty.textContent='No runtime actor frames yet. A live swactor runtime publishing runtime.actors will populate this page.';elEmpty.hidden=false;}
|
||||||
|
render(rt);
|
||||||
|
}catch(e){
|
||||||
|
elEmpty.className='toast err';
|
||||||
|
elEmpty.textContent='Failed to load actor frames: '+e.message;
|
||||||
|
elEmpty.hidden=false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
function setSort(key){if(sortKey===key)sortDir=sortDir==='asc'?'desc':'asc';else{sortKey=key;sortDir=DEFDIR[key]||'asc';}renderHead();renderRows(pickRuntime(runtimes));}
|
||||||
|
elRuntime.addEventListener('change',e=>{runtimeKey=e.target.value;render(pickRuntime(runtimes));});
|
||||||
|
elSearch.addEventListener('input',e=>{fSearch=e.target.value.toLowerCase();renderRows(pickRuntime(runtimes));});
|
||||||
|
elState.addEventListener('change',e=>{fState=e.target.value;renderRows(pickRuntime(runtimes));});
|
||||||
|
elWorker.addEventListener('change',e=>{fWorker=e.target.value;renderRows(pickRuntime(runtimes));});
|
||||||
|
elHead.addEventListener('click',e=>{const th=e.target.closest('th[data-key]');if(th)setSort(th.getAttribute('data-key'));});
|
||||||
|
document.addEventListener('click',e=>{const el=e.target.closest('[data-addr]');if(el)location.href='/view/swactor/actor-dossier?actor='+encodeURIComponent(el.getAttribute('data-addr'));});
|
||||||
|
renderHead();
|
||||||
|
refresh();
|
||||||
|
setInterval(refresh,POLL_MS);
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
635
crates/dashboard/src/swactor/actor_view.rs
Normal file
635
crates/dashboard/src/swactor/actor_view.rs
Normal file
|
|
@ -0,0 +1,635 @@
|
||||||
|
//! Actor-centric read-only view over `runtime.actors` / `runtime.stats` frames.
|
||||||
|
//!
|
||||||
|
//! Pure frame consumer: it folds incoming per-actor snapshots into local state
|
||||||
|
//! and exposes JSON + HTML. It sends nothing back to observed runtimes. Parsing
|
||||||
|
//! is intentionally tolerant of publisher shape — a frame is a flat record that
|
||||||
|
//! may come from the per-worker `DatastreamStatsHook` envelope
|
||||||
|
//! (`{worker_id, actors:[...]}`) or from a process that publishes a merged
|
||||||
|
//! `{actors:[...]}` payload (the dashboard dummy node, app runtimes). Fields the
|
||||||
|
//! publisher includes (name, worker_id, lifecycle flags) are displayed; ones it
|
||||||
|
//! omits are left blank rather than fabricated.
|
||||||
|
//!
|
||||||
|
//! Reliably on the frame today: address, mailbox depth, throughput, last
|
||||||
|
//! message, poisoned flag, and the message-type diet. `worker_id` and `name`
|
||||||
|
//! appear when the publisher sends them. Finer lifecycle granularity
|
||||||
|
//! (new/suspended/stopping), actor Rust type, and spawn age are not on the
|
||||||
|
//! current frame and are therefore not shown — enriching the feed is a separate
|
||||||
|
//! core concern, not a dashboard one.
|
||||||
|
//!
|
||||||
|
//! Two pages share one data model: the fused overview+roster
|
||||||
|
//! (`/view/swactor/actor-overview`) and the per-actor dossier
|
||||||
|
//! (`/view/swactor/actor-dossier`). Each is a self-contained `DashboardView`
|
||||||
|
//! instance ingesting the same channels.
|
||||||
|
|
||||||
|
use std::collections::{BTreeMap, VecDeque};
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use datastream::frame::{Frame, StreamId};
|
||||||
|
use parking_lot::RwLock;
|
||||||
|
use serde::Serialize;
|
||||||
|
use serde_json::{Value, json};
|
||||||
|
|
||||||
|
use crate::swactor::{RUNTIME_ACTORS, RUNTIME_STATS};
|
||||||
|
use crate::view::DashboardView;
|
||||||
|
use crate::{FrameEvent, StreamEvent};
|
||||||
|
|
||||||
|
const CHANNELS: &[&str] = &[RUNTIME_ACTORS, RUNTIME_STATS];
|
||||||
|
const HISTORY_CAP: usize = 512;
|
||||||
|
const HISTORY_MIN_INTERVAL: Duration = Duration::from_millis(250);
|
||||||
|
const PER_ACTOR_HISTORY_CAP: usize = 120;
|
||||||
|
const GROWTH_WINDOW: usize = 12;
|
||||||
|
const LIVE_TTL: Duration = Duration::from_secs(8);
|
||||||
|
|
||||||
|
/// Read-only actor panel. One instance per served page so each page owns its
|
||||||
|
/// state independently; both ingest the same frames.
|
||||||
|
pub struct ActorPanelView {
|
||||||
|
id: &'static str,
|
||||||
|
title: &'static str,
|
||||||
|
path: &'static str,
|
||||||
|
html: &'static str,
|
||||||
|
state: RwLock<PanelState>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ActorPanelView {
|
||||||
|
/// Fused overview + roster at `/view/swactor/actor-overview`.
|
||||||
|
pub fn overview() -> Self {
|
||||||
|
Self::new(
|
||||||
|
"swactor-actor-overview",
|
||||||
|
"Actor overview",
|
||||||
|
"swactor/actor-overview",
|
||||||
|
include_str!("actor_overview.html"),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Per-actor dossier at `/view/swactor/actor-dossier`.
|
||||||
|
pub fn dossier() -> Self {
|
||||||
|
Self::new(
|
||||||
|
"swactor-actor-dossier",
|
||||||
|
"Actor dossier",
|
||||||
|
"swactor/actor-dossier",
|
||||||
|
include_str!("actor_dossier.html"),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn new(id: &'static str, title: &'static str, path: &'static str, html: &'static str) -> Self {
|
||||||
|
Self {
|
||||||
|
id,
|
||||||
|
title,
|
||||||
|
path,
|
||||||
|
html,
|
||||||
|
state: RwLock::new(PanelState::default()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct PanelState {
|
||||||
|
runtimes: BTreeMap<String, RuntimeState>,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct RuntimeState {
|
||||||
|
stream: StreamEvent,
|
||||||
|
last_seen: Instant,
|
||||||
|
num_workers: Option<u32>,
|
||||||
|
uptime_ms: Option<u64>,
|
||||||
|
actors: BTreeMap<String, ActorState>,
|
||||||
|
history: VecDeque<HistorySample>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RuntimeState {
|
||||||
|
fn new(stream: StreamEvent, now: Instant) -> Self {
|
||||||
|
Self {
|
||||||
|
stream,
|
||||||
|
last_seen: now,
|
||||||
|
num_workers: None,
|
||||||
|
uptime_ms: None,
|
||||||
|
actors: BTreeMap::new(),
|
||||||
|
history: VecDeque::with_capacity(HISTORY_CAP),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn update(&mut self, channel: &str, payload: &[u8], now: Instant) {
|
||||||
|
self.last_seen = now;
|
||||||
|
let Ok(value) = serde_json::from_slice::<Value>(payload) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
match channel {
|
||||||
|
RUNTIME_ACTORS => self.apply_actors(&value, now),
|
||||||
|
RUNTIME_STATS => self.apply_stats(&value, now),
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
self.push_history(now);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn apply_actors(&mut self, value: &Value, now: Instant) {
|
||||||
|
// Per-worker `worker_id` wrapper (DatastreamStatsHook shape) is the
|
||||||
|
// default placement for actors that do not carry one inline.
|
||||||
|
let wrapper_worker = u32_field(value, &["worker_id", "worker"]);
|
||||||
|
if let Some(actors) = value.get("actors").and_then(Value::as_array) {
|
||||||
|
for actor in actors {
|
||||||
|
self.apply_actor(actor, now, wrapper_worker);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// A bare actor object per frame (no envelope).
|
||||||
|
self.apply_actor(value, now, wrapper_worker);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn apply_actor(&mut self, value: &Value, now: Instant, default_worker: Option<u32>) {
|
||||||
|
let Some(address) = string_field(value, &["address", "addr", "actor_addr"]) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let actor = self
|
||||||
|
.actors
|
||||||
|
.entry(address.clone())
|
||||||
|
.or_insert_with(|| ActorState::new(address));
|
||||||
|
actor.apply_json(value, now, default_worker);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn apply_stats(&mut self, value: &Value, now: Instant) {
|
||||||
|
if let Some(num_workers) = u32_field(value, &["num_workers", "workers_live"]) {
|
||||||
|
self.num_workers = Some(num_workers);
|
||||||
|
}
|
||||||
|
if let Some(uptime_ms) = u64_field(value, &["uptime_ms"]) {
|
||||||
|
self.uptime_ms = Some(uptime_ms);
|
||||||
|
}
|
||||||
|
// address -> worker_id mapping; runtime.stats `actors` is [[addr, wid]].
|
||||||
|
if let Some(actors) = value.get("actors").and_then(Value::as_array) {
|
||||||
|
for entry in actors {
|
||||||
|
if let Some(items) = entry.as_array()
|
||||||
|
&& items.len() >= 2
|
||||||
|
&& let (Some(address), Some(worker_id)) =
|
||||||
|
(value_to_string(&items[0]), value_to_u32(&items[1]))
|
||||||
|
{
|
||||||
|
self.actors
|
||||||
|
.entry(address.clone())
|
||||||
|
.or_insert_with(|| ActorState::new(address))
|
||||||
|
.worker_id = Some(worker_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Some publishers carry full per-actor detail under `actor_details`.
|
||||||
|
if let Some(details) = value.get("actor_details").and_then(Value::as_array) {
|
||||||
|
for actor in details {
|
||||||
|
self.apply_actor(actor, now, None);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_history(&mut self, now: Instant) {
|
||||||
|
let totals = self.totals();
|
||||||
|
if let Some(last) = self.history.back_mut()
|
||||||
|
&& now.duration_since(last.at) < HISTORY_MIN_INTERVAL
|
||||||
|
{
|
||||||
|
last.mailbox_depth = totals.mailbox_depth;
|
||||||
|
last.msg_per_sec = totals.msg_per_sec;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if self.history.len() == HISTORY_CAP {
|
||||||
|
self.history.pop_front();
|
||||||
|
}
|
||||||
|
self.history.push_back(HistorySample {
|
||||||
|
at: now,
|
||||||
|
mailbox_depth: totals.mailbox_depth,
|
||||||
|
msg_per_sec: totals.msg_per_sec,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
fn totals(&self) -> Totals {
|
||||||
|
let mut totals = Totals::default();
|
||||||
|
totals.actors = self.actors.len().min(u32::MAX as usize) as u32;
|
||||||
|
for actor in self.actors.values() {
|
||||||
|
totals.mailbox_depth = totals.mailbox_depth.saturating_add(actor.mailbox_depth);
|
||||||
|
totals.msg_per_sec += actor.msg_per_sec;
|
||||||
|
if actor.poisoned {
|
||||||
|
totals.poisoned = totals.poisoned.saturating_add(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
totals
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct ActorState {
|
||||||
|
address: String,
|
||||||
|
name: Option<String>,
|
||||||
|
actor_type: Option<String>,
|
||||||
|
message_type: Option<String>,
|
||||||
|
worker_id: Option<u32>,
|
||||||
|
mailbox_depth: u32,
|
||||||
|
mailbox_growth: f64,
|
||||||
|
messages_processed: u64,
|
||||||
|
msg_per_sec: f64,
|
||||||
|
last_msg_type: Option<String>,
|
||||||
|
poisoned: bool,
|
||||||
|
message_type_counts: Vec<(String, u64)>,
|
||||||
|
history: VecDeque<ActorHistorySample>,
|
||||||
|
last_update: Option<Instant>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ActorState {
|
||||||
|
fn new(address: String) -> Self {
|
||||||
|
Self {
|
||||||
|
address,
|
||||||
|
name: None,
|
||||||
|
actor_type: None,
|
||||||
|
message_type: None,
|
||||||
|
worker_id: None,
|
||||||
|
mailbox_depth: 0,
|
||||||
|
mailbox_growth: 0.0,
|
||||||
|
messages_processed: 0,
|
||||||
|
msg_per_sec: 0.0,
|
||||||
|
last_msg_type: None,
|
||||||
|
poisoned: false,
|
||||||
|
message_type_counts: Vec::new(),
|
||||||
|
history: VecDeque::new(),
|
||||||
|
last_update: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn apply_json(&mut self, value: &Value, now: Instant, default_worker: Option<u32>) {
|
||||||
|
let elapsed = self
|
||||||
|
.last_update
|
||||||
|
.map(|then| now.duration_since(then).as_secs_f64())
|
||||||
|
.unwrap_or(0.0);
|
||||||
|
if let Some(name) = string_field(value, &["name"]).filter(|name| !name.is_empty()) {
|
||||||
|
self.name = Some(name);
|
||||||
|
}
|
||||||
|
if let Some(actor_type) = string_field(value, &["actor_type"]).filter(|t| !t.is_empty()) {
|
||||||
|
self.actor_type = Some(actor_type);
|
||||||
|
}
|
||||||
|
if let Some(message_type) = string_field(value, &["message_type"]).filter(|t| !t.is_empty()) {
|
||||||
|
self.message_type = Some(message_type);
|
||||||
|
}
|
||||||
|
if let Some(worker_id) = u32_field(value, &["worker_id", "worker"]) {
|
||||||
|
self.worker_id = Some(worker_id);
|
||||||
|
} else if self.worker_id.is_none() {
|
||||||
|
self.worker_id = default_worker;
|
||||||
|
}
|
||||||
|
assign_u32(&mut self.mailbox_depth, value, &["mailbox_depth", "queued"]);
|
||||||
|
assign_u64_rate(
|
||||||
|
&mut self.messages_processed,
|
||||||
|
&mut self.msg_per_sec,
|
||||||
|
value,
|
||||||
|
&["messages_processed", "processed", "messages_handled"],
|
||||||
|
elapsed,
|
||||||
|
);
|
||||||
|
if let Some(last) = string_field(
|
||||||
|
value,
|
||||||
|
&["last_msg_type", "last_message", "last_message_type"],
|
||||||
|
)
|
||||||
|
.filter(|last| !last.is_empty())
|
||||||
|
{
|
||||||
|
self.last_msg_type = Some(last);
|
||||||
|
}
|
||||||
|
if let Some(poisoned) = value.get("poisoned").and_then(Value::as_bool) {
|
||||||
|
self.poisoned = poisoned;
|
||||||
|
}
|
||||||
|
if let Some(counts) = parse_message_type_counts(value.get("message_type_counts")) {
|
||||||
|
self.message_type_counts = counts;
|
||||||
|
}
|
||||||
|
self.last_update = Some(now);
|
||||||
|
self.push_history(now);
|
||||||
|
self.recompute_growth();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_history(&mut self, now: Instant) {
|
||||||
|
if let Some(last) = self.history.back_mut()
|
||||||
|
&& now.duration_since(last.at) < HISTORY_MIN_INTERVAL
|
||||||
|
{
|
||||||
|
last.mailbox_depth = self.mailbox_depth;
|
||||||
|
last.msg_per_sec = self.msg_per_sec;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if self.history.len() == PER_ACTOR_HISTORY_CAP {
|
||||||
|
self.history.pop_front();
|
||||||
|
}
|
||||||
|
self.history.push_back(ActorHistorySample {
|
||||||
|
at: now,
|
||||||
|
mailbox_depth: self.mailbox_depth,
|
||||||
|
msg_per_sec: self.msg_per_sec,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Mailbox depth/s over the recent window, for trend colouring.
|
||||||
|
fn recompute_growth(&mut self) {
|
||||||
|
let n = self.history.len();
|
||||||
|
if n < 2 {
|
||||||
|
self.mailbox_growth = 0.0;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let start = n - n.min(GROWTH_WINDOW);
|
||||||
|
let first = &self.history[start];
|
||||||
|
let last = &self.history[n - 1];
|
||||||
|
let span = last.at.duration_since(first.at).as_secs_f64();
|
||||||
|
self.mailbox_growth = if span > 0.0 {
|
||||||
|
(last.mailbox_depth as f64 - first.mailbox_depth as f64) / span
|
||||||
|
} else {
|
||||||
|
0.0
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
struct HistorySample {
|
||||||
|
at: Instant,
|
||||||
|
mailbox_depth: u32,
|
||||||
|
msg_per_sec: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
struct ActorHistorySample {
|
||||||
|
at: Instant,
|
||||||
|
mailbox_depth: u32,
|
||||||
|
msg_per_sec: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct Totals {
|
||||||
|
actors: u32,
|
||||||
|
mailbox_depth: u32,
|
||||||
|
msg_per_sec: f64,
|
||||||
|
poisoned: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct PanelSnapshot {
|
||||||
|
runtimes: Vec<RuntimeSnapshot>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct RuntimeSnapshot {
|
||||||
|
stream: StreamSnapshot,
|
||||||
|
live: bool,
|
||||||
|
last_seen_ms_ago: u64,
|
||||||
|
summary: SummarySnapshot,
|
||||||
|
actors: Vec<ActorSnapshot>,
|
||||||
|
history: Vec<HistorySnapshot>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct StreamSnapshot {
|
||||||
|
key: String,
|
||||||
|
node: String,
|
||||||
|
life: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct SummarySnapshot {
|
||||||
|
actors: u32,
|
||||||
|
msg_per_sec: f64,
|
||||||
|
mailbox_depth: u32,
|
||||||
|
poisoned: u32,
|
||||||
|
uptime_ms: Option<u64>,
|
||||||
|
num_workers: Option<u32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct ActorSnapshot {
|
||||||
|
address: String,
|
||||||
|
name: Option<String>,
|
||||||
|
actor_type: Option<String>,
|
||||||
|
message_type: Option<String>,
|
||||||
|
worker_id: Option<u32>,
|
||||||
|
/// Single derived display state. The frame carries only `poisoned`, so the
|
||||||
|
/// granularity is poisoned | running until the feed is enriched.
|
||||||
|
state: &'static str,
|
||||||
|
mailbox_depth: u32,
|
||||||
|
mailbox_growth: f64,
|
||||||
|
messages_processed: u64,
|
||||||
|
msg_per_sec: f64,
|
||||||
|
last_msg_type: Option<String>,
|
||||||
|
poisoned: bool,
|
||||||
|
message_type_counts: Vec<MessageTypeCountSnapshot>,
|
||||||
|
history: Vec<ActorHistorySnapshot>,
|
||||||
|
last_seen_ms_ago: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct MessageTypeCountSnapshot {
|
||||||
|
message_type: String,
|
||||||
|
count: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct HistorySnapshot {
|
||||||
|
ms_ago: u64,
|
||||||
|
msg_per_sec: f64,
|
||||||
|
mailbox_depth: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct ActorHistorySnapshot {
|
||||||
|
ms_ago: u64,
|
||||||
|
mailbox_depth: u32,
|
||||||
|
msg_per_sec: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl DashboardView for ActorPanelView {
|
||||||
|
fn id(&self) -> &'static str {
|
||||||
|
self.id
|
||||||
|
}
|
||||||
|
|
||||||
|
fn title(&self) -> &'static str {
|
||||||
|
self.title
|
||||||
|
}
|
||||||
|
|
||||||
|
fn path(&self) -> &'static str {
|
||||||
|
self.path
|
||||||
|
}
|
||||||
|
|
||||||
|
fn channels(&self) -> &'static [&'static str] {
|
||||||
|
CHANNELS
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ingest(&self, _stream: &StreamId, _frame: &Frame, event: &FrameEvent) {
|
||||||
|
let now = Instant::now();
|
||||||
|
let mut state = self.state.write();
|
||||||
|
let key = stream_key(&event.stream);
|
||||||
|
state
|
||||||
|
.runtimes
|
||||||
|
.entry(key)
|
||||||
|
.or_insert_with(|| RuntimeState::new(event.stream.clone(), now))
|
||||||
|
.update(&event.channel, &event.payload, now);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn snapshot_json(&self) -> Value {
|
||||||
|
let now = Instant::now();
|
||||||
|
let snapshot = PanelSnapshot {
|
||||||
|
runtimes: self
|
||||||
|
.state
|
||||||
|
.read()
|
||||||
|
.runtimes
|
||||||
|
.values()
|
||||||
|
.map(|runtime| runtime_snapshot(runtime, now))
|
||||||
|
.collect(),
|
||||||
|
};
|
||||||
|
serde_json::to_value(snapshot).unwrap_or_else(|_| json!({ "runtimes": [] }))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn html(&self) -> Option<&'static str> {
|
||||||
|
Some(self.html)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn runtime_snapshot(runtime: &RuntimeState, now: Instant) -> RuntimeSnapshot {
|
||||||
|
let totals = runtime.totals();
|
||||||
|
RuntimeSnapshot {
|
||||||
|
stream: StreamSnapshot {
|
||||||
|
key: stream_key(&runtime.stream),
|
||||||
|
node: runtime.stream.node.clone(),
|
||||||
|
life: runtime.stream.life,
|
||||||
|
},
|
||||||
|
live: now.duration_since(runtime.last_seen) <= LIVE_TTL,
|
||||||
|
last_seen_ms_ago: now.duration_since(runtime.last_seen).as_millis() as u64,
|
||||||
|
summary: SummarySnapshot {
|
||||||
|
actors: totals.actors,
|
||||||
|
msg_per_sec: totals.msg_per_sec,
|
||||||
|
mailbox_depth: totals.mailbox_depth,
|
||||||
|
poisoned: totals.poisoned,
|
||||||
|
uptime_ms: runtime.uptime_ms,
|
||||||
|
num_workers: runtime.num_workers,
|
||||||
|
},
|
||||||
|
actors: runtime
|
||||||
|
.actors
|
||||||
|
.values()
|
||||||
|
.map(|actor| ActorSnapshot {
|
||||||
|
address: actor.address.clone(),
|
||||||
|
name: actor.name.clone(),
|
||||||
|
actor_type: actor.actor_type.clone(),
|
||||||
|
message_type: actor.message_type.clone(),
|
||||||
|
worker_id: actor.worker_id,
|
||||||
|
state: if actor.poisoned { "poisoned" } else { "running" },
|
||||||
|
mailbox_depth: actor.mailbox_depth,
|
||||||
|
mailbox_growth: actor.mailbox_growth,
|
||||||
|
messages_processed: actor.messages_processed,
|
||||||
|
msg_per_sec: actor.msg_per_sec,
|
||||||
|
last_msg_type: actor.last_msg_type.clone(),
|
||||||
|
poisoned: actor.poisoned,
|
||||||
|
message_type_counts: actor
|
||||||
|
.message_type_counts
|
||||||
|
.iter()
|
||||||
|
.map(|(message_type, count)| MessageTypeCountSnapshot {
|
||||||
|
message_type: message_type.clone(),
|
||||||
|
count: *count,
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
history: actor
|
||||||
|
.history
|
||||||
|
.iter()
|
||||||
|
.map(|sample| ActorHistorySnapshot {
|
||||||
|
ms_ago: now.duration_since(sample.at).as_millis() as u64,
|
||||||
|
mailbox_depth: sample.mailbox_depth,
|
||||||
|
msg_per_sec: sample.msg_per_sec,
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
last_seen_ms_ago: actor
|
||||||
|
.last_update
|
||||||
|
.map(|then| now.duration_since(then).as_millis() as u64)
|
||||||
|
.unwrap_or(0),
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
history: runtime
|
||||||
|
.history
|
||||||
|
.iter()
|
||||||
|
.map(|sample| HistorySnapshot {
|
||||||
|
ms_ago: now.duration_since(sample.at).as_millis() as u64,
|
||||||
|
msg_per_sec: sample.msg_per_sec,
|
||||||
|
mailbox_depth: sample.mailbox_depth,
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- tolerant JSON helpers (publisher-shape-agnostic readers) ---------------
|
||||||
|
|
||||||
|
fn assign_u32(slot: &mut u32, value: &Value, names: &[&str]) {
|
||||||
|
if let Some(v) = u32_field(value, names) {
|
||||||
|
*slot = v;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assign_u64_rate(slot: &mut u64, rate: &mut f64, value: &Value, names: &[&str], elapsed: f64) {
|
||||||
|
if let Some(next) = u64_field(value, names) {
|
||||||
|
if elapsed > 0.0 && next > *slot {
|
||||||
|
*rate = (next - *slot) as f64 / elapsed;
|
||||||
|
} else if next < *slot {
|
||||||
|
*rate = 0.0;
|
||||||
|
}
|
||||||
|
*slot = next;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn u32_field(value: &Value, names: &[&str]) -> Option<u32> {
|
||||||
|
u64_field(value, names).and_then(|v| u32::try_from(v).ok())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn u64_field(value: &Value, names: &[&str]) -> Option<u64> {
|
||||||
|
names
|
||||||
|
.iter()
|
||||||
|
.find_map(|name| value.get(*name).and_then(value_to_u64))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn string_field(value: &Value, names: &[&str]) -> Option<String> {
|
||||||
|
names
|
||||||
|
.iter()
|
||||||
|
.find_map(|name| value.get(*name).and_then(value_to_string))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn value_to_u32(value: &Value) -> Option<u32> {
|
||||||
|
value_to_u64(value).and_then(|v| u32::try_from(v).ok())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn value_to_u64(value: &Value) -> Option<u64> {
|
||||||
|
value
|
||||||
|
.as_u64()
|
||||||
|
.or_else(|| value.as_str().and_then(|s| s.parse::<u64>().ok()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn value_to_string(value: &Value) -> Option<String> {
|
||||||
|
value.as_str().map(ToOwned::to_owned).or_else(|| {
|
||||||
|
if value.is_null() {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(value.to_string())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Accepts pair form `["Type", N]`, object form `{"ty": "Type", "count": N}`,
|
||||||
|
/// and map form `{"Type": N}`.
|
||||||
|
fn parse_message_type_counts(value: Option<&Value>) -> Option<Vec<(String, u64)>> {
|
||||||
|
let value = value?;
|
||||||
|
if let Some(items) = value.as_array() {
|
||||||
|
let mut out = Vec::new();
|
||||||
|
for item in items {
|
||||||
|
if let Some(pair) = item.as_array()
|
||||||
|
&& pair.len() >= 2
|
||||||
|
&& let (Some(name), Some(count)) =
|
||||||
|
(value_to_string(&pair[0]), value_to_u64(&pair[1]))
|
||||||
|
{
|
||||||
|
out.push((name, count));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if let Some(name) = string_field(item, &["ty", "message_type", "type", "name"])
|
||||||
|
&& let Some(count) = u64_field(item, &["count"])
|
||||||
|
{
|
||||||
|
out.push((name, count));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Some(out);
|
||||||
|
}
|
||||||
|
if let Some(map) = value.as_object() {
|
||||||
|
let mut out: Vec<(String, u64)> = map
|
||||||
|
.iter()
|
||||||
|
.filter_map(|(name, count)| value_to_u64(count).map(|count| (name.clone(), count)))
|
||||||
|
.collect();
|
||||||
|
out.sort_by(|a, b| b.1.cmp(&a.1));
|
||||||
|
return Some(out);
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stream_key(stream: &StreamEvent) -> String {
|
||||||
|
format!("{}#{}", stream.node, stream.life)
|
||||||
|
}
|
||||||
|
|
@ -2,10 +2,12 @@ use std::sync::Arc;
|
||||||
|
|
||||||
use crate::view::DashboardView;
|
use crate::view::DashboardView;
|
||||||
|
|
||||||
|
mod actor_view;
|
||||||
mod worker_page;
|
mod worker_page;
|
||||||
mod worker_view;
|
mod worker_view;
|
||||||
|
|
||||||
pub use worker_view::SwactorWorkerView;
|
pub use worker_view::SwactorWorkerView;
|
||||||
|
pub use actor_view::ActorPanelView;
|
||||||
|
|
||||||
pub const RUNTIME_STATS: &str = "runtime.stats";
|
pub const RUNTIME_STATS: &str = "runtime.stats";
|
||||||
pub const RUNTIME_WORKERS: &str = "runtime.workers";
|
pub const RUNTIME_WORKERS: &str = "runtime.workers";
|
||||||
|
|
@ -14,3 +16,13 @@ pub const RUNTIME_ACTORS: &str = "runtime.actors";
|
||||||
pub fn worker_view() -> Arc<dyn DashboardView> {
|
pub fn worker_view() -> Arc<dyn DashboardView> {
|
||||||
Arc::new(SwactorWorkerView::default())
|
Arc::new(SwactorWorkerView::default())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Built-in actor overview + roster view (`/view/swactor/actor-overview`).
|
||||||
|
pub fn actor_overview_view() -> Arc<dyn DashboardView> {
|
||||||
|
Arc::new(ActorPanelView::overview())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Built-in actor dossier view (`/view/swactor/actor-dossier`).
|
||||||
|
pub fn actor_dossier_view() -> Arc<dyn DashboardView> {
|
||||||
|
Arc::new(ActorPanelView::dossier())
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -758,9 +758,11 @@ impl<'a> RuntimeActorStatsRecord<'a> {
|
||||||
actors: snapshots
|
actors: snapshots
|
||||||
.iter()
|
.iter()
|
||||||
.map(|snapshot| RuntimeActorSnapshotRecord {
|
.map(|snapshot| RuntimeActorSnapshotRecord {
|
||||||
address: snapshot.address.to_string(),
|
address: snapshot.address.to_full_hex(),
|
||||||
mailbox_depth: snapshot.mailbox_depth,
|
mailbox_depth: snapshot.mailbox_depth,
|
||||||
last_msg_type: snapshot.last_msg_type,
|
last_msg_type: snapshot.last_msg_type,
|
||||||
|
actor_type: snapshot.actor_type,
|
||||||
|
message_type: snapshot.message_type,
|
||||||
messages_processed: snapshot.messages_processed,
|
messages_processed: snapshot.messages_processed,
|
||||||
poisoned: snapshot.poisoned,
|
poisoned: snapshot.poisoned,
|
||||||
message_type_counts: snapshot
|
message_type_counts: snapshot
|
||||||
|
|
@ -779,6 +781,8 @@ struct RuntimeActorSnapshotRecord<'a> {
|
||||||
address: String,
|
address: String,
|
||||||
mailbox_depth: usize,
|
mailbox_depth: usize,
|
||||||
last_msg_type: Option<&'static str>,
|
last_msg_type: Option<&'static str>,
|
||||||
|
actor_type: Option<&'static str>,
|
||||||
|
message_type: Option<&'static str>,
|
||||||
messages_processed: u64,
|
messages_processed: u64,
|
||||||
poisoned: bool,
|
poisoned: bool,
|
||||||
message_type_counts: Vec<RuntimeMessageTypeCount<'a>>,
|
message_type_counts: Vec<RuntimeMessageTypeCount<'a>>,
|
||||||
|
|
|
||||||
|
|
@ -300,6 +300,8 @@ fn stats_hook_adapter_submits_worker_snapshot_json() {
|
||||||
address: actor,
|
address: actor,
|
||||||
mailbox_depth: 3,
|
mailbox_depth: 3,
|
||||||
last_msg_type: Some("Ping"),
|
last_msg_type: Some("Ping"),
|
||||||
|
actor_type: Some("TestActor"),
|
||||||
|
message_type: Some("Ping"),
|
||||||
messages_processed: 5,
|
messages_processed: 5,
|
||||||
poisoned: false,
|
poisoned: false,
|
||||||
message_type_counts: vec![("Ping", 5)],
|
message_type_counts: vec![("Ping", 5)],
|
||||||
|
|
@ -313,11 +315,13 @@ fn stats_hook_adapter_submits_worker_snapshot_json() {
|
||||||
assert_eq!(delivery.channel.channel, runtime);
|
assert_eq!(delivery.channel.channel, runtime);
|
||||||
let json: Value = serde_json::from_slice(&delivery.payload).unwrap();
|
let json: Value = serde_json::from_slice(&delivery.payload).unwrap();
|
||||||
assert_eq!(json["worker_id"], 2);
|
assert_eq!(json["worker_id"], 2);
|
||||||
assert_eq!(json["actors"][0]["address"], actor.to_string());
|
assert_eq!(json["actors"][0]["address"], actor.to_full_hex());
|
||||||
assert_eq!(json["actors"][0]["mailbox_depth"], 3);
|
assert_eq!(json["actors"][0]["mailbox_depth"], 3);
|
||||||
assert_eq!(json["actors"][0]["last_msg_type"], "Ping");
|
assert_eq!(json["actors"][0]["last_msg_type"], "Ping");
|
||||||
assert_eq!(json["actors"][0]["messages_processed"], 5);
|
assert_eq!(json["actors"][0]["messages_processed"], 5);
|
||||||
assert_eq!(json["actors"][0]["message_type_counts"][0]["ty"], "Ping");
|
assert_eq!(json["actors"][0]["message_type_counts"][0]["ty"], "Ping");
|
||||||
|
assert_eq!(json["actors"][0]["actor_type"], "TestActor");
|
||||||
|
assert_eq!(json["actors"][0]["message_type"], "Ping");
|
||||||
}
|
}
|
||||||
|
|
||||||
fn positions(events: &[DatastreamEvent]) -> Vec<u64> {
|
fn positions(events: &[DatastreamEvent]) -> Vec<u64> {
|
||||||
|
|
|
||||||
|
|
@ -19,9 +19,8 @@ iroh = "0.98"
|
||||||
# `test-utils` exposes `CaRootsConfig::insecure_skip_verify()` so clients can
|
# `test-utils` exposes `CaRootsConfig::insecure_skip_verify()` so clients can
|
||||||
# trust operator-controlled custom relays with self-signed QAD certs.
|
# trust operator-controlled custom relays with self-signed QAD certs.
|
||||||
iroh-relay = { version = "0.98", features = ["test-utils"] }
|
iroh-relay = { version = "0.98", features = ["test-utils"] }
|
||||||
tokio = { version = "1", features = ["rt-multi-thread", "time"] }
|
tokio.workspace = true
|
||||||
parking_lot = "0.12"
|
parking_lot = "0.12"
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
iroh-relay = { version = "0.98", features = ["server", "test-utils"] }
|
iroh-relay = { version = "0.98", features = ["server", "test-utils"] }
|
||||||
tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "io-util", "time"] }
|
|
||||||
|
|
|
||||||
2
rust-toolchain.toml
Normal file
2
rust-toolchain.toml
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
[toolchain]
|
||||||
|
channel = "nightly-2026-02-07"
|
||||||
12
src/actor.rs
12
src/actor.rs
|
|
@ -162,6 +162,18 @@ impl ActorAddress {
|
||||||
crate::get_random(&mut bytes);
|
crate::get_random(&mut bytes);
|
||||||
Self(bytes)
|
Self(bytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Full 64-character hex encoding of all 32 bytes, for displays that need
|
||||||
|
/// the untruncated address (dashboards). [`Display`](std::fmt::Display)
|
||||||
|
/// stays short for logs.
|
||||||
|
pub fn to_full_hex(&self) -> String {
|
||||||
|
use std::fmt::Write;
|
||||||
|
let mut s = String::with_capacity(64);
|
||||||
|
for b in &self.0 {
|
||||||
|
let _ = write!(s, "{:02x}", b);
|
||||||
|
}
|
||||||
|
s
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─── Environment ─────────────────────────────────────────────────────────────
|
// ─── Environment ─────────────────────────────────────────────────────────────
|
||||||
|
|
|
||||||
|
|
@ -124,6 +124,8 @@ pub struct ActorSnapshot {
|
||||||
pub address: ActorAddress,
|
pub address: ActorAddress,
|
||||||
pub mailbox_depth: usize,
|
pub mailbox_depth: usize,
|
||||||
pub last_msg_type: Option<&'static str>,
|
pub last_msg_type: Option<&'static str>,
|
||||||
|
pub actor_type: Option<&'static str>,
|
||||||
|
pub message_type: Option<&'static str>,
|
||||||
pub messages_processed: u64,
|
pub messages_processed: u64,
|
||||||
pub poisoned: bool,
|
pub poisoned: bool,
|
||||||
/// Per-message-type counts, sorted descending by count.
|
/// Per-message-type counts, sorted descending by count.
|
||||||
|
|
|
||||||
|
|
@ -998,10 +998,13 @@ impl ActorPool {
|
||||||
let mut type_counts: Vec<(&'static str, u64)> =
|
let mut type_counts: Vec<(&'static str, u64)> =
|
||||||
slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect();
|
slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect();
|
||||||
type_counts.sort_by(|a, b| b.1.cmp(&a.1));
|
type_counts.sort_by(|a, b| b.1.cmp(&a.1));
|
||||||
|
let metadata = slot.actor.metadata();
|
||||||
ActorSnapshot {
|
ActorSnapshot {
|
||||||
address: addr,
|
address: addr,
|
||||||
mailbox_depth: slot.mailbox.len(),
|
mailbox_depth: slot.mailbox.len(),
|
||||||
last_msg_type: slot.last_msg_type,
|
last_msg_type: slot.last_msg_type,
|
||||||
|
actor_type: Some(metadata.actor_type_name),
|
||||||
|
message_type: Some(metadata.message_type_name),
|
||||||
messages_processed: slot.messages_processed,
|
messages_processed: slot.messages_processed,
|
||||||
poisoned: slot.poisoned,
|
poisoned: slot.poisoned,
|
||||||
message_type_counts: type_counts,
|
message_type_counts: type_counts,
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@ publish = false
|
||||||
reqwest = { version = "0.12", features = ["json"] }
|
reqwest = { version = "0.12", features = ["json"] }
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
tokio = { version = "1", features = ["macros", "rt", "time"] }
|
tokio.workspace = true
|
||||||
urlencoding = "2"
|
urlencoding = "2"
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
|
|
|
||||||
|
|
@ -280,7 +280,7 @@ OPTIONS:
|
||||||
Select the runtime provider
|
Select the runtime provider
|
||||||
--config <path> Load config overlay
|
--config <path> Load config overlay
|
||||||
--pipeline-stages <count> Number of pipeline stages
|
--pipeline-stages <count> Number of pipeline stages
|
||||||
--cached-model[=<path>] Use discovered or explicit cached GGUF model
|
--cached-model[=<path>] Use discovered or explicit cached GGUF model (default for --process)
|
||||||
--dump-logs[=<path>] Write datastream frame log
|
--dump-logs[=<path>] Write datastream frame log
|
||||||
--run-id <id> Override run id
|
--run-id <id> Override run id
|
||||||
--skip-rebuild Reuse existing Cargo artifacts
|
--skip-rebuild Reuse existing Cargo artifacts
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue