feat(telemetry): add sequenced runtime dashboard signals
Rate-limit actor census and activity telemetry while preserving immediate lifecycle transitions. Route managed-process output into Fleet node tails, bound explorer retention, aggregate SWIM probes, and make sampler health transition-based. Restore stable explorer synchronization and refine Fleet bulk controls.
This commit is contained in:
parent
0411c20b94
commit
b95163823e
32 changed files with 2017 additions and 388 deletions
1
Cargo.lock
generated
1
Cargo.lock
generated
|
|
@ -4368,6 +4368,7 @@ dependencies = [
|
||||||
"swactor-process",
|
"swactor-process",
|
||||||
"swactor-transport",
|
"swactor-transport",
|
||||||
"tar",
|
"tar",
|
||||||
|
"telemetry",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
"tokio",
|
"tokio",
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -58,7 +58,7 @@ use parking_lot::Mutex;
|
||||||
use serde_json::{Value, json};
|
use serde_json::{Value, json};
|
||||||
use swactor::actor::{ActorAddress, ActorInterface};
|
use swactor::actor::{ActorAddress, ActorInterface};
|
||||||
use swactor::runtime::{Ctx, ExternalSender, Inbox, Runtime};
|
use swactor::runtime::{Ctx, ExternalSender, Inbox, Runtime};
|
||||||
use swactor::stats::{ActorSnapshot, StatsHook};
|
use swactor::stats::{ActorSnapshot, StatsHook, StatsSnapshotKind};
|
||||||
use swactor_engine::{ActorCompletion, Engine, EngineHandle, TokioBackend, TokioConfig};
|
use swactor_engine::{ActorCompletion, Engine, EngineHandle, TokioBackend, TokioConfig};
|
||||||
use swactor_job_runner::{NodeJobActor, register_job_codecs};
|
use swactor_job_runner::{NodeJobActor, register_job_codecs};
|
||||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt};
|
use tokio::io::{AsyncBufReadExt, AsyncWriteExt};
|
||||||
|
|
@ -209,7 +209,7 @@ struct InspectableStatsHook {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl StatsHook for InspectableStatsHook {
|
impl StatsHook for InspectableStatsHook {
|
||||||
fn on_tick(&self, worker_id: usize, snapshots: &[ActorSnapshot]) {
|
fn on_snapshot(&self, worker_id: usize, snapshots: &[ActorSnapshot], kind: StatsSnapshotKind) {
|
||||||
self.inspector.latest.lock().insert(
|
self.inspector.latest.lock().insert(
|
||||||
worker_id,
|
worker_id,
|
||||||
snapshots
|
snapshots
|
||||||
|
|
@ -222,7 +222,7 @@ impl StatsHook for InspectableStatsHook {
|
||||||
})
|
})
|
||||||
.collect(),
|
.collect(),
|
||||||
);
|
);
|
||||||
self.inner.on_tick(worker_id, snapshots);
|
self.inner.on_snapshot(worker_id, snapshots, kind);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -599,6 +599,13 @@ impl SamplerHealthContext {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||||
|
enum SamplerHealthState {
|
||||||
|
Waiting,
|
||||||
|
Ready,
|
||||||
|
Failed,
|
||||||
|
}
|
||||||
|
|
||||||
fn sampler_health_payload(
|
fn sampler_health_payload(
|
||||||
context: SamplerHealthContext,
|
context: SamplerHealthContext,
|
||||||
sampler: &str,
|
sampler: &str,
|
||||||
|
|
@ -681,18 +688,33 @@ fn submit_sampler_sample_health(
|
||||||
producer: &TelemetryProducer,
|
producer: &TelemetryProducer,
|
||||||
health_channel: ChannelId,
|
health_channel: ChannelId,
|
||||||
context: SamplerHealthContext,
|
context: SamplerHealthContext,
|
||||||
sampler: &str,
|
identity: (&str, &str),
|
||||||
sample_channel: &str,
|
|
||||||
seq: u64,
|
seq: u64,
|
||||||
error: Option<&str>,
|
error: Option<&str>,
|
||||||
|
state: &Mutex<SamplerHealthState>,
|
||||||
) {
|
) {
|
||||||
let (status, detail) = match error {
|
let (sampler, sample_channel) = identity;
|
||||||
Some(error) => (
|
let mut state = state.lock();
|
||||||
|
let (status, detail, next) = match (*state, error) {
|
||||||
|
(SamplerHealthState::Waiting, None) => (
|
||||||
|
"ready",
|
||||||
|
json!({"state":"sample_observed","sample_seq":seq}),
|
||||||
|
SamplerHealthState::Ready,
|
||||||
|
),
|
||||||
|
(SamplerHealthState::Failed, None) => (
|
||||||
|
"recovered",
|
||||||
|
json!({"state":"sample_observed","sample_seq":seq}),
|
||||||
|
SamplerHealthState::Ready,
|
||||||
|
),
|
||||||
|
(SamplerHealthState::Waiting | SamplerHealthState::Ready, Some(error)) => (
|
||||||
"failed",
|
"failed",
|
||||||
json!({"state":"error","sample_seq":seq,"error":error}),
|
json!({"state":"error","sample_seq":seq,"error":error}),
|
||||||
|
SamplerHealthState::Failed,
|
||||||
),
|
),
|
||||||
None => ("ready", json!({"state":"sample_observed","sample_seq":seq})),
|
(SamplerHealthState::Ready, None) | (SamplerHealthState::Failed, Some(_)) => return,
|
||||||
};
|
};
|
||||||
|
*state = next;
|
||||||
|
drop(state);
|
||||||
submit_sampler_health(
|
submit_sampler_health(
|
||||||
producer,
|
producer,
|
||||||
health_channel,
|
health_channel,
|
||||||
|
|
@ -738,6 +760,7 @@ fn spawn_blocking_sampler_task<State, Sample>(
|
||||||
error_of,
|
error_of,
|
||||||
} = config;
|
} = config;
|
||||||
let started_producer = producer.clone();
|
let started_producer = producer.clone();
|
||||||
|
let health_state = Arc::new(Mutex::new(SamplerHealthState::Waiting));
|
||||||
telemetry::hardware::spawn_blocking_sampler(
|
telemetry::hardware::spawn_blocking_sampler(
|
||||||
engine,
|
engine,
|
||||||
interval,
|
interval,
|
||||||
|
|
@ -758,10 +781,10 @@ fn spawn_blocking_sampler_task<State, Sample>(
|
||||||
&producer,
|
&producer,
|
||||||
health_channel,
|
health_channel,
|
||||||
health_context,
|
health_context,
|
||||||
sampler,
|
(sampler, sample_channel),
|
||||||
sample_channel,
|
|
||||||
seq,
|
seq,
|
||||||
error_of(&sample),
|
error_of(&sample),
|
||||||
|
&health_state,
|
||||||
);
|
);
|
||||||
producer.submit_record(channel, &sample);
|
producer.submit_record(channel, &sample);
|
||||||
},
|
},
|
||||||
|
|
@ -2001,6 +2024,7 @@ fn run() -> Result<(), String> {
|
||||||
.spawn(
|
.spawn(
|
||||||
NodeJobActor::unbound(workdir, stack.runtime.create_sender())
|
NodeJobActor::unbound(workdir, stack.runtime.create_sender())
|
||||||
.with_actor_timers(engine.handle())
|
.with_actor_timers(engine.handle())
|
||||||
|
.with_process_telemetry(telemetry.producer.clone())
|
||||||
.with_route_registrar(job_route_registrar)
|
.with_route_registrar(job_route_registrar)
|
||||||
.with_data_plane(Arc::new(data_plane.clone())),
|
.with_data_plane(Arc::new(data_plane.clone())),
|
||||||
)
|
)
|
||||||
|
|
@ -2769,6 +2793,12 @@ fn emit_swim_telemetry(
|
||||||
.producer
|
.producer
|
||||||
.submit_record(telemetry.channels.swim_probes, &record);
|
.submit_record(telemetry.channels.swim_probes, &record);
|
||||||
}
|
}
|
||||||
|
if let Some(summary) = stack.drain_swim_probe_summary() {
|
||||||
|
let record = stack.swim_probe_summary_record(summary, local_phase);
|
||||||
|
telemetry
|
||||||
|
.producer
|
||||||
|
.submit_record(telemetry.channels.swim_probes, &record);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy)]
|
#[derive(Clone, Copy)]
|
||||||
|
|
@ -5378,6 +5408,104 @@ mod control_flow_properties {
|
||||||
|
|
||||||
const DRIVE_PER_ACTION: usize = 16;
|
const DRIVE_PER_ACTION: usize = 16;
|
||||||
const FINAL_DRIVE_BUDGET: usize = 256;
|
const FINAL_DRIVE_BUDGET: usize = 256;
|
||||||
|
#[test]
|
||||||
|
fn sampler_health_emits_only_state_transitions() {
|
||||||
|
let endpoint = TelemetryEndpoint::with_descriptor(
|
||||||
|
StreamDescriptor {
|
||||||
|
stream: StreamId::new(NodeId::new("sampler-health-test"), Lifetime(1)),
|
||||||
|
label: None,
|
||||||
|
origin: StreamOrigin::RemoteNode,
|
||||||
|
},
|
||||||
|
16,
|
||||||
|
16,
|
||||||
|
);
|
||||||
|
let producer = endpoint.producer();
|
||||||
|
let health_channel = producer.register_channel(
|
||||||
|
NODE_SAMPLER_CHANNEL,
|
||||||
|
ChannelContent::JsonRecord {
|
||||||
|
schema: Some(NODE_SAMPLER_CHANNEL.to_owned()),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let subscription = endpoint.subscribe_all("sampler-health-transitions");
|
||||||
|
let context = SamplerHealthContext {
|
||||||
|
run_id: 1,
|
||||||
|
node_id: 2,
|
||||||
|
stage_index: 0,
|
||||||
|
};
|
||||||
|
let state = Mutex::new(SamplerHealthState::Waiting);
|
||||||
|
|
||||||
|
submit_sampler_started(
|
||||||
|
&producer,
|
||||||
|
health_channel,
|
||||||
|
context,
|
||||||
|
"cpu",
|
||||||
|
"host.cpu",
|
||||||
|
Duration::from_secs(1),
|
||||||
|
);
|
||||||
|
submit_sampler_sample_health(
|
||||||
|
&producer,
|
||||||
|
health_channel,
|
||||||
|
context,
|
||||||
|
("cpu", "host.cpu"),
|
||||||
|
0,
|
||||||
|
None,
|
||||||
|
&state,
|
||||||
|
);
|
||||||
|
submit_sampler_sample_health(
|
||||||
|
&producer,
|
||||||
|
health_channel,
|
||||||
|
context,
|
||||||
|
("cpu", "host.cpu"),
|
||||||
|
1,
|
||||||
|
None,
|
||||||
|
&state,
|
||||||
|
);
|
||||||
|
submit_sampler_sample_health(
|
||||||
|
&producer,
|
||||||
|
health_channel,
|
||||||
|
context,
|
||||||
|
("cpu", "host.cpu"),
|
||||||
|
2,
|
||||||
|
Some("unavailable"),
|
||||||
|
&state,
|
||||||
|
);
|
||||||
|
submit_sampler_sample_health(
|
||||||
|
&producer,
|
||||||
|
health_channel,
|
||||||
|
context,
|
||||||
|
("cpu", "host.cpu"),
|
||||||
|
3,
|
||||||
|
Some("still unavailable"),
|
||||||
|
&state,
|
||||||
|
);
|
||||||
|
submit_sampler_sample_health(
|
||||||
|
&producer,
|
||||||
|
health_channel,
|
||||||
|
context,
|
||||||
|
("cpu", "host.cpu"),
|
||||||
|
4,
|
||||||
|
None,
|
||||||
|
&state,
|
||||||
|
);
|
||||||
|
endpoint.tick();
|
||||||
|
|
||||||
|
let statuses = subscription
|
||||||
|
.drain_available()
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|event| match event {
|
||||||
|
TelemetryEvent::Frame(delivery) if delivery.channel.channel == health_channel => {
|
||||||
|
serde_json::from_slice::<Value>(&delivery.payload)
|
||||||
|
.ok()
|
||||||
|
.and_then(|value| value["status"].as_str().map(str::to_owned))
|
||||||
|
}
|
||||||
|
_ => None,
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
assert_eq!(
|
||||||
|
statuses,
|
||||||
|
["started", "waiting", "ready", "failed", "recovered"]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn hardware_sampler_runs_as_engine_task_not_actor() {
|
fn hardware_sampler_runs_as_engine_task_not_actor() {
|
||||||
|
|
|
||||||
|
|
@ -2354,6 +2354,7 @@ fn serve_cluster_flush_reply(reply: ManualControlReply) -> Option<ServeClusterMs
|
||||||
ManualControlReply::Accepted(_)
|
ManualControlReply::Accepted(_)
|
||||||
| ManualControlReply::Provider(_)
|
| ManualControlReply::Provider(_)
|
||||||
| ManualControlReply::Status(_)
|
| ManualControlReply::Status(_)
|
||||||
|
| ManualControlReply::FleetStatus(_)
|
||||||
| ManualControlReply::Offers(_)
|
| ManualControlReply::Offers(_)
|
||||||
| ManualControlReply::Rejoined(_) => None,
|
| ManualControlReply::Rejoined(_) => None,
|
||||||
}
|
}
|
||||||
|
|
@ -2788,6 +2789,10 @@ fn emit_swim_probe_events(
|
||||||
let record = stack.swim_probe_event_record(event, local_phase);
|
let record = stack.swim_probe_event_record(event, local_phase);
|
||||||
orch_telemetry.emit_record(dashboard, &record);
|
orch_telemetry.emit_record(dashboard, &record);
|
||||||
}
|
}
|
||||||
|
if let Some(summary) = stack.drain_swim_probe_summary() {
|
||||||
|
let record = stack.swim_probe_summary_record(summary, local_phase);
|
||||||
|
orch_telemetry.emit_record(dashboard, &record);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn env_optional(name: &str) -> Option<String> {
|
pub(crate) fn env_optional(name: &str) -> Option<String> {
|
||||||
|
|
|
||||||
|
|
@ -429,6 +429,7 @@ pub(crate) fn plugin(
|
||||||
let routes = Router::new()
|
let routes = Router::new()
|
||||||
.route(FLEET_CONTROL_SCRIPT_URL, get(fleet_control_script))
|
.route(FLEET_CONTROL_SCRIPT_URL, get(fleet_control_script))
|
||||||
.route("/api/control/status", get(status))
|
.route("/api/control/status", get(status))
|
||||||
|
.route("/api/control/fleet", get(fleet_status))
|
||||||
.route("/api/control/actors", get(actor_stats))
|
.route("/api/control/actors", get(actor_stats))
|
||||||
.route("/api/control/provision", post(provision))
|
.route("/api/control/provision", post(provision))
|
||||||
.route("/api/control/kill", post(kill))
|
.route("/api/control/kill", post(kill))
|
||||||
|
|
@ -645,6 +646,37 @@ async fn status(State(state): State<ControlHttpState>) -> Response {
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn fleet_status(State(state): State<ControlHttpState>) -> Response {
|
||||||
|
let response_rx = match begin_request_reply(&state, CONTROL_REPLY_TIMEOUT, |reply_to| {
|
||||||
|
ManualControlMsg::QueryFleet { reply_to }
|
||||||
|
}) {
|
||||||
|
Ok(response_rx) => response_rx,
|
||||||
|
Err(response) => return *response,
|
||||||
|
};
|
||||||
|
match response_rx.await {
|
||||||
|
Ok(ManualControlReply::FleetStatus(model)) => {
|
||||||
|
Json(ManualControlReply::FleetStatus(model)).into_response()
|
||||||
|
}
|
||||||
|
Ok(ManualControlReply::Rejected(error)) => {
|
||||||
|
(StatusCode::CONFLICT, Json(ErrorResponse { error })).into_response()
|
||||||
|
}
|
||||||
|
Ok(_) => (
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
Json(ErrorResponse {
|
||||||
|
error: "manual control returned an unexpected Fleet status reply".to_owned(),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
|
Err(error) => (
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
Json(ErrorResponse {
|
||||||
|
error: format!("control reply observer stopped: {error}"),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn actor_stats(State(state): State<ControlHttpState>) -> impl IntoResponse {
|
async fn actor_stats(State(state): State<ControlHttpState>) -> impl IntoResponse {
|
||||||
Json(state.runtime.stats())
|
Json(state.runtime.stats())
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -29,9 +29,10 @@ use distribution::registry_actor::{RegistryActor, RegistryIn, RegistryView};
|
||||||
use distribution::swim::actor::{MembershipChanged, SwimActor, SwimIn};
|
use distribution::swim::actor::{MembershipChanged, SwimActor, SwimIn};
|
||||||
use distribution::swim::member_list::MemberList;
|
use distribution::swim::member_list::MemberList;
|
||||||
use distribution::swim::probe::SwimConfig;
|
use distribution::swim::probe::SwimConfig;
|
||||||
use distribution::swim::telemetry::{ObservedProbeEvent, ObservedTransition, SwimTelemetry};
|
use distribution::swim::telemetry::{
|
||||||
use distribution::telemetry::MembershipTransition;
|
ObservedProbeEvent, ObservedProbeSummary, ObservedTransition, SwimTelemetry,
|
||||||
use distribution::telemetry::SwimProbeEvent;
|
};
|
||||||
|
use distribution::telemetry::{MembershipTransition, SwimProbeEvent, SwimProbeSummary};
|
||||||
use distribution::transport_bridge::{
|
use distribution::transport_bridge::{
|
||||||
Outbox, OutboxPeerDirectory, OutboxRouteBinder, RelayMirror, RouteView, RouteViewTransport,
|
Outbox, OutboxPeerDirectory, OutboxRouteBinder, RelayMirror, RouteView, RouteViewTransport,
|
||||||
};
|
};
|
||||||
|
|
@ -314,6 +315,10 @@ impl DistributionRuntimeStack {
|
||||||
self.swim_telemetry.drain_probe_events()
|
self.swim_telemetry.drain_probe_events()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn drain_swim_probe_summary(&self) -> Option<ObservedProbeSummary> {
|
||||||
|
self.swim_telemetry.drain_probe_summary()
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn swim_recent_probe_targets(&self) -> Vec<String> {
|
pub(crate) fn swim_recent_probe_targets(&self) -> Vec<String> {
|
||||||
self.swim_telemetry
|
self.swim_telemetry
|
||||||
.recent_targets()
|
.recent_targets()
|
||||||
|
|
@ -353,6 +358,28 @@ impl DistributionRuntimeStack {
|
||||||
lifeguard_enabled: config.lifeguard.is_some(),
|
lifeguard_enabled: config.lifeguard.is_some(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn swim_probe_summary_record(
|
||||||
|
&self,
|
||||||
|
summary: ObservedProbeSummary,
|
||||||
|
local_phase: &str,
|
||||||
|
) -> SwimProbeSummary {
|
||||||
|
SwimProbeSummary {
|
||||||
|
event: "summary".to_owned(),
|
||||||
|
interval_ms: summary.interval_ms,
|
||||||
|
sent: summary.sent,
|
||||||
|
acked: summary.acked,
|
||||||
|
direct_sent: summary.direct_sent,
|
||||||
|
indirect_sent: summary.indirect_sent,
|
||||||
|
rtt_samples: summary.rtt_samples,
|
||||||
|
rtt_ms_min: summary.rtt_ms_min,
|
||||||
|
rtt_ms_p50: summary.rtt_ms_p50,
|
||||||
|
rtt_ms_p95: summary.rtt_ms_p95,
|
||||||
|
rtt_ms_max: summary.rtt_ms_max,
|
||||||
|
local_phase: local_phase.to_owned(),
|
||||||
|
probe_interval_ms: duration_ms_u64(self.swim_config.probe_interval),
|
||||||
|
}
|
||||||
|
}
|
||||||
pub(crate) fn membership_transition(
|
pub(crate) fn membership_transition(
|
||||||
&self,
|
&self,
|
||||||
transition: &ObservedTransition,
|
transition: &ObservedTransition,
|
||||||
|
|
|
||||||
|
|
@ -1,15 +1,28 @@
|
||||||
(() => {
|
(() => {
|
||||||
const CONTROL_ID = 'myelin-fleet-control';
|
const NODE_CONTROL_ID = 'myelin-fleet-control';
|
||||||
|
const BULK_CONTROL_ID = 'myelin-fleet-bulk-control';
|
||||||
const CONFIRM_ID = 'myelin-confirm-dialog';
|
const CONFIRM_ID = 'myelin-confirm-dialog';
|
||||||
const CONFIRM_STYLE_ID = 'myelin-confirm-dialog-style';
|
const STYLE_ID = 'myelin-fleet-control-style';
|
||||||
const selectedJobs = new Map();
|
const selectedJobs = new Map();
|
||||||
|
const selectedNodes = new Set();
|
||||||
|
let lastModel = null;
|
||||||
|
let syncing = false;
|
||||||
|
const page = document.getElementById('page');
|
||||||
|
if (!page) return;
|
||||||
|
|
||||||
function confirmKill(logicalNodeId) {
|
function installUi() {
|
||||||
let dialog = document.getElementById(CONFIRM_ID);
|
if (document.getElementById(STYLE_ID)) return;
|
||||||
if (!dialog) {
|
|
||||||
const style = document.createElement('style');
|
const style = document.createElement('style');
|
||||||
style.id = CONFIRM_STYLE_ID;
|
style.id = STYLE_ID;
|
||||||
style.textContent = `
|
style.textContent = `
|
||||||
|
.myelin-control { display:flex;align-items:center;flex-wrap:wrap;gap:10px;margin:0 0 12px }
|
||||||
|
.myelin-control button { border:1px solid var(--cyan);background:transparent;color:var(--cyan);border-radius:var(--r);padding:6px 10px;font:600 12px var(--mono);cursor:pointer }
|
||||||
|
.myelin-control button.danger { border-color:var(--bad);color:var(--bad) }
|
||||||
|
.myelin-control button:disabled { cursor:not-allowed;opacity:.55 }
|
||||||
|
.myelin-control-message { color:var(--muted);font:12px var(--mono) }
|
||||||
|
.myelin-bulk-control { width:fit-content;margin:0 0 12px auto;padding:6px 8px;border:1px solid var(--divider);background:transparent;border-radius:var(--r) }
|
||||||
|
.node-card[data-bulk-selected="true"] { border-color:var(--bad);background:var(--selected);box-shadow:inset 3px 0 0 var(--bad) }
|
||||||
|
.node-card[data-bulk-killable="true"] { user-select:none }
|
||||||
.myelin-confirm { width:min(440px,calc(100vw - 32px));padding:0;color:var(--text);background:var(--panel);border:1px solid var(--bad);border-radius:var(--r);box-shadow:0 18px 60px rgba(0,0,0,.55) }
|
.myelin-confirm { width:min(440px,calc(100vw - 32px));padding:0;color:var(--text);background:var(--panel);border:1px solid var(--bad);border-radius:var(--r);box-shadow:0 18px 60px rgba(0,0,0,.55) }
|
||||||
.myelin-confirm::backdrop { background:rgba(0,6,12,.78) }
|
.myelin-confirm::backdrop { background:rgba(0,6,12,.78) }
|
||||||
.myelin-confirm form { display:grid;gap:14px;padding:18px }
|
.myelin-confirm form { display:grid;gap:14px;padding:18px }
|
||||||
|
|
@ -20,23 +33,29 @@
|
||||||
.myelin-confirm button[value="confirm"] { color:var(--danger-ink);background:var(--danger-fill);border-color:var(--danger-border) }
|
.myelin-confirm button[value="confirm"] { color:var(--danger-ink);background:var(--danger-fill);border-color:var(--danger-border) }
|
||||||
`;
|
`;
|
||||||
document.head.append(style);
|
document.head.append(style);
|
||||||
|
}
|
||||||
|
|
||||||
|
function confirmTermination(title, message, action) {
|
||||||
|
let dialog = document.getElementById(CONFIRM_ID);
|
||||||
|
if (!dialog) {
|
||||||
dialog = document.createElement('dialog');
|
dialog = document.createElement('dialog');
|
||||||
dialog.id = CONFIRM_ID;
|
dialog.id = CONFIRM_ID;
|
||||||
dialog.className = 'myelin-confirm';
|
dialog.className = 'myelin-confirm';
|
||||||
dialog.setAttribute('aria-labelledby', 'myelin-confirm-title');
|
dialog.setAttribute('aria-labelledby', 'myelin-confirm-title');
|
||||||
dialog.setAttribute('aria-describedby', 'myelin-confirm-message');
|
dialog.setAttribute('aria-describedby', 'myelin-confirm-message');
|
||||||
dialog.innerHTML = `<form method="dialog">
|
dialog.innerHTML = `<form method="dialog">
|
||||||
<h2 id="myelin-confirm-title">Terminate managed node?</h2>
|
<h2 id="myelin-confirm-title"></h2>
|
||||||
<p id="myelin-confirm-message"></p>
|
<p id="myelin-confirm-message"></p>
|
||||||
<div class="myelin-confirm-actions">
|
<div class="myelin-confirm-actions">
|
||||||
<button value="cancel" autofocus>Cancel</button>
|
<button value="cancel" autofocus>Cancel</button>
|
||||||
<button value="confirm">Terminate node</button>
|
<button value="confirm"></button>
|
||||||
</div>
|
</div>
|
||||||
</form>`;
|
</form>`;
|
||||||
document.body.append(dialog);
|
document.body.append(dialog);
|
||||||
}
|
}
|
||||||
dialog.querySelector('#myelin-confirm-message').textContent =
|
dialog.querySelector('#myelin-confirm-title').textContent = title;
|
||||||
`Terminate managed node ${logicalNodeId}? Vast.ai contracts are destroyed and billing stops.`;
|
dialog.querySelector('#myelin-confirm-message').textContent = message;
|
||||||
|
dialog.querySelector('[value="confirm"]').textContent = action;
|
||||||
dialog.returnValue = 'cancel';
|
dialog.returnValue = 'cancel';
|
||||||
return new Promise(resolve => {
|
return new Promise(resolve => {
|
||||||
dialog.addEventListener('close', () => resolve(dialog.returnValue === 'confirm'), { once: true });
|
dialog.addEventListener('close', () => resolve(dialog.returnValue === 'confirm'), { once: true });
|
||||||
|
|
@ -44,28 +63,152 @@
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async function syncControl() {
|
function killable(node) {
|
||||||
let model;
|
return node && !['kill_requested', 'stopping', 'stopped', 'orphan'].includes(node.phase);
|
||||||
|
}
|
||||||
|
|
||||||
|
function managedNodes(model) {
|
||||||
|
return Array.isArray(model?.nodes) ? model.nodes : [];
|
||||||
|
}
|
||||||
|
|
||||||
|
function commandId(logicalNodeId) {
|
||||||
|
return `fleet-kill-${logicalNodeId}-${globalThis.crypto?.randomUUID?.() || Date.now()}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function submitKills(logicalNodeIds) {
|
||||||
|
return Promise.all(logicalNodeIds.map(async logicalNodeId => {
|
||||||
try {
|
try {
|
||||||
const response = await fetch('/api/control/status', { cache: 'no-store' });
|
const response = await fetch(`/api/control/nodes/${logicalNodeId}/kill`, {
|
||||||
if (!response.ok) return;
|
method: 'POST',
|
||||||
const payload = await response.json();
|
headers: { 'content-type': 'application/json' },
|
||||||
model = payload.Status;
|
body: JSON.stringify({ command_id: commandId(logicalNodeId) }),
|
||||||
} catch (_) {
|
});
|
||||||
|
if (!response.ok) {
|
||||||
|
const body = await response.json().catch(() => ({}));
|
||||||
|
throw new Error(body.error || `HTTP ${response.status}`);
|
||||||
|
}
|
||||||
|
return { logicalNodeId, error: null };
|
||||||
|
} catch (error) {
|
||||||
|
return { logicalNodeId, error: error.message };
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
function resultMessage(results) {
|
||||||
|
const failed = results.filter(result => result.error);
|
||||||
|
if (!failed.length) return `Termination accepted for ${results.length} node${results.length === 1 ? '' : 's'}`;
|
||||||
|
const accepted = results.length - failed.length;
|
||||||
|
const failures = failed.map(result => `${result.logicalNodeId}: ${result.error}`).join('; ');
|
||||||
|
return `${accepted} accepted; ${failed.length} failed — ${failures}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function syncFleetControl(model) {
|
||||||
|
const page = document.getElementById('page');
|
||||||
|
const cards = [...page.querySelectorAll('a.node-card[data-node]')];
|
||||||
|
if (!cards.length) {
|
||||||
|
document.getElementById(BULK_CONTROL_ID)?.remove();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
window.dispatchEvent(new CustomEvent('dashboard-hardware-source', {
|
const byId = new Map(managedNodes(model).map(node => [node.logical_node_id, node]));
|
||||||
detail: {
|
for (const logicalNodeId of [...selectedNodes]) {
|
||||||
source: model?.provider?.provisioning_mode === 'mock' ? 'orchestrator' : 'node',
|
if (!killable(byId.get(logicalNodeId))) selectedNodes.delete(logicalNodeId);
|
||||||
},
|
}
|
||||||
}));
|
for (const card of cards) {
|
||||||
|
const logicalNodeId = Number(card.dataset.node);
|
||||||
|
const selectable = card.dataset.origin !== 'orchestrator'
|
||||||
|
&& Number.isSafeInteger(logicalNodeId)
|
||||||
|
&& killable(byId.get(logicalNodeId));
|
||||||
|
card.dataset.bulkKillable = String(selectable);
|
||||||
|
card.dataset.bulkSelected = String(selectable && selectedNodes.has(logicalNodeId));
|
||||||
|
card.setAttribute('aria-selected', String(selectable && selectedNodes.has(logicalNodeId)));
|
||||||
|
card.title = selectable ? 'Open node; Shift+click to select for termination' : '';
|
||||||
|
}
|
||||||
|
|
||||||
const nodeView = document.querySelector('.node-view[data-node]');
|
const count = selectedNodes.size;
|
||||||
if (!nodeView) return;
|
let control = document.getElementById(BULK_CONTROL_ID);
|
||||||
const rawNodeId = nodeView.getAttribute('data-node') || '';
|
if (!count) {
|
||||||
if (!/^\d+$/.test(rawNodeId)) return;
|
control?.remove();
|
||||||
const logicalNodeId = Number(rawNodeId);
|
return;
|
||||||
const node = model?.nodes?.find(candidate => candidate.logical_node_id === logicalNodeId);
|
}
|
||||||
|
if (!control) {
|
||||||
|
control = document.createElement('div');
|
||||||
|
control.id = BULK_CONTROL_ID;
|
||||||
|
control.className = 'myelin-control myelin-bulk-control';
|
||||||
|
control.innerHTML = `<span data-selection></span>
|
||||||
|
<button type="button" class="danger" data-action="kill-selected">Terminate selected</button>
|
||||||
|
<span class="myelin-control-message" data-message role="status"></span>`;
|
||||||
|
page.querySelector('.grid').before(control);
|
||||||
|
control.querySelector('[data-action="kill-selected"]').onclick = killSelected;
|
||||||
|
}
|
||||||
|
control.querySelector('[data-selection]').textContent =
|
||||||
|
`${count} worker node${count === 1 ? '' : 's'} selected`;
|
||||||
|
const button = control.querySelector('[data-action="kill-selected"]');
|
||||||
|
button.disabled = false;
|
||||||
|
button.textContent = `Terminate ${count} selected`;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async function killSelected() {
|
||||||
|
const control = document.getElementById(BULK_CONTROL_ID);
|
||||||
|
const button = control?.querySelector('[data-action="kill-selected"]');
|
||||||
|
const message = control?.querySelector('[data-message]');
|
||||||
|
const byId = new Map(managedNodes(lastModel).map(node => [node.logical_node_id, node]));
|
||||||
|
const ids = [...selectedNodes].filter(logicalNodeId => killable(byId.get(logicalNodeId)));
|
||||||
|
if (!ids.length) return;
|
||||||
|
const label = ids.join(', ');
|
||||||
|
if (!await confirmTermination(
|
||||||
|
`Terminate ${ids.length} managed node${ids.length === 1 ? '' : 's'}?`,
|
||||||
|
`Terminate managed node${ids.length === 1 ? '' : 's'} ${label}? Vast.ai contracts are destroyed and billing stops.`,
|
||||||
|
`Terminate ${ids.length}`,
|
||||||
|
)) return;
|
||||||
|
button.disabled = true;
|
||||||
|
message.textContent = 'Submitting termination requests…';
|
||||||
|
const results = await submitKills(ids);
|
||||||
|
for (const result of results) {
|
||||||
|
if (!result.error) selectedNodes.delete(result.logicalNodeId);
|
||||||
|
}
|
||||||
|
message.textContent = resultMessage(results);
|
||||||
|
syncFleetControl(lastModel);
|
||||||
|
}
|
||||||
|
|
||||||
|
function ensureOrchestratorControl(nodeView, model) {
|
||||||
|
const heading = nodeView.querySelector('section.panel > h2');
|
||||||
|
if (!heading) return;
|
||||||
|
let control = document.getElementById(NODE_CONTROL_ID);
|
||||||
|
if (!control) {
|
||||||
|
control = document.createElement('div');
|
||||||
|
control.id = NODE_CONTROL_ID;
|
||||||
|
control.className = 'myelin-control';
|
||||||
|
control.innerHTML = `<button type="button" class="danger" data-action="shutdown-all">Shutdown all</button>
|
||||||
|
<span class="myelin-control-message" data-message role="status"></span>`;
|
||||||
|
heading.after(control);
|
||||||
|
control.querySelector('[data-action="shutdown-all"]').onclick = shutdownAll;
|
||||||
|
}
|
||||||
|
const count = managedNodes(model).filter(killable).length;
|
||||||
|
const button = control.querySelector('[data-action="shutdown-all"]');
|
||||||
|
button.disabled = count === 0;
|
||||||
|
button.textContent = count ? `Shutdown all (${count})` : 'No managed nodes to shut down';
|
||||||
|
}
|
||||||
|
|
||||||
|
async function shutdownAll() {
|
||||||
|
const control = document.getElementById(NODE_CONTROL_ID);
|
||||||
|
const button = control?.querySelector('[data-action="shutdown-all"]');
|
||||||
|
const message = control?.querySelector('[data-message]');
|
||||||
|
const ids = managedNodes(lastModel).filter(killable).map(node => node.logical_node_id);
|
||||||
|
if (!ids.length) return;
|
||||||
|
if (!await confirmTermination(
|
||||||
|
`Shut down all ${ids.length} managed node${ids.length === 1 ? '' : 's'}?`,
|
||||||
|
`This terminates managed nodes ${ids.join(', ')}. Vast.ai contracts are destroyed and billing stops.`,
|
||||||
|
'Shutdown all',
|
||||||
|
)) return;
|
||||||
|
button.disabled = true;
|
||||||
|
message.textContent = 'Submitting termination requests…';
|
||||||
|
const results = await submitKills(ids);
|
||||||
|
message.textContent = resultMessage(results);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function syncManagedNodeControl(nodeView, model, logicalNodeId) {
|
||||||
|
const node = managedNodes(model).find(candidate => candidate.logical_node_id === logicalNodeId);
|
||||||
if (!node) return;
|
if (!node) return;
|
||||||
|
|
||||||
let job = { state: 'idle', message: 'No job submitted' };
|
let job = { state: 'idle', message: 'No job submitted' };
|
||||||
|
|
@ -76,34 +219,16 @@
|
||||||
|
|
||||||
const heading = nodeView.querySelector('section.panel > h2');
|
const heading = nodeView.querySelector('section.panel > h2');
|
||||||
if (!heading) return;
|
if (!heading) return;
|
||||||
let control = document.getElementById(CONTROL_ID);
|
let control = document.getElementById(NODE_CONTROL_ID);
|
||||||
if (!control) {
|
if (!control) {
|
||||||
control = document.createElement('div');
|
control = document.createElement('div');
|
||||||
control.id = CONTROL_ID;
|
control.id = NODE_CONTROL_ID;
|
||||||
control.style.cssText = 'display:flex;align-items:center;flex-wrap:wrap;gap:10px;margin:0 0 12px';
|
control.className = 'myelin-control';
|
||||||
|
control.innerHTML = `<input type="file" accept=".toml,text/plain,application/toml" data-job-file>
|
||||||
const fileInput = document.createElement('input');
|
<button type="button" data-action="job">Submit job</button>
|
||||||
fileInput.type = 'file';
|
<button type="button" class="danger" data-action="kill">Kill</button>
|
||||||
fileInput.accept = '.toml,text/plain,application/toml';
|
<span class="myelin-control-message" data-message role="status"></span>`;
|
||||||
fileInput.dataset.jobFile = '';
|
control.querySelector('[data-job-file]').style.cssText = 'max-width:260px;color:var(--muted);font:12px var(--mono)';
|
||||||
fileInput.style.cssText = 'max-width:260px;color:var(--muted);font:12px ui-monospace,SFMono-Regular,Menlo,Consolas,monospace';
|
|
||||||
|
|
||||||
const jobButton = document.createElement('button');
|
|
||||||
jobButton.type = 'button';
|
|
||||||
jobButton.dataset.action = 'job';
|
|
||||||
jobButton.textContent = 'Submit job';
|
|
||||||
jobButton.style.cssText = 'border:1px solid #5cd5ff;background:transparent;color:#5cd5ff;border-radius:3px;padding:6px 10px;font:600 12px ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;cursor:pointer';
|
|
||||||
|
|
||||||
const killButton = document.createElement('button');
|
|
||||||
killButton.type = 'button';
|
|
||||||
killButton.dataset.action = 'kill';
|
|
||||||
killButton.style.cssText = 'border:1px solid #ef4444;background:transparent;color:#ef4444;border-radius:3px;padding:6px 10px;font:600 12px ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;cursor:pointer';
|
|
||||||
|
|
||||||
const message = document.createElement('span');
|
|
||||||
message.dataset.message = '';
|
|
||||||
message.style.cssText = 'font:12px ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;color:var(--muted)';
|
|
||||||
|
|
||||||
control.append(fileInput, jobButton, killButton, message);
|
|
||||||
heading.after(control);
|
heading.after(control);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -124,17 +249,11 @@
|
||||||
killButton.hidden = terminal;
|
killButton.hidden = terminal;
|
||||||
killButton.disabled = pending;
|
killButton.disabled = pending;
|
||||||
killButton.textContent = pending ? 'Kill requested' : 'Kill';
|
killButton.textContent = pending ? 'Kill requested' : 'Kill';
|
||||||
killButton.style.opacity = pending ? '.55' : '1';
|
|
||||||
|
|
||||||
if (job.state !== 'idle') {
|
if (job.state !== 'idle') message.textContent = job.message;
|
||||||
message.textContent = job.message;
|
else if (selectedJob) message.textContent = `Selected ${selectedJob.name}`;
|
||||||
} else if (selectedJob) {
|
else if (terminal) message.textContent = `managed node ${logicalNodeId}: ${node.phase}`;
|
||||||
message.textContent = `Selected ${selectedJob.name}`;
|
else message.textContent = 'Select a job TOML file';
|
||||||
} else if (terminal) {
|
|
||||||
message.textContent = `managed node ${logicalNodeId}: ${node.phase}`;
|
|
||||||
} else {
|
|
||||||
message.textContent = 'Select a job TOML file';
|
|
||||||
}
|
|
||||||
|
|
||||||
fileInput.onchange = async () => {
|
fileInput.onchange = async () => {
|
||||||
const file = fileInput.files?.[0];
|
const file = fileInput.files?.[0];
|
||||||
|
|
@ -145,8 +264,7 @@
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
const text = await file.text();
|
selectedJobs.set(logicalNodeId, { name: file.name, text: await file.text() });
|
||||||
selectedJobs.set(logicalNodeId, { name: file.name, text });
|
|
||||||
jobButton.disabled = jobActive;
|
jobButton.disabled = jobActive;
|
||||||
message.textContent = `Selected ${file.name}`;
|
message.textContent = `Selected ${file.name}`;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|
@ -177,30 +295,76 @@
|
||||||
};
|
};
|
||||||
|
|
||||||
killButton.onclick = async () => {
|
killButton.onclick = async () => {
|
||||||
if (!await confirmKill(logicalNodeId)) return;
|
if (!await confirmTermination(
|
||||||
|
'Terminate managed node?',
|
||||||
|
`Terminate managed node ${logicalNodeId}? Vast.ai contracts are destroyed and billing stops.`,
|
||||||
|
'Terminate node',
|
||||||
|
)) return;
|
||||||
killButton.disabled = true;
|
killButton.disabled = true;
|
||||||
message.textContent = 'Submitting kill…';
|
message.textContent = 'Submitting kill…';
|
||||||
const commandId = `fleet-kill-${globalThis.crypto?.randomUUID?.() || Date.now()}`;
|
const [result] = await submitKills([logicalNodeId]);
|
||||||
try {
|
message.textContent = result.error ? result.error : 'Kill accepted';
|
||||||
const response = await fetch(`/api/control/nodes/${logicalNodeId}/kill`, {
|
if (result.error) killButton.disabled = false;
|
||||||
method: 'POST',
|
|
||||||
headers: { 'content-type': 'application/json' },
|
|
||||||
body: JSON.stringify({ command_id: commandId }),
|
|
||||||
});
|
|
||||||
if (!response.ok) {
|
|
||||||
const body = await response.json().catch(() => ({}));
|
|
||||||
throw new Error(body.error || `HTTP ${response.status}`);
|
|
||||||
}
|
|
||||||
message.textContent = 'Kill accepted';
|
|
||||||
} catch (error) {
|
|
||||||
killButton.disabled = false;
|
|
||||||
message.textContent = error.message;
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const page = document.getElementById('page');
|
async function syncControl() {
|
||||||
if (page) new MutationObserver(syncControl).observe(page, { childList: true, subtree: true });
|
if (syncing) return;
|
||||||
|
syncing = true;
|
||||||
|
try {
|
||||||
|
let model;
|
||||||
|
try {
|
||||||
|
const response = await fetch('/api/control/fleet', { cache: 'no-store' });
|
||||||
|
if (!response.ok) return;
|
||||||
|
model = (await response.json()).FleetStatus;
|
||||||
|
} catch (_) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
lastModel = model;
|
||||||
|
window.dispatchEvent(new CustomEvent('dashboard-hardware-source', {
|
||||||
|
detail: { source: model?.provider?.provisioning_mode === 'mock' ? 'orchestrator' : 'node' },
|
||||||
|
}));
|
||||||
|
|
||||||
|
const nodeView = document.querySelector('.node-view[data-node]');
|
||||||
|
if (!nodeView) {
|
||||||
|
syncFleetControl(model);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (nodeView.dataset.origin === 'orchestrator') {
|
||||||
|
ensureOrchestratorControl(nodeView, model);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const rawNodeId = nodeView.dataset.node || '';
|
||||||
|
if (/^\d+$/.test(rawNodeId)) await syncManagedNodeControl(nodeView, model, Number(rawNodeId));
|
||||||
|
} finally {
|
||||||
|
syncing = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
installUi();
|
||||||
|
page.addEventListener('click', event => {
|
||||||
|
const card = event.target.closest('a.node-card[data-node]');
|
||||||
|
if (!card || !page.contains(card) || !event.shiftKey) return;
|
||||||
|
const logicalNodeId = Number(card.dataset.node);
|
||||||
|
const node = managedNodes(lastModel).find(candidate => candidate.logical_node_id === logicalNodeId);
|
||||||
|
const selectable = card.dataset.origin !== 'orchestrator'
|
||||||
|
&& Number.isSafeInteger(logicalNodeId)
|
||||||
|
&& killable(node);
|
||||||
|
if (!selectable) return;
|
||||||
|
event.preventDefault();
|
||||||
|
if (selectedNodes.has(logicalNodeId)) selectedNodes.delete(logicalNodeId);
|
||||||
|
else selectedNodes.add(logicalNodeId);
|
||||||
|
syncFleetControl(lastModel);
|
||||||
|
});
|
||||||
|
new MutationObserver(mutations => {
|
||||||
|
const dashboardChanged = mutations.some(mutation => {
|
||||||
|
const element = mutation.target.nodeType === Node.ELEMENT_NODE
|
||||||
|
? mutation.target
|
||||||
|
: mutation.target.parentElement;
|
||||||
|
return !element?.closest(`#${NODE_CONTROL_ID}, #${BULK_CONTROL_ID}`);
|
||||||
|
});
|
||||||
|
if (dashboardChanged) syncControl();
|
||||||
|
}).observe(page, { childList: true, subtree: true });
|
||||||
syncControl();
|
syncControl();
|
||||||
setInterval(syncControl, 1000);
|
setInterval(syncControl, 1000);
|
||||||
})();
|
})();
|
||||||
|
|
|
||||||
|
|
@ -262,6 +262,12 @@ pub(crate) struct ManualReadModel {
|
||||||
pub nodes: Vec<SnapshotNode>,
|
pub nodes: Vec<SnapshotNode>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub(crate) struct FleetReadModel {
|
||||||
|
pub provider: ProviderReadiness,
|
||||||
|
pub nodes: Vec<SnapshotNode>,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub(crate) enum EffectKind {
|
pub(crate) enum EffectKind {
|
||||||
Create,
|
Create,
|
||||||
|
|
@ -430,6 +436,13 @@ impl ManualControl {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn fleet_read_model(&self) -> FleetReadModel {
|
||||||
|
FleetReadModel {
|
||||||
|
provider: self.provider.clone(),
|
||||||
|
nodes: self.snapshot.nodes.clone(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn request_provision<F>(
|
pub(crate) fn request_provision<F>(
|
||||||
&mut self,
|
&mut self,
|
||||||
request: ProvisionRequest,
|
request: ProvisionRequest,
|
||||||
|
|
@ -1297,6 +1310,9 @@ pub(crate) enum ManualControlMsg {
|
||||||
Query {
|
Query {
|
||||||
reply_to: ActorAddress,
|
reply_to: ActorAddress,
|
||||||
},
|
},
|
||||||
|
QueryFleet {
|
||||||
|
reply_to: ActorAddress,
|
||||||
|
},
|
||||||
Flush {
|
Flush {
|
||||||
reply_to: ActorAddress,
|
reply_to: ActorAddress,
|
||||||
},
|
},
|
||||||
|
|
@ -1329,6 +1345,7 @@ pub(crate) enum ManualControlReply {
|
||||||
Accepted(CommandRecord),
|
Accepted(CommandRecord),
|
||||||
Provider(ProviderReadiness),
|
Provider(ProviderReadiness),
|
||||||
Status(ManualReadModel),
|
Status(ManualReadModel),
|
||||||
|
FleetStatus(FleetReadModel),
|
||||||
Offers(Vec<OfferDto>),
|
Offers(Vec<OfferDto>),
|
||||||
Rejoined(RejoinBinding),
|
Rejoined(RejoinBinding),
|
||||||
Flushed,
|
Flushed,
|
||||||
|
|
@ -1702,6 +1719,12 @@ impl ManualActorControl {
|
||||||
ManualControlMsg::Query { reply_to } => {
|
ManualControlMsg::Query { reply_to } => {
|
||||||
let _ = ctx.send(reply_to, ManualControlReply::Status(self.core.read_model()));
|
let _ = ctx.send(reply_to, ManualControlReply::Status(self.core.read_model()));
|
||||||
}
|
}
|
||||||
|
ManualControlMsg::QueryFleet { reply_to } => {
|
||||||
|
let _ = ctx.send(
|
||||||
|
reply_to,
|
||||||
|
ManualControlReply::FleetStatus(self.core.fleet_read_model()),
|
||||||
|
);
|
||||||
|
}
|
||||||
ManualControlMsg::PersistenceFinished { generation, error } => {
|
ManualControlMsg::PersistenceFinished { generation, error } => {
|
||||||
if self.persistence_in_flight != Some(generation) {
|
if self.persistence_in_flight != Some(generation) {
|
||||||
return;
|
return;
|
||||||
|
|
|
||||||
|
|
@ -8,4 +8,4 @@ Keep this crate read-only with respect to observed programs.
|
||||||
- It must not send control signals to observed runtimes.
|
- It must not send control signals to observed runtimes.
|
||||||
- It must not require changes outside `crates/dashboard` for dashboard-only work.
|
- It must not require changes outside `crates/dashboard` for dashboard-only work.
|
||||||
|
|
||||||
Main built-in view: the fused control plane at `/` and `/view/fleet` (node cards with machine + actor rollup, per-node roster, per-actor dossier via `/api/view/fleet/detail`), backed by `host.*`, `proc.<label>.lifecycle`, `runtime.stats`, and `runtime.actors` frames when present. It is a pure frame consumer, tolerant of publisher shape. Message history is folded view-side from `messages_processed` deltas - no producer changes. The unified navbar is injected server-side from the view and application plugin-page registries; pages opt in with a `<!--swactor:nav-->` placeholder. Plugin routers remain application-owned composition inputs and must not add dashboard-owned control state.
|
Main built-in view: the fused control plane at `/` and `/view/fleet` (node cards with machine + actor rollup, per-node output tail and roster, per-actor dossier via `/api/view/fleet/detail`), backed by `host.*`, `proc.<label>.*`, `runtime.stats`, and sequenced `runtime.actors` census/vital/activity frames when present. It is a pure frame consumer, tolerant of publisher shape. Message history is folded view-side from `messages_processed` totals and deltas. The unified navbar is injected server-side from the view and application plugin-page registries; pages opt in with a `<!--swactor:nav-->` placeholder. Plugin routers remain application-owned composition inputs and must not add dashboard-owned control state.
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,14 @@ Read-only HTML/SSE dashboard over incoming telemetry frames.
|
||||||
|
|
||||||
The crate owns the Axum server, bounded raw frame window, view registry, and application plugin-page registry. Component crates can keep telemetry views beside their code and register them through `DashboardHandle::register_view`. An embedding application can pass `DashboardPlugin` values to `DashboardHandle::with_plugins`; each plugin contributes an application-owned router and optional `PluginPage` metadata/HTML. The built-in control-plane view is hosted here because worker/actor/message processing is universal to swactor programs.
|
The crate owns the Axum server, bounded raw frame window, view registry, and application plugin-page registry. Component crates can keep telemetry views beside their code and register them through `DashboardHandle::register_view`. An embedding application can pass `DashboardPlugin` values to `DashboardHandle::with_plugins`; each plugin contributes an application-owned router and optional `PluginPage` metadata/HTML. The built-in control-plane view is hosted here because worker/actor/message processing is universal to swactor programs.
|
||||||
|
|
||||||
The control-plane page fuses machine stats and actor stats per node stream: node cards (CPU/GPU/net + actor rollup) → per-node actor roster → per-actor dossier (identity, message diet, sampled message history). The Rust type name is the actor's display name; the address is the unique key. Stale streams (silent beyond the liveness window) render in a separate collapsed pool, superseded `life` generations are evicted immediately, and the stale pool is hard-capped.
|
The control-plane page fuses machine stats, actor telemetry, and a 50-line
|
||||||
|
stdout/stderr tail per node stream: node cards (CPU/GPU/net + actor rollup) →
|
||||||
|
per-node actor roster and output cue → per-actor dossier. Pre-join provisioning
|
||||||
|
output is keyed by run/node identity and merges into the joined runtime card.
|
||||||
|
The Rust type name is the actor's display name; the address is the unique key.
|
||||||
|
Stale streams (silent beyond the liveness window) render in a separate collapsed
|
||||||
|
pool, superseded `life` generations are evicted immediately, and the stale pool
|
||||||
|
is hard-capped at 50.
|
||||||
|
|
||||||
## Routes
|
## Routes
|
||||||
|
|
||||||
|
|
@ -13,7 +20,7 @@ The control-plane page fuses machine stats and actor stats per node stream: node
|
||||||
- `GET /api/frames` — recent raw frame window
|
- `GET /api/frames` — recent raw frame window
|
||||||
- `GET /api/views` — registered view metadata
|
- `GET /api/views` — registered view metadata
|
||||||
- `GET /view/telemetry/live` — generic live explorer over retained and incoming telemetry frames
|
- `GET /view/telemetry/live` — generic live explorer over retained and incoming telemetry frames
|
||||||
- `GET /api/view/telemetry/live` — bounded per-stream/channel explorer snapshot
|
- `GET /api/view/telemetry/live` — 2,000 frames per stream/channel, newest lifetime per node, all live streams plus 50 stale streams
|
||||||
- `GET /view/fleet` — fused control-plane page (machine + actors per node)
|
- `GET /view/fleet` — fused control-plane page (machine + actors per node)
|
||||||
- `GET /api/view/fleet` — live/stale pools with per-node machine and roster snapshot
|
- `GET /api/view/fleet` — live/stale pools with per-node machine and roster snapshot
|
||||||
- `GET /api/view/fleet/detail?stream=<node#life>&actor=<addr>` — bounded per-actor dossier detail (diet, history, sampled receipts)
|
- `GET /api/view/fleet/detail?stream=<node#life>&actor=<addr>` — bounded per-actor dossier detail (diet, history, sampled receipts)
|
||||||
|
|
|
||||||
|
|
@ -13,7 +13,7 @@
|
||||||
//! [`STALE_POOL_CAP`]. A newer `life` generation for the same node evicts
|
//! [`STALE_POOL_CAP`]. A newer `life` generation for the same node evicts
|
||||||
//! older generations immediately — restarts stop accumulating.
|
//! older generations immediately — restarts stop accumulating.
|
||||||
|
|
||||||
use std::collections::BTreeMap;
|
use std::collections::{BTreeMap, VecDeque};
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use parking_lot::RwLock;
|
use parking_lot::RwLock;
|
||||||
|
|
@ -37,6 +37,9 @@ const STALE_POOL_CAP: usize = 50;
|
||||||
/// Stream origin of the process hosting this dashboard. Its card can never
|
/// Stream origin of the process hosting this dashboard. Its card can never
|
||||||
/// go meaningfully stale: if that publisher were silent, no page would render.
|
/// go meaningfully stale: if that publisher were silent, no page would render.
|
||||||
const ORIGIN_ORCHESTRATOR: &str = "orchestrator";
|
const ORIGIN_ORCHESTRATOR: &str = "orchestrator";
|
||||||
|
const OUTPUT_TAIL_CAP: usize = 50;
|
||||||
|
const PROVISIONING_EVENTS: &str = "myelin.provisioning.events";
|
||||||
|
const PROVISIONING_LOG_PREFIX: &str = "myelin.provisioning.logs.node.";
|
||||||
|
|
||||||
/// Fused control-plane view serving `/` and `/view/fleet`.
|
/// Fused control-plane view serving `/` and `/view/fleet`.
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
|
|
@ -54,6 +57,7 @@ struct FusedNode {
|
||||||
last_seen: Instant,
|
last_seen: Instant,
|
||||||
hardware: NodeHardwareState,
|
hardware: NodeHardwareState,
|
||||||
actors: RuntimeState,
|
actors: RuntimeState,
|
||||||
|
output: NodeOutputState,
|
||||||
origin: Option<String>,
|
origin: Option<String>,
|
||||||
label: Option<String>,
|
label: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
@ -64,6 +68,76 @@ impl FusedNode {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct NodeOutputState {
|
||||||
|
lines: VecDeque<NodeOutputLine>,
|
||||||
|
phase: Option<String>,
|
||||||
|
partials: BTreeMap<(String, String), String>,
|
||||||
|
last_output: Option<Instant>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Serialize)]
|
||||||
|
struct NodeOutputLine {
|
||||||
|
source: String,
|
||||||
|
phase: String,
|
||||||
|
text: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl NodeOutputState {
|
||||||
|
fn set_phase(&mut self, phase: impl Into<String>) {
|
||||||
|
self.phase = Some(phase.into());
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_line(&mut self, source: &str, phase: &str, text: impl Into<String>, now: Instant) {
|
||||||
|
if self.lines.len() == OUTPUT_TAIL_CAP {
|
||||||
|
self.lines.pop_front();
|
||||||
|
}
|
||||||
|
self.lines.push_back(NodeOutputLine {
|
||||||
|
source: source.to_owned(),
|
||||||
|
phase: phase.to_owned(),
|
||||||
|
text: text.into(),
|
||||||
|
});
|
||||||
|
self.phase = Some(phase.to_owned());
|
||||||
|
self.last_output = Some(now);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_chunk(&mut self, source: &str, phase: &str, payload: &[u8], now: Instant) {
|
||||||
|
let key = (source.to_owned(), phase.to_owned());
|
||||||
|
let mut buffered = self.partials.remove(&key).unwrap_or_default();
|
||||||
|
buffered.push_str(&String::from_utf8_lossy(payload));
|
||||||
|
while let Some(newline) = buffered.find('\n') {
|
||||||
|
let mut line = buffered.drain(..=newline).collect::<String>();
|
||||||
|
line.truncate(line.trim_end_matches(['\r', '\n']).len());
|
||||||
|
self.push_line(source, phase, line, now);
|
||||||
|
}
|
||||||
|
if !buffered.is_empty() {
|
||||||
|
self.partials.insert(key, buffered);
|
||||||
|
}
|
||||||
|
if !payload.is_empty() {
|
||||||
|
self.phase = Some(phase.to_owned());
|
||||||
|
self.last_output = Some(now);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn snapshot_lines(&self) -> Vec<NodeOutputLine> {
|
||||||
|
let mut lines = self.lines.iter().cloned().collect::<Vec<_>>();
|
||||||
|
lines.extend(
|
||||||
|
self.partials
|
||||||
|
.iter()
|
||||||
|
.filter(|(_, text)| !text.is_empty())
|
||||||
|
.map(|((source, phase), text)| NodeOutputLine {
|
||||||
|
source: source.clone(),
|
||||||
|
phase: phase.clone(),
|
||||||
|
text: text.clone(),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
if lines.len() > OUTPUT_TAIL_CAP {
|
||||||
|
lines.drain(..lines.len() - OUTPUT_TAIL_CAP);
|
||||||
|
}
|
||||||
|
lines
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl DashboardView for ControlPlaneView {
|
impl DashboardView for ControlPlaneView {
|
||||||
fn id(&self) -> &'static str {
|
fn id(&self) -> &'static str {
|
||||||
"fleet"
|
"fleet"
|
||||||
|
|
@ -86,15 +160,21 @@ impl DashboardView for ControlPlaneView {
|
||||||
fn ingest(&self, _stream: &StreamId, _frame: &Frame, event: &FrameEvent) {
|
fn ingest(&self, _stream: &StreamId, _frame: &Frame, event: &FrameEvent) {
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
let mut state = self.state.write();
|
let mut state = self.state.write();
|
||||||
let key = stream_key(&event.stream);
|
|
||||||
let node = state.streams.entry(key).or_insert_with(|| FusedNode {
|
if let Some(routed) = provisioning_output(event) {
|
||||||
stream: event.stream.clone(),
|
{
|
||||||
last_seen: now,
|
let pending = ensure_node(&mut state.streams, &routed.stream, now);
|
||||||
hardware: NodeHardwareState::new(now),
|
pending.last_seen = now;
|
||||||
actors: RuntimeState::new(now),
|
pending.output.set_phase(&routed.phase);
|
||||||
origin: event.stream.origin.clone(),
|
if let (Some(source), Some(text)) = (routed.source, routed.text) {
|
||||||
label: event.stream.label.clone(),
|
pending.output.push_line(&source, &routed.phase, text, now);
|
||||||
});
|
}
|
||||||
|
}
|
||||||
|
prune(&mut state.streams, &routed.stream, now);
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
let node = ensure_node(&mut state.streams, &event.stream, now);
|
||||||
node.last_seen = now;
|
node.last_seen = now;
|
||||||
// Later events may carry descriptor metadata the first lacked.
|
// Later events may carry descriptor metadata the first lacked.
|
||||||
if let Some(origin) = &event.stream.origin {
|
if let Some(origin) = &event.stream.origin {
|
||||||
|
|
@ -105,6 +185,10 @@ impl DashboardView for ControlPlaneView {
|
||||||
}
|
}
|
||||||
node.hardware.update(&event.channel, &event.payload, now);
|
node.hardware.update(&event.channel, &event.payload, now);
|
||||||
node.actors.update(&event.channel, &event.payload, now);
|
node.actors.update(&event.channel, &event.payload, now);
|
||||||
|
if let Some((source, phase)) = process_output_channel(&event.channel) {
|
||||||
|
node.output.push_chunk(source, &phase, &event.payload, now);
|
||||||
|
}
|
||||||
|
}
|
||||||
prune(&mut state.streams, &event.stream, now);
|
prune(&mut state.streams, &event.stream, now);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -185,6 +269,96 @@ impl DashboardView for ControlPlaneView {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct RoutedProvisionOutput {
|
||||||
|
stream: StreamEvent,
|
||||||
|
phase: String,
|
||||||
|
source: Option<String>,
|
||||||
|
text: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ensure_node<'a>(
|
||||||
|
streams: &'a mut BTreeMap<String, FusedNode>,
|
||||||
|
stream: &StreamEvent,
|
||||||
|
now: Instant,
|
||||||
|
) -> &'a mut FusedNode {
|
||||||
|
streams
|
||||||
|
.entry(stream_key(stream))
|
||||||
|
.or_insert_with(|| FusedNode {
|
||||||
|
stream: stream.clone(),
|
||||||
|
last_seen: now,
|
||||||
|
hardware: NodeHardwareState::new(now),
|
||||||
|
actors: RuntimeState::new(now),
|
||||||
|
output: NodeOutputState::default(),
|
||||||
|
origin: stream.origin.clone(),
|
||||||
|
label: stream.label.clone(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn provisioning_output(event: &FrameEvent) -> Option<RoutedProvisionOutput> {
|
||||||
|
let value = serde_json::from_slice::<Value>(&event.payload).ok()?;
|
||||||
|
let (run_id, node_id, phase, source, text) = if event.channel == PROVISIONING_EVENTS {
|
||||||
|
let event = value.get("event")?;
|
||||||
|
let kind = event.get("kind").and_then(Value::as_str)?;
|
||||||
|
let phase = match kind {
|
||||||
|
"ProvisionStart" => "provisioning",
|
||||||
|
"NodeLive" => "joining",
|
||||||
|
"ProvisionFailed" => "failed",
|
||||||
|
"NodeStopped" => "stopped",
|
||||||
|
_ => "provisioning",
|
||||||
|
};
|
||||||
|
(
|
||||||
|
event.get("run_id").and_then(Value::as_u64)?,
|
||||||
|
event.get("node_id").and_then(Value::as_u64)?,
|
||||||
|
phase.to_owned(),
|
||||||
|
event
|
||||||
|
.get("message")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(|_| "provider".to_owned()),
|
||||||
|
event
|
||||||
|
.get("message")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(str::to_owned),
|
||||||
|
)
|
||||||
|
} else if event.channel.starts_with(PROVISIONING_LOG_PREFIX) {
|
||||||
|
let line = value.get("line")?;
|
||||||
|
let source = line
|
||||||
|
.get("stream")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.unwrap_or("Provider")
|
||||||
|
.to_ascii_lowercase();
|
||||||
|
(
|
||||||
|
line.get("run_id").and_then(Value::as_u64)?,
|
||||||
|
line.get("node_id").and_then(Value::as_u64)?,
|
||||||
|
"provisioning".to_owned(),
|
||||||
|
Some(source),
|
||||||
|
line.get("line").and_then(Value::as_str).map(str::to_owned),
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
return None;
|
||||||
|
};
|
||||||
|
Some(RoutedProvisionOutput {
|
||||||
|
stream: StreamEvent {
|
||||||
|
node: node_id.to_string(),
|
||||||
|
life: run_id,
|
||||||
|
origin: Some("bootstrap".to_owned()),
|
||||||
|
label: Some("pending node".to_owned()),
|
||||||
|
},
|
||||||
|
phase,
|
||||||
|
source,
|
||||||
|
text,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn process_output_channel(channel: &str) -> Option<(&'static str, String)> {
|
||||||
|
let label = channel.strip_prefix("proc.")?;
|
||||||
|
if let Some(phase) = label.strip_suffix(".stdout") {
|
||||||
|
return Some(("stdout", phase.to_owned()));
|
||||||
|
}
|
||||||
|
label
|
||||||
|
.strip_suffix(".stderr")
|
||||||
|
.map(|phase| ("stderr", phase.to_owned()))
|
||||||
|
}
|
||||||
|
|
||||||
/// Evict superseded life generations and enforce the stale-pool cap.
|
/// Evict superseded life generations and enforce the stale-pool cap.
|
||||||
fn prune(streams: &mut BTreeMap<String, FusedNode>, fresh: &StreamEvent, now: Instant) {
|
fn prune(streams: &mut BTreeMap<String, FusedNode>, fresh: &StreamEvent, now: Instant) {
|
||||||
// A newer life generation for the same node replaces older ones: the old
|
// A newer life generation for the same node replaces older ones: the old
|
||||||
|
|
@ -241,6 +415,7 @@ struct NodeCard {
|
||||||
storage: Option<telemetry::hardware::storage::HostStorageSample>,
|
storage: Option<telemetry::hardware::storage::HostStorageSample>,
|
||||||
process: Option<crate::hardware_view::ProcessSnapshot>,
|
process: Option<crate::hardware_view::ProcessSnapshot>,
|
||||||
history: Vec<HardwareHistorySnapshot>,
|
history: Vec<HardwareHistorySnapshot>,
|
||||||
|
output: NodeOutputSnapshot,
|
||||||
actor_summary: ActorSummarySnapshot,
|
actor_summary: ActorSummarySnapshot,
|
||||||
/// Aggregate-only roster rows; heavy per-actor detail lives behind the
|
/// Aggregate-only roster rows; heavy per-actor detail lives behind the
|
||||||
/// detail endpoint so snapshot payload stays independent of ring sizes.
|
/// detail endpoint so snapshot payload stays independent of ring sizes.
|
||||||
|
|
@ -268,6 +443,13 @@ struct ActorSummarySnapshot {
|
||||||
num_workers: Option<u32>,
|
num_workers: Option<u32>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct NodeOutputSnapshot {
|
||||||
|
phase: Option<String>,
|
||||||
|
last_output_ms_ago: Option<u64>,
|
||||||
|
lines: Vec<NodeOutputLine>,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
struct RosterRow {
|
struct RosterRow {
|
||||||
address: String,
|
address: String,
|
||||||
|
|
@ -281,6 +463,7 @@ struct RosterRow {
|
||||||
worker_id: Option<u32>,
|
worker_id: Option<u32>,
|
||||||
poisoned: bool,
|
poisoned: bool,
|
||||||
last_msg_type: Option<String>,
|
last_msg_type: Option<String>,
|
||||||
|
last_active_ms_ago: Option<u64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
|
|
@ -300,6 +483,7 @@ struct ActorDetail {
|
||||||
last_msg_type: Option<String>,
|
last_msg_type: Option<String>,
|
||||||
message_type_counts: Vec<MessageTypeCountSnapshot>,
|
message_type_counts: Vec<MessageTypeCountSnapshot>,
|
||||||
history: Vec<ActorHistoryPoint>,
|
history: Vec<ActorHistoryPoint>,
|
||||||
|
last_active_ms_ago: Option<u64>,
|
||||||
receipts: Vec<ReceiptSnapshot>,
|
receipts: Vec<ReceiptSnapshot>,
|
||||||
sampled_out: u64,
|
sampled_out: u64,
|
||||||
}
|
}
|
||||||
|
|
@ -330,10 +514,15 @@ fn stream_key(stream: &StreamEvent) -> String {
|
||||||
|
|
||||||
fn node_card(node: &FusedNode, now: Instant) -> NodeCard {
|
fn node_card(node: &FusedNode, now: Instant) -> NodeCard {
|
||||||
let summary = node.hardware.summary();
|
let summary = node.hardware.summary();
|
||||||
let totals = node.actors.totals();
|
let totals = node.actors.totals_at(now);
|
||||||
// Address-keyed map order is the stable default. Volatile telemetry must
|
// Address-keyed map order is the stable default. Volatile telemetry must
|
||||||
// not move a row out from under the pointer; the page offers explicit sorts.
|
// not move a row out from under the pointer; the page offers explicit sorts.
|
||||||
let roster: Vec<RosterRow> = node.actors.actors.values().map(roster_row).collect();
|
let roster: Vec<RosterRow> = node
|
||||||
|
.actors
|
||||||
|
.actors
|
||||||
|
.values()
|
||||||
|
.map(|actor| roster_row(actor, now))
|
||||||
|
.collect();
|
||||||
NodeCard {
|
NodeCard {
|
||||||
stream: StreamKeySnapshot {
|
stream: StreamKeySnapshot {
|
||||||
key: stream_key(&node.stream),
|
key: stream_key(&node.stream),
|
||||||
|
|
@ -372,6 +561,14 @@ fn node_card(node: &FusedNode, now: Instant) -> NodeCard {
|
||||||
io_pressure_some_avg10: sample.io_pressure_some_avg10,
|
io_pressure_some_avg10: sample.io_pressure_some_avg10,
|
||||||
})
|
})
|
||||||
.collect(),
|
.collect(),
|
||||||
|
output: NodeOutputSnapshot {
|
||||||
|
phase: node.output.phase.clone(),
|
||||||
|
last_output_ms_ago: node
|
||||||
|
.output
|
||||||
|
.last_output
|
||||||
|
.map(|last| duration_ms(now.duration_since(last))),
|
||||||
|
lines: node.output.snapshot_lines(),
|
||||||
|
},
|
||||||
actor_summary: ActorSummarySnapshot {
|
actor_summary: ActorSummarySnapshot {
|
||||||
actors: totals.actors,
|
actors: totals.actors,
|
||||||
msg_per_sec: totals.msg_per_sec,
|
msg_per_sec: totals.msg_per_sec,
|
||||||
|
|
@ -384,7 +581,7 @@ fn node_card(node: &FusedNode, now: Instant) -> NodeCard {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn roster_row(actor: &ActorState) -> RosterRow {
|
fn roster_row(actor: &ActorState, now: Instant) -> RosterRow {
|
||||||
RosterRow {
|
RosterRow {
|
||||||
address: actor.address.clone(),
|
address: actor.address.clone(),
|
||||||
name: actor.name.clone(),
|
name: actor.name.clone(),
|
||||||
|
|
@ -397,9 +594,12 @@ fn roster_row(actor: &ActorState) -> RosterRow {
|
||||||
},
|
},
|
||||||
mailbox_depth: actor.mailbox_depth,
|
mailbox_depth: actor.mailbox_depth,
|
||||||
messages_processed: actor.messages_processed,
|
messages_processed: actor.messages_processed,
|
||||||
msg_per_sec: actor.msg_per_sec,
|
msg_per_sec: actor.rate_at(now),
|
||||||
worker_id: actor.worker_id,
|
worker_id: actor.worker_id,
|
||||||
poisoned: actor.poisoned,
|
poisoned: actor.poisoned,
|
||||||
|
last_active_ms_ago: actor
|
||||||
|
.last_active
|
||||||
|
.map(|last| duration_ms(now.duration_since(last))),
|
||||||
last_msg_type: actor.last_msg_type.clone(),
|
last_msg_type: actor.last_msg_type.clone(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -434,8 +634,11 @@ fn actor_detail(
|
||||||
mailbox_depth: actor.mailbox_depth,
|
mailbox_depth: actor.mailbox_depth,
|
||||||
mailbox_growth: actor.mailbox_growth,
|
mailbox_growth: actor.mailbox_growth,
|
||||||
messages_processed: actor.messages_processed,
|
messages_processed: actor.messages_processed,
|
||||||
msg_per_sec: actor.msg_per_sec,
|
msg_per_sec: actor.rate_at(now),
|
||||||
last_msg_type: actor.last_msg_type.clone(),
|
last_msg_type: actor.last_msg_type.clone(),
|
||||||
|
last_active_ms_ago: actor
|
||||||
|
.last_active
|
||||||
|
.map(|last| duration_ms(now.duration_since(last))),
|
||||||
message_type_counts: actor
|
message_type_counts: actor
|
||||||
.message_type_counts
|
.message_type_counts
|
||||||
.iter()
|
.iter()
|
||||||
|
|
@ -559,6 +762,73 @@ mod tests {
|
||||||
assert_eq!(card["label"], json!("provisioning supervisor"));
|
assert_eq!(card["label"], json!("provisioning supervisor"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn provisioning_tail_merges_into_joined_node_card() {
|
||||||
|
let view = ControlPlaneView::default();
|
||||||
|
let orchestrator = StreamId::new(NodeId::new("orchestrator"), Lifetime(42));
|
||||||
|
ingest_json(
|
||||||
|
&view,
|
||||||
|
&orchestrator,
|
||||||
|
0,
|
||||||
|
PROVISIONING_EVENTS,
|
||||||
|
serde_json::to_vec(&json!({
|
||||||
|
"event": {
|
||||||
|
"run_id": 42,
|
||||||
|
"node_id": 7,
|
||||||
|
"kind": "ProvisionStart",
|
||||||
|
"message": "leasing GPU"
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
let pending = view.snapshot_json();
|
||||||
|
let pending_card = pending["live"]
|
||||||
|
.as_array()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.find(|card| card["stream"]["node"] == "7")
|
||||||
|
.expect("pending card");
|
||||||
|
assert_eq!(pending_card["stream"]["origin"], "bootstrap");
|
||||||
|
assert_eq!(pending_card["output"]["phase"], "provisioning");
|
||||||
|
|
||||||
|
ingest_json(
|
||||||
|
&view,
|
||||||
|
&orchestrator,
|
||||||
|
1,
|
||||||
|
"myelin.provisioning.logs.node.7.stdout",
|
||||||
|
serde_json::to_vec(&json!({
|
||||||
|
"line": {
|
||||||
|
"run_id": 42,
|
||||||
|
"node_id": 7,
|
||||||
|
"stream": "Stdout",
|
||||||
|
"line": "runtime starting"
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
let remote = StreamId::new(NodeId::new("7"), Lifetime(42));
|
||||||
|
ingest_json(
|
||||||
|
&view,
|
||||||
|
&remote,
|
||||||
|
2,
|
||||||
|
"proc.job-runner-run.stdout",
|
||||||
|
b"job output\n".to_vec(),
|
||||||
|
);
|
||||||
|
|
||||||
|
let joined = view.snapshot_json();
|
||||||
|
let cards = joined["live"]
|
||||||
|
.as_array()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.filter(|card| card["stream"]["node"] == "7")
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
assert_eq!(cards.len(), 1, "join keeps one visual identity");
|
||||||
|
assert_eq!(cards[0]["output"]["phase"], "job-runner-run");
|
||||||
|
assert_eq!(cards[0]["output"]["lines"].as_array().unwrap().len(), 3);
|
||||||
|
assert_eq!(cards[0]["output"]["lines"][2]["source"], "stdout");
|
||||||
|
assert_eq!(cards[0]["output"]["lines"][2]["text"], "job output");
|
||||||
|
}
|
||||||
|
|
||||||
fn actors_payload(worker: u32, actors: serde_json::Value) -> Vec<u8> {
|
fn actors_payload(worker: u32, actors: serde_json::Value) -> Vec<u8> {
|
||||||
serde_json::json!({ "worker_id": worker, "actors": actors })
|
serde_json::json!({ "worker_id": worker, "actors": actors })
|
||||||
.to_string()
|
.to_string()
|
||||||
|
|
|
||||||
|
|
@ -108,6 +108,8 @@
|
||||||
.empty { padding: 24px; border: 1px dashed var(--border); border-radius: var(--r); color: var(--muted); font-family: var(--mono); font-size: 12px; }
|
.empty { padding: 24px; border: 1px dashed var(--border); border-radius: var(--r); color: var(--muted); font-family: var(--mono); font-size: 12px; }
|
||||||
.notice { font-size: 12px; color: var(--muted); margin: 6px 0; }
|
.notice { font-size: 12px; color: var(--muted); margin: 6px 0; }
|
||||||
canvas { width: 100%; height: 64px; background: var(--inset); border: 1px solid var(--border); border-radius: var(--r); }
|
canvas { width: 100%; height: 64px; background: var(--inset); border: 1px solid var(--border); border-radius: var(--r); }
|
||||||
|
.output-head { display:flex; justify-content:space-between; gap:12px; align-items:baseline; margin-bottom:8px; }
|
||||||
|
.output-tail { margin:0; height:180px; overflow:auto; padding:10px; background:var(--inset); border:1px solid var(--divider); border-radius:var(--r); color:var(--text); font:11px/1.45 var(--mono); white-space:pre-wrap; overflow-wrap:anywhere; }
|
||||||
.hardware-source { margin-bottom: 8px; }
|
.hardware-source { margin-bottom: 8px; }
|
||||||
.hardware-grid { display: grid; grid-template-columns: repeat(12, minmax(0, 1fr)); gap: 8px; }
|
.hardware-grid { display: grid; grid-template-columns: repeat(12, minmax(0, 1fr)); gap: 8px; }
|
||||||
.hw-card { min-width: 0; padding: 10px; background: var(--inset); border: 1px solid var(--divider); border-radius: var(--r); }
|
.hw-card { min-width: 0; padding: 10px; background: var(--inset); border: 1px solid var(--divider); border-radius: var(--r); }
|
||||||
|
|
@ -316,7 +318,7 @@ function nodeCard(node) {
|
||||||
// The stats line (actors · msg/s · uptime) jitters every frame on busy
|
// The stats line (actors · msg/s · uptime) jitters every frame on busy
|
||||||
// supervisors; it renders as a pass-filled span so the card markup — and
|
// supervisors; it renders as a pass-filled span so the card markup — and
|
||||||
// with it the whole grid — stays byte-identical between polls.
|
// with it the whole grid — stays byte-identical between polls.
|
||||||
return `<a class="node-card${isOrchestrator ? ' orchestrator' : ''}" data-stream="${esc(node.stream.key)}" data-live="${node.live ? 'true' : 'false'}" href="?stream=${encodeURIComponent(node.stream.key)}">
|
return `<a class="node-card${isOrchestrator ? ' orchestrator' : ''}" data-stream="${esc(node.stream.key)}" data-node="${esc(node.stream.node)}" data-origin="${esc(origin)}" data-live="${node.live ? 'true' : 'false'}" href="?stream=${encodeURIComponent(node.stream.key)}">
|
||||||
${role}
|
${role}
|
||||||
<h3>${esc(node.stream.node)} <span class="muted" style="font-size:12px">life ${fmt(node.stream.life)}</span></h3>
|
<h3>${esc(node.stream.node)} <span class="muted" style="font-size:12px">life ${fmt(node.stream.life)}</span></h3>
|
||||||
<div class="meta">${proc}seen <span data-seen></span> ago · <span data-stats></span>${label}</div>
|
<div class="meta">${proc}seen <span data-seen></span> ago · <span data-stats></span>${label}</div>
|
||||||
|
|
@ -362,11 +364,15 @@ function renderNode(page, node, live, stale) {
|
||||||
const html = `
|
const html = `
|
||||||
<a class="focus-back" href="/">← Fleet (<span data-live-count></span> live, <span data-stale-count></span> stale)</a>
|
<a class="focus-back" href="/">← Fleet (<span data-live-count></span> live, <span data-stale-count></span> stale)</a>
|
||||||
<div class="layout ${selectedActor ? 'with-dossier' : ''}">
|
<div class="layout ${selectedActor ? 'with-dossier' : ''}">
|
||||||
<div class="node-view" data-node="${esc(node.stream.node)}">
|
<div class="node-view" data-node="${esc(node.stream.node)}" data-origin="${esc(node.stream.origin || '')}">
|
||||||
<section class="panel">
|
<section class="panel">
|
||||||
<h2 style="margin:0 0 8px">${esc(node.stream.node)}${node.stream.label ? ` <span class="muted" style="font-size:13px">${esc(node.stream.label)}</span>` : ''} <span class="muted" style="font-size:13px">life ${fmt(node.stream.life)} · <span data-seen></span></span></h2>
|
<h2 style="margin:0 0 8px">${esc(node.stream.node)}${node.stream.label ? ` <span class="muted" style="font-size:13px">${esc(node.stream.label)}</span>` : ''} <span class="muted" style="font-size:13px">life ${fmt(node.stream.life)} · <span data-seen></span></span></h2>
|
||||||
<div id="machine-detail"></div>
|
<div id="machine-detail"></div>
|
||||||
</section>
|
</section>
|
||||||
|
<section class="panel">
|
||||||
|
<div class="output-head"><h2 style="margin:0">Node output</h2><span class="muted mono" id="output-meta"></span></div>
|
||||||
|
<pre class="output-tail" id="node-output" aria-label="Latest node stdout and stderr"></pre>
|
||||||
|
</section>
|
||||||
<section class="panel">
|
<section class="panel">
|
||||||
<h2 style="margin:0 0 8px">Actors (<span data-actor-count></span>)</h2>
|
<h2 style="margin:0 0 8px">Actors (<span data-actor-count></span>)</h2>
|
||||||
<input type="search" id="roster-filter" aria-label="Filter actors" placeholder="filter by type, address, worker…" value="${esc(rosterFilter)}">
|
<input type="search" id="roster-filter" aria-label="Filter actors" placeholder="filter by type, address, worker…" value="${esc(rosterFilter)}">
|
||||||
|
|
@ -402,6 +408,7 @@ function renderNode(page, node, live, stale) {
|
||||||
machineSlot.innerHTML = machine;
|
machineSlot.innerHTML = machine;
|
||||||
lastMachineHtml = machine;
|
lastMachineHtml = machine;
|
||||||
}
|
}
|
||||||
|
renderNodeOutput(node);
|
||||||
renderRoster(node);
|
renderRoster(node);
|
||||||
if (selectedActor) {
|
if (selectedActor) {
|
||||||
if (!detailTimer) startDetailPolling();
|
if (!detailTimer) startDetailPolling();
|
||||||
|
|
@ -410,6 +417,24 @@ function renderNode(page, node, live, stale) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function renderNodeOutput(node) {
|
||||||
|
const tail = document.getElementById('node-output');
|
||||||
|
const meta = document.getElementById('output-meta');
|
||||||
|
if (!tail || !meta) return;
|
||||||
|
const output = node.output || {};
|
||||||
|
const lines = output.lines || [];
|
||||||
|
const wasAtBottom = tail.dataset.initialized !== 'true'
|
||||||
|
|| tail.scrollHeight - tail.clientHeight - tail.scrollTop < 8;
|
||||||
|
tail.textContent = lines.length
|
||||||
|
? lines.map(line => `[${line.source || '?'} · ${line.phase || '?'}] ${line.text || ''}`).join('\n')
|
||||||
|
: 'No output observed yet.';
|
||||||
|
tail.dataset.initialized = 'true';
|
||||||
|
if (wasAtBottom) tail.scrollTop = tail.scrollHeight;
|
||||||
|
const phase = output.phase || 'waiting';
|
||||||
|
const age = output.last_output_ms_ago == null ? 'no output yet' : `last output ${ago(output.last_output_ms_ago)} ago`;
|
||||||
|
meta.textContent = `${phase} · ${age}`;
|
||||||
|
}
|
||||||
|
|
||||||
function percentOf(used, total) {
|
function percentOf(used, total) {
|
||||||
const numerator = Number(used), denominator = Number(total);
|
const numerator = Number(used), denominator = Number(total);
|
||||||
return Number.isFinite(numerator) && Number.isFinite(denominator) && denominator > 0
|
return Number.isFinite(numerator) && Number.isFinite(denominator) && denominator > 0
|
||||||
|
|
@ -614,7 +639,7 @@ function renderRoster(node) {
|
||||||
<td data-rate></td>
|
<td data-rate></td>
|
||||||
<td data-processed></td>
|
<td data-processed></td>
|
||||||
<td>${a.worker_id == null ? '—' : fmt(a.worker_id)}</td>
|
<td>${a.worker_id == null ? '—' : fmt(a.worker_id)}</td>
|
||||||
<td><span class="muted" data-last-message></span></td>
|
<td><span class="muted" data-last-message></span><br><span class="muted" data-last-active></span></td>
|
||||||
</tr>`).join('')}</tbody>
|
</tr>`).join('')}</tbody>
|
||||||
</table>`;
|
</table>`;
|
||||||
}
|
}
|
||||||
|
|
@ -638,6 +663,9 @@ function updateRosterTexts(wrap, actors) {
|
||||||
last.title = messageType;
|
last.title = messageType;
|
||||||
const parts = String(messageType).split('::');
|
const parts = String(messageType).split('::');
|
||||||
last.textContent = messageType ? parts[parts.length - 1] || messageType : '—';
|
last.textContent = messageType ? parts[parts.length - 1] || messageType : '—';
|
||||||
|
row.querySelector('[data-last-active]').textContent = actor.last_active_ms_ago == null
|
||||||
|
? 'never active'
|
||||||
|
: `${ago(actor.last_active_ms_ago)} ago`;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
@ -707,6 +735,7 @@ function renderDossier(d) {
|
||||||
<span class="k">throughput</span><span>${fmtRate(d.msg_per_sec)} msg/s</span>
|
<span class="k">throughput</span><span>${fmtRate(d.msg_per_sec)} msg/s</span>
|
||||||
<span class="k">processed</span><span>${fmt(d.messages_processed)}</span>
|
<span class="k">processed</span><span>${fmt(d.messages_processed)}</span>
|
||||||
<span class="k">last message</span><span class="mono" title="${esc(d.last_msg_type || '')}">${esc(d.last_msg_type || '—')}</span>
|
<span class="k">last message</span><span class="mono" title="${esc(d.last_msg_type || '')}">${esc(d.last_msg_type || '—')}</span>
|
||||||
|
<span class="k">last active</span><span>${d.last_active_ms_ago == null ? 'never' : `${ago(d.last_active_ms_ago)} ago`}</span>
|
||||||
</div>
|
</div>
|
||||||
<h2>Throughput</h2>
|
<h2>Throughput</h2>
|
||||||
<canvas id="dossier-spark" width="380" height="64"></canvas>
|
<canvas id="dossier-spark" width="380" height="64"></canvas>
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
use std::collections::{BTreeMap, VecDeque};
|
use std::collections::{BTreeMap, VecDeque};
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use parking_lot::RwLock;
|
use parking_lot::RwLock;
|
||||||
use serde_json::{Value, json};
|
use serde_json::{Value, json};
|
||||||
|
|
@ -8,12 +9,93 @@ use crate::FrameEvent;
|
||||||
use crate::view::DashboardView;
|
use crate::view::DashboardView;
|
||||||
|
|
||||||
const LIVE_EXPLORER_HTML: &str = include_str!("live_explorer_page.html");
|
const LIVE_EXPLORER_HTML: &str = include_str!("live_explorer_page.html");
|
||||||
|
const FRAME_HISTORY_CAP: usize = 2_000;
|
||||||
const FRAME_HISTORY_CAP: usize = 500;
|
const LIVE_TTL: Duration = Duration::from_secs(8);
|
||||||
|
const STALE_STREAM_CAP: usize = 50;
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
pub struct LiveTelemetryExplorer {
|
pub struct LiveTelemetryExplorer {
|
||||||
state: RwLock<BTreeMap<(String, u64, String), VecDeque<FrameEvent>>>,
|
state: RwLock<ExplorerState>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct ExplorerState {
|
||||||
|
streams: BTreeMap<(String, u64), ExplorerStream>,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ExplorerStream {
|
||||||
|
last_seen: Instant,
|
||||||
|
channels: BTreeMap<String, VecDeque<FrameEvent>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LiveTelemetryExplorer {
|
||||||
|
fn ingest_at(&self, event: &FrameEvent, now: Instant) {
|
||||||
|
let mut state = self.state.write();
|
||||||
|
let newest_life = state
|
||||||
|
.streams
|
||||||
|
.keys()
|
||||||
|
.filter(|(node, _)| node == &event.stream.node)
|
||||||
|
.map(|(_, life)| *life)
|
||||||
|
.max();
|
||||||
|
if newest_life.is_some_and(|life| event.stream.life < life) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if newest_life.is_none_or(|life| event.stream.life > life) {
|
||||||
|
state
|
||||||
|
.streams
|
||||||
|
.retain(|(node, _), _| node != &event.stream.node);
|
||||||
|
}
|
||||||
|
|
||||||
|
let stream = state
|
||||||
|
.streams
|
||||||
|
.entry((event.stream.node.clone(), event.stream.life))
|
||||||
|
.or_insert_with(|| ExplorerStream {
|
||||||
|
last_seen: now,
|
||||||
|
channels: BTreeMap::new(),
|
||||||
|
});
|
||||||
|
stream.last_seen = now;
|
||||||
|
let frames = stream.channels.entry(event.channel.clone()).or_default();
|
||||||
|
if frames.len() >= FRAME_HISTORY_CAP {
|
||||||
|
frames.pop_front();
|
||||||
|
}
|
||||||
|
frames.push_back(event.clone());
|
||||||
|
prune_stale(&mut state.streams, now);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn snapshot_at(&self, now: Instant) -> Value {
|
||||||
|
let mut state = self.state.write();
|
||||||
|
prune_stale(&mut state.streams, now);
|
||||||
|
let mut frames = state
|
||||||
|
.streams
|
||||||
|
.values()
|
||||||
|
.flat_map(|stream| stream.channels.values())
|
||||||
|
.flat_map(|frames| frames.iter())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
frames.sort_by(|left, right| {
|
||||||
|
left.stream
|
||||||
|
.node
|
||||||
|
.cmp(&right.stream.node)
|
||||||
|
.then_with(|| left.stream.life.cmp(&right.stream.life))
|
||||||
|
.then_with(|| left.position.cmp(&right.position))
|
||||||
|
});
|
||||||
|
json!({ "frames": frames })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn prune_stale(streams: &mut BTreeMap<(String, u64), ExplorerStream>, now: Instant) {
|
||||||
|
let mut stale = streams
|
||||||
|
.iter()
|
||||||
|
.filter(|(_, stream)| now.duration_since(stream.last_seen) > LIVE_TTL)
|
||||||
|
.map(|(key, stream)| (key.clone(), stream.last_seen))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
if stale.len() <= STALE_STREAM_CAP {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let excess = stale.len() - STALE_STREAM_CAP;
|
||||||
|
stale.sort_by_key(|(_, last_seen)| *last_seen);
|
||||||
|
for (key, _) in stale.into_iter().take(excess) {
|
||||||
|
streams.remove(&key);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl DashboardView for LiveTelemetryExplorer {
|
impl DashboardView for LiveTelemetryExplorer {
|
||||||
|
|
@ -30,34 +112,11 @@ impl DashboardView for LiveTelemetryExplorer {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn ingest(&self, _stream: &StreamId, _frame: &Frame, event: &FrameEvent) {
|
fn ingest(&self, _stream: &StreamId, _frame: &Frame, event: &FrameEvent) {
|
||||||
let key = (
|
self.ingest_at(event, Instant::now());
|
||||||
event.stream.node.clone(),
|
|
||||||
event.stream.life,
|
|
||||||
event.channel.clone(),
|
|
||||||
);
|
|
||||||
let mut state = self.state.write();
|
|
||||||
let frames = state.entry(key).or_default();
|
|
||||||
if frames.len() >= FRAME_HISTORY_CAP {
|
|
||||||
frames.pop_front();
|
|
||||||
}
|
|
||||||
frames.push_back(event.clone());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn snapshot_json(&self) -> Value {
|
fn snapshot_json(&self) -> Value {
|
||||||
let state = self.state.read();
|
self.snapshot_at(Instant::now())
|
||||||
let mut frames = state
|
|
||||||
.values()
|
|
||||||
.flat_map(|frames| frames.iter())
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
frames.sort_by(|left, right| {
|
|
||||||
left.stream
|
|
||||||
.node
|
|
||||||
.cmp(&right.stream.node)
|
|
||||||
.then_with(|| left.stream.life.cmp(&right.stream.life))
|
|
||||||
.then_with(|| left.position.cmp(&right.position))
|
|
||||||
});
|
|
||||||
|
|
||||||
json!({ "frames": frames })
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn html(&self) -> Option<&'static str> {
|
fn html(&self) -> Option<&'static str> {
|
||||||
|
|
@ -104,29 +163,54 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn retains_same_channel_separately_across_stream_lifetimes() {
|
fn newer_lifetime_evicts_older_lifetime_and_rejects_late_frames() {
|
||||||
let view = LiveTelemetryExplorer::default();
|
let view = LiveTelemetryExplorer::default();
|
||||||
let first = StreamId::new(NodeId::new("node-a"), Lifetime(1));
|
let first = StreamId::new(NodeId::new("node-a"), Lifetime(1));
|
||||||
let second = StreamId::new(NodeId::new("node-a"), Lifetime(2));
|
let second = StreamId::new(NodeId::new("node-a"), Lifetime(2));
|
||||||
ingest(&view, &first, "host.cpu", 7);
|
ingest(&view, &first, "host.cpu", 7);
|
||||||
ingest(&view, &second, "host.cpu", 0);
|
ingest(&view, &second, "host.cpu", 0);
|
||||||
|
ingest(&view, &first, "host.cpu", 8);
|
||||||
|
|
||||||
let snapshot = view.snapshot_json();
|
let snapshot = view.snapshot_json();
|
||||||
let frames = snapshot["frames"].as_array().expect("frames array");
|
let frames = snapshot["frames"].as_array().expect("frames array");
|
||||||
assert_eq!(frames.len(), 2);
|
assert_eq!(frames.len(), 1);
|
||||||
assert_eq!(frames[0]["stream"]["life"].as_u64(), Some(1));
|
assert_eq!(frames[0]["stream"]["life"].as_u64(), Some(2));
|
||||||
assert_eq!(frames[0]["position"].as_u64(), Some(7));
|
assert_eq!(frames[0]["position"].as_u64(), Some(0));
|
||||||
assert_eq!(frames[1]["stream"]["life"].as_u64(), Some(2));
|
|
||||||
assert_eq!(frames[1]["position"].as_u64(), Some(0));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn ingest(view: &LiveTelemetryExplorer, stream: &StreamId, channel: &str, position: u64) {
|
#[test]
|
||||||
let frame = Frame::new(
|
fn retains_all_live_streams_and_only_fifty_stale_streams() {
|
||||||
|
let view = LiveTelemetryExplorer::default();
|
||||||
|
let start = Instant::now();
|
||||||
|
for node in 0..60 {
|
||||||
|
let stream = StreamId::new(NodeId::new(format!("node-{node}")), Lifetime(1));
|
||||||
|
let frame = test_frame(node);
|
||||||
|
let event = test_event(&stream, "host.cpu", node, &frame);
|
||||||
|
view.ingest_at(&event, start + Duration::from_millis(node));
|
||||||
|
}
|
||||||
|
|
||||||
|
let snapshot = view.snapshot_at(start + LIVE_TTL + Duration::from_secs(1));
|
||||||
|
let streams = snapshot["frames"]
|
||||||
|
.as_array()
|
||||||
|
.expect("frames array")
|
||||||
|
.iter()
|
||||||
|
.map(|frame| frame["stream"]["node"].as_str().unwrap())
|
||||||
|
.collect::<std::collections::BTreeSet<_>>();
|
||||||
|
assert_eq!(streams.len(), STALE_STREAM_CAP);
|
||||||
|
assert!(!streams.contains("node-0"));
|
||||||
|
assert!(streams.contains("node-59"));
|
||||||
|
}
|
||||||
|
|
||||||
|
fn test_frame(position: u64) -> Frame {
|
||||||
|
Frame::new(
|
||||||
ChannelId(position as u32 + 1),
|
ChannelId(position as u32 + 1),
|
||||||
Position(position),
|
Position(position),
|
||||||
position.to_le_bytes().to_vec(),
|
position.to_le_bytes().to_vec(),
|
||||||
);
|
)
|
||||||
let event = FrameEvent {
|
}
|
||||||
|
|
||||||
|
fn test_event(stream: &StreamId, channel: &str, position: u64, frame: &Frame) -> FrameEvent {
|
||||||
|
FrameEvent {
|
||||||
stream: crate::StreamEvent {
|
stream: crate::StreamEvent {
|
||||||
node: stream.node.as_str().to_string(),
|
node: stream.node.as_str().to_string(),
|
||||||
life: stream.life.0,
|
life: stream.life.0,
|
||||||
|
|
@ -136,7 +220,12 @@ mod tests {
|
||||||
channel: channel.to_owned(),
|
channel: channel.to_owned(),
|
||||||
position,
|
position,
|
||||||
payload: frame.payload.clone(),
|
payload: frame.payload.clone(),
|
||||||
};
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ingest(view: &LiveTelemetryExplorer, stream: &StreamId, channel: &str, position: u64) {
|
||||||
|
let frame = test_frame(position);
|
||||||
|
let event = test_event(stream, channel, position, &frame);
|
||||||
view.ingest(stream, &frame, &event);
|
view.ingest(stream, &frame, &event);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -48,6 +48,8 @@ tr[data-active="true"]{background:var(--selected);box-shadow:inset 2px 0 0 var(-
|
||||||
.tag.binary{color:var(--magenta);border-color:var(--magenta)}
|
.tag.binary{color:var(--magenta);border-color:var(--magenta)}
|
||||||
.json{color:var(--ok)}.text{color:var(--amber)}.binary{color:var(--magenta)}
|
.json{color:var(--ok)}.text{color:var(--amber)}.binary{color:var(--magenta)}
|
||||||
.ok{color:var(--ok)}.warn{color:var(--amber)}.bad{color:var(--bad)}
|
.ok{color:var(--ok)}.warn{color:var(--amber)}.bad{color:var(--bad)}
|
||||||
|
.connection-state{display:grid;justify-items:end;gap:4px;max-width:min(420px,45vw)}
|
||||||
|
.status-detail{font:11px/1.3 var(--mono);text-align:right;overflow-wrap:anywhere}
|
||||||
.kv{display:grid;grid-template-columns:115px 1fr;gap:8px;border-bottom:1px solid var(--divider);padding:7px 0;font-size:12px;font-family:var(--mono)}
|
.kv{display:grid;grid-template-columns:115px 1fr;gap:8px;border-bottom:1px solid var(--divider);padding:7px 0;font-size:12px;font-family:var(--mono)}
|
||||||
pre{white-space:pre-wrap;word-break:break-word;font-family:var(--mono);background:var(--inset);border:1px solid var(--border);border-radius:var(--r);padding:12px;max-height:560px;overflow:auto;font-size:12px}
|
pre{white-space:pre-wrap;word-break:break-word;font-family:var(--mono);background:var(--inset);border:1px solid var(--border);border-radius:var(--r);padding:12px;max-height:560px;overflow:auto;font-size:12px}
|
||||||
.empty{border:1px dashed var(--border);border-radius:var(--r);padding:22px;color:var(--muted);font-family:var(--mono);font-size:12px}
|
.empty{border:1px dashed var(--border);border-radius:var(--r);padding:22px;color:var(--muted);font-family:var(--mono);font-size:12px}
|
||||||
|
|
@ -71,13 +73,15 @@ RISK: terminal cosplay if decoration creeps past data; held by the palette law.
|
||||||
<h1>Telemetry live explorer</h1>
|
<h1>Telemetry live explorer</h1>
|
||||||
<div class="muted">Auto-connected. Retained channel history is restored from /api/view/telemetry/live, then live frames continue from /events.</div>
|
<div class="muted">Auto-connected. Retained channel history is restored from /api/view/telemetry/live, then live frames continue from /events.</div>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="connection-state">
|
||||||
<span class="pill warn" id="status" role="status" aria-live="polite">starting</span>
|
<span class="pill warn" id="status" role="status" aria-live="polite">starting</span>
|
||||||
|
<span class="status-detail muted" id="statusDetail"></span>
|
||||||
|
</div>
|
||||||
</header>
|
</header>
|
||||||
<section class="top">
|
<section class="top">
|
||||||
<button data-level="streams" aria-pressed="true">Streams</button>
|
<button data-level="streams" aria-pressed="true">Streams</button>
|
||||||
<button data-level="channels" aria-pressed="false">Channels</button>
|
<button data-level="channels" aria-pressed="false">Channels</button>
|
||||||
<button data-level="frames" aria-pressed="false">Frames</button>
|
<button data-level="frames" aria-pressed="false">Frames</button>
|
||||||
<button id="clearScope" aria-label="Reset scope to all streams">all streams</button>
|
|
||||||
<input id="filter" aria-label="Filter visible rows" placeholder="filter visible rows" spellcheck="false">
|
<input id="filter" aria-label="Filter visible rows" placeholder="filter visible rows" spellcheck="false">
|
||||||
<select id="kind"><option value="">all payloads</option><option value="json">json</option><option value="text">text</option><option value="binary">binary</option></select>
|
<select id="kind"><option value="">all payloads</option><option value="json">json</option><option value="text">text</option><option value="binary">binary</option></select>
|
||||||
<label class="muted" for="limit">keep/channel</label>
|
<label class="muted" for="limit">keep/channel</label>
|
||||||
|
|
@ -136,6 +140,7 @@ RISK: terminal cosplay if decoration creeps past data; held by the palette law.
|
||||||
selectedFrame: '',
|
selectedFrame: '',
|
||||||
connected: false,
|
connected: false,
|
||||||
error: '',
|
error: '',
|
||||||
|
notice: '',
|
||||||
messageVersion: 0,
|
messageVersion: 0,
|
||||||
totalFrames: 0,
|
totalFrames: 0,
|
||||||
jsonFrames: 0,
|
jsonFrames: 0,
|
||||||
|
|
@ -338,9 +343,12 @@ RISK: terminal cosplay if decoration creeps past data; held by the palette law.
|
||||||
}
|
}
|
||||||
|
|
||||||
function updateMetrics() {
|
function updateMetrics() {
|
||||||
const status = state.error ? 'error' : state.connected ? 'live' : 'connecting';
|
const status = state.error ? 'reconnecting' : state.notice ? 'resyncing' : state.connected ? 'live' : 'connecting';
|
||||||
|
const detail = state.error || state.notice;
|
||||||
nodes.status.textContent = status;
|
nodes.status.textContent = status;
|
||||||
nodes.status.className = `pill ${status === 'live' ? 'ok' : status === 'error' ? 'bad' : 'warn'}`;
|
nodes.status.className = `pill ${status === 'live' ? 'ok' : 'warn'}`;
|
||||||
|
nodes.statusDetail.textContent = detail;
|
||||||
|
nodes.status.title = detail;
|
||||||
setSeg('metricFramesSeg', state.totalFrames);
|
setSeg('metricFramesSeg', state.totalFrames);
|
||||||
setSeg('metricStreamsSeg', state.streams.size);
|
setSeg('metricStreamsSeg', state.streams.size);
|
||||||
setSeg('metricChannelsSeg', state.channels.size);
|
setSeg('metricChannelsSeg', state.channels.size);
|
||||||
|
|
@ -511,21 +519,20 @@ RISK: terminal cosplay if decoration creeps past data; held by the palette law.
|
||||||
|
|
||||||
async function reconcileSnapshot({ notice = '', minimumNoticeMs = 0 } = {}) {
|
async function reconcileSnapshot({ notice = '', minimumNoticeMs = 0 } = {}) {
|
||||||
if (notice) {
|
if (notice) {
|
||||||
state.error = notice;
|
state.notice = notice;
|
||||||
state.messageVersion++;
|
state.messageVersion++;
|
||||||
updateMetrics();
|
updateMetrics();
|
||||||
renderDetail();
|
|
||||||
}
|
}
|
||||||
const messageVersion = state.messageVersion;
|
const messageVersion = state.messageVersion;
|
||||||
const startedAt = performance.now();
|
const startedAt = performance.now();
|
||||||
try {
|
try {
|
||||||
const response = await fetch('/api/view/telemetry/live', { cache: 'no-store' });
|
const response = await fetch('/api/view/telemetry/live', { cache: 'no-store' });
|
||||||
if (!response.ok) throw new Error(`/api/view/telemetry/live HTTP ${response.status}`);
|
if (!response.ok) throw new Error(`history restore failed: HTTP ${response.status}`);
|
||||||
const snapshot = await response.json();
|
const snapshot = await response.json();
|
||||||
if (!snapshot || !Array.isArray(snapshot.frames)) {
|
if (!snapshot || !Array.isArray(snapshot.frames)) {
|
||||||
throw new Error('/api/view/telemetry/live returned an invalid snapshot');
|
throw new Error('history restore returned invalid data');
|
||||||
}
|
}
|
||||||
for (const event of snapshot.frames) addFrame(event);
|
for (const frame of snapshot.frames) addFrame(frame);
|
||||||
flushNow();
|
flushNow();
|
||||||
|
|
||||||
const remainingNoticeMs = minimumNoticeMs - (performance.now() - startedAt);
|
const remainingNoticeMs = minimumNoticeMs - (performance.now() - startedAt);
|
||||||
|
|
@ -534,13 +541,15 @@ RISK: terminal cosplay if decoration creeps past data; held by the palette law.
|
||||||
}
|
}
|
||||||
if (state.messageVersion === messageVersion) {
|
if (state.messageVersion === messageVersion) {
|
||||||
state.error = '';
|
state.error = '';
|
||||||
|
state.notice = '';
|
||||||
state.messageVersion++;
|
state.messageVersion++;
|
||||||
updateMetrics();
|
updateMetrics();
|
||||||
renderDetail();
|
renderDetail();
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (state.messageVersion === messageVersion) {
|
if (state.messageVersion === messageVersion) {
|
||||||
state.error = error.message;
|
state.error = error.message || 'history restore failed';
|
||||||
|
state.notice = '';
|
||||||
state.messageVersion++;
|
state.messageVersion++;
|
||||||
updateMetrics();
|
updateMetrics();
|
||||||
renderDetail();
|
renderDetail();
|
||||||
|
|
@ -557,6 +566,7 @@ RISK: terminal cosplay if decoration creeps past data; held by the palette law.
|
||||||
if (state.source !== source) return;
|
if (state.source !== source) return;
|
||||||
state.connected = true;
|
state.connected = true;
|
||||||
state.error = '';
|
state.error = '';
|
||||||
|
state.notice = '';
|
||||||
state.messageVersion++;
|
state.messageVersion++;
|
||||||
updateMetrics();
|
updateMetrics();
|
||||||
reconcileSnapshot();
|
reconcileSnapshot();
|
||||||
|
|
@ -578,10 +588,8 @@ RISK: terminal cosplay if decoration creeps past data; held by the palette law.
|
||||||
source.onerror = () => {
|
source.onerror = () => {
|
||||||
if (state.source !== source) return;
|
if (state.source !== source) return;
|
||||||
state.connected = false;
|
state.connected = false;
|
||||||
if (!state.totalFrames) {
|
state.notice = 'Telemetry stream disconnected; retrying automatically.';
|
||||||
state.error = 'waiting for /events';
|
|
||||||
state.messageVersion++;
|
state.messageVersion++;
|
||||||
}
|
|
||||||
updateMetrics();
|
updateMetrics();
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
@ -621,12 +629,12 @@ RISK: terminal cosplay if decoration creeps past data; held by the palette law.
|
||||||
if (row) selectFrame(row.dataset.frame);
|
if (row) selectFrame(row.dataset.frame);
|
||||||
});
|
});
|
||||||
|
|
||||||
nodes.clearScope.addEventListener('click', () => selectStream(''));
|
|
||||||
nodes.filter.addEventListener('input', applyFilter);
|
nodes.filter.addEventListener('input', applyFilter);
|
||||||
nodes.kind.addEventListener('change', renderSelectedFrames);
|
nodes.kind.addEventListener('change', renderSelectedFrames);
|
||||||
nodes.limit.addEventListener('change', trimAllChannels);
|
nodes.limit.addEventListener('change', trimAllChannels);
|
||||||
nodes.reconnect.addEventListener('click', () => {
|
nodes.reconnect.addEventListener('click', () => {
|
||||||
state.error = '';
|
state.error = '';
|
||||||
|
state.notice = '';
|
||||||
state.messageVersion++;
|
state.messageVersion++;
|
||||||
connectEvents();
|
connectEvents();
|
||||||
updateMetrics();
|
updateMetrics();
|
||||||
|
|
|
||||||
|
|
@ -32,6 +32,7 @@ const HISTORY_CAP: usize = 512;
|
||||||
const HISTORY_MIN_INTERVAL: Duration = Duration::from_millis(250);
|
const HISTORY_MIN_INTERVAL: Duration = Duration::from_millis(250);
|
||||||
const PER_ACTOR_HISTORY_CAP: usize = 120;
|
const PER_ACTOR_HISTORY_CAP: usize = 120;
|
||||||
const GROWTH_WINDOW: usize = 12;
|
const GROWTH_WINDOW: usize = 12;
|
||||||
|
const ACTIVITY_RATE_WINDOW: Duration = Duration::from_secs(3);
|
||||||
/// Sampled message-receipt ring bound (view side).
|
/// Sampled message-receipt ring bound (view side).
|
||||||
pub(crate) const RECEIPT_CAP: usize = 16;
|
pub(crate) const RECEIPT_CAP: usize = 16;
|
||||||
/// Minimum spacing between receipts; bounds ring churn for noisy actors.
|
/// Minimum spacing between receipts; bounds ring churn for noisy actors.
|
||||||
|
|
@ -54,6 +55,9 @@ pub(crate) struct RuntimeState {
|
||||||
pub(crate) actors: BTreeMap<String, ActorState>,
|
pub(crate) actors: BTreeMap<String, ActorState>,
|
||||||
pub(crate) history: VecDeque<HistorySample>,
|
pub(crate) history: VecDeque<HistorySample>,
|
||||||
actor_snapshot_generation: u64,
|
actor_snapshot_generation: u64,
|
||||||
|
telemetry_generation: Option<u64>,
|
||||||
|
telemetry_sequence: Option<u64>,
|
||||||
|
telemetry_discontinuities: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RuntimeState {
|
impl RuntimeState {
|
||||||
|
|
@ -65,6 +69,9 @@ impl RuntimeState {
|
||||||
actors: BTreeMap::new(),
|
actors: BTreeMap::new(),
|
||||||
history: VecDeque::with_capacity(HISTORY_CAP),
|
history: VecDeque::with_capacity(HISTORY_CAP),
|
||||||
actor_snapshot_generation: 0,
|
actor_snapshot_generation: 0,
|
||||||
|
telemetry_generation: None,
|
||||||
|
telemetry_sequence: None,
|
||||||
|
telemetry_discontinuities: 0,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -74,13 +81,85 @@ impl RuntimeState {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
match channel {
|
match channel {
|
||||||
RUNTIME_ACTORS => self.apply_actors(&value, now),
|
RUNTIME_ACTORS if self.accept_protocol_frame(&value) => {
|
||||||
|
match value.get("kind").and_then(Value::as_str) {
|
||||||
|
Some("census") => self.apply_actors(&value, now),
|
||||||
|
Some("activity") => self.apply_activity(&value, now),
|
||||||
|
Some("vital") => self.apply_vital(&value, now),
|
||||||
|
_ => self.apply_actors(&value, now),
|
||||||
|
}
|
||||||
|
}
|
||||||
RUNTIME_STATS => self.apply_stats(&value, now),
|
RUNTIME_STATS => self.apply_stats(&value, now),
|
||||||
_ => {}
|
_ => {}
|
||||||
}
|
}
|
||||||
self.push_history(now);
|
self.push_history(now);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn accept_protocol_frame(&mut self, value: &Value) -> bool {
|
||||||
|
let Some(generation) = value.get("generation").and_then(Value::as_u64) else {
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
match self.telemetry_generation {
|
||||||
|
Some(current) if generation < current => return false,
|
||||||
|
Some(current) if generation > current => {
|
||||||
|
self.actors.clear();
|
||||||
|
self.telemetry_sequence = None;
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
self.telemetry_generation = Some(generation);
|
||||||
|
|
||||||
|
let Some(sequence) = value.get("sequence").and_then(Value::as_u64) else {
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
if let Some(previous) = self.telemetry_sequence {
|
||||||
|
if sequence <= previous {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if sequence != previous.wrapping_add(1) {
|
||||||
|
self.telemetry_discontinuities = self.telemetry_discontinuities.saturating_add(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.telemetry_sequence = Some(sequence);
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
fn apply_vital(&mut self, value: &Value, now: Instant) {
|
||||||
|
let Some(actor) = value.get("actor") else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let address = string_field(actor, &["address", "addr", "actor_addr"]);
|
||||||
|
if value.get("event").and_then(Value::as_str) == Some("stopped") {
|
||||||
|
if let Some(address) = address {
|
||||||
|
self.actors.remove(&address);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let worker = u32_field(value, &["worker_id", "worker"]);
|
||||||
|
let _ = self.apply_actor(actor, now, worker);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn apply_activity(&mut self, value: &Value, now: Instant) {
|
||||||
|
let worker = u32_field(value, &["worker_id", "worker"]);
|
||||||
|
let interval = value
|
||||||
|
.get("interval_ms")
|
||||||
|
.and_then(Value::as_u64)
|
||||||
|
.map(|ms| Duration::from_millis(ms).as_secs_f64())
|
||||||
|
.unwrap_or(0.0);
|
||||||
|
let Some(actors) = value.get("actors").and_then(Value::as_array) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
for record in actors {
|
||||||
|
let Some(address) = string_field(record, &["address", "addr", "actor_addr"]) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
self.actors
|
||||||
|
.entry(address.clone())
|
||||||
|
.or_insert_with(|| ActorState::new(address))
|
||||||
|
.apply_activity(record, now, worker, interval);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn apply_actors(&mut self, value: &Value, now: Instant) {
|
fn apply_actors(&mut self, value: &Value, now: Instant) {
|
||||||
// A wrapped actor list is a complete snapshot for one worker. A list
|
// A wrapped actor list is a complete snapshot for one worker. A list
|
||||||
// without a worker is a complete merged-runtime snapshot. Bare actor
|
// without a worker is a complete merged-runtime snapshot. Bare actor
|
||||||
|
|
@ -186,6 +265,12 @@ impl RuntimeState {
|
||||||
}
|
}
|
||||||
totals
|
totals
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn totals_at(&self, now: Instant) -> Totals {
|
||||||
|
let mut totals = self.totals();
|
||||||
|
totals.msg_per_sec = self.actors.values().map(|actor| actor.rate_at(now)).sum();
|
||||||
|
totals
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
|
|
@ -207,9 +292,18 @@ pub(crate) struct ActorState {
|
||||||
/// Messages folded away by the receipt sampling interval.
|
/// Messages folded away by the receipt sampling interval.
|
||||||
pub(crate) sampled_out: u64,
|
pub(crate) sampled_out: u64,
|
||||||
pub(crate) last_update: Option<Instant>,
|
pub(crate) last_update: Option<Instant>,
|
||||||
|
pub(crate) last_active: Option<Instant>,
|
||||||
|
activity_rates: VecDeque<ActivityRateSample>,
|
||||||
snapshot_generation: u64,
|
snapshot_generation: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct ActivityRateSample {
|
||||||
|
at: Instant,
|
||||||
|
interval: f64,
|
||||||
|
delta: u64,
|
||||||
|
}
|
||||||
|
|
||||||
impl ActorState {
|
impl ActorState {
|
||||||
fn new(address: String) -> Self {
|
fn new(address: String) -> Self {
|
||||||
Self {
|
Self {
|
||||||
|
|
@ -229,6 +323,8 @@ impl ActorState {
|
||||||
receipts: VecDeque::with_capacity(RECEIPT_CAP),
|
receipts: VecDeque::with_capacity(RECEIPT_CAP),
|
||||||
sampled_out: 0,
|
sampled_out: 0,
|
||||||
last_update: None,
|
last_update: None,
|
||||||
|
last_active: None,
|
||||||
|
activity_rates: VecDeque::new(),
|
||||||
snapshot_generation: 0,
|
snapshot_generation: 0,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -282,9 +378,81 @@ impl ActorState {
|
||||||
self.messages_processed.saturating_sub(processed_before),
|
self.messages_processed.saturating_sub(processed_before),
|
||||||
);
|
);
|
||||||
self.push_history(now);
|
self.push_history(now);
|
||||||
|
if self.messages_processed > processed_before {
|
||||||
|
self.last_active = Some(now);
|
||||||
|
}
|
||||||
self.recompute_growth();
|
self.recompute_growth();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn apply_activity(
|
||||||
|
&mut self,
|
||||||
|
value: &Value,
|
||||||
|
now: Instant,
|
||||||
|
default_worker: Option<u32>,
|
||||||
|
interval: f64,
|
||||||
|
) {
|
||||||
|
if let Some(worker_id) = default_worker {
|
||||||
|
self.worker_id = Some(worker_id);
|
||||||
|
}
|
||||||
|
let delta = u64_field(value, &["messages_processed_delta"]).unwrap_or(0);
|
||||||
|
self.messages_processed = self.messages_processed.saturating_add(delta);
|
||||||
|
if delta > 0 {
|
||||||
|
self.last_active = Some(now);
|
||||||
|
}
|
||||||
|
if interval > 0.0 {
|
||||||
|
self.activity_rates.push_back(ActivityRateSample {
|
||||||
|
at: now,
|
||||||
|
interval,
|
||||||
|
delta,
|
||||||
|
});
|
||||||
|
while self
|
||||||
|
.activity_rates
|
||||||
|
.front()
|
||||||
|
.is_some_and(|sample| now.duration_since(sample.at) > ACTIVITY_RATE_WINDOW)
|
||||||
|
{
|
||||||
|
self.activity_rates.pop_front();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.msg_per_sec = self.rate_at(now);
|
||||||
|
assign_u32(&mut self.mailbox_depth, value, &["mailbox_depth", "queued"]);
|
||||||
|
if let Some(last) = string_field(
|
||||||
|
value,
|
||||||
|
&["last_msg_type", "last_message", "last_message_type"],
|
||||||
|
)
|
||||||
|
.filter(|last| !last.is_empty())
|
||||||
|
{
|
||||||
|
self.last_msg_type = Some(last);
|
||||||
|
}
|
||||||
|
if let Some(poisoned) = value.get("poisoned").and_then(Value::as_bool) {
|
||||||
|
self.poisoned = poisoned;
|
||||||
|
}
|
||||||
|
self.last_update = Some(now);
|
||||||
|
self.fold_receipt(now, delta);
|
||||||
|
self.push_history(now);
|
||||||
|
self.recompute_growth();
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn rate_at(&self, now: Instant) -> f64 {
|
||||||
|
if self.activity_rates.is_empty() {
|
||||||
|
return self.msg_per_sec;
|
||||||
|
}
|
||||||
|
let mut messages = 0_u64;
|
||||||
|
let mut seconds = 0.0;
|
||||||
|
for sample in self
|
||||||
|
.activity_rates
|
||||||
|
.iter()
|
||||||
|
.filter(|sample| now.duration_since(sample.at) <= ACTIVITY_RATE_WINDOW)
|
||||||
|
{
|
||||||
|
messages = messages.saturating_add(sample.delta);
|
||||||
|
seconds += sample.interval;
|
||||||
|
}
|
||||||
|
if seconds > 0.0 {
|
||||||
|
messages as f64 / seconds
|
||||||
|
} else {
|
||||||
|
0.0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Fold a `messages_processed` jump into the sampled receipt ring. The
|
/// Fold a `messages_processed` jump into the sampled receipt ring. The
|
||||||
/// receipt carries the last message type visible on the frame; messages
|
/// receipt carries the last message type visible on the frame; messages
|
||||||
/// hidden inside the jump (or skipped by the interval) are counted, not
|
/// hidden inside the jump (or skipped by the interval) are counted, not
|
||||||
|
|
@ -464,6 +632,10 @@ mod tests {
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
fn apply_protocol(runtime: &mut RuntimeState, value: Value, at: Instant) {
|
||||||
|
runtime.update(RUNTIME_ACTORS, value.to_string().as_bytes(), at);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn per_worker_snapshots_remove_stopped_actors_without_touching_other_workers() {
|
fn per_worker_snapshots_remove_stopped_actors_without_touching_other_workers() {
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
|
|
@ -518,4 +690,80 @@ mod tests {
|
||||||
vec!["other-worker"]
|
vec!["other-worker"]
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn folds_census_activity_and_vital_records_in_sequence() {
|
||||||
|
let now = Instant::now();
|
||||||
|
let mut runtime = RuntimeState::new(now);
|
||||||
|
apply_protocol(
|
||||||
|
&mut runtime,
|
||||||
|
json!({
|
||||||
|
"kind": "census",
|
||||||
|
"generation": 7,
|
||||||
|
"sequence": 1,
|
||||||
|
"worker_id": 0,
|
||||||
|
"actors": [{
|
||||||
|
"address": "actor-a",
|
||||||
|
"actor_type": "ActorA",
|
||||||
|
"message_type": "Ping",
|
||||||
|
"messages_processed": 10,
|
||||||
|
"mailbox_depth": 1,
|
||||||
|
"poisoned": false
|
||||||
|
}]
|
||||||
|
}),
|
||||||
|
now,
|
||||||
|
);
|
||||||
|
apply_protocol(
|
||||||
|
&mut runtime,
|
||||||
|
json!({
|
||||||
|
"kind": "activity",
|
||||||
|
"generation": 7,
|
||||||
|
"sequence": 2,
|
||||||
|
"worker_id": 0,
|
||||||
|
"interval_ms": 500,
|
||||||
|
"actors": [{
|
||||||
|
"address": "actor-a",
|
||||||
|
"messages_processed_delta": 4,
|
||||||
|
"mailbox_depth": 2,
|
||||||
|
"mailbox_max_depth": 5,
|
||||||
|
"last_msg_type": "Ping"
|
||||||
|
}]
|
||||||
|
}),
|
||||||
|
now + Duration::from_millis(500),
|
||||||
|
);
|
||||||
|
let actor = runtime.actors.get("actor-a").unwrap();
|
||||||
|
assert_eq!(actor.messages_processed, 14);
|
||||||
|
assert_eq!(actor.mailbox_depth, 2);
|
||||||
|
assert_eq!(actor.msg_per_sec, 8.0);
|
||||||
|
assert_eq!(actor.rate_at(now + Duration::from_secs(4)), 0.0);
|
||||||
|
assert_eq!(actor.last_active, Some(now + Duration::from_millis(500)));
|
||||||
|
|
||||||
|
apply_protocol(
|
||||||
|
&mut runtime,
|
||||||
|
json!({
|
||||||
|
"kind": "vital",
|
||||||
|
"generation": 7,
|
||||||
|
"sequence": 3,
|
||||||
|
"worker_id": 0,
|
||||||
|
"event": "stopped",
|
||||||
|
"actor": {"address": "actor-a"}
|
||||||
|
}),
|
||||||
|
now + Duration::from_millis(600),
|
||||||
|
);
|
||||||
|
assert!(runtime.actors.is_empty());
|
||||||
|
|
||||||
|
apply_protocol(
|
||||||
|
&mut runtime,
|
||||||
|
json!({
|
||||||
|
"kind": "activity",
|
||||||
|
"generation": 7,
|
||||||
|
"sequence": 2,
|
||||||
|
"worker_id": 0,
|
||||||
|
"interval_ms": 250,
|
||||||
|
"actors": [{"address": "actor-a", "messages_processed_delta": 99}]
|
||||||
|
}),
|
||||||
|
now + Duration::from_millis(700),
|
||||||
|
);
|
||||||
|
assert!(runtime.actors.is_empty(), "stale sequence is ignored");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -62,6 +62,21 @@ pub struct ObservedProbeEvent {
|
||||||
pub consecutive_timeouts: u32,
|
pub consecutive_timeouts: u32,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// One-second aggregate of routine successful probe traffic.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct ObservedProbeSummary {
|
||||||
|
pub interval_ms: u64,
|
||||||
|
pub sent: u64,
|
||||||
|
pub acked: u64,
|
||||||
|
pub direct_sent: u64,
|
||||||
|
pub indirect_sent: u64,
|
||||||
|
pub rtt_samples: u64,
|
||||||
|
pub rtt_ms_min: Option<u32>,
|
||||||
|
pub rtt_ms_p50: Option<u32>,
|
||||||
|
pub rtt_ms_p95: Option<u32>,
|
||||||
|
pub rtt_ms_max: Option<u32>,
|
||||||
|
}
|
||||||
|
|
||||||
/// Current probe state for one peer.
|
/// Current probe state for one peer.
|
||||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||||
pub struct PeerProbeState {
|
pub struct PeerProbeState {
|
||||||
|
|
@ -85,6 +100,12 @@ struct Inner {
|
||||||
probe_events: VecDeque<ObservedProbeEvent>,
|
probe_events: VecDeque<ObservedProbeEvent>,
|
||||||
/// Transitions awaiting drain by the node's telemetry tick.
|
/// Transitions awaiting drain by the node's telemetry tick.
|
||||||
transitions: VecDeque<ObservedTransition>,
|
transitions: VecDeque<ObservedTransition>,
|
||||||
|
summary_started: Option<Instant>,
|
||||||
|
summary_sent: u64,
|
||||||
|
summary_acked: u64,
|
||||||
|
summary_direct_sent: u64,
|
||||||
|
summary_indirect_sent: u64,
|
||||||
|
summary_rtts: Vec<u32>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The installed SWIM observer plus the readouts the node consumes each tick.
|
/// The installed SWIM observer plus the readouts the node consumes each tick.
|
||||||
|
|
@ -147,6 +168,50 @@ impl SwimTelemetry {
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Drain one aggregate after at least one second. Routine sent/acked
|
||||||
|
/// observations never enter the immediate-event queue.
|
||||||
|
pub fn drain_probe_summary(&self) -> Option<ObservedProbeSummary> {
|
||||||
|
let now = Instant::now();
|
||||||
|
let mut inner = self.inner.lock().expect("swim telemetry poisoned");
|
||||||
|
let started = inner.summary_started.get_or_insert(now);
|
||||||
|
let elapsed = now.duration_since(*started);
|
||||||
|
if elapsed < Duration::from_secs(1) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
inner.summary_started = Some(now);
|
||||||
|
if inner.summary_sent == 0 && inner.summary_acked == 0 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
inner.summary_rtts.sort_unstable();
|
||||||
|
let percentile = |numerator: usize, denominator: usize| {
|
||||||
|
if inner.summary_rtts.is_empty() {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
let last = inner.summary_rtts.len() - 1;
|
||||||
|
Some(inner.summary_rtts[last * numerator / denominator])
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let summary = ObservedProbeSummary {
|
||||||
|
interval_ms: elapsed.as_millis().min(u64::MAX as u128) as u64,
|
||||||
|
sent: inner.summary_sent,
|
||||||
|
acked: inner.summary_acked,
|
||||||
|
direct_sent: inner.summary_direct_sent,
|
||||||
|
indirect_sent: inner.summary_indirect_sent,
|
||||||
|
rtt_samples: inner.summary_rtts.len() as u64,
|
||||||
|
rtt_ms_min: inner.summary_rtts.first().copied(),
|
||||||
|
rtt_ms_p50: percentile(1, 2),
|
||||||
|
rtt_ms_p95: percentile(95, 100),
|
||||||
|
rtt_ms_max: inner.summary_rtts.last().copied(),
|
||||||
|
};
|
||||||
|
inner.summary_sent = 0;
|
||||||
|
inner.summary_acked = 0;
|
||||||
|
inner.summary_direct_sent = 0;
|
||||||
|
inner.summary_indirect_sent = 0;
|
||||||
|
inner.summary_rtts.clear();
|
||||||
|
Some(summary)
|
||||||
|
}
|
||||||
|
|
||||||
/// Snapshot the probe state for one peer without draining events.
|
/// Snapshot the probe state for one peer without draining events.
|
||||||
pub fn peer_probe_state(&self, peer: NodeId) -> PeerProbeState {
|
pub fn peer_probe_state(&self, peer: NodeId) -> PeerProbeState {
|
||||||
let inner = self.inner.lock().expect("swim telemetry poisoned");
|
let inner = self.inner.lock().expect("swim telemetry poisoned");
|
||||||
|
|
@ -184,27 +249,23 @@ impl SwimTelemetry {
|
||||||
inner.targets.pop_front();
|
inner.targets.pop_front();
|
||||||
}
|
}
|
||||||
inner.targets.push_back(target);
|
inner.targets.push_back(target);
|
||||||
let state = Self::peer_probe_state_locked(&inner, target);
|
inner.summary_started.get_or_insert_with(Instant::now);
|
||||||
Self::push_probe_event(
|
inner.summary_sent = inner.summary_sent.saturating_add(1);
|
||||||
&mut inner,
|
match kind {
|
||||||
ObservedProbeEvent {
|
"direct" => {
|
||||||
event: "sent",
|
inner.summary_direct_sent = inner.summary_direct_sent.saturating_add(1);
|
||||||
target,
|
}
|
||||||
sequence,
|
"indirect" => {
|
||||||
kind,
|
inner.summary_indirect_sent = inner.summary_indirect_sent.saturating_add(1);
|
||||||
rtt_ms: None,
|
}
|
||||||
budget_ms: None,
|
_ => {}
|
||||||
last_ack_age: state.last_ack_age,
|
}
|
||||||
consecutive_timeouts: state.consecutive_timeouts,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
SwimObservation::ProbeAcked {
|
SwimObservation::ProbeAcked {
|
||||||
target,
|
target,
|
||||||
sequence,
|
sequence,
|
||||||
kind,
|
kind: _,
|
||||||
} => {
|
} => {
|
||||||
let state = Self::peer_probe_state_locked(&inner, target);
|
|
||||||
let rtt_ms = inner
|
let rtt_ms = inner
|
||||||
.in_flight
|
.in_flight
|
||||||
.remove(&(target, sequence))
|
.remove(&(target, sequence))
|
||||||
|
|
@ -217,19 +278,10 @@ impl SwimTelemetry {
|
||||||
}
|
}
|
||||||
inner.last_ack.insert(target, Instant::now());
|
inner.last_ack.insert(target, Instant::now());
|
||||||
inner.consecutive_timeouts.remove(&target);
|
inner.consecutive_timeouts.remove(&target);
|
||||||
Self::push_probe_event(
|
inner.summary_acked = inner.summary_acked.saturating_add(1);
|
||||||
&mut inner,
|
if let Some(rtt) = rtt_ms {
|
||||||
ObservedProbeEvent {
|
inner.summary_rtts.push(rtt);
|
||||||
event: "acked",
|
}
|
||||||
target,
|
|
||||||
sequence,
|
|
||||||
kind,
|
|
||||||
rtt_ms,
|
|
||||||
budget_ms: None,
|
|
||||||
last_ack_age: state.last_ack_age,
|
|
||||||
consecutive_timeouts: state.consecutive_timeouts,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
SwimObservation::ProbeTimedOut {
|
SwimObservation::ProbeTimedOut {
|
||||||
target,
|
target,
|
||||||
|
|
|
||||||
|
|
@ -86,6 +86,26 @@ pub struct SwimProbeEvent {
|
||||||
pub lifeguard_enabled: bool,
|
pub lifeguard_enabled: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// One-second aggregate of routine SWIM sent/acked traffic.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct SwimProbeSummary {
|
||||||
|
pub event: String,
|
||||||
|
pub interval_ms: u64,
|
||||||
|
pub sent: u64,
|
||||||
|
pub acked: u64,
|
||||||
|
pub direct_sent: u64,
|
||||||
|
pub indirect_sent: u64,
|
||||||
|
pub rtt_samples: u64,
|
||||||
|
pub rtt_ms_min: Option<u32>,
|
||||||
|
pub rtt_ms_p50: Option<u32>,
|
||||||
|
pub rtt_ms_p95: Option<u32>,
|
||||||
|
pub rtt_ms_max: Option<u32>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub local_phase: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pub probe_interval_ms: u64,
|
||||||
|
}
|
||||||
|
|
||||||
/// Consolidated distribution-subsystem state.
|
/// Consolidated distribution-subsystem state.
|
||||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct DistributionState {
|
pub struct DistributionState {
|
||||||
|
|
@ -142,6 +162,9 @@ impl Record for MembershipTransition {
|
||||||
impl Record for SwimProbeEvent {
|
impl Record for SwimProbeEvent {
|
||||||
const CHANNEL: &'static str = SWIM_PROBES;
|
const CHANNEL: &'static str = SWIM_PROBES;
|
||||||
}
|
}
|
||||||
|
impl Record for SwimProbeSummary {
|
||||||
|
const CHANNEL: &'static str = SWIM_PROBES;
|
||||||
|
}
|
||||||
impl Record for DistributionState {
|
impl Record for DistributionState {
|
||||||
const CHANNEL: &'static str = DIST_STATE;
|
const CHANNEL: &'static str = DIST_STATE;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -249,15 +249,21 @@ mod snapshot_and_swim_telemetry {
|
||||||
});
|
});
|
||||||
|
|
||||||
let events = telemetry.drain_probe_events();
|
let events = telemetry.drain_probe_events();
|
||||||
assert_eq!(events.len(), 4);
|
assert_eq!(events.len(), 1, "only timeout remains immediate");
|
||||||
assert_eq!(events[0].event, "sent");
|
assert_eq!(events[0].event, "timed_out");
|
||||||
assert_eq!(events[1].event, "acked");
|
assert_eq!(events[0].budget_ms, Some(15_000));
|
||||||
assert!(events[1].rtt_ms.is_some());
|
assert_eq!(events[0].rtt_ms, None);
|
||||||
assert_eq!(events[2].event, "sent");
|
assert_eq!(events[0].consecutive_timeouts, 1);
|
||||||
assert_eq!(events[3].event, "timed_out");
|
|
||||||
assert_eq!(events[3].budget_ms, Some(15_000));
|
|
||||||
assert_eq!(events[3].rtt_ms, None);
|
|
||||||
assert_eq!(events[3].consecutive_timeouts, 1);
|
|
||||||
assert!(telemetry.drain_probe_events().is_empty());
|
assert!(telemetry.drain_probe_events().is_empty());
|
||||||
|
|
||||||
|
std::thread::sleep(Duration::from_millis(1_010));
|
||||||
|
let summary = telemetry
|
||||||
|
.drain_probe_summary()
|
||||||
|
.expect("one-second routine probe aggregate");
|
||||||
|
assert_eq!(summary.sent, 2);
|
||||||
|
assert_eq!(summary.acked, 1);
|
||||||
|
assert_eq!(summary.direct_sent, 2);
|
||||||
|
assert_eq!(summary.rtt_samples, 1);
|
||||||
|
assert!(summary.rtt_ms_p50.is_some());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -14,6 +14,7 @@ swactor = { path = "../..", features = ["serde"] }
|
||||||
swactor-transport = { path = "../transport" }
|
swactor-transport = { path = "../transport" }
|
||||||
swactor-process = { path = "../process" }
|
swactor-process = { path = "../process" }
|
||||||
swactor-engine = { path = "../engine" }
|
swactor-engine = { path = "../engine" }
|
||||||
|
telemetry = { path = "../telemetry" }
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
tar = "0.4"
|
tar = "0.4"
|
||||||
|
|
|
||||||
|
|
@ -81,6 +81,7 @@ pub struct NodeJobActor {
|
||||||
output_sink: Option<SharedJobEdgeSink>,
|
output_sink: Option<SharedJobEdgeSink>,
|
||||||
data_plane: Option<Arc<dyn JobDataPlanePort>>,
|
data_plane: Option<Arc<dyn JobDataPlanePort>>,
|
||||||
route_registrar: Option<Arc<dyn JobRouteRegistrar>>,
|
route_registrar: Option<Arc<dyn JobRouteRegistrar>>,
|
||||||
|
process_telemetry: Option<telemetry::TelemetryProducer>,
|
||||||
data_plane_env: BTreeMap<String, String>,
|
data_plane_env: BTreeMap<String, String>,
|
||||||
data_plane_error: Option<String>,
|
data_plane_error: Option<String>,
|
||||||
data_plane_pending: bool,
|
data_plane_pending: bool,
|
||||||
|
|
@ -117,6 +118,7 @@ impl NodeJobActor {
|
||||||
output_sink: None,
|
output_sink: None,
|
||||||
data_plane: None,
|
data_plane: None,
|
||||||
route_registrar: None,
|
route_registrar: None,
|
||||||
|
process_telemetry: None,
|
||||||
data_plane_env: BTreeMap::new(),
|
data_plane_env: BTreeMap::new(),
|
||||||
data_plane_error: None,
|
data_plane_error: None,
|
||||||
data_plane_pending: false,
|
data_plane_pending: false,
|
||||||
|
|
@ -143,6 +145,11 @@ impl NodeJobActor {
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn with_process_telemetry(mut self, producer: telemetry::TelemetryProducer) -> Self {
|
||||||
|
self.process_telemetry = Some(producer);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
/// Edge mode: workspace bytes arrive over EDGE_ALPN and are extracted by the
|
/// Edge mode: workspace bytes arrive over EDGE_ALPN and are extracted by the
|
||||||
/// integration layer, which sets `flag` once the workspace is on disk.
|
/// integration layer, which sets `flag` once the workspace is on disk.
|
||||||
pub fn with_workspace_ready(mut self, flag: Arc<AtomicBool>) -> Self {
|
pub fn with_workspace_ready(mut self, flag: Arc<AtomicBool>) -> Self {
|
||||||
|
|
@ -246,12 +253,11 @@ impl NodeJobActor {
|
||||||
working_dir: Some(self.workdir.clone()),
|
working_dir: Some(self.workdir.clone()),
|
||||||
label: Some(format!("job-runner-{:?}", phase).to_lowercase()),
|
label: Some(format!("job-runner-{:?}", phase).to_lowercase()),
|
||||||
};
|
};
|
||||||
if let Err(e) = spawn_local_process(
|
let output = self.process_telemetry.clone().map_or_else(
|
||||||
ctx,
|
|| ProcessOutputConfig::disabled(relay),
|
||||||
&self.sender,
|
|producer| ProcessOutputConfig::telemetry_mirror(relay, producer),
|
||||||
spec,
|
);
|
||||||
ProcessOutputConfig::disabled(relay),
|
if let Err(e) = spawn_local_process(ctx, &self.sender, spec, output) {
|
||||||
) {
|
|
||||||
self.emit(
|
self.emit(
|
||||||
ctx,
|
ctx,
|
||||||
NodeJobEvent::NodeFault {
|
NodeJobEvent::NodeFault {
|
||||||
|
|
|
||||||
|
|
@ -71,8 +71,9 @@ ProcessOutputConfig::observability(&self) -> ProcessLifecycleObservability
|
||||||
|
|
||||||
`disabled` sends only upstream `ProcessOutput`.
|
`disabled` sends only upstream `ProcessOutput`.
|
||||||
|
|
||||||
`telemetry_mirror` sends upstream `ProcessOutput` and also mirrors each
|
`telemetry_mirror` sends every `ProcessOutput` upstream, mirrors lifecycle
|
||||||
lifecycle/control output to one telemetry channel.
|
records to `proc.<label>.lifecycle`, and mirrors raw child bytes to
|
||||||
|
`proc.<label>.stdout` and `proc.<label>.stderr`.
|
||||||
|
|
||||||
### 1.3 ProcessLifecycleObservability
|
### 1.3 ProcessLifecycleObservability
|
||||||
|
|
||||||
|
|
@ -81,8 +82,8 @@ ProcessLifecycleObservability::Disabled
|
||||||
ProcessLifecycleObservability::TelemetryMirror
|
ProcessLifecycleObservability::TelemetryMirror
|
||||||
```
|
```
|
||||||
|
|
||||||
This setting controls lifecycle/control mirroring only. It does not enable child
|
This setting controls telemetry mirroring. Child stdout/stderr are always
|
||||||
stdin/stdout/stderr handling.
|
delivered upstream; the mirror additionally publishes those bytes.
|
||||||
|
|
||||||
### 1.4 ProcessCommand
|
### 1.4 ProcessCommand
|
||||||
|
|
||||||
|
|
@ -98,6 +99,8 @@ present, is the grace duration before kill escalation.
|
||||||
### 1.5 ProcessOutput
|
### 1.5 ProcessOutput
|
||||||
|
|
||||||
```text
|
```text
|
||||||
|
ProcessOutput::Stdout(Vec<u8>)
|
||||||
|
ProcessOutput::Stderr(Vec<u8>)
|
||||||
ProcessOutput::Started { pid: u32 }
|
ProcessOutput::Started { pid: u32 }
|
||||||
ProcessOutput::SpawnFailed { error: String }
|
ProcessOutput::SpawnFailed { error: String }
|
||||||
ProcessOutput::Exited { status: ExitStatus }
|
ProcessOutput::Exited { status: ExitStatus }
|
||||||
|
|
@ -187,17 +190,18 @@ cmd.args(&spec.args)
|
||||||
cmd.env(key, value) for each spec.env entry
|
cmd.env(key, value) for each spec.env entry
|
||||||
cmd.current_dir(dir) when spec.working_dir is Some(dir)
|
cmd.current_dir(dir) when spec.working_dir is Some(dir)
|
||||||
cmd.stdin(Stdio::null())
|
cmd.stdin(Stdio::null())
|
||||||
cmd.stdout(Stdio::null())
|
cmd.stdout(Stdio::piped())
|
||||||
cmd.stderr(Stdio::null())
|
cmd.stderr(Stdio::piped())
|
||||||
cmd.spawn()
|
cmd.spawn()
|
||||||
```
|
```
|
||||||
|
|
||||||
The crate does not invoke a shell unless the caller explicitly sets `command` to
|
The crate does not invoke a shell unless the caller explicitly sets `command` to
|
||||||
a shell executable and supplies shell arguments.
|
a shell executable and supplies shell arguments.
|
||||||
|
|
||||||
Child stdin/stdout/stderr are connected to null handles. The managed-process
|
Child stdin is connected to a null handle. Stdout and stderr are piped to
|
||||||
protocol does not expose stdin writes, stdout/stderr output events, PTY resize,
|
bounded-read threads and delivered as `ProcessOutput::Stdout` /
|
||||||
or arbitrary signal commands.
|
`ProcessOutput::Stderr` before the terminal output. The protocol does not expose
|
||||||
|
stdin writes, PTY resize, or arbitrary signal commands.
|
||||||
|
|
||||||
If `cmd.spawn()` fails, the actor emits exactly one terminal
|
If `cmd.spawn()` fails, the actor emits exactly one terminal
|
||||||
`ProcessOutput::SpawnFailed { error }` and does not emit `Started`, `Exited`, or
|
`ProcessOutput::SpawnFailed { error }` and does not emit `Started`, `Exited`, or
|
||||||
|
|
@ -209,13 +213,14 @@ If `cmd.spawn()` fails, the actor emits exactly one terminal
|
||||||
|
|
||||||
The actor sends each public `ProcessOutput` to the configured upstream actor.
|
The actor sends each public `ProcessOutput` to the configured upstream actor.
|
||||||
|
|
||||||
When `ProcessOutputConfig::telemetry_mirror` is used, the actor also mirrors
|
When `ProcessOutputConfig::telemetry_mirror` is used, lifecycle outputs are
|
||||||
each output to the configured telemetry producer. Telemetry submit failure is
|
encoded as JSON on `proc.<label>.lifecycle`; stdout/stderr retain their real byte
|
||||||
ignored and does not suppress upstream output or emit `ProcessOutput::Error`.
|
content on text-stream channels. Telemetry submit failure is ignored and does
|
||||||
|
not suppress upstream output or emit `ProcessOutput::Error`.
|
||||||
|
|
||||||
Public lifecycle/control output order follows observed lifecycle:
|
Public lifecycle/control output order follows observed lifecycle:
|
||||||
|
|
||||||
- successful spawn emits `Started` before any terminal `Exited`;
|
- successful spawn emits `Started`, then any stdout/stderr chunks, before terminal `Exited`;
|
||||||
- spawn failure emits `SpawnFailed` without `Started` or `Exited`;
|
- spawn failure emits `SpawnFailed` without `Started` or `Exited`;
|
||||||
- supervisor failure emits `Error`;
|
- supervisor failure emits `Error`;
|
||||||
- after a terminal output, later public stop attempts emit no additional
|
- after a terminal output, later public stop attempts emit no additional
|
||||||
|
|
|
||||||
|
|
@ -167,6 +167,16 @@ impl ProcessActor {
|
||||||
self.state = ProcessActorState::Done(ProcessDoneState::Exited(status));
|
self.state = ProcessActorState::Done(ProcessDoneState::Exited(status));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
ThreadEvent::Output { stderr, bytes } => {
|
||||||
|
self.emit_process_output(
|
||||||
|
ctx,
|
||||||
|
if stderr {
|
||||||
|
ProcessOutput::Stderr(bytes)
|
||||||
|
} else {
|
||||||
|
ProcessOutput::Stdout(bytes)
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
ThreadEvent::Error { error } => {
|
ThreadEvent::Error { error } => {
|
||||||
if !matches!(self.state, ProcessActorState::Done(_)) {
|
if !matches!(self.state, ProcessActorState::Done(_)) {
|
||||||
self.emit_process_output(
|
self.emit_process_output(
|
||||||
|
|
|
||||||
|
|
@ -54,12 +54,22 @@ pub(crate) struct PreparedProcessOutput {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) struct LifecycleTelemetryMirror {
|
pub(crate) struct LifecycleTelemetryMirror {
|
||||||
channel: telemetry::ChannelId,
|
lifecycle: telemetry::ChannelId,
|
||||||
|
stdout: telemetry::ChannelId,
|
||||||
|
stderr: telemetry::ChannelId,
|
||||||
producer: TelemetryProducer,
|
producer: TelemetryProducer,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl LifecycleTelemetryMirror {
|
impl LifecycleTelemetryMirror {
|
||||||
pub(crate) fn submit(&self, output: &ProcessOutput) {
|
pub(crate) fn submit(&self, output: &ProcessOutput) {
|
||||||
|
match output {
|
||||||
|
ProcessOutput::Stdout(bytes) => {
|
||||||
|
let _ = self.producer.submit_bytes(self.stdout, bytes.clone());
|
||||||
|
}
|
||||||
|
ProcessOutput::Stderr(bytes) => {
|
||||||
|
let _ = self.producer.submit_bytes(self.stderr, bytes.clone());
|
||||||
|
}
|
||||||
|
output => {
|
||||||
let payload = match output {
|
let payload = match output {
|
||||||
ProcessOutput::Started { pid } => json!({"event": "started", "pid": pid}),
|
ProcessOutput::Started { pid } => json!({"event": "started", "pid": pid}),
|
||||||
ProcessOutput::SpawnFailed { error } => {
|
ProcessOutput::SpawnFailed { error } => {
|
||||||
|
|
@ -72,12 +82,26 @@ impl LifecycleTelemetryMirror {
|
||||||
ExitStatus::Signal(value) => {
|
ExitStatus::Signal(value) => {
|
||||||
json!({"event": "exited", "status": {"kind": "signal", "value": value}})
|
json!({"event": "exited", "status": {"kind": "signal", "value": value}})
|
||||||
}
|
}
|
||||||
ExitStatus::Unknown => json!({"event": "exited", "status": {"kind": "unknown"}}),
|
ExitStatus::Unknown => {
|
||||||
|
json!({"event": "exited", "status": {"kind": "unknown"}})
|
||||||
|
}
|
||||||
},
|
},
|
||||||
ProcessOutput::Error { error } => json!({"event": "error", "error": error}),
|
ProcessOutput::Error { error } => json!({"event": "error", "error": error}),
|
||||||
|
ProcessOutput::Stdout(_) | ProcessOutput::Stderr(_) => unreachable!(),
|
||||||
};
|
};
|
||||||
let bytes = serde_json::to_vec(&payload).expect("process lifecycle record serializes");
|
let bytes =
|
||||||
let _ = self.producer.submit_bytes(self.channel, bytes);
|
serde_json::to_vec(&payload).expect("process lifecycle record serializes");
|
||||||
|
let _ = self.producer.submit_bytes(self.lifecycle, bytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn channel_registration_error(error: telemetry::ChannelRegistrationError) -> swactor::Error {
|
||||||
|
match error {
|
||||||
|
telemetry::ChannelRegistrationError::ConflictingName { name } => swactor::Error::from(
|
||||||
|
format!("conflicting telemetry channel registration for {name}"),
|
||||||
|
),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -98,25 +122,29 @@ pub(crate) fn prepare_process_output(
|
||||||
.expect("telemetry mirror config stores producer");
|
.expect("telemetry mirror config stores producer");
|
||||||
let reservation =
|
let reservation =
|
||||||
LifecycleLabelReservation::reserve(producer.stream_id().clone(), &label)?;
|
LifecycleLabelReservation::reserve(producer.stream_id().clone(), &label)?;
|
||||||
let channel_name = label.channel_name();
|
let lifecycle = producer
|
||||||
let channel = producer
|
|
||||||
.try_register_channel(
|
.try_register_channel(
|
||||||
channel_name,
|
label.channel_name(),
|
||||||
ChannelContent::JsonRecord {
|
ChannelContent::JsonRecord {
|
||||||
schema: Some("swactor_process.lifecycle.v1".to_owned()),
|
schema: Some("swactor_process.lifecycle.v1".to_owned()),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.map_err(|err| match err {
|
.map_err(channel_registration_error)?;
|
||||||
telemetry::ChannelRegistrationError::ConflictingName { name } => {
|
let stdout = producer
|
||||||
swactor::Error::from(format!(
|
.try_register_channel(label.stdout_channel_name(), ChannelContent::TextStream)
|
||||||
"conflicting telemetry channel registration for {name}"
|
.map_err(channel_registration_error)?;
|
||||||
))
|
let stderr = producer
|
||||||
}
|
.try_register_channel(label.stderr_channel_name(), ChannelContent::TextStream)
|
||||||
})?;
|
.map_err(channel_registration_error)?;
|
||||||
|
|
||||||
Ok(PreparedProcessOutput {
|
Ok(PreparedProcessOutput {
|
||||||
upstream: config.upstream,
|
upstream: config.upstream,
|
||||||
mirror: Some(LifecycleTelemetryMirror { channel, producer }),
|
mirror: Some(LifecycleTelemetryMirror {
|
||||||
|
lifecycle,
|
||||||
|
stdout,
|
||||||
|
stderr,
|
||||||
|
producer,
|
||||||
|
}),
|
||||||
_label_reservation: Some(reservation),
|
_label_reservation: Some(reservation),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
@ -134,6 +162,14 @@ impl LifecycleLabel {
|
||||||
pub(crate) fn channel_name(&self) -> String {
|
pub(crate) fn channel_name(&self) -> String {
|
||||||
format!("proc.{}.lifecycle", self.0)
|
format!("proc.{}.lifecycle", self.0)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn stdout_channel_name(&self) -> String {
|
||||||
|
format!("proc.{}.stdout", self.0)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn stderr_channel_name(&self) -> String {
|
||||||
|
format!("proc.{}.stderr", self.0)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn derive_lifecycle_label(spec: &ProcessSpec) -> Result<LifecycleLabel, swactor::Error> {
|
pub(crate) fn derive_lifecycle_label(spec: &ProcessSpec) -> Result<LifecycleLabel, swactor::Error> {
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,8 @@ pub enum ProcessCommand {
|
||||||
/// Public managed-process outputs.
|
/// Public managed-process outputs.
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub enum ProcessOutput {
|
pub enum ProcessOutput {
|
||||||
|
Stdout(Vec<u8>),
|
||||||
|
Stderr(Vec<u8>),
|
||||||
Started { pid: u32 },
|
Started { pid: u32 },
|
||||||
SpawnFailed { error: String },
|
SpawnFailed { error: String },
|
||||||
Exited { status: ExitStatus },
|
Exited { status: ExitStatus },
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
use std::io;
|
use std::io::{self, Read};
|
||||||
use std::mem;
|
use std::mem;
|
||||||
use std::os::fd::RawFd;
|
use std::os::fd::RawFd;
|
||||||
use std::process::{Child, Command, Stdio};
|
use std::process::{Child, Command, Stdio};
|
||||||
|
|
@ -21,6 +21,7 @@ pub(crate) enum ThreadEvent {
|
||||||
Started { pid: u32 },
|
Started { pid: u32 },
|
||||||
SpawnFailed { error: String },
|
SpawnFailed { error: String },
|
||||||
Exited { status: ExitStatus },
|
Exited { status: ExitStatus },
|
||||||
|
Output { stderr: bool, bytes: Vec<u8> },
|
||||||
Error { error: String },
|
Error { error: String },
|
||||||
ThreadFinished,
|
ThreadFinished,
|
||||||
}
|
}
|
||||||
|
|
@ -308,6 +309,7 @@ fn supervisor_thread_main(
|
||||||
let pid: Option<u32>;
|
let pid: Option<u32>;
|
||||||
let mut kill_deadline: Option<Instant> = None;
|
let mut kill_deadline: Option<Instant> = None;
|
||||||
let mut kill_sent = false;
|
let mut kill_sent = false;
|
||||||
|
let mut output_threads = Vec::new();
|
||||||
debug_assert!(matches!(state, SupervisorState::Spawning));
|
debug_assert!(matches!(state, SupervisorState::Spawning));
|
||||||
|
|
||||||
let mut cmd = Command::new(&spec.command);
|
let mut cmd = Command::new(&spec.command);
|
||||||
|
|
@ -319,12 +321,18 @@ fn supervisor_thread_main(
|
||||||
cmd.current_dir(dir);
|
cmd.current_dir(dir);
|
||||||
}
|
}
|
||||||
cmd.stdin(Stdio::null())
|
cmd.stdin(Stdio::null())
|
||||||
.stdout(Stdio::null())
|
.stdout(Stdio::piped())
|
||||||
.stderr(Stdio::null());
|
.stderr(Stdio::piped());
|
||||||
|
|
||||||
match cmd.spawn() {
|
match cmd.spawn() {
|
||||||
Ok(spawned_child) => {
|
Ok(mut spawned_child) => {
|
||||||
let child_pid = spawned_child.id();
|
let child_pid = spawned_child.id();
|
||||||
|
if let Some(stdout) = spawned_child.stdout.take() {
|
||||||
|
output_threads.push(spawn_output_reader(stdout, events.clone(), false));
|
||||||
|
}
|
||||||
|
if let Some(stderr) = spawned_child.stderr.take() {
|
||||||
|
output_threads.push(spawn_output_reader(stderr, events.clone(), true));
|
||||||
|
}
|
||||||
pid = Some(child_pid);
|
pid = Some(child_pid);
|
||||||
child = Some(spawned_child);
|
child = Some(spawned_child);
|
||||||
events.push(ThreadEvent::Started { pid: child_pid });
|
events.push(ThreadEvent::Started { pid: child_pid });
|
||||||
|
|
@ -339,11 +347,11 @@ fn supervisor_thread_main(
|
||||||
) {
|
) {
|
||||||
CommandOutcome::Continue => {}
|
CommandOutcome::Continue => {}
|
||||||
CommandOutcome::Exited(status) => {
|
CommandOutcome::Exited(status) => {
|
||||||
finish_with_exit(&events, &mut state, &mut child, status);
|
finish_with_exit(&events, &mut state, &mut child, &mut output_threads, status);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
CommandOutcome::Error(error) => {
|
CommandOutcome::Error(error) => {
|
||||||
finish_with_error(&events, &mut state, &mut child, error);
|
finish_with_error(&events, &mut state, &mut child, &mut output_threads, error);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -367,6 +375,7 @@ fn supervisor_thread_main(
|
||||||
&events,
|
&events,
|
||||||
&mut state,
|
&mut state,
|
||||||
&mut child,
|
&mut child,
|
||||||
|
&mut output_threads,
|
||||||
format!("process supervisor command wake failed: {err}"),
|
format!("process supervisor command wake failed: {err}"),
|
||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
|
|
@ -381,11 +390,23 @@ fn supervisor_thread_main(
|
||||||
) {
|
) {
|
||||||
CommandOutcome::Continue => {}
|
CommandOutcome::Continue => {}
|
||||||
CommandOutcome::Exited(status) => {
|
CommandOutcome::Exited(status) => {
|
||||||
finish_with_exit(&events, &mut state, &mut child, status);
|
finish_with_exit(
|
||||||
|
&events,
|
||||||
|
&mut state,
|
||||||
|
&mut child,
|
||||||
|
&mut output_threads,
|
||||||
|
status,
|
||||||
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
CommandOutcome::Error(error) => {
|
CommandOutcome::Error(error) => {
|
||||||
finish_with_error(&events, &mut state, &mut child, error);
|
finish_with_error(
|
||||||
|
&events,
|
||||||
|
&mut state,
|
||||||
|
&mut child,
|
||||||
|
&mut output_threads,
|
||||||
|
error,
|
||||||
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -396,6 +417,7 @@ fn supervisor_thread_main(
|
||||||
&events,
|
&events,
|
||||||
&mut state,
|
&mut state,
|
||||||
&mut child,
|
&mut child,
|
||||||
|
&mut output_threads,
|
||||||
format!("process supervisor command wake failed: {err}"),
|
format!("process supervisor command wake failed: {err}"),
|
||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
|
|
@ -404,12 +426,12 @@ fn supervisor_thread_main(
|
||||||
|
|
||||||
match try_wait_pid(child_pid) {
|
match try_wait_pid(child_pid) {
|
||||||
Ok(Some(status)) => {
|
Ok(Some(status)) => {
|
||||||
finish_with_exit(&events, &mut state, &mut child, status);
|
finish_with_exit(&events, &mut state, &mut child, &mut output_threads, status);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
Ok(None) => {}
|
Ok(None) => {}
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
finish_with_error(&events, &mut state, &mut child, error);
|
finish_with_error(&events, &mut state, &mut child, &mut output_threads, error);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -422,11 +444,11 @@ fn supervisor_thread_main(
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
CommandOutcome::Exited(status) => {
|
CommandOutcome::Exited(status) => {
|
||||||
finish_with_exit(&events, &mut state, &mut child, status);
|
finish_with_exit(&events, &mut state, &mut child, &mut output_threads, status);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
CommandOutcome::Error(error) => {
|
CommandOutcome::Error(error) => {
|
||||||
finish_with_error(&events, &mut state, &mut child, error);
|
finish_with_error(&events, &mut state, &mut child, &mut output_threads, error);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -539,15 +561,44 @@ fn send_kill_or_observe_exit(pid: u32) -> CommandOutcome {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn spawn_output_reader(
|
||||||
|
mut reader: impl Read + Send + 'static,
|
||||||
|
events: ThreadEventSink,
|
||||||
|
stderr: bool,
|
||||||
|
) -> JoinHandle<()> {
|
||||||
|
thread::spawn(move || {
|
||||||
|
let mut buffer = vec![0_u8; 4_096];
|
||||||
|
loop {
|
||||||
|
match reader.read(&mut buffer) {
|
||||||
|
Ok(0) => break,
|
||||||
|
Ok(read) => events.push(ThreadEvent::Output {
|
||||||
|
stderr,
|
||||||
|
bytes: buffer[..read].to_vec(),
|
||||||
|
}),
|
||||||
|
Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,
|
||||||
|
Err(_) => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn join_output_readers(readers: &mut Vec<JoinHandle<()>>) {
|
||||||
|
for reader in readers.drain(..) {
|
||||||
|
let _ = reader.join();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn finish_with_exit(
|
fn finish_with_exit(
|
||||||
events: &ThreadEventSink,
|
events: &ThreadEventSink,
|
||||||
state: &mut SupervisorState,
|
state: &mut SupervisorState,
|
||||||
child: &mut Option<Child>,
|
child: &mut Option<Child>,
|
||||||
|
output_threads: &mut Vec<JoinHandle<()>>,
|
||||||
status: ExitStatus,
|
status: ExitStatus,
|
||||||
) {
|
) {
|
||||||
*state = SupervisorState::Done;
|
*state = SupervisorState::Done;
|
||||||
debug_assert!(matches!(*state, SupervisorState::Done));
|
debug_assert!(matches!(*state, SupervisorState::Done));
|
||||||
let _ = child.take();
|
let _ = child.take();
|
||||||
|
join_output_readers(output_threads);
|
||||||
events.push(ThreadEvent::Exited { status });
|
events.push(ThreadEvent::Exited { status });
|
||||||
events.push(ThreadEvent::ThreadFinished);
|
events.push(ThreadEvent::ThreadFinished);
|
||||||
}
|
}
|
||||||
|
|
@ -556,11 +607,17 @@ fn finish_with_error(
|
||||||
events: &ThreadEventSink,
|
events: &ThreadEventSink,
|
||||||
state: &mut SupervisorState,
|
state: &mut SupervisorState,
|
||||||
child: &mut Option<Child>,
|
child: &mut Option<Child>,
|
||||||
|
output_threads: &mut Vec<JoinHandle<()>>,
|
||||||
error: String,
|
error: String,
|
||||||
) {
|
) {
|
||||||
*state = SupervisorState::Done;
|
*state = SupervisorState::Done;
|
||||||
debug_assert!(matches!(*state, SupervisorState::Done));
|
debug_assert!(matches!(*state, SupervisorState::Done));
|
||||||
|
if let Some(child) = child.as_mut() {
|
||||||
|
let _ = child.kill();
|
||||||
|
let _ = child.wait();
|
||||||
|
}
|
||||||
let _ = child.take();
|
let _ = child.take();
|
||||||
|
join_output_readers(output_threads);
|
||||||
events.push(ThreadEvent::Error { error });
|
events.push(ThreadEvent::Error { error });
|
||||||
events.push(ThreadEvent::ThreadFinished);
|
events.push(ThreadEvent::ThreadFinished);
|
||||||
}
|
}
|
||||||
|
|
@ -877,7 +934,7 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn supervisor_uses_null_stdio_and_reports_only_lifecycle() {
|
fn supervisor_captures_stdout_and_stderr_before_lifecycle_completion() {
|
||||||
let (sink, receiver) = thread_event_channel(|| {});
|
let (sink, receiver) = thread_event_channel(|| {});
|
||||||
let mut handle = ProcessSupervisorThread::start(
|
let mut handle = ProcessSupervisorThread::start(
|
||||||
shell_spec("echo stdout; echo stderr >&2; exit 0"),
|
shell_spec("echo stdout; echo stderr >&2; exit 0"),
|
||||||
|
|
@ -888,13 +945,22 @@ mod tests {
|
||||||
let events = collect_until_finished(&receiver, Duration::from_secs(2));
|
let events = collect_until_finished(&receiver, Duration::from_secs(2));
|
||||||
join_finished(&mut handle);
|
join_finished(&mut handle);
|
||||||
|
|
||||||
assert_eq!(events.len(), 3);
|
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
events.first(),
|
events.first(),
|
||||||
Some(ThreadEvent::Started { pid }) if *pid > 0
|
Some(ThreadEvent::Started { pid }) if *pid > 0
|
||||||
));
|
));
|
||||||
|
let exit = events
|
||||||
|
.iter()
|
||||||
|
.position(|event| matches!(event, ThreadEvent::Exited { .. }))
|
||||||
|
.expect("exited event");
|
||||||
|
assert!(events[..exit]
|
||||||
|
.iter()
|
||||||
|
.any(|event| matches!(event, ThreadEvent::Output { stderr: false, bytes } if bytes == b"stdout\n")));
|
||||||
|
assert!(events[..exit]
|
||||||
|
.iter()
|
||||||
|
.any(|event| matches!(event, ThreadEvent::Output { stderr: true, bytes } if bytes == b"stderr\n")));
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
events.get(1),
|
events.get(exit),
|
||||||
Some(&ThreadEvent::Exited {
|
Some(&ThreadEvent::Exited {
|
||||||
status: ExitStatus::Code(0)
|
status: ExitStatus::Code(0)
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -291,12 +291,8 @@ fn lifecycle_outputs_are_sent_upstream_and_mirrored_to_telemetry() {
|
||||||
.any(|name| name == "proc.trainer_0_foo.lifecycle"),
|
.any(|name| name == "proc.trainer_0_foo.lifecycle"),
|
||||||
"catalog should contain lifecycle channel, got {names:?}"
|
"catalog should contain lifecycle channel, got {names:?}"
|
||||||
);
|
);
|
||||||
assert!(
|
assert!(names.iter().any(|name| name == "proc.trainer_0_foo.stdout"));
|
||||||
names
|
assert!(names.iter().any(|name| name == "proc.trainer_0_foo.stderr"));
|
||||||
.iter()
|
|
||||||
.all(|name| !name.contains("stdout") && !name.contains("stderr")),
|
|
||||||
"process core should not register stdout/stderr channels: {names:?}"
|
|
||||||
);
|
|
||||||
|
|
||||||
let payloads: Vec<Value> = telemetry_events
|
let payloads: Vec<Value> = telemetry_events
|
||||||
.iter()
|
.iter()
|
||||||
|
|
@ -322,13 +318,15 @@ fn lifecycle_outputs_are_sent_upstream_and_mirrored_to_telemetry() {
|
||||||
}),
|
}),
|
||||||
"lifecycle mirror should include exited JSON, got {payloads:?}"
|
"lifecycle mirror should include exited JSON, got {payloads:?}"
|
||||||
);
|
);
|
||||||
assert!(
|
let text_payloads = telemetry_events
|
||||||
payloads.iter().all(|payload| {
|
.iter()
|
||||||
payload.get("event") != Some(&Value::String("stdout".to_owned()))
|
.filter_map(|event| match event {
|
||||||
&& payload.get("event") != Some(&Value::String("stderr".to_owned()))
|
TelemetryEvent::Frame(delivery) => String::from_utf8(delivery.payload.clone()).ok(),
|
||||||
}),
|
_ => None,
|
||||||
"lifecycle mirror should not include child output events: {payloads:?}"
|
})
|
||||||
);
|
.collect::<Vec<_>>();
|
||||||
|
assert!(text_payloads.iter().any(|payload| payload == "stdout\n"));
|
||||||
|
assert!(text_payloads.iter().any(|payload| payload == "stderr\n"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
@ -948,7 +946,7 @@ fn lifecycle_mirror_submit_failure_does_not_suppress_upstream_or_emit_error() {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn stdout_and_stderr_writes_do_not_affect_lifecycle() {
|
fn stdout_and_stderr_are_delivered_without_affecting_lifecycle() {
|
||||||
let (rt, mut host) = runtime_host();
|
let (rt, mut host) = runtime_host();
|
||||||
let sender = rt.create_sender();
|
let sender = rt.create_sender();
|
||||||
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
|
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
|
||||||
|
|
@ -983,11 +981,26 @@ fn stdout_and_stderr_writes_do_not_affect_lifecycle() {
|
||||||
has_exited(outputs, ExitStatus::Code(0))
|
has_exited(outputs, ExitStatus::Code(0))
|
||||||
});
|
});
|
||||||
|
|
||||||
assert_eq!(
|
let stdout = outputs
|
||||||
outputs.len(),
|
.iter()
|
||||||
2,
|
.filter_map(|output| match output {
|
||||||
"process core should emit only lifecycle outputs: {outputs:?}"
|
ProcessOutput::Stdout(bytes) => Some(bytes.as_slice()),
|
||||||
);
|
_ => None,
|
||||||
|
})
|
||||||
|
.flatten()
|
||||||
|
.copied()
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let stderr = outputs
|
||||||
|
.iter()
|
||||||
|
.filter_map(|output| match output {
|
||||||
|
ProcessOutput::Stderr(bytes) => Some(bytes.as_slice()),
|
||||||
|
_ => None,
|
||||||
|
})
|
||||||
|
.flatten()
|
||||||
|
.copied()
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
assert_eq!(String::from_utf8(stdout).unwrap().lines().count(), 1_000);
|
||||||
|
assert_eq!(String::from_utf8(stderr).unwrap().lines().count(), 1_000);
|
||||||
assert!(started_index(&outputs) < exited_index(&outputs));
|
assert!(started_index(&outputs) < exited_index(&outputs));
|
||||||
assert!(
|
assert!(
|
||||||
has_exited(&outputs, ExitStatus::Code(0)),
|
has_exited(&outputs, ExitStatus::Code(0)),
|
||||||
|
|
|
||||||
|
|
@ -408,14 +408,32 @@ trait ProcessOutputObserver {
|
||||||
}
|
}
|
||||||
|
|
||||||
trait StatsHook {
|
trait StatsHook {
|
||||||
fn on_tick(&self, worker_id: usize, snapshots: &[ActorSnapshot]);
|
fn on_snapshot(
|
||||||
|
&self,
|
||||||
|
worker_id: usize,
|
||||||
|
snapshots: &[ActorSnapshot],
|
||||||
|
kind: StatsSnapshotKind,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
Process output uses a caller-provided closure from `(label, is_stderr)` to an
|
Process output uses a caller-provided closure from `(label, is_stderr)` to an
|
||||||
already registered `ChannelId`. Runtime stats default to `runtime.actors` as a
|
already registered `ChannelId`. Runtime actor telemetry defaults to
|
||||||
JSON record channel. Other subsystems follow the same rule: register or reuse a
|
`runtime.actors`. Its JSON records use `swactor.actor-telemetry.v1` and carry
|
||||||
channel id, encode their own payload, submit bytes.
|
the stream lifetime as `generation` plus a monotonic `sequence`:
|
||||||
|
|
||||||
|
- `census` is a complete per-worker actor list, emitted initially and every
|
||||||
|
15 seconds for reconciliation;
|
||||||
|
- `vital` immediately reports actor start/stop, poison/recovery, and mailbox
|
||||||
|
pressure transitions;
|
||||||
|
- `activity` contains only changed actors, processed-message deltas, current
|
||||||
|
and interval-maximum mailbox depth, and the last message type, at no more
|
||||||
|
than four records per second per worker.
|
||||||
|
|
||||||
|
The hook receives a complete in-memory worker snapshot only when one of those
|
||||||
|
lanes is due; actors never receive reporting messages. Other subsystems follow
|
||||||
|
the same rule: register or reuse a channel id, encode their own payload, and
|
||||||
|
submit bytes.
|
||||||
|
|
||||||
### 4.4 The mux: single position authority
|
### 4.4 The mux: single position authority
|
||||||
|
|
||||||
|
|
@ -633,8 +651,8 @@ not be contradicted:
|
||||||
|----------------|-----------------|
|
|----------------|-----------------|
|
||||||
| `telemetry.health` | telemetry self-health counters |
|
| `telemetry.health` | telemetry self-health counters |
|
||||||
| `host.cpu`, `host.gpu`, `host.net` | host hardware samples |
|
| `host.cpu`, `host.gpu`, `host.net` | host hardware samples |
|
||||||
| `runtime.actors` | runtime actor stats hook output |
|
| `runtime.actors` | sequenced actor census, vital events, and sampled activity |
|
||||||
| `proc.<label>.stdout`, `proc.<label>.stderr` | managed process output |
|
| `proc.<label>.lifecycle`, `.stdout`, `.stderr` | managed process lifecycle and real output |
|
||||||
| `mvp.lifecycle` | MVP lifecycle facts |
|
| `mvp.lifecycle` | MVP lifecycle facts |
|
||||||
| `mvp.provisioning.events` | MVP provisioning lifecycle facts |
|
| `mvp.provisioning.events` | MVP provisioning lifecycle facts |
|
||||||
| `mvp.provisioning.logs.node.<id>.<stream>` | MVP provisioning stdout/stderr/provider lines |
|
| `mvp.provisioning.logs.node.<id>.<stream>` | MVP provisioning stdout/stderr/provider lines |
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,7 @@
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
use std::sync::atomic::{AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
use crossbeam_channel::{
|
use crossbeam_channel::{
|
||||||
Receiver, RecvError, RecvTimeoutError, Sender, TryRecvError, TrySendError, bounded,
|
Receiver, RecvError, RecvTimeoutError, Sender, TryRecvError, TrySendError, bounded,
|
||||||
|
|
@ -14,7 +15,7 @@ use crossbeam_channel::{
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use swactor::process_observer::ProcessOutputObserver;
|
use swactor::process_observer::ProcessOutputObserver;
|
||||||
use swactor::stats::{ActorSnapshot, StatsHook};
|
use swactor::stats::{ActorSnapshot, StatsHook, StatsSnapshotKind};
|
||||||
|
|
||||||
use crate::emit::ProcessChannelRouter;
|
use crate::emit::ProcessChannelRouter;
|
||||||
use crate::frame::{
|
use crate::frame::{
|
||||||
|
|
@ -28,6 +29,8 @@ use crate::transport::Delivery;
|
||||||
const DEFAULT_MUX_CAPACITY: usize = 4096;
|
const DEFAULT_MUX_CAPACITY: usize = 4096;
|
||||||
const DEFAULT_SUBSCRIBER_CAPACITY: usize = 1024;
|
const DEFAULT_SUBSCRIBER_CAPACITY: usize = 1024;
|
||||||
const DEFAULT_STATS_CHANNEL: &str = "runtime.actors";
|
const DEFAULT_STATS_CHANNEL: &str = "runtime.actors";
|
||||||
|
const ACTOR_MAILBOX_WARNING: usize = 1_024;
|
||||||
|
const ACTOR_MAILBOX_RECOVERY: usize = 512;
|
||||||
|
|
||||||
/// Stable handle identifying a local telemetry subscription.
|
/// Stable handle identifying a local telemetry subscription.
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||||
|
|
@ -686,6 +689,7 @@ impl TelemetryProducer {
|
||||||
Arc::new(TelemetryStatsHook {
|
Arc::new(TelemetryStatsHook {
|
||||||
producer: self.clone(),
|
producer: self.clone(),
|
||||||
channel,
|
channel,
|
||||||
|
state: Mutex::new(TelemetryStatsState::default()),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -732,53 +736,256 @@ impl ProcessOutputObserver for TelemetryProcessObserver {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Runtime stats hook that submits one JSON record per productive worker tick.
|
/// Runtime actor observer implementing the census/vital/activity protocol.
|
||||||
pub struct TelemetryStatsHook {
|
pub struct TelemetryStatsHook {
|
||||||
producer: TelemetryProducer,
|
producer: TelemetryProducer,
|
||||||
channel: ChannelId,
|
channel: ChannelId,
|
||||||
|
state: Mutex<TelemetryStatsState>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl StatsHook for TelemetryStatsHook {
|
#[derive(Default)]
|
||||||
fn on_tick(&self, worker_id: usize, snapshots: &[ActorSnapshot]) {
|
struct TelemetryStatsState {
|
||||||
let payload = RuntimeActorStatsRecord::from_snapshots(worker_id, snapshots);
|
sequence: u64,
|
||||||
let bytes = serde_json::to_vec(&payload).expect("runtime stats record serializes");
|
workers: BTreeMap<usize, WorkerTelemetryState>,
|
||||||
self.producer.submit_bytes(self.channel, bytes);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
#[derive(Default)]
|
||||||
struct RuntimeActorStatsRecord<'a> {
|
struct WorkerTelemetryState {
|
||||||
worker_id: usize,
|
actors: BTreeMap<String, RetainedActor>,
|
||||||
actors: Vec<RuntimeActorSnapshotRecord<'a>>,
|
last_activity: Option<Instant>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<'a> RuntimeActorStatsRecord<'a> {
|
#[derive(Clone)]
|
||||||
fn from_snapshots(worker_id: usize, snapshots: &'a [ActorSnapshot]) -> Self {
|
struct RetainedActor {
|
||||||
|
mailbox_depth: usize,
|
||||||
|
mailbox_max_depth: usize,
|
||||||
|
last_msg_type: Option<&'static str>,
|
||||||
|
actor_type: Option<&'static str>,
|
||||||
|
message_type: Option<&'static str>,
|
||||||
|
messages_processed: u64,
|
||||||
|
poisoned: bool,
|
||||||
|
under_pressure: bool,
|
||||||
|
activity_baseline: u64,
|
||||||
|
message_type_counts: Vec<(&'static str, u64)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RetainedActor {
|
||||||
|
fn from_snapshot(snapshot: &ActorSnapshot, previous: Option<&Self>) -> Self {
|
||||||
|
let under_pressure = match previous {
|
||||||
|
Some(previous) if previous.under_pressure => {
|
||||||
|
snapshot.mailbox_depth > ACTOR_MAILBOX_RECOVERY
|
||||||
|
}
|
||||||
|
_ => snapshot.mailbox_max_depth >= ACTOR_MAILBOX_WARNING,
|
||||||
|
};
|
||||||
Self {
|
Self {
|
||||||
worker_id,
|
|
||||||
actors: snapshots
|
|
||||||
.iter()
|
|
||||||
.map(|snapshot| RuntimeActorSnapshotRecord {
|
|
||||||
address: snapshot.address.to_full_hex(),
|
|
||||||
mailbox_depth: snapshot.mailbox_depth,
|
mailbox_depth: snapshot.mailbox_depth,
|
||||||
|
mailbox_max_depth: snapshot.mailbox_max_depth,
|
||||||
last_msg_type: snapshot.last_msg_type,
|
last_msg_type: snapshot.last_msg_type,
|
||||||
actor_type: snapshot.actor_type,
|
actor_type: snapshot.actor_type,
|
||||||
message_type: snapshot.message_type,
|
message_type: snapshot.message_type,
|
||||||
messages_processed: snapshot.messages_processed,
|
messages_processed: snapshot.messages_processed,
|
||||||
poisoned: snapshot.poisoned,
|
poisoned: snapshot.poisoned,
|
||||||
message_type_counts: snapshot
|
under_pressure,
|
||||||
|
activity_baseline: previous
|
||||||
|
.map(|actor| actor.activity_baseline)
|
||||||
|
.unwrap_or(snapshot.messages_processed),
|
||||||
|
message_type_counts: snapshot.message_type_counts.clone(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn full_record(&self, address: String) -> RuntimeActorSnapshotRecord {
|
||||||
|
RuntimeActorSnapshotRecord {
|
||||||
|
address,
|
||||||
|
mailbox_depth: self.mailbox_depth,
|
||||||
|
last_msg_type: self.last_msg_type,
|
||||||
|
actor_type: self.actor_type,
|
||||||
|
message_type: self.message_type,
|
||||||
|
messages_processed: self.messages_processed,
|
||||||
|
poisoned: self.poisoned,
|
||||||
|
message_type_counts: self
|
||||||
.message_type_counts
|
.message_type_counts
|
||||||
.iter()
|
.iter()
|
||||||
.map(|(ty, count)| RuntimeMessageTypeCount { ty, count: *count })
|
.map(|(ty, count)| RuntimeMessageTypeCount { ty, count: *count })
|
||||||
.collect(),
|
.collect(),
|
||||||
})
|
|
||||||
.collect(),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl StatsHook for TelemetryStatsHook {
|
||||||
|
fn on_snapshot(&self, worker_id: usize, snapshots: &[ActorSnapshot], kind: StatsSnapshotKind) {
|
||||||
|
let now = Instant::now();
|
||||||
|
let generation = self.producer.stream_id().life.0;
|
||||||
|
let mut state = self.state.lock().expect("actor telemetry state poisoned");
|
||||||
|
let mut payloads = Vec::new();
|
||||||
|
|
||||||
|
{
|
||||||
|
let worker = state.workers.entry(worker_id).or_default();
|
||||||
|
let previous = std::mem::take(&mut worker.actors);
|
||||||
|
let mut current = BTreeMap::new();
|
||||||
|
let mut full_state_emitted = BTreeMap::new();
|
||||||
|
|
||||||
|
for snapshot in snapshots {
|
||||||
|
let address = snapshot.address.to_full_hex();
|
||||||
|
let before = previous.get(&address);
|
||||||
|
let actor = RetainedActor::from_snapshot(snapshot, before);
|
||||||
|
match before {
|
||||||
|
None => {
|
||||||
|
payloads.push(json_record(
|
||||||
|
"vital",
|
||||||
|
worker_id,
|
||||||
|
serde_json::json!({
|
||||||
|
"event": "started",
|
||||||
|
"actor": actor.full_record(address.clone()),
|
||||||
|
}),
|
||||||
|
));
|
||||||
|
full_state_emitted.insert(address.clone(), ());
|
||||||
|
}
|
||||||
|
Some(before) => {
|
||||||
|
if before.poisoned != actor.poisoned {
|
||||||
|
payloads.push(json_record(
|
||||||
|
"vital",
|
||||||
|
worker_id,
|
||||||
|
serde_json::json!({
|
||||||
|
"event": if actor.poisoned { "poisoned" } else { "recovered" },
|
||||||
|
"actor": actor.full_record(address.clone()),
|
||||||
|
}),
|
||||||
|
));
|
||||||
|
full_state_emitted.insert(address.clone(), ());
|
||||||
|
}
|
||||||
|
if before.under_pressure != actor.under_pressure {
|
||||||
|
payloads.push(json_record(
|
||||||
|
"vital",
|
||||||
|
worker_id,
|
||||||
|
serde_json::json!({
|
||||||
|
"event": if actor.under_pressure {
|
||||||
|
"mailbox_pressure"
|
||||||
|
} else {
|
||||||
|
"mailbox_recovered"
|
||||||
|
},
|
||||||
|
"actor": actor.full_record(address.clone()),
|
||||||
|
"warning_depth": ACTOR_MAILBOX_WARNING,
|
||||||
|
"recovery_depth": ACTOR_MAILBOX_RECOVERY,
|
||||||
|
}),
|
||||||
|
));
|
||||||
|
full_state_emitted.insert(address.clone(), ());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
current.insert(address, actor);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (address, actor) in &previous {
|
||||||
|
if !current.contains_key(address) {
|
||||||
|
payloads.push(json_record(
|
||||||
|
"vital",
|
||||||
|
worker_id,
|
||||||
|
serde_json::json!({
|
||||||
|
"event": "stopped",
|
||||||
|
"actor": actor.full_record(address.clone()),
|
||||||
|
}),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if kind.census {
|
||||||
|
payloads.push(json_record(
|
||||||
|
"census",
|
||||||
|
worker_id,
|
||||||
|
serde_json::json!({
|
||||||
|
"actors": current
|
||||||
|
.iter()
|
||||||
|
.map(|(address, actor)| actor.full_record(address.clone()))
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
}),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
if kind.activity && !kind.census {
|
||||||
|
let interval_ms = worker
|
||||||
|
.last_activity
|
||||||
|
.map(|last| now.duration_since(last).as_millis().min(u64::MAX as u128) as u64)
|
||||||
|
.unwrap_or(0);
|
||||||
|
let mut actors = Vec::new();
|
||||||
|
for (address, actor) in ¤t {
|
||||||
|
if full_state_emitted.contains_key(address) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let before = previous.get(address);
|
||||||
|
let delta = actor
|
||||||
|
.messages_processed
|
||||||
|
.saturating_sub(actor.activity_baseline);
|
||||||
|
let mailbox_changed =
|
||||||
|
before.is_none_or(|before| before.mailbox_depth != actor.mailbox_depth);
|
||||||
|
let max_depth = actor.mailbox_max_depth;
|
||||||
|
if delta > 0 || mailbox_changed || max_depth > actor.mailbox_depth {
|
||||||
|
actors.push(RuntimeActorActivityRecord {
|
||||||
|
address: address.clone(),
|
||||||
|
messages_processed_delta: delta,
|
||||||
|
mailbox_depth: actor.mailbox_depth,
|
||||||
|
mailbox_max_depth: max_depth,
|
||||||
|
last_msg_type: actor.last_msg_type,
|
||||||
|
poisoned: actor.poisoned.then_some(true),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !actors.is_empty() {
|
||||||
|
payloads.push(json_record(
|
||||||
|
"activity",
|
||||||
|
worker_id,
|
||||||
|
serde_json::json!({
|
||||||
|
"interval_ms": interval_ms,
|
||||||
|
"actors": actors,
|
||||||
|
}),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if kind.activity {
|
||||||
|
worker.last_activity = Some(now);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (address, actor) in &mut current {
|
||||||
|
if kind.census
|
||||||
|
|| kind.activity
|
||||||
|
|| full_state_emitted.contains_key(address)
|
||||||
|
|| !previous.contains_key(address)
|
||||||
|
{
|
||||||
|
actor.activity_baseline = actor.messages_processed;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
worker.actors = current;
|
||||||
|
}
|
||||||
|
|
||||||
|
for mut payload in payloads {
|
||||||
|
state.sequence = state.sequence.wrapping_add(1);
|
||||||
|
let object = payload
|
||||||
|
.as_object_mut()
|
||||||
|
.expect("actor telemetry record is an object");
|
||||||
|
object.insert("generation".to_owned(), generation.into());
|
||||||
|
object.insert("sequence".to_owned(), state.sequence.into());
|
||||||
|
let bytes = serde_json::to_vec(&payload).expect("actor telemetry record serializes");
|
||||||
|
self.producer.submit_bytes(self.channel, bytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn json_record(
|
||||||
|
kind: &'static str,
|
||||||
|
worker_id: usize,
|
||||||
|
fields: serde_json::Value,
|
||||||
|
) -> serde_json::Value {
|
||||||
|
let mut record = serde_json::json!({
|
||||||
|
"protocol": "swactor.actor-telemetry.v1",
|
||||||
|
"kind": kind,
|
||||||
|
"worker_id": worker_id,
|
||||||
|
});
|
||||||
|
if let (Some(record), Some(fields)) = (record.as_object_mut(), fields.as_object()) {
|
||||||
|
record.extend(fields.clone());
|
||||||
|
}
|
||||||
|
record
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
#[derive(Debug, Serialize)]
|
||||||
struct RuntimeActorSnapshotRecord<'a> {
|
struct RuntimeActorSnapshotRecord {
|
||||||
address: String,
|
address: String,
|
||||||
mailbox_depth: usize,
|
mailbox_depth: usize,
|
||||||
last_msg_type: Option<&'static str>,
|
last_msg_type: Option<&'static str>,
|
||||||
|
|
@ -786,12 +993,23 @@ struct RuntimeActorSnapshotRecord<'a> {
|
||||||
message_type: Option<&'static str>,
|
message_type: Option<&'static str>,
|
||||||
messages_processed: u64,
|
messages_processed: u64,
|
||||||
poisoned: bool,
|
poisoned: bool,
|
||||||
message_type_counts: Vec<RuntimeMessageTypeCount<'a>>,
|
message_type_counts: Vec<RuntimeMessageTypeCount>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
#[derive(Debug, Serialize)]
|
||||||
struct RuntimeMessageTypeCount<'a> {
|
struct RuntimeActorActivityRecord {
|
||||||
ty: &'a str,
|
address: String,
|
||||||
|
messages_processed_delta: u64,
|
||||||
|
mailbox_depth: usize,
|
||||||
|
mailbox_max_depth: usize,
|
||||||
|
last_msg_type: Option<&'static str>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
poisoned: Option<bool>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
struct RuntimeMessageTypeCount {
|
||||||
|
ty: &'static str,
|
||||||
count: u64,
|
count: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@ use std::time::Duration;
|
||||||
|
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use swactor::actor::ActorAddress;
|
use swactor::actor::ActorAddress;
|
||||||
use swactor::stats::ActorSnapshot;
|
use swactor::stats::{ActorSnapshot, StatsSnapshotKind};
|
||||||
use telemetry::frame::{FrameDelivery, TelemetryEvent};
|
use telemetry::frame::{FrameDelivery, TelemetryEvent};
|
||||||
use telemetry::{
|
use telemetry::{
|
||||||
ChannelContent, ChannelContentKind, ChannelFilter, ChannelId, Lifetime, NodeId, Position,
|
ChannelContent, ChannelContentKind, ChannelFilter, ChannelId, Lifetime, NodeId, Position,
|
||||||
|
|
@ -299,6 +299,7 @@ fn stats_hook_adapter_submits_worker_snapshot_json() {
|
||||||
let snapshots = [ActorSnapshot {
|
let snapshots = [ActorSnapshot {
|
||||||
address: actor,
|
address: actor,
|
||||||
mailbox_depth: 3,
|
mailbox_depth: 3,
|
||||||
|
mailbox_max_depth: 4,
|
||||||
last_msg_type: Some("Ping"),
|
last_msg_type: Some("Ping"),
|
||||||
actor_type: Some("TestActor"),
|
actor_type: Some("TestActor"),
|
||||||
message_type: Some("Ping"),
|
message_type: Some("Ping"),
|
||||||
|
|
@ -307,14 +308,23 @@ fn stats_hook_adapter_submits_worker_snapshot_json() {
|
||||||
message_type_counts: vec![("Ping", 5)],
|
message_type_counts: vec![("Ping", 5)],
|
||||||
}];
|
}];
|
||||||
|
|
||||||
hook.on_tick(2, &snapshots);
|
hook.on_snapshot(2, &snapshots, StatsSnapshotKind::CENSUS);
|
||||||
endpoint.tick();
|
endpoint.tick();
|
||||||
|
|
||||||
let event = sub.recv_timeout(Duration::from_millis(50)).unwrap();
|
let events = sub.drain_available();
|
||||||
let delivery = frame_event(&event);
|
assert_eq!(events.len(), 2, "started vital plus recovery census");
|
||||||
|
let delivery = events
|
||||||
|
.iter()
|
||||||
|
.map(frame_event)
|
||||||
|
.find(|delivery| {
|
||||||
|
serde_json::from_slice::<Value>(&delivery.payload)
|
||||||
|
.is_ok_and(|json| json["kind"] == "census")
|
||||||
|
})
|
||||||
|
.expect("census frame");
|
||||||
assert_eq!(delivery.channel.channel, runtime);
|
assert_eq!(delivery.channel.channel, runtime);
|
||||||
let json: Value = serde_json::from_slice(&delivery.payload).unwrap();
|
let json: Value = serde_json::from_slice(&delivery.payload).unwrap();
|
||||||
assert_eq!(json["worker_id"], 2);
|
assert_eq!(json["worker_id"], 2);
|
||||||
|
assert_eq!(json["generation"], 7);
|
||||||
assert_eq!(json["actors"][0]["address"], actor.to_full_hex());
|
assert_eq!(json["actors"][0]["address"], actor.to_full_hex());
|
||||||
assert_eq!(json["actors"][0]["mailbox_depth"], 3);
|
assert_eq!(json["actors"][0]["mailbox_depth"], 3);
|
||||||
assert_eq!(json["actors"][0]["last_msg_type"], "Ping");
|
assert_eq!(json["actors"][0]["last_msg_type"], "Ping");
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@ use crate::Error;
|
||||||
use crate::actor::{ActorAddress, Message, SpawnRequest};
|
use crate::actor::{ActorAddress, Message, SpawnRequest};
|
||||||
use crate::channel::{AsyncSender, Sender};
|
use crate::channel::{AsyncSender, Sender};
|
||||||
use crate::config::RuntimeConfig;
|
use crate::config::RuntimeConfig;
|
||||||
use crate::stats::{StatsHook, WorkerStats};
|
use crate::stats::WorkerStats;
|
||||||
use parking_lot::RwLock;
|
use parking_lot::RwLock;
|
||||||
use std::any::Any;
|
use std::any::Any;
|
||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{HashMap, HashSet};
|
||||||
|
|
@ -209,7 +209,6 @@ pub(crate) struct TickContext<'a> {
|
||||||
pub(crate) extension: Option<&'a dyn crate::extension::RuntimeExtension>,
|
pub(crate) extension: Option<&'a dyn crate::extension::RuntimeExtension>,
|
||||||
pub(crate) process_output_observer:
|
pub(crate) process_output_observer:
|
||||||
Option<&'a Arc<dyn crate::process_observer::ProcessOutputObserver>>,
|
Option<&'a Arc<dyn crate::process_observer::ProcessOutputObserver>>,
|
||||||
pub(crate) stats_hook: Option<&'a dyn StatsHook>,
|
|
||||||
pub(crate) worker_stats: &'a WorkerStats,
|
pub(crate) worker_stats: &'a WorkerStats,
|
||||||
pub(crate) num_workers: usize,
|
pub(crate) num_workers: usize,
|
||||||
pub(crate) worker_id: WorkerId,
|
pub(crate) worker_id: WorkerId,
|
||||||
|
|
|
||||||
44
src/stats.rs
44
src/stats.rs
|
|
@ -105,10 +105,40 @@ pub struct WorkerInfo {
|
||||||
pub stops: u64,
|
pub stops: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Per-actor snapshot transferred from worker to runtime (not serialized).
|
/// Why a worker is publishing a complete in-memory snapshot to its observer.
|
||||||
|
///
|
||||||
|
/// The reason lets observers emit sparse activity without losing immediate
|
||||||
|
/// lifecycle changes or periodic recovery censuses.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub struct StatsSnapshotKind {
|
||||||
|
pub activity: bool,
|
||||||
|
pub census: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StatsSnapshotKind {
|
||||||
|
pub const VITAL: Self = Self {
|
||||||
|
activity: false,
|
||||||
|
census: false,
|
||||||
|
};
|
||||||
|
pub const ACTIVITY: Self = Self {
|
||||||
|
activity: true,
|
||||||
|
census: false,
|
||||||
|
};
|
||||||
|
pub const CENSUS: Self = Self {
|
||||||
|
activity: false,
|
||||||
|
census: true,
|
||||||
|
};
|
||||||
|
pub const ACTIVITY_AND_CENSUS: Self = Self {
|
||||||
|
activity: true,
|
||||||
|
census: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Snapshot of one actor transferred from worker to observer (not serialized).
|
||||||
pub struct ActorSnapshot {
|
pub struct ActorSnapshot {
|
||||||
pub address: ActorAddress,
|
pub address: ActorAddress,
|
||||||
pub mailbox_depth: usize,
|
pub mailbox_depth: usize,
|
||||||
|
pub mailbox_max_depth: usize,
|
||||||
pub last_msg_type: Option<&'static str>,
|
pub last_msg_type: Option<&'static str>,
|
||||||
pub actor_type: Option<&'static str>,
|
pub actor_type: Option<&'static str>,
|
||||||
pub message_type: Option<&'static str>,
|
pub message_type: Option<&'static str>,
|
||||||
|
|
@ -118,16 +148,16 @@ pub struct ActorSnapshot {
|
||||||
pub message_type_counts: Vec<(&'static str, u64)>,
|
pub message_type_counts: Vec<(&'static str, u64)>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Observer hook called by workers after productive ticks.
|
/// Observer hook called with rate-limited, worker-owned actor snapshots.
|
||||||
///
|
///
|
||||||
/// Implement this to collect per-actor snapshot data outside the runtime.
|
/// The runtime stores no observer state. Snapshots are ephemeral and borrowed;
|
||||||
/// The runtime itself stores nothing — snapshots are ephemeral and passed by reference.
|
/// observers copy only what they retain.
|
||||||
pub trait StatsHook: Send + Sync {
|
pub trait StatsHook: Send + Sync {
|
||||||
/// Called once per worker after a productive tick.
|
/// Called for immediate vital changes, activity sampling, periodic census,
|
||||||
|
/// or a combined activity/census deadline.
|
||||||
///
|
///
|
||||||
/// `worker_id` is the index of the worker (0..num_workers).
|
/// `worker_id` is the index of the worker (0..num_workers).
|
||||||
/// `snapshots` borrows the worker's scratch buffer — copy what you need.
|
fn on_snapshot(&self, worker_id: usize, snapshots: &[ActorSnapshot], kind: StatsSnapshotKind);
|
||||||
fn on_tick(&self, worker_id: usize, snapshots: &[ActorSnapshot]);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Per-actor runtime stats.
|
/// Per-actor runtime stats.
|
||||||
|
|
|
||||||
146
src/worker.rs
146
src/worker.rs
|
|
@ -5,6 +5,7 @@ use std::cmp::Reverse;
|
||||||
use std::collections::{HashMap, VecDeque};
|
use std::collections::{HashMap, VecDeque};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::sync::atomic::Ordering;
|
use std::sync::atomic::Ordering;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
use crate::Error;
|
use crate::Error;
|
||||||
use crate::actor::{
|
use crate::actor::{
|
||||||
|
|
@ -17,10 +18,12 @@ use crate::admin::{
|
||||||
};
|
};
|
||||||
use crate::channel::Receiver;
|
use crate::channel::Receiver;
|
||||||
use crate::delivery::{AddrBuildHasher, AddrMap, Envelope, TickContext, WorkerId};
|
use crate::delivery::{AddrBuildHasher, AddrMap, Envelope, TickContext, WorkerId};
|
||||||
use crate::stats::{ActorSnapshot, TickTiming, WorkerStats};
|
use crate::stats::{ActorSnapshot, StatsHook, StatsSnapshotKind, TickTiming, WorkerStats};
|
||||||
|
|
||||||
use crate::extension::WorkerExtension;
|
use crate::extension::WorkerExtension;
|
||||||
use crate::runtime::RuntimeShared;
|
use crate::runtime::RuntimeShared;
|
||||||
|
const ACTOR_ACTIVITY_INTERVAL: Duration = Duration::from_millis(250);
|
||||||
|
const ACTOR_CENSUS_INTERVAL: Duration = Duration::from_secs(15);
|
||||||
|
|
||||||
/// Whether an actor should be skipped during `tick_all`.
|
/// Whether an actor should be skipped during `tick_all`.
|
||||||
pub(crate) fn should_skip_actor(poisoned: bool, stopping: bool, suspended: bool) -> bool {
|
pub(crate) fn should_skip_actor(poisoned: bool, stopping: bool, suspended: bool) -> bool {
|
||||||
|
|
@ -78,6 +81,10 @@ pub struct Worker {
|
||||||
pub(crate) stats: Arc<WorkerStats>,
|
pub(crate) stats: Arc<WorkerStats>,
|
||||||
/// Reusable scratch buffer for building per-actor snapshots.
|
/// Reusable scratch buffer for building per-actor snapshots.
|
||||||
snapshot_buf: Vec<ActorSnapshot>,
|
snapshot_buf: Vec<ActorSnapshot>,
|
||||||
|
/// Last activity snapshot handed to the observer.
|
||||||
|
last_activity_snapshot: Option<Instant>,
|
||||||
|
/// Last periodic reconciliation census handed to the observer.
|
||||||
|
last_census_snapshot: Option<Instant>,
|
||||||
/// Per-worker extension (e.g., timer wheel). Created by RuntimeExtension factory.
|
/// Per-worker extension (e.g., timer wheel). Created by RuntimeExtension factory.
|
||||||
pub(crate) worker_ext: Option<Box<dyn WorkerExtension>>,
|
pub(crate) worker_ext: Option<Box<dyn WorkerExtension>>,
|
||||||
/// True if the previous tick did work — ensures one full tick follows a productive
|
/// True if the previous tick did work — ensures one full tick follows a productive
|
||||||
|
|
@ -108,6 +115,8 @@ impl Worker {
|
||||||
admin_rx,
|
admin_rx,
|
||||||
stats,
|
stats,
|
||||||
snapshot_buf: Vec::new(),
|
snapshot_buf: Vec::new(),
|
||||||
|
last_activity_snapshot: None,
|
||||||
|
last_census_snapshot: None,
|
||||||
worker_ext: None,
|
worker_ext: None,
|
||||||
has_backlog: false,
|
has_backlog: false,
|
||||||
deferred_transfers: VecDeque::new(),
|
deferred_transfers: VecDeque::new(),
|
||||||
|
|
@ -135,9 +144,21 @@ impl Worker {
|
||||||
pub fn try_tick(&mut self) -> bool {
|
pub fn try_tick(&mut self) -> bool {
|
||||||
let wid = self.id;
|
let wid = self.id;
|
||||||
|
|
||||||
// Fast idle path: skip the entire tick when nothing could have changed.
|
// Fast idle path: only the infrequent reconciliation census can be due.
|
||||||
// Cost: ~3 atomic loads, zero syscalls, zero actor iteration.
|
// Between censuses this adds no actor iteration or allocation.
|
||||||
if !self.has_work() {
|
if !self.has_work() {
|
||||||
|
Self::publish_actor_stats(
|
||||||
|
self.id,
|
||||||
|
self.shared.stats_hook.get().map(Arc::as_ref),
|
||||||
|
&mut self.pool,
|
||||||
|
&mut self.snapshot_buf,
|
||||||
|
(
|
||||||
|
&mut self.last_activity_snapshot,
|
||||||
|
&mut self.last_census_snapshot,
|
||||||
|
),
|
||||||
|
Instant::now(),
|
||||||
|
false,
|
||||||
|
);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -152,7 +173,6 @@ impl Worker {
|
||||||
config: &shared.config,
|
config: &shared.config,
|
||||||
extension: shared.extension.get().map(|a| a.as_ref()),
|
extension: shared.extension.get().map(|a| a.as_ref()),
|
||||||
process_output_observer: shared.process_output_observer.get(),
|
process_output_observer: shared.process_output_observer.get(),
|
||||||
stats_hook: shared.stats_hook.get().map(|a| a.as_ref()),
|
|
||||||
worker_stats: &self.stats,
|
worker_stats: &self.stats,
|
||||||
num_workers: shared.worker_stats.len(),
|
num_workers: shared.worker_stats.len(),
|
||||||
worker_id: wid,
|
worker_id: wid,
|
||||||
|
|
@ -277,7 +297,8 @@ impl Worker {
|
||||||
|
|
||||||
let t5 = Instant::now();
|
let t5 = Instant::now();
|
||||||
|
|
||||||
// 8. Publish stats (skip entirely when idle to avoid allocation + mutex)
|
// 8. Publish cheap aggregate stats. Per-actor telemetry is emitted once
|
||||||
|
// cleanup has made the worker transition complete.
|
||||||
if did_work {
|
if did_work {
|
||||||
self.stats
|
self.stats
|
||||||
.num_actors
|
.num_actors
|
||||||
|
|
@ -288,11 +309,6 @@ impl Worker {
|
||||||
self.stats
|
self.stats
|
||||||
.messages_processed
|
.messages_processed
|
||||||
.fetch_add(processed as u64, Ordering::Relaxed);
|
.fetch_add(processed as u64, Ordering::Relaxed);
|
||||||
|
|
||||||
if let Some(hook) = tc.stats_hook {
|
|
||||||
self.pool.mailbox_depths_into(&mut self.snapshot_buf);
|
|
||||||
hook.on_tick(wid.index(), &self.snapshot_buf);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let t6 = Instant::now();
|
let t6 = Instant::now();
|
||||||
|
|
@ -323,10 +339,23 @@ impl Worker {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 9. Clean up poisoned and stopping actors. Cleanup changes the pool after
|
// 9. Clean up poisoned and stopping actors before publishing per-actor
|
||||||
// phase 8 published its snapshot, so republish cardinality and mailbox
|
if self.pool.has_vital_dirty() {
|
||||||
// state when actors were removed rather than leaving observability stale
|
Self::publish_actor_stats(
|
||||||
// until unrelated work reaches this worker.
|
self.id,
|
||||||
|
shared.stats_hook.get().map(Arc::as_ref),
|
||||||
|
&mut self.pool,
|
||||||
|
&mut self.snapshot_buf,
|
||||||
|
(
|
||||||
|
&mut self.last_activity_snapshot,
|
||||||
|
&mut self.last_census_snapshot,
|
||||||
|
),
|
||||||
|
t6,
|
||||||
|
did_work,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// state, so observers see one complete transition.
|
||||||
let cleaned_dead = Self::cleanup_dead_actors(
|
let cleaned_dead = Self::cleanup_dead_actors(
|
||||||
&mut self.pool,
|
&mut self.pool,
|
||||||
&mut self.worker_ext,
|
&mut self.worker_ext,
|
||||||
|
|
@ -341,16 +370,64 @@ impl Worker {
|
||||||
self.stats
|
self.stats
|
||||||
.total_mailbox_depth
|
.total_mailbox_depth
|
||||||
.store(self.pool.total_mailbox_depth(), Ordering::Relaxed);
|
.store(self.pool.total_mailbox_depth(), Ordering::Relaxed);
|
||||||
if let Some(hook) = tc.stats_hook {
|
|
||||||
self.pool.mailbox_depths_into(&mut self.snapshot_buf);
|
|
||||||
hook.on_tick(wid.index(), &self.snapshot_buf);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Self::publish_actor_stats(
|
||||||
|
self.id,
|
||||||
|
shared.stats_hook.get().map(Arc::as_ref),
|
||||||
|
&mut self.pool,
|
||||||
|
&mut self.snapshot_buf,
|
||||||
|
(
|
||||||
|
&mut self.last_activity_snapshot,
|
||||||
|
&mut self.last_census_snapshot,
|
||||||
|
),
|
||||||
|
t6,
|
||||||
|
did_work,
|
||||||
|
);
|
||||||
|
|
||||||
self.has_backlog = did_work;
|
self.has_backlog = did_work;
|
||||||
did_work
|
did_work
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn publish_actor_stats(
|
||||||
|
worker_id: WorkerId,
|
||||||
|
hook: Option<&dyn StatsHook>,
|
||||||
|
pool: &mut ActorPool,
|
||||||
|
snapshot_buf: &mut Vec<ActorSnapshot>,
|
||||||
|
deadlines: (&mut Option<Instant>, &mut Option<Instant>),
|
||||||
|
now: Instant,
|
||||||
|
activity_possible: bool,
|
||||||
|
) {
|
||||||
|
let (last_activity_snapshot, last_census_snapshot) = deadlines;
|
||||||
|
let Some(hook) = hook else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let census = last_census_snapshot
|
||||||
|
.is_none_or(|last| now.duration_since(last) >= ACTOR_CENSUS_INTERVAL);
|
||||||
|
let activity = activity_possible
|
||||||
|
&& last_activity_snapshot
|
||||||
|
.is_none_or(|last| now.duration_since(last) >= ACTOR_ACTIVITY_INTERVAL);
|
||||||
|
let vital = pool.take_vital_dirty();
|
||||||
|
if !census && !activity && !vital {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
pool.actor_snapshots_into(snapshot_buf, activity);
|
||||||
|
let kind = match (activity, census) {
|
||||||
|
(true, true) => StatsSnapshotKind::ACTIVITY_AND_CENSUS,
|
||||||
|
(true, false) => StatsSnapshotKind::ACTIVITY,
|
||||||
|
(false, true) => StatsSnapshotKind::CENSUS,
|
||||||
|
(false, false) => StatsSnapshotKind::VITAL,
|
||||||
|
};
|
||||||
|
hook.on_snapshot(worker_id.index(), snapshot_buf, kind);
|
||||||
|
if activity {
|
||||||
|
*last_activity_snapshot = Some(now);
|
||||||
|
}
|
||||||
|
if census {
|
||||||
|
*last_census_snapshot = Some(now);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn drain_spawns(
|
fn drain_spawns(
|
||||||
pool: &mut ActorPool,
|
pool: &mut ActorPool,
|
||||||
spawn_rx: &Receiver<SpawnRequest>,
|
spawn_rx: &Receiver<SpawnRequest>,
|
||||||
|
|
@ -738,6 +815,8 @@ struct ActorSlot {
|
||||||
suspended: bool,
|
suspended: bool,
|
||||||
last_msg_type: Option<&'static str>,
|
last_msg_type: Option<&'static str>,
|
||||||
messages_processed: u64,
|
messages_processed: u64,
|
||||||
|
/// Highest mailbox depth observed since the previous activity snapshot.
|
||||||
|
mailbox_max_depth: usize,
|
||||||
/// Per-message-type counters (bounded to 32 entries).
|
/// Per-message-type counters (bounded to 32 entries).
|
||||||
msg_type_counts: HashMap<&'static str, u64>,
|
msg_type_counts: HashMap<&'static str, u64>,
|
||||||
/// Address of the actor that spawned this one, or `None` for externally-spawned actors.
|
/// Address of the actor that spawned this one, or `None` for externally-spawned actors.
|
||||||
|
|
@ -751,12 +830,15 @@ struct ActorSlot {
|
||||||
/// Per-worker actor storage. Owns per-actor mailboxes.
|
/// Per-worker actor storage. Owns per-actor mailboxes.
|
||||||
pub(crate) struct ActorPool {
|
pub(crate) struct ActorPool {
|
||||||
actors: AddrMap<ActorSlot>,
|
actors: AddrMap<ActorSlot>,
|
||||||
|
/// Population or poison state changed since the previous observer call.
|
||||||
|
vital_dirty: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ActorPool {
|
impl ActorPool {
|
||||||
pub fn new() -> Self {
|
pub fn new() -> Self {
|
||||||
Self {
|
Self {
|
||||||
actors: HashMap::with_hasher(AddrBuildHasher),
|
actors: HashMap::with_hasher(AddrBuildHasher),
|
||||||
|
vital_dirty: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -772,12 +854,14 @@ impl ActorPool {
|
||||||
suspended: false,
|
suspended: false,
|
||||||
last_msg_type: None,
|
last_msg_type: None,
|
||||||
messages_processed: 0,
|
messages_processed: 0,
|
||||||
|
mailbox_max_depth: 0,
|
||||||
msg_type_counts: HashMap::new(),
|
msg_type_counts: HashMap::new(),
|
||||||
parent_addr: req.parent,
|
parent_addr: req.parent,
|
||||||
env: req.env,
|
env: req.env,
|
||||||
exit_value: None,
|
exit_value: None,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
self.vital_dirty = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Deliver a type-erased message to the actor at `addr`.
|
/// Deliver a type-erased message to the actor at `addr`.
|
||||||
|
|
@ -806,6 +890,7 @@ impl ActorPool {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
slot.mailbox.push_back(msg);
|
slot.mailbox.push_back(msg);
|
||||||
|
slot.mailbox_max_depth = slot.mailbox_max_depth.max(slot.mailbox.len());
|
||||||
true
|
true
|
||||||
} else {
|
} else {
|
||||||
false
|
false
|
||||||
|
|
@ -953,6 +1038,7 @@ impl ActorPool {
|
||||||
#[cfg(feature = "tracing")]
|
#[cfg(feature = "tracing")]
|
||||||
tracing::error!(actor_addr = %addr, "actor.on_start_panicked");
|
tracing::error!(actor_addr = %addr, "actor.on_start_panicked");
|
||||||
slot.poisoned = true;
|
slot.poisoned = true;
|
||||||
|
self.vital_dirty = true;
|
||||||
slot.mailbox.clear();
|
slot.mailbox.clear();
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
@ -1035,6 +1121,7 @@ impl ActorPool {
|
||||||
#[cfg(feature = "tracing")]
|
#[cfg(feature = "tracing")]
|
||||||
tracing::error!(actor_addr = %addr, "actor.panicked");
|
tracing::error!(actor_addr = %addr, "actor.panicked");
|
||||||
slot.poisoned = true;
|
slot.poisoned = true;
|
||||||
|
self.vital_dirty = true;
|
||||||
slot.mailbox.clear();
|
slot.mailbox.clear();
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
@ -1124,6 +1211,7 @@ impl ActorPool {
|
||||||
let mut dead = Vec::with_capacity(dead_addrs.len());
|
let mut dead = Vec::with_capacity(dead_addrs.len());
|
||||||
for addr in dead_addrs {
|
for addr in dead_addrs {
|
||||||
if let Some(mut slot) = self.actors.remove(&addr) {
|
if let Some(mut slot) = self.actors.remove(&addr) {
|
||||||
|
self.vital_dirty = true;
|
||||||
let reason = determine_stop_reason(slot.poisoned, slot.exit_value.is_some());
|
let reason = determine_stop_reason(slot.poisoned, slot.exit_value.is_some());
|
||||||
// Call on_stop for gracefully stopping actors only
|
// Call on_stop for gracefully stopping actors only
|
||||||
debug_assert!(
|
debug_assert!(
|
||||||
|
|
@ -1154,26 +1242,40 @@ impl ActorPool {
|
||||||
dead
|
dead
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Fill `out` with per-actor snapshots, reusing the existing allocation.
|
/// Fill `out` with complete per-actor state, reusing its top-level
|
||||||
pub fn mailbox_depths_into(&self, out: &mut Vec<ActorSnapshot>) {
|
/// allocation. Activity snapshots begin a new mailbox-maximum window.
|
||||||
|
pub fn actor_snapshots_into(&mut self, out: &mut Vec<ActorSnapshot>, reset_mailbox_max: bool) {
|
||||||
out.clear();
|
out.clear();
|
||||||
out.extend(self.actors.iter().map(|(&addr, slot)| {
|
out.extend(self.actors.iter_mut().map(|(&addr, slot)| {
|
||||||
let mut type_counts: Vec<(&'static str, u64)> =
|
let mut type_counts: Vec<(&'static str, u64)> =
|
||||||
slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect();
|
slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect();
|
||||||
type_counts.sort_by_key(|&(_, count)| Reverse(count));
|
type_counts.sort_by_key(|&(_, count)| Reverse(count));
|
||||||
let metadata = slot.actor.metadata();
|
let metadata = slot.actor.metadata();
|
||||||
ActorSnapshot {
|
let snapshot = ActorSnapshot {
|
||||||
address: addr,
|
address: addr,
|
||||||
mailbox_depth: slot.mailbox.len(),
|
mailbox_depth: slot.mailbox.len(),
|
||||||
|
mailbox_max_depth: slot.mailbox_max_depth.max(slot.mailbox.len()),
|
||||||
last_msg_type: slot.last_msg_type,
|
last_msg_type: slot.last_msg_type,
|
||||||
actor_type: Some(metadata.actor_type_name),
|
actor_type: Some(metadata.actor_type_name),
|
||||||
message_type: Some(metadata.message_type_name),
|
message_type: Some(metadata.message_type_name),
|
||||||
messages_processed: slot.messages_processed,
|
messages_processed: slot.messages_processed,
|
||||||
poisoned: slot.poisoned,
|
poisoned: slot.poisoned,
|
||||||
message_type_counts: type_counts,
|
message_type_counts: type_counts,
|
||||||
|
};
|
||||||
|
if reset_mailbox_max {
|
||||||
|
slot.mailbox_max_depth = slot.mailbox.len();
|
||||||
}
|
}
|
||||||
|
snapshot
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn has_vital_dirty(&self) -> bool {
|
||||||
|
self.vital_dirty
|
||||||
|
}
|
||||||
|
|
||||||
|
fn take_vital_dirty(&mut self) -> bool {
|
||||||
|
std::mem::take(&mut self.vital_dirty)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|
|
||||||
|
|
@ -349,6 +349,7 @@ impl ActorInterface for NodeRelayActor {
|
||||||
|
|
||||||
fn handle(&mut self, _ctx: &Ctx, output: ProcessOutput) {
|
fn handle(&mut self, _ctx: &Ctx, output: ProcessOutput) {
|
||||||
match output {
|
match output {
|
||||||
|
ProcessOutput::Stdout(_) | ProcessOutput::Stderr(_) => {}
|
||||||
ProcessOutput::Started { pid } => self.manager.set_pid(self.attempt, pid),
|
ProcessOutput::Started { pid } => self.manager.set_pid(self.attempt, pid),
|
||||||
ProcessOutput::Exited { status } => self.manager.set_exited(self.attempt, status),
|
ProcessOutput::Exited { status } => self.manager.set_exited(self.attempt, status),
|
||||||
ProcessOutput::SpawnFailed { error } => {
|
ProcessOutput::SpawnFailed { error } => {
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue