From d19dd913244d06cded895d56d1b1f2ec6ed5a162 Mon Sep 17 00:00:00 2001 From: Zachery Aaron Shores-Chmielewski Date: Sat, 15 Aug 2026 18:10:52 +0400 Subject: [PATCH] feat(xtask): provisioning-reconciler-demo with live fleet control MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `cargo xtask provisioning-reconciler-demo [--port n] [--nodes n]` boots a lightweight orchestrator for visual, human-checked E2E confirmation of the provisioning reconciler: swactor engine + real ClusterDriver + demo provider, with node children re-exec'ing the same xtask binary in node role and joining the supervisor over real iroh connections. - supervisor actor owns driver/provider/shape on a 250ms wall-clock tick, mirroring the production ClusterReconciler poll semantics; emits prov.reconciler.events/snapshot plus per-node lifecycle/status streams - k8s-styled reconciler view: ready/desired header, node stage cards, commands-out and events-in feeds - dashboard `demo-control` feature: POST /control/{kill,provision,remove} + Fleet Control view; regular builds compile none of it (symbol-verified) - fleet cards fold proc..lifecycle and node.status heartbeats into per-node pid/state pills that stay live - hardening: exe resolution survives binary replacement by rebuilds, spawn failures feed back as BootstrapFailed so the reconciler retries instead of wedging at SshReady, teardown skips exit waits for never-started children Verified in-browser: boot 3/3 converged with real joins; dashboard kill dips and fully recovers with a replacement; provision +1 → 4/4; remove −2 graceful teardown → 2/2; child process count matches reconciler nodes. --- Cargo.lock | 56 +- crates/dashboard/Cargo.toml | 5 + crates/dashboard/src/control.rs | 44 + crates/dashboard/src/demo_control.rs | 112 +++ crates/dashboard/src/demo_control_page.html | 148 ++++ crates/dashboard/src/hardware_page.html | 1 + crates/dashboard/src/hardware_view.rs | 52 +- crates/dashboard/src/lib.rs | 9 +- crates/dashboard/src/server.rs | 59 +- xtask/Cargo.toml | 13 + xtask/src/main.rs | 9 + xtask/src/provisioning_demo/control.rs | 36 + xtask/src/provisioning_demo/feed.rs | 811 ++++++++++++++++++ xtask/src/provisioning_demo/mod.rs | 359 ++++++++ xtask/src/provisioning_demo/node.rs | 119 +++ xtask/src/provisioning_demo/provider.rs | 469 ++++++++++ .../provisioning_demo/reconciler_page.html | 156 ++++ xtask/src/provisioning_demo/view.rs | 170 ++++ 18 files changed, 2599 insertions(+), 29 deletions(-) create mode 100644 crates/dashboard/src/control.rs create mode 100644 crates/dashboard/src/demo_control.rs create mode 100644 crates/dashboard/src/demo_control_page.html create mode 100644 xtask/src/provisioning_demo/control.rs create mode 100644 xtask/src/provisioning_demo/feed.rs create mode 100644 xtask/src/provisioning_demo/mod.rs create mode 100644 xtask/src/provisioning_demo/node.rs create mode 100644 xtask/src/provisioning_demo/provider.rs create mode 100644 xtask/src/provisioning_demo/reconciler_page.html create mode 100644 xtask/src/provisioning_demo/view.rs diff --git a/Cargo.lock b/Cargo.lock index cc66e65..3f34102 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -899,11 +899,11 @@ name = "dashboard" version = "0.1.0" dependencies = [ "axum", - "telemetry", "parking_lot", "serde", "serde_json", "swactor", + "telemetry", "tokio", "tokio-stream", ] @@ -952,23 +952,10 @@ dependencies = [ name = "data-plane" version = "0.1.0" dependencies = [ + "libc", + "serde", + "swactor", "telemetry", - "libc", - "serde", - "swactor", -] - -[[package]] -name = "telemetry" -version = "0.1.0" -dependencies = [ - "crossbeam-channel", - "iroh", - "libc", - "serde", - "serde_json", - "swactor", - "swactor-transport", ] [[package]] @@ -1138,13 +1125,13 @@ dependencies = [ name = "distribution" version = "0.1.0" dependencies = [ - "telemetry", "libc", "proptest", "serde", "serde_json", "swactor", "swactor-transport", + "telemetry", "uuid", ] @@ -2141,7 +2128,6 @@ name = "iroh-driver" version = "0.1.0" dependencies = [ "crossbeam-channel", - "telemetry", "distribution", "iroh", "iroh-relay", @@ -2151,6 +2137,7 @@ dependencies = [ "swactor", "swactor-engine", "swactor-transport", + "telemetry", "tokio", ] @@ -2524,7 +2511,6 @@ dependencies = [ "blake3", "dashboard", "data-plane", - "telemetry", "distribution", "iroh", "iroh-driver", @@ -2539,6 +2525,7 @@ dependencies = [ "swactor-process", "swactor-transport", "swactor-vastai", + "telemetry", "tokio", "toml 0.8.23", "ureq", @@ -3362,6 +3349,7 @@ dependencies = [ name = "provisioning" version = "0.1.0" dependencies = [ + "parking_lot", "serde", "serde_json", ] @@ -4418,12 +4406,12 @@ name = "swactor-process" version = "0.1.0" dependencies = [ "crossbeam-queue", - "telemetry", "libc", "serde", "serde_json", "serde_yaml", "swactor", + "telemetry", ] [[package]] @@ -4513,6 +4501,19 @@ version = "0.12.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" +[[package]] +name = "telemetry" +version = "0.1.0" +dependencies = [ + "crossbeam-channel", + "iroh", + "libc", + "serde", + "serde_json", + "swactor", + "swactor-transport", +] + [[package]] name = "tempfile" version = "3.27.0" @@ -5837,8 +5838,21 @@ name = "xtask" version = "0.1.0" dependencies = [ "blake3", + "dashboard", + "distribution", + "iroh", + "iroh-driver", "libc", + "parking_lot", + "provisioning", + "serde", "serde_json", + "swactor", + "swactor-engine", + "swactor-process", + "swactor-transport", + "telemetry", + "tokio", ] [[package]] diff --git a/crates/dashboard/Cargo.toml b/crates/dashboard/Cargo.toml index c10b09d..3a3ba2a 100644 --- a/crates/dashboard/Cargo.toml +++ b/crates/dashboard/Cargo.toml @@ -13,3 +13,8 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" tokio.workspace = true tokio-stream = "0.1" + +[features] +# Live control actions (kill processes, provision nodes) for the +# provisioning-reconciler demo. Never enabled in shipping builds. +demo-control = [] diff --git a/crates/dashboard/src/control.rs b/crates/dashboard/src/control.rs new file mode 100644 index 0000000..a36b59c --- /dev/null +++ b/crates/dashboard/src/control.rs @@ -0,0 +1,44 @@ +//! Demo-only control plane: write actions out of the dashboard. +//! +//! This module exists only under the `demo-control` feature. The dashboard's +//! data path stays read-only in every regular build; the provisioning +//! reconciler demo turns this feature on so a human can kill provisioned +//! processes and request new ones from the fleet view. +//! +//! Routes (only present when the feature is enabled and a control sink is +//! installed): +//! - `POST /control/kill` body `{"node": ""}` +//! - `POST /control/provision` body `{"count": 1}` + +use std::sync::mpsc::Sender; +use std::sync::OnceLock; + +use serde::Deserialize; + +/// A control command issued from the dashboard UI. +#[derive(Clone, Debug, Deserialize)] +pub enum ControlCommand { + /// Kill the process backing the fleet card identified by its stream node. + Kill { node: String }, + /// Ask the reconciler to provision `count` additional nodes. + Provision { count: u32 }, + /// Lower the desired cluster size by `count` nodes (graceful scale + /// down: teardown through the reconciler, not a kill). + Remove { count: u32 }, +} + +static CONTROL_SENDER: OnceLock> = OnceLock::new(); + +/// Install the sink that receives dashboard-issued control commands. +/// +/// Called once by the embedding demo before the HTTP server starts. Without a +/// sink the control routes answer `503 Service Unavailable`. +pub fn set_control_sender(sender: Sender) { + let _ = CONTROL_SENDER.set(sender); +} + +pub(crate) fn dispatch(command: ControlCommand) -> bool { + CONTROL_SENDER + .get() + .is_some_and(|sender| sender.send(command).is_ok()) +} diff --git a/crates/dashboard/src/demo_control.rs b/crates/dashboard/src/demo_control.rs new file mode 100644 index 0000000..deaee00 --- /dev/null +++ b/crates/dashboard/src/demo_control.rs @@ -0,0 +1,112 @@ +//! Demo-only fleet control view (`demo-control` feature). +//! +//! A k8s-style control panel beside the fleet view: one row per provisioned +//! process (streamed on `proc..lifecycle`), each with a kill action, +//! plus a provision action that asks the reconciler for more nodes. Inert in +//! regular builds — this module compiles only under `demo-control`. + +use std::collections::BTreeMap; +use std::time::Instant; + +use parking_lot::Mutex; +use serde::Serialize; +use serde_json::Value; +use telemetry::frame::{Frame, StreamId}; + +use crate::view::DashboardView; +use crate::FrameEvent; + +#[derive(Clone, Serialize)] +struct ProcessEntry { + node: String, + pid: Option, + state: String, + seen_ms_ago: u64, +} + +#[derive(Serialize)] +struct ControlSnapshot { + nodes: Vec, +} + +#[derive(Default)] +struct ProcessState { + pid: Option, + state: String, + seen: Option, +} + +#[derive(Default)] +pub struct DemoControlView { + processes: Mutex>, +} + +fn is_lifecycle_channel(channel: &str) -> bool { + channel.starts_with("proc.") && channel.ends_with(".lifecycle") +} + +impl DashboardView for DemoControlView { + fn id(&self) -> &'static str { + "demo-control" + } + + fn title(&self) -> &'static str { + "Fleet Control" + } + + fn path(&self) -> &'static str { + "demo-control" + } + + fn channels(&self) -> &'static [&'static str] { + &[] + } + + fn ingest(&self, _stream: &StreamId, _frame: &Frame, event: &FrameEvent) { + if !is_lifecycle_channel(&event.channel) && event.channel != "node.status" { + return; + } + let Ok(payload) = serde_json::from_slice::(&event.payload) else { + return; + }; + let node = event.stream.node.clone(); + let mut processes = self.processes.lock(); + let entry = processes.entry(node).or_default(); + entry.seen = Some(Instant::now()); + if let Some(pid) = payload.get("pid").and_then(Value::as_u64) { + entry.pid = Some(pid as u32); + } + if let Some(kind) = payload.get("event").and_then(Value::as_str) { + entry.state = match kind { + "started" => "running".to_owned(), + "exited" => "exited".to_owned(), + "spawn_failed" | "error" => "failed".to_owned(), + other => other.to_owned(), + }; + } + } + + fn snapshot_json(&self) -> Value { + let now = Instant::now(); + let nodes: Vec = self + .processes + .lock() + .iter() + .map(|(node, state)| ProcessEntry { + node: node.clone(), + pid: state.pid, + state: state.state.clone(), + seen_ms_ago: state + .seen + .map(|seen| now.duration_since(seen).as_millis() as u64) + .unwrap_or(u64::MAX), + }) + .collect(); + serde_json::to_value(ControlSnapshot { nodes }) + .unwrap_or_else(|_| serde_json::json!({ "nodes": [] })) + } + + fn html(&self) -> Option<&'static str> { + Some(include_str!("demo_control_page.html")) + } +} diff --git a/crates/dashboard/src/demo_control_page.html b/crates/dashboard/src/demo_control_page.html new file mode 100644 index 0000000..80014d4 --- /dev/null +++ b/crates/dashboard/src/demo_control_page.html @@ -0,0 +1,148 @@ + + + + + + Fleet Control + + + + +
+

Fleet Control

+
+ + + + + +
+
+
+ + + + + +
nodepidstateseen
+ +
+ + + diff --git a/crates/dashboard/src/hardware_page.html b/crates/dashboard/src/hardware_page.html index 32c58ad..5033b9e 100644 --- a/crates/dashboard/src/hardware_page.html +++ b/crates/dashboard/src/hardware_page.html @@ -664,6 +664,7 @@
Seen ${escapeHtml(fmtDuration(node.last_seen_ms_ago))} ago + ${node.process ? `pid ${escapeHtml(node.process.pid ?? '—')} · ${escapeHtml(node.process.state)}` : ''} ${escapeHtml(fmtNumber(errors.length))} error${errors.length === 1 ? '' : 's'}
diff --git a/crates/dashboard/src/hardware_view.rs b/crates/dashboard/src/hardware_view.rs index c0974bf..179db10 100644 --- a/crates/dashboard/src/hardware_view.rs +++ b/crates/dashboard/src/hardware_view.rs @@ -40,9 +40,17 @@ struct NodeHardwareState { cpu: Option, gpu: Option, net: Option, + process: Option, history: VecDeque, } +/// Managed-process state folded from `proc.