zacheryasc/swactor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

stash dashboard prune and match mvp to spec

Browse source
This commit is contained in:
Zachery Aaron Shores-Chmielewski 2026-06-25 11:29:16 +04:00
parent 915d548e41
commit 6ccc6ed666
77 changed files with 5784 additions and 8256 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.deploy/deploy.example.toml
View file

@ -3,7 +3,6 @@
# and fill in your machine details.
[defaults]
dashboard_port = 9090
relay_port = 3340
# image = "swactor" # Required for --docker mode
# container = "swactor" # Required for --docker mode

87
Cargo.lock generated
View file

@ -267,58 +267,6 @@ version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
[[package]]
name = "axum"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90"
dependencies = [
"axum-core",
"bytes",
"form_urlencoded",
"futures-util",
"http",
"http-body",
"http-body-util",
"hyper",
"hyper-util",
"itoa",
"matchit",
"memchr",
"mime",
"percent-encoding",
"pin-project-lite",
"serde_core",
"serde_json",
"serde_path_to_error",
"serde_urlencoded",
"sync_wrapper",
"tokio",
"tower",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "axum-core"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1"
dependencies = [
"bytes",
"futures-core",
"http",
"http-body",
"http-body-util",
"mime",
"pin-project-lite",
"sync_wrapper",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "backon"
version = "1.6.0"
@ -968,22 +916,6 @@ dependencies = [
"syn",
]
[[package]]
name = "dashboard"
version = "0.1.0"
dependencies = [
"axum",
"crossbeam-queue",
"datastream",
"distribution",
"libc",
"serde",
"serde_json",
"swactor",
"tokio",
"tokio-stream",
]
[[package]]
name = "dashmap"
version = "6.2.1"
@ -2642,12 +2574,6 @@ dependencies = [
"regex-automata",
]
[[package]]
name = "matchit"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
[[package]]
name = "md5"
version = "0.7.0"
@ -2723,7 +2649,6 @@ dependencies = [
name = "mvp-system"
version = "0.1.0"
dependencies = [
"dashboard",
"datastream",
"distribution",
"iroh",
@ -4661,17 +4586,6 @@ dependencies = [
"zmij",
]
[[package]]
name = "serde_path_to_error"
version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457"
dependencies = [
"itoa",
"serde",
"serde_core",
]
[[package]]
name = "serde_spanned"
version = "1.1.1"
@ -5432,7 +5346,6 @@ dependencies = [
"tokio",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]

1
Cargo.toml
View file

@ -3,7 +3,6 @@ members = [
".",
"crates/bindings/python",
"crates/bindings/wasm-runtime",
"crates/dashboard",
"crates/process",
"crates/transport",
"crates/distribution",

88
apps/pipeline-parallel-inference/Cargo.lock generated
View file

@ -150,58 +150,6 @@ version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
[[package]]
name = "axum"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90"
dependencies = [
"axum-core",
"bytes",
"form_urlencoded",
"futures-util",
"http",
"http-body",
"http-body-util",
"hyper",
"hyper-util",
"itoa",
"matchit",
"memchr",
"mime",
"percent-encoding",
"pin-project-lite",
"serde_core",
"serde_json",
"serde_path_to_error",
"serde_urlencoded",
"sync_wrapper",
"tokio",
"tower",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "axum-core"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1"
dependencies = [
"bytes",
"futures-core",
"http",
"http-body",
"http-body-util",
"mime",
"pin-project-lite",
"sync_wrapper",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "backon"
version = "1.6.0"
@ -690,22 +638,6 @@ dependencies = [
"syn",
]
[[package]]
name = "dashboard"
version = "0.1.0"
dependencies = [
"axum",
"crossbeam-queue",
"datastream",
"distribution",
"libc",
"serde",
"serde_json",
"swactor",
"tokio",
"tokio-stream",
]
[[package]]
name = "dashmap"
version = "6.2.1"
@ -2279,12 +2211,6 @@ dependencies = [
"regex-automata",
]
[[package]]
name = "matchit"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
[[package]]
name = "md5"
version = "0.7.0"
@ -2994,7 +2920,6 @@ name = "pipeline-parallel-inference"
version = "0.1.0"
dependencies = [
"base64",
"dashboard",
"datastream",
"distribution",
"futures-util",
@ -3898,17 +3823,6 @@ dependencies = [
"zmij",
]
[[package]]
name = "serde_path_to_error"
version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457"
dependencies = [
"itoa",
"serde",
"serde_core",
]
[[package]]
name = "serde_urlencoded"
version = "0.7.1"
@ -4216,6 +4130,7 @@ dependencies = [
"crossbeam-queue",
"crossbeam-utils",
"getrandom 0.2.17",
"parking_lot",
"serde",
]
@ -4576,7 +4491,6 @@ dependencies = [
"tokio",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]

4
apps/pipeline-parallel-inference/Cargo.toml
View file

@ -5,7 +5,6 @@ name = "pipeline-parallel-inference"
version = "0.1.0"
edition = "2024"
publish = false
[dependencies]
swactor = { path = "../..", features = ["transport", "serde", "std"] }
swactor-transport = { path = "../../crates/transport" }
@ -20,9 +19,6 @@ tokio = { version = "1", features = ["full"] }
distribution = { path = "../../crates/distribution" }
iroh-driver = { path = "../../crates/iroh-driver" }
datastream = { path = "../../crates/datastream" }
# Live runtime dashboard (HTTP overview/actors/topology pages, served on
# localhost when PP_DASHBOARD is set).
dashboard = { path = "../../crates/dashboard" }
iroh = "0.98"
urlencoding = "2"
base64 = "0.22"

85
apps/pipeline-parallel-inference/scripts/demo-fleet.sh
View file

@ -1,22 +1,11 @@
#!/usr/bin/env bash
# demo-fleet.sh — one-command local mock of a vast.ai fleet, watchable live.
# demo-fleet.sh — one-command local mock of a vast.ai fleet.
#
# Brings up, from a single command, a self-contained demo of the production
# topology with NO off-box collector: the orchestrator runs locally, hosts the
# FULL swactor dashboard, AND hosts the fleet view in-process. Each stage ships
# its telemetry over the datastream (identity + host.resource frames) to the
# orchestrator's in-process FleetView sink; the orchestrator's Fleet tab renders
# it beside its own live actor / topology / distribution views.
#
# - pp-orchestrator on the HOST in --seed mode (PP_DASHBOARD on), spawning N
# pp-worker containers (one per stage) via docker-gpu-node.sh, each on
# --network host, and serving the full dashboard at http://127.0.0.1:9095/.
# It binds a UDP datastream sink (PP_FLEET_SINK) and folds every stage's
# frames into the live Fleet table.
# - Each stage pp-worker emits its fleet frames (~every few seconds) to that
# sink, so the Fleet tab animates in real time.
# - PP_HOLD=1 keeps the cluster up after the first drive, so the stages keep
# streaming and the dashboard stays live for inspection.
# topology with NO off-box collector. The orchestrator runs locally in seed
# mode and spawns N pp-worker containers via docker-gpu-node.sh on
# --network host. PP_HOLD=1 keeps the cluster up after the first drive for
# manual inspection.
#
# Ctrl+C (or any exit) tears everything down: stage containers, orchestrator,
# and all temp files.
@ -34,10 +23,6 @@
# GPU gauges populate once a real GPU source is wired.
# PP_PROMPT inference prompt (default: "fleet demo")
# PP_MAX_TOKENS decode token cap (default: 4)
# PP_BIND_HOST dashboard + fleet-sink bind host (default: 127.0.0.1)
# PP_DASHBOARD_PORT orchestrator dashboard HTTP port (default: 9095)
# PP_FLEET_PORT orchestrator fleet UDP sink port (default: 9096)
# PP_NO_OPEN if set, don't try to open the dashboard in a browser
# PP_STAGE_NETWORK docker network for stages (default: host)
set -euo pipefail
@ -46,14 +31,6 @@ IMAGE="${PP_DIAG_IMAGE:-swactor-pp-gpu:latest}"
BASE_IMAGE="${PP_BASE_IMAGE:-swactor-pp-base:cuda12.6}"
PROMPT="${PP_PROMPT:-fleet demo}"
MAX_TOKENS="${PP_MAX_TOKENS:-4}"
BIND_HOST="${PP_BIND_HOST:-127.0.0.1}"
DASH_PORT="${PP_DASHBOARD_PORT:-9095}"
FLEET_PORT="${PP_FLEET_PORT:-9096}"
CONTAINER_PREFIX="demo-fleet-stage"
RUN_ID="demo-fleet-$(date +%s)"
# The full swactor dashboard — including the in-process Fleet tab — is served by
# the orchestrator at "/".
DASH_URL="http://${BIND_HOST}:${DASH_PORT}/"
if ! [[ "$NUM_STAGES" =~ ^[0-9]+$ ]] || [ "$NUM_STAGES" -lt 2 ]; then
echo "demo-fleet: N must be an integer >= 2 (seed mode needs >=2 stages), got '$NUM_STAGES'" >&2
@ -65,16 +42,6 @@ fi
if ! docker info >/dev/null 2>&1; then
echo "demo-fleet: docker daemon unreachable" >&2; exit 2
fi
# Fail loudly on a clash for the orchestrator dashboard port. Its HTTP server is
# spawned on the driver's tokio runtime and `.expect()`s its bind; a collision
# panics that task silently and the run carries on with no dashboard. Catch it
# here so the user can pick a free one.
if (exec 3<>"/dev/tcp/${BIND_HOST}/${DASH_PORT}") 2>/dev/null; then
exec 3>&- 3<&-
echo "demo-fleet: dashboard port ${DASH_PORT} is already in use." \
"Pick a free one: PP_DASHBOARD_PORT=9097 $0 ${NUM_STAGES}" >&2
exit 2
fi
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
CRATE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
@ -137,25 +104,18 @@ cleanup() {
trap cleanup EXIT
trap 'exit 130' INT TERM
# ── Step 3: orchestrator (hosts the full dashboard + fleet, holds cluster) ─
# ── Step 3: orchestrator (holds cluster) ──────────────────────────────────
# stdin is the FIFO; we hold its write end open on fd 3 so hold_open() never
# sees EOF and the cluster stays up until we tear down.
exec 3<>"$FIFO"
echo "demo-fleet: launching orchestrator + ${NUM_STAGES} stage containers (run_id=$RUN_ID)"
# The orchestrator (and, via inheritance, docker-gpu-node.sh) read these from
# the environment. PP_HOLD keeps the cluster up; PP_DASHBOARD makes the
# orchestrator host the full swactor dashboard locally; PP_FLEET_SINK is the
# UDP address the orchestrator binds for its in-process FleetView and that each
# stage ships its datastream frames to (reachable from the --network host stage
# containers via the shared loopback).
# the environment. PP_HOLD keeps the cluster up after the first drive.
export PP_HOLD=1
export PP_WORKER_STUB=1
export PP_DEV=CPU
export PP_IMAGE="$IMAGE"
export PP_CONTAINER_PREFIX="$CONTAINER_PREFIX"
export PP_DASHBOARD=1
export PP_DASHBOARD_PORT="$DASH_PORT"
export PP_FLEET_SINK="${BIND_HOST}:${FLEET_PORT}"
[ -n "${PP_GPUS:-}" ] && export PP_GPUS
[ -n "${PP_STAGE_NETWORK:-}" ] && export PP_STAGE_NETWORK
"$ORCHESTRATOR_BIN" \
@ -168,32 +128,7 @@ export PP_FLEET_SINK="${BIND_HOST}:${FLEET_PORT}"
<"$FIFO" >"$ORCH_LOG" 2>&1 &
ORCH_PID=$!
# ── Step 4: wait for the orchestrator's dashboard to bind, then announce + open
WAITED=0
until (echo > "/dev/tcp/${BIND_HOST}/${DASH_PORT}") >/dev/null 2>&1; do
if ! kill -0 "$ORCH_PID" >/dev/null 2>&1; then
echo "demo-fleet: orchestrator exited before its dashboard came up." >&2
tail -n 40 "$ORCH_LOG" >&2 || true
exit 1
fi
WAITED=$((WAITED + 1))
[ "$WAITED" -ge 30 ] && { echo "demo-fleet: orchestrator dashboard did not bind :${DASH_PORT} in 30s" >&2; tail -n 40 "$ORCH_LOG" >&2; exit 1; }
sleep 1
done
echo
echo " ┌─────────────────────────────────────────────────────────────┐"
echo " │ Full swactor dashboard: $DASH_URL"
echo " │ (overview / actors / topology / distribution / netmap / fleet)"
echo " │ Fleet datastream sink: ${BIND_HOST}:${FLEET_PORT} (UDP, in-process)"
echo " └─────────────────────────────────────────────────────────────┘"
echo
if [ -z "${PP_NO_OPEN:-}" ]; then
if command -v xdg-open >/dev/null 2>&1; then (xdg-open "$DASH_URL" >/dev/null 2>&1 &) || true
elif command -v open >/dev/null 2>&1; then (open "$DASH_URL" >/dev/null 2>&1 &) || true
fi
fi
# ── Step 5: wait until the cluster is converged + held open ────────────────
# ── Step 4: wait until the cluster is converged + held open ────────────────
echo "demo-fleet: waiting for the cluster to converge (first inference drive)…"
WAITED=0
until grep -q "holding cluster open" "$ORCH_LOG" 2>/dev/null; do
@ -208,10 +143,10 @@ until grep -q "holding cluster open" "$ORCH_LOG" 2>/dev/null; do
sleep 1
done
RUNNING=$(docker ps -q --filter "name=^${CONTAINER_PREFIX}-[0-9]+$" | wc -l | tr -d ' ')
echo
echo "demo-fleet: ✅ fleet up — ${RUNNING}/${NUM_STAGES} stage containers streaming real metrics."
echo "demo-fleet: watch live at $DASH_URL"
echo "demo-fleet: fleet up — ${RUNNING}/${NUM_STAGES} stage containers are running."
echo "demo-fleet: press Ctrl+C to tear everything down."
echo

148
apps/pipeline-parallel-inference/scripts/docker-dashboard-e2e.sh
View file

@ -1,148 +0,0 @@
#!/usr/bin/env bash
# docker-dashboard-e2e.sh — the docker-e2e run, held open under the live
# swactor dashboard, one dashboard PER STAGE.
#
# Brings up `N` stub-mode `pp-worker` containers on localhost and drives
# one InferenceRequest through them, exactly like `docker-e2e.sh` — but each
# stage serves the live swactor dashboard (PP_STAGE_DASHBOARD) and the
# orchestrator HOLDS after the drive (PP_HOLD). The stage containers run on
# `--network host`, so each stage's dashboard is reachable on the host at
# http://localhost:<BASE + stage> (BASE default 9100)
# i.e. stage 0 → 9100, stage 1 → 9101, … Each board shows that stage's
# StageActor + bridge actors and live message activity as tokens flow.
#
# This targets the STAGE runtimes deliberately: the orchestrator's own
# runtime is near-empty (it sends one request and waits), so there is nothing
# to see there — the actors that do the work live inside the stage processes.
#
# The cluster stays up until you press Enter in this terminal, at which point
# the orchestrator unwinds and tears everything down.
#
# Usage:
# apps/pipeline-parallel-inference/scripts/docker-dashboard-e2e.sh [N]
#
# Environment overrides:
# PP_STAGE_DASHBOARD_PORT_BASE base port; stage K serves BASE+K (default 9100)
# PP_IMAGE code image tag (default: swactor-pp-gpu:latest)
# PP_BASE_IMAGE base image tag (default: swactor-pp-base:cuda12.6)
# PP_CONTAINER_PREFIX container name prefix (default: pp-stage)
# PP_MAX_TOKENS max decode tokens (default: 4)
# PP_PROMPT inference prompt (default: "Say hello")
# PP_SKIP_BUILD skip cargo build (use existing target/)
# PP_SKIP_IMAGE_BUILD skip docker image build (use existing tag)
set -euo pipefail
NUM_STAGES="${1:-3}"
PREFIX="${PP_CONTAINER_PREFIX:-pp-stage}"
MAX_TOKENS="${PP_MAX_TOKENS:-4}"
PROMPT="${PP_PROMPT:-Say hello}"
PORT_BASE="${PP_STAGE_DASHBOARD_PORT_BASE:-9100}"
ORCH_PORT="${PP_DASHBOARD_PORT:-9099}"
IMAGE="${PP_IMAGE:-swactor-pp-gpu:latest}"
BASE_IMAGE="${PP_BASE_IMAGE:-swactor-pp-base:cuda12.6}"
if ! [[ "$NUM_STAGES" =~ ^[0-9]+$ ]] || [ "$NUM_STAGES" -lt 2 ]; then
echo "docker-dashboard-e2e: NUM_STAGES must be an integer >= 2, got '$NUM_STAGES'" >&2
exit 2
fi
if ! command -v docker >/dev/null 2>&1; then
echo "docker-dashboard-e2e: docker not on PATH" >&2
exit 2
fi
if ! docker info >/dev/null 2>&1; then
echo "docker-dashboard-e2e: docker daemon unreachable" >&2
exit 2
fi
# Fail loudly on a port clash for ANY stage port. The dashboard's HTTP server
# is spawned on the driver's tokio runtime and `.expect()`s its bind; a
# collision panics that task silently and the stage keeps running, so the
# browser just shows whatever already owns the port. Catch it here instead.
if (exec 3<>"/dev/tcp/127.0.0.1/${ORCH_PORT}") 2>/dev/null; then
exec 3>&- 3<&-
echo "docker-dashboard-e2e: orchestrator port ${ORCH_PORT} is already in use." \
"Pick a free one: PP_DASHBOARD_PORT=9098 $0 ${NUM_STAGES}" >&2
exit 2
fi
for ((k = 0; k < NUM_STAGES; k++)); do
p=$((PORT_BASE + k))
if (exec 3<>"/dev/tcp/127.0.0.1/${p}") 2>/dev/null; then
exec 3>&- 3<&-
echo "docker-dashboard-e2e: port ${p} (stage ${k}) is already in use." \
"Pick a free base: PP_STAGE_DASHBOARD_PORT_BASE=9200 $0 ${NUM_STAGES}" >&2
exit 2
fi
done
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
CRATE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
WORKSPACE_DIR="$(cd "$CRATE_DIR/../.." && pwd)"
ORCHESTRATOR_BIN="$CRATE_DIR/target/release/pp-orchestrator"
WORKER_BIN="$CRATE_DIR/target/release/pp-worker"
WORKER_PY="$CRATE_DIR/pp_tinygrad_worker.py"
# Step 1: build the release artifacts the docker image packages.
if [ -z "${PP_SKIP_BUILD:-}" ]; then
echo "docker-dashboard-e2e: building pp-worker + pp-orchestrator (release)"
cargo build --manifest-path "$CRATE_DIR/Cargo.toml" --release \
--bin pp-worker --bin pp-orchestrator
fi
for f in "$ORCHESTRATOR_BIN" "$WORKER_BIN" "$WORKER_PY"; do
[ -f "$f" ] || { echo "docker-dashboard-e2e: missing $f" >&2; exit 1; }
done
# Step 2: build the layered image (heavy CUDA base, then thin code layer).
# The stage dashboard lives in the pp-worker binary baked into this image,
# so a stale image without it will show nothing — rebuild unless you know the
# current image already carries the dashboard-enabled binary.
if [ -z "${PP_SKIP_IMAGE_BUILD:-}" ]; then
echo "docker-dashboard-e2e: building $BASE_IMAGE (base)"
docker build -f "$CRATE_DIR/Dockerfile.base" -t "$BASE_IMAGE" "$WORKSPACE_DIR"
echo "docker-dashboard-e2e: building $IMAGE (code)"
docker build -f "$CRATE_DIR/Dockerfile" \
--build-arg "BASE_IMAGE=$BASE_IMAGE" -t "$IMAGE" "$WORKSPACE_DIR"
fi
# Step 3: clean up stage containers from prior runs, and on exit (the
# orchestrator's ChainGuard kills its own children, but a Ctrl-C mid-run can
# leave strays).
cleanup_containers() {
local ids
ids=$(docker ps -aq --filter "name=^${PREFIX}-[0-9]+$" || true)
if [ -n "$ids" ]; then
# shellcheck disable=SC2086
docker rm -f $ids >/dev/null 2>&1 || true
fi
}
cleanup_containers
trap cleanup_containers EXIT
echo "docker-dashboard-e2e: dashboards will come up at:"
echo " orchestrator: http://localhost:${ORCH_PORT} (overview / actors / topology / distribution)"
for ((k = 0; k < NUM_STAGES; k++)); do
echo " stage ${k}: http://localhost:$((PORT_BASE + k))"
done
# Step 4: drive pp-orchestrator with the docker shim. The orchestrator serves its
# own dashboard (PP_DASHBOARD) — including the live SWIM distribution graph and
# message tallies — and each stage serves its own (PP_STAGE_DASHBOARD). PP_HOLD
# makes the orchestrator block at the end, ticking the driver so the
# distribution view keeps updating. stdin/stdout stay on this terminal so the
# hold can read your Enter.
PP_WORKER_STUB=1 \
PP_IMAGE="$IMAGE" \
PP_CONTAINER_PREFIX="$PREFIX" \
PP_DEV=CPU \
PP_HOLD=1 \
PP_DASHBOARD=1 \
PP_DASHBOARD_PORT="$ORCH_PORT" \
PP_STAGE_DASHBOARD=1 \
PP_STAGE_DASHBOARD_PORT_BASE="$PORT_BASE" \
"$ORCHESTRATOR_BIN" \
--seed \
--num-stages "$NUM_STAGES" \
--gpu-node "$SCRIPT_DIR/docker-gpu-node.sh" \
--worker "$WORKER_PY" \
--prompt "$PROMPT" \
--max-tokens "$MAX_TOKENS"

2
apps/pipeline-parallel-inference/scripts/docker-gpu-node.sh
View file

@ -102,8 +102,6 @@ exec docker run --rm --init \
-e FIRST_PEER_DIRECT \
-e PP_BOOT_DELAY_STAGE \
-e PP_BOOT_DELAY_SECS \
-e PP_STAGE_DASHBOARD \
-e PP_STAGE_DASHBOARD_PORT_BASE \
-e PP_FLEET_SINK \
-e DEV="$DEV" \
-e WORKER_SCRIPT=/usr/local/share/pp_tinygrad_worker.py \

351
apps/pipeline-parallel-inference/src/bin/pp_orchestrator.rs
View file

@ -29,31 +29,27 @@
//! 5. Kills any spawned child processes and (on `--vastai`) destroys all
//! rented instances regardless of success or failure.
use std::net::SocketAddr;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::sync::Arc;
use std::time::{Duration, Instant};
use serde::{Deserialize, Serialize};
use datastream::{DATASTREAM_SINK_NAME, DatastreamSink};
use datastream::transport::Delivery;
use datastream::{Consumer, DATASTREAM_SINK_NAME, DatastreamSink};
use distribution::node::DistributedNodeConfig;
use distribution::snapshot::DistributionNodeSnapshot;
use distribution::registry::RegistryConfig;
use distribution::swim::probe::SwimConfig;
use iroh::{PublicKey, RelayMode, SecretKey};
use iroh_driver::IrohDriverConfig;
use swactor::actor::ActorAddress;
use swactor::runtime::Inbox;
use pipeline_parallel_inference::cluster::ClusterNode;
use dashboard::datastream_source::{FleetView, distribution_cache_plugin, fleet_cache_plugin};
use dashboard::{DashboardConfig, start_dashboard};
use pipeline_parallel_inference::iroh_transport::{
ACTOR_ALPN, ActorMessagePump, IrohActorTransport,
};
@ -66,7 +62,6 @@ use pipeline_parallel_inference::orchestrator::{
use pipeline_parallel_inference::topology::{ENTRY_NAME, stage_name};
const ORCHESTRATOR_NAME: &str = "pp-orchestrator";
type SharedSnapshot = Arc<Mutex<Option<DistributionNodeSnapshot>>>;
fn node_config() -> DistributedNodeConfig {
DistributedNodeConfig {
@ -380,50 +375,13 @@ fn check_child_death(guard: &mut ChainGuard) -> Result<(), String> {
Ok(())
}
/// Register the Fleet tab and spawn the orchestrator-hosted datastream
/// consumer behind it: the [`DatastreamSink`] actor, which every node's
/// `ClusterFrameSink` resolves (under [`DATASTREAM_SINK_NAME`]) and ships its
/// telemetry to over the regular swactor transport (no dedicated channel). The
/// actor folds each delivery into an in-process `FleetView` and caches the
/// fleet JSON the tab serves. Returns the sink's address so the boot-phase
/// provisioner can ship the orchestrator's own (and each booting node's) frames
/// to it in-process.
///
/// The actor is spawned here, but the [`DATASTREAM_SINK_NAME`] cluster-name
/// registration is deliberately NOT done here — see [`register_fleet_sink_name`]
/// and the [`ORCHESTRATOR_NAME`] registration: a name published before the
/// cluster is non-empty sizes its SWIM dissemination budget for a one-node
/// cluster and exhausts it before any stage can observe the entry via piggyback
/// gossip, so the stages never resolve the sink and the Fleet tab stays empty.
/// The caller must register the returned address post-convergence.
fn wire_fleet_sink(
cluster: &ClusterNode,
handle: Arc<dashboard::DashboardHandle>,
) -> Option<ActorAddress> {
let fleet_cache: Arc<Mutex<Option<String>>> = Arc::new(Mutex::new(None));
let dist_cache: Arc<Mutex<Option<String>>> = Arc::new(Mutex::new(None));
handle.register_plugin(fleet_cache_plugin(Arc::clone(&fleet_cache)));
handle.register_plugin(distribution_cache_plugin(Arc::clone(&dist_cache)));
// Fold every received delivery into a FleetView, caching datastream-derived
// fleet/distribution JSON and pushing synthesized stats/activity into the
// dashboard. This keeps the dashboard off the local swactor runtime.
let mut view = FleetView::new(None);
let fleet_cache = Arc::clone(&fleet_cache);
let dist_cache = Arc::clone(&dist_cache);
let handle_for_updates = Arc::clone(&handle);
/// Spawn the local datastream consumer and return its actor address. The fold
/// keeps the raw stream in a datastream store without coupling this binary to
/// any presentation layer.
fn wire_datastream_sink(cluster: &ClusterNode) -> Option<ActorAddress> {
let mut consumer = Consumer::new();
let sink = DatastreamSink::new(move |stream, frame| {
let update = view.ingest(&stream, &frame);
*fleet_cache.lock().unwrap() = Some(update.fleet_json);
if let Some(dist_json) = update.dist_json {
*dist_cache.lock().unwrap() = Some(dist_json);
}
if let Some(stats) = update.stats {
handle_for_updates.set_stats(stats);
}
for (is_warn, message) in update.logs {
handle_for_updates.push_activity(is_warn, message);
}
let _ = consumer.accept(Delivery { stream, frame });
});
match cluster.rt.spawn(sink) {
Ok(addr) => {
@ -437,20 +395,16 @@ fn wire_fleet_sink(
}
}
/// Publish the datastream-sink actor under [`DATASTREAM_SINK_NAME`] so the
/// stages can resolve it and ship their fleet telemetry. MUST be called
/// post-convergence (see [`wire_fleet_sink`]): registering it earlier would
/// size the SWIM dissemination budget for a one-node cluster and the entry
/// would exhaust its budget before any stage could observe it via piggyback
/// gossip — leaving the Fleet tab empty. Mirrors the [`ORCHESTRATOR_NAME`]
/// post-convergence registration.
fn register_fleet_sink_name(cluster: &ClusterNode, addr: Option<ActorAddress>) {
/// Publish the datastream sink after convergence so the name-dissemination
/// budget is sized for the real cluster, matching the orchestrator inbox.
fn register_datastream_sink_name(cluster: &ClusterNode, addr: Option<ActorAddress>) {
if let Some(addr) = addr {
cluster.register_name(DATASTREAM_SINK_NAME, addr);
eprintln!("pp-orchestrator: datastream-sink registered -> {addr:?}");
}
}
fn run_seed(args: &Args) -> i32 {
let gpu_node_bin = resolve_gpu_node_path(args);
if !gpu_node_bin.exists() {
@ -469,21 +423,6 @@ fn run_seed(args: &Args) -> i32 {
return 1;
}
// Build the dashboard handle up front. Runtime stats/distribution panels are
// fed only by datastream updates folded through `wire_fleet_sink`.
let dashboard = if std::env::var_os("PP_DASHBOARD").is_some() {
let port: u16 = std::env::var("PP_DASHBOARD_PORT")
.ok()
.and_then(|s| s.trim().parse().ok())
.unwrap_or(9090);
let handle = Arc::new(start_dashboard(DashboardConfig {
port,
..Default::default()
}));
Some((handle, port))
} else {
None
};
let mut cluster = match ClusterNode::new(
IrohDriverConfig {
@ -504,9 +443,6 @@ fn run_seed(args: &Args) -> i32 {
}
};
// Keep the existing snapshot cache only for hold/prompt loop refresh paths;
// dashboard Distribution is fed from datastream-derived FleetView updates.
let want_dist = std::env::var_os("PP_DASHBOARD").is_some();
let my_id = cluster.node_id();
let my_hex: String = my_id.0.iter().map(|b| format!("{:02x}", b)).collect();
@ -520,6 +456,7 @@ fn run_seed(args: &Args) -> i32 {
"pp-orchestrator (--seed --num-stages {n}): orchestrator node {my_hex}, direct={direct:?}",
n = args.num_stages,
);
let seed_datastream_sink_addr = wire_datastream_sink(&cluster);
// Run inside a labelled block so every failure point can `break`
// with both an exit code and a stable exit-reason string; the
@ -529,34 +466,6 @@ fn run_seed(args: &Args) -> i32 {
let rt = Arc::clone(&cluster.rt);
let router = Arc::clone(&cluster.transport_router);
// Shared distribution snapshot cell for hold-loop refresh paths. The
// dashboard's Distribution plugin is datastream-backed via FleetView.
let dist_cached: Option<SharedSnapshot> = if want_dist {
Some(Arc::new(Mutex::new(Some(cluster.snapshot()))))
} else {
None
};
// Datastream-sink actor address, set when the dashboard wires the Fleet
// tab. Its cluster-name publish is deferred to post-convergence (see
// `register_fleet_sink_name`).
let mut seed_fleet_sink_addr: Option<ActorAddress> = None;
// When the in-process dashboard is on, register only datastream-backed
// plugins. FleetView updates fill both Fleet and Distribution caches.
if let Some((handle, port)) = &dashboard {
// Fleet tab ("Fleet" nav): the orchestrator-hosted datastream
// consumer. Each stage resolves the `datastream-sink` actor and
// ships its identity + host.resource frames over the swactor
// transport; the actor folds them into a live FleetView and serves
// the cross-node telemetry table beside the orchestrator's own views.
seed_fleet_sink_addr = wire_fleet_sink(&cluster, Arc::clone(handle));
handle.start_http(cluster.driver.tokio_handle());
eprintln!(
"pp-orchestrator: live dashboard on http://localhost:{port} \
(overview / actors / topology / distribution / fleet)"
);
}
let response_inbox = match rt.new_inbox::<InferenceResponse>() {
Ok(i) => i,
@ -612,9 +521,7 @@ fn run_seed(args: &Args) -> i32 {
// via SWIM piggyback gossip. Doing it post-convergence gives the registry
// a budget sized for the real cluster.
cluster.register_name(ORCHESTRATOR_NAME, inbox_addr);
// Publish the datastream-sink now, for the same budget reason: stages
// resolve this name to ship their fleet telemetry to the Fleet tab.
register_fleet_sink_name(&cluster, seed_fleet_sink_addr);
register_datastream_sink_name(&cluster, seed_datastream_sink_addr);
eprintln!("pp-orchestrator: registered {ORCHESTRATOR_NAME} -> {inbox_addr:?}");
if let Err(e) = conv_res {
eprintln!("pp-orchestrator: {e}");
@ -752,14 +659,9 @@ fn run_seed(args: &Args) -> i32 {
println!("{text}");
println!("============================================");
// The ChainGuard is still in scope here, so the stage
// containers stay up while we hold — letting the dashboard
// show a live, converged cluster rather than a torn-down one.
if dashboard.is_some() || std::env::var_os("PP_HOLD").is_some() {
hold_open(
&mut cluster,
dist_cached.as_ref(),
dashboard.as_ref().map(|(_, p)| *p),
);
// containers stay up while we hold.
if std::env::var_os("PP_HOLD").is_some() {
hold_open(&mut cluster);
}
(0, "ok")
}
@ -890,25 +792,11 @@ impl AwaitError {
/// `dead`. When that happens, a `pp_drive_dead_member` diagnostic
/// event is emitted identifying the stage and the dead member's
/// `node_id_short` before returning [`AwaitError::ForwardPathDead`].
/// Block the orchestrator after a successful drive so the live dashboard —
/// and the stage containers, whose `ChainGuard` is still in scope — stay up
/// for inspection. Returns when the operator presses Enter or closes stdin
/// (Ctrl-D), at which point the run unwinds and tears the cluster down.
/// Hold the cluster open after a successful drive. The `ChainGuard` is still
/// in scope (containers stay up), and we keep ticking the driver so SWIM stays
/// converged. The optional snapshot cache is retained for non-dashboard hold-loop
/// refresh paths; dashboard panels are updated through datastream FleetView.
/// Returns when the operator presses Enter or closes stdin (Ctrl-D).
fn hold_open(cluster: &mut ClusterNode, dist_cached: Option<&SharedSnapshot>, port: Option<u16>) {
match port {
Some(p) => eprintln!(
"pp-orchestrator: holding cluster open — orchestrator dashboard at \
http://localhost:{p}. Press Enter (or Ctrl-D) to tear down."
),
None => eprintln!(
"pp-orchestrator: holding cluster open. Press Enter (or Ctrl-D) to tear down."
),
}
/// converged. Returns when the operator presses Enter or closes stdin (Ctrl-D).
fn hold_open(cluster: &mut ClusterNode) {
eprintln!("pp-orchestrator: holding cluster open. Press Enter (or Ctrl-D) to tear down.");
// Read stdin on a side thread so the main thread can keep pumping the
// driver; a blocking read here would freeze SWIM and the live snapshot.
let stop = Arc::new(AtomicBool::new(false));
@ -920,16 +808,10 @@ fn hold_open(cluster: &mut ClusterNode, dist_cached: Option<&SharedSnapshot>, po
stop.store(true, Ordering::SeqCst);
});
}
// Drain inbound ACTOR_ALPN traffic so fleet `DatastreamFrame`s from the
// stages reach the `datastream-sink` actor while the cluster is held open;
// `pump_once` only services SWIM/protocol gossip, not app messages.
let msg_pump = ActorMessagePump::new();
while !stop.load(Ordering::SeqCst) {
cluster.pump_once();
msg_pump.pump(&cluster.driver, &cluster.codecs, &cluster.rt);
if let Some(cached) = dist_cached {
*cached.lock().unwrap() = Some(cluster.snapshot());
}
std::thread::sleep(Duration::from_millis(200));
}
}
@ -1084,13 +966,12 @@ fn drive_once(
result
}
/// Live multi-prompt loop for vast.ai mode (dashboard or `--hold`). Keeps the
/// cluster converged and the dashboard SSE fed while the operator drives more
/// prompts. A stdin-reader side thread feeds prompt lines so the main thread
/// can keep pumping the driver; each non-empty line drives one more inference at
/// the next `drive_seq`. A blank line, `quit`, or EOF (Ctrl-D) ends the loop,
/// after which the caller's finalize + teardown tail runs.
#[allow(clippy::too_many_arguments)]
/// Live multi-prompt loop for vast.ai `--hold` mode. Keeps the cluster
/// converged while the operator drives more prompts. A stdin-reader side thread
/// feeds prompt lines so the main thread can keep pumping the driver; each
/// non-empty line drives one more inference at the next `drive_seq`. A blank
/// line, `quit`, or EOF (Ctrl-D) ends the loop, after which the caller's
/// finalize + teardown tail runs.
fn prompt_loop(
cluster: &mut ClusterNode,
response_inbox: &Inbox<InferenceResponse>,
@ -1100,19 +981,11 @@ fn prompt_loop(
label: &str,
max_tokens: u32,
first_drive_seq: u32,
dist_cached: Option<&SharedSnapshot>,
port: Option<u16>,
) {
match port {
Some(p) => eprintln!(
"pp-orchestrator: cluster live — dashboard at http://localhost:{p}. \
Type a prompt + Enter to drive again; blank line / Ctrl-D / `quit` to tear down."
),
None => eprintln!(
"pp-orchestrator: cluster live. Type a prompt + Enter to drive again; \
blank line / Ctrl-D / `quit` to tear down."
),
}
eprintln!(
"pp-orchestrator: cluster live. Type a prompt + Enter to drive again; \
blank line / Ctrl-D / `quit` to tear down."
);
// Read prompts on a side thread so the main thread keeps pumping the driver;
// a blocking stdin read here would freeze SWIM and the live snapshot. The
@ -1139,16 +1012,13 @@ fn prompt_loop(
});
}
// Drain inbound ACTOR_ALPN traffic between drives so fleet telemetry keeps
// flowing into the `datastream-sink` while the operator is idle at the prompt.
// Drain inbound ACTOR_ALPN traffic between drives while the operator is
// idle at the prompt.
let msg_pump = ActorMessagePump::new();
let mut drive_seq = first_drive_seq;
while !stop.load(Ordering::SeqCst) {
cluster.pump_once();
msg_pump.pump(&cluster.driver, &cluster.codecs, &cluster.rt);
if let Some(cached) = dist_cached {
*cached.lock().unwrap() = Some(cluster.snapshot());
}
// Drive any prompts that arrived since the last tick. drive_once pumps
// the driver itself while awaiting each response.
while let Ok(prompt) = rx.try_recv() {
@ -1427,21 +1297,6 @@ fn run_vastai(args: &Args) -> i32 {
let num_stages = cluster.num_stages;
let label = cluster.label.clone();
// Build the dashboard handle up front. Runtime stats/distribution panels are
// fed only by datastream updates folded through `wire_fleet_sink`.
let dashboard = if std::env::var_os("PP_DASHBOARD").is_some() {
let port: u16 = std::env::var("PP_DASHBOARD_PORT")
.ok()
.and_then(|s| s.trim().parse().ok())
.unwrap_or(9090);
let handle = Arc::new(start_dashboard(DashboardConfig {
port,
..Default::default()
}));
Some((handle, port))
} else {
None
};
let mut cluster_node = match ClusterNode::new(
IrohDriverConfig {
@ -1462,10 +1317,6 @@ fn run_vastai(args: &Args) -> i32 {
}
};
// The Distribution tab renders from the orchestrator's own SWIM view; its
// snapshot cell exists whenever the in-process dashboard is on.
let want_dist = std::env::var_os("PP_DASHBOARD").is_some();
// Per-cluster drive counter, emitted on pp_drive_start / pp_drive_end so
// the bundle reader can slice the interleaved event stream by attempt.
// One-shot and --hold each drive exactly once per process, so this is 1.
@ -1511,47 +1362,9 @@ fn run_vastai(args: &Args) -> i32 {
eprintln!("pp-orchestrator: no relay URL after 20s — vastai mode usually requires one");
}
// Per-stage host telemetry rides the swactor cluster transport: each rented
// stage resolves the `datastream-sink` actor and ships its frames there,
// and the orchestrator folds the booting node's SSH output into the same
// sink. `fleet_sink_addr` is filled when the dashboard wiring spawns the
// sink below; the provisioner ships boot frames to it in-process.
let mut fleet_sink_addr: Option<ActorAddress> = None;
// ── Live dashboard + runtime — started BEFORE the lease ──────────────
// The lease + image-load phase is the slow, failure-prone part the operator
// most needs to watch, so the HTTP server binds here (start_http) rather
// than after convergence. Plugins populate as the cluster comes up. The
// ClusterNode already owns the runtime + codec + router (shared with the
// protocol actors); the dashboard handle was built up front above.
let rt = Arc::clone(&cluster_node.rt);
let router = Arc::clone(&cluster_node.transport_router);
// Shared distribution snapshot cell for prompt-loop refresh paths. The
// dashboard's Distribution plugin is datastream-backed via FleetView; this
// cache remains for existing loop bookkeeping and is skipped when off.
let dist_cached: Option<SharedSnapshot> = if want_dist {
Some(Arc::new(Mutex::new(Some(cluster_node.snapshot()))))
} else {
None
};
// When the in-process dashboard is on, register only datastream-backed
// plugins. FleetView updates fill both Fleet and Distribution caches.
if let Some((handle, port)) = &dashboard {
// Fleet tab: the orchestrator-hosted datastream consumer. Each rented
// stage resolves the `datastream-sink` actor and ships its telemetry
// over the swactor transport; the actor folds them into a live FleetView
// and serves the cross-node table. The returned address also receives
// each booting node's SSH output (boot phase) and the orchestrator's own
// frames, in-process.
fleet_sink_addr = wire_fleet_sink(&cluster_node, Arc::clone(handle));
handle.start_http(cluster_node.driver.tokio_handle());
eprintln!(
"pp-orchestrator: live dashboard on http://localhost:{port} \
(overview / actors / topology / distribution / fleet)"
);
}
// ── Acquire the running cluster ──────────────────────────────────
// Lease N fresh instances and (on --hold) persist the handle.
@ -1611,11 +1424,7 @@ fn run_vastai(args: &Args) -> i32 {
num_stages,
);
// Run the lease in 200ms slices instead of one blocking call, so the
// main thread can pump SWIM and refresh the dashboard snapshot while
// instances come up. Otherwise the membership/topology/net-map panels
// freeze at the empty startup snapshot for the entire (multi-minute,
// CDI-retrying) lease — even though stages are already joining SWIM —
// which defeats the point of binding the dashboard before the lease.
// main thread can pump SWIM while instances come up.
let lease_result = {
let mut lease_fut = Box::pin(pipeline_parallel_inference::vastai::lease_chain(
&http,
@ -1646,9 +1455,6 @@ fn run_vastai(args: &Args) -> i32 {
Ok(res) => break res,
Err(_elapsed) => {
cluster_node.pump_once();
if let Some(cached) = dist_cached.as_ref() {
*cached.lock().unwrap() = Some(cluster_node.snapshot());
}
}
}
}
@ -1693,62 +1499,7 @@ fn run_vastai(args: &Args) -> i32 {
};
eprintln!("pp-orchestrator: cluster contracts {contract_ids:?}");
// ── Boot-phase telemetry (best-effort, opt-in) ───────────────────────
// With a deploy SSH key configured (PP_DEPLOY_KEY) and the fleet sink live,
// SSH into each rented node and stream pp-worker's boot log onto the
// orchestrator's own datastream (proc.boot.<stage>.*) until the node's
// swactor telemetry takes over the cluster transport. No deploy key →
// skipped; the container entrypoint still launches the worker, so the run is
// unchanged. This is the "ssh signal until the node runs swactor" half.
use pipeline_parallel_inference::{provision, vastai};
if let (Some(sink_addr), Some(key_file)) = (fleet_sink_addr, provision::deploy_key_path()) {
match tokio_rt.block_on(vastai::list_instances_by_label(
&http, base_url, &api_key, &label,
)) {
Ok(list) => {
provision::install_boot_telemetry(&rt, &my_hex, 0, sink_addr);
match rt.spawn(provision::ProvisionActor::new(
rt.create_sender(),
cluster_node.driver.tokio_handle(),
)) {
Ok(prov_addr) => {
for (stage, &cid) in contract_ids.iter().enumerate() {
let Some(inst) = list.iter().find(|i| i.contract_id == cid) else {
continue;
};
let host = if !inst.ssh_host.is_empty() {
inst.ssh_host.clone()
} else {
inst.public_ipaddr.clone()
};
if host.is_empty() || inst.ssh_port == 0 {
eprintln!(
"pp-orchestrator: stage {stage} has no SSH endpoint yet; boot tail skipped"
);
continue;
}
let _ = rt.send_to(
prov_addr,
provision::ProvisionMsg::TailStage {
stage: stage as u32,
ssh: provision::SshTarget {
host,
port: inst.ssh_port,
username: "root".to_string(),
key_file: key_file.clone(),
},
},
);
}
}
Err(e) => eprintln!("pp-orchestrator: could not spawn ProvisionActor: {e}"),
}
}
Err(e) => eprintln!(
"pp-orchestrator: boot telemetry skipped (SSH endpoint discovery failed: {e})"
),
}
}
let datastream_sink_addr = wire_datastream_sink(&cluster_node);
// Drive the run inside a labelled block returning `(code, reason)` so
// every failure point can name the reason it bailed; the orchestrator's
@ -1780,9 +1531,7 @@ fn run_vastai(args: &Args) -> i32 {
"pp-orchestrator: waiting for SWIM convergence ({} alive peers, {}s budget)...",
num_stages, orch_converge_secs,
);
// Drain inbound ACTOR_ALPN throughout convergence + pipeline wiring so
// each stage's fleet `DatastreamFrame`s reach the `datastream-sink` as it
// joins (pump_once only services SWIM/protocol gossip, not app messages).
// Drain inbound ACTOR_ALPN throughout convergence + pipeline wiring.
let fleet_pump = ActorMessagePump::new();
let conv_res = await_convergence(
num_stages as usize,
@ -1792,11 +1541,6 @@ fn run_vastai(args: &Args) -> i32 {
cluster_node.pump_once();
fleet_pump.pump(&cluster_node.driver, &cluster_node.codecs, &cluster_node.rt);
let snap = cluster_node.snapshot();
// Keep the dashboard membership/topology panels live as peers
// join during convergence.
if let Some(cached) = dist_cached.as_ref() {
*cached.lock().unwrap() = Some(snap.clone());
}
snap.members.iter().filter(|m| m.state == "alive").count()
},
);
@ -1806,10 +1550,8 @@ fn run_vastai(args: &Args) -> i32 {
}
cluster_node.register_name(ORCHESTRATOR_NAME, inbox_addr);
register_datastream_sink_name(&cluster_node, datastream_sink_addr);
eprintln!("pp-orchestrator: registered {ORCHESTRATOR_NAME} -> {inbox_addr:?}");
// Publish the datastream-sink now, for the same budget reason: rented
// stages resolve this name to ship their fleet telemetry to the Fleet tab.
register_fleet_sink_name(&cluster_node, fleet_sink_addr);
// Spec §4.5 + §4.6: gate the drive on (a) every pp-stage-K
// resolvable and (b) pp-entry resolvable. Both are proxies for
@ -1828,10 +1570,7 @@ fn run_vastai(args: &Args) -> i32 {
let (stage0_addr, stage0_node_id) = loop {
cluster_node.pump_once();
fleet_pump.pump(&cluster_node.driver, &cluster_node.codecs, &cluster_node.rt);
// Keep the dashboard panels live while the pipeline wires up.
if let Some(cached) = dist_cached.as_ref() {
*cached.lock().unwrap() = Some(cluster_node.snapshot());
}
// Keep app messages flowing while the pipeline wires up.
for k in 0..num_stages {
if roster_hex[k as usize].is_some() {
continue;
@ -1920,13 +1659,9 @@ fn run_vastai(args: &Args) -> i32 {
Err(e) => (1, e.exit_reason()),
};
// Live multi-prompt loop, gated on the dashboard or --hold. A plain
// one-shot (neither set) keeps today's single-drive-then-exit behaviour.
// Modeled on hold_open: a stdin-reader side thread feeds prompt lines
// while the main thread pumps the driver (~200 ms) and refreshes the
// distribution snapshot between prompts. Dashboard panels are fed by
// datastream FleetView updates; EOF / blank line / `quit` leaves the loop.
if std::env::var_os("PP_DASHBOARD").is_some() || args.hold {
// Live multi-prompt loop for held clusters. A plain one-shot keeps
// today's single-drive-then-exit behaviour.
if args.hold {
prompt_loop(
&mut cluster_node,
&response_inbox,
@ -1936,8 +1671,6 @@ fn run_vastai(args: &Args) -> i32 {
&label,
args.max_tokens,
drive_seq,
dist_cached.as_ref(),
dashboard.as_ref().map(|(_, p)| *p),
);
}

44
crates/dashboard/AGENTS.md
View file

@ -1,39 +1,11 @@
# Datastream Dashboard — Agent Interface
# Dashboard crate contract
## Data Flow
Keep this crate read-only with respect to observed programs.
The dashboard is an HTTP/SSE consumer of datastream-derived models. Agents should inspect the browser endpoints and JSON plugin endpoints.
- It may ingest datastream frames.
- It may retain bounded raw-frame and view state for HTML/API rendering.
- It may host universal swactor runtime views.
- It must not send control signals to observed runtimes.
- It must not require changes outside `crates/dashboard` for dashboard-only work.
`datastream_source::FleetView` is the canonical fold from delivered frames to dashboard models. It produces:
- fleet JSON for the node/fleet page
- distribution JSON for the distribution page
- `RuntimeStats` for the selected node overview and actors table
- activity messages for the event stream
## HTTP Pages
- `GET /` — selected node overview
- `GET /actors` — selected node actor rows
- `GET /topology` — topology derived from the latest selected-node stats
- `GET /plugin/distribution` — distribution graph and peer/cache state
- `GET /plugin/vastai` — fleet view
- `GET /events` — server-sent events for stats, activity, history, and plugin updates
## JSON Endpoints
- `GET /api/stats` — latest selected-node `RuntimeStats`, or `{}` before the first selected-node frame
- `GET /api/topology` — topology derived from latest stats, or `{}`
- `GET /api/history` — in-memory worker history
- `GET /api/logs` — retained activity events, optionally filtered by query params
- `GET /api/plugin/vastai` — current fleet JSON cache
- `GET /api/plugin/distribution` — current distribution JSON cache
The same plugin names are used on the SSE stream for incremental browser
updates. Distribution page buttons post to `/api/plugin/distribution/rejoin`
and `/api/plugin/distribution/clear_status`; the datastream-backed plugin
acknowledges them as read-only no-ops.
## Extension Rule
Extend the dashboard through plugins backed by datastream-folded caches. A producer may add telemetry records, a fold may update shared JSON, and a plugin may serve that JSON/page over HTTP and SSE.
Main built-in view: `/view/swactor/workers`, backed by `runtime.stats`, `runtime.workers`, and `runtime.actors` frames when present.

849
crates/dashboard/Cargo.lock generated Normal file
View file

@ -0,0 +1,849 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "atomic-waker"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
[[package]]
name = "axum"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90"
dependencies = [
"axum-core",
"bytes",
"form_urlencoded",
"futures-util",
"http",
"http-body",
"http-body-util",
"hyper",
"hyper-util",
"itoa",
"matchit",
"memchr",
"mime",
"percent-encoding",
"pin-project-lite",
"serde_core",
"serde_json",
"serde_path_to_error",
"serde_urlencoded",
"sync_wrapper",
"tokio",
"tower",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "axum-core"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1"
dependencies = [
"bytes",
"futures-core",
"http",
"http-body",
"http-body-util",
"mime",
"pin-project-lite",
"sync_wrapper",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "base64ct"
version = "1.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06"
[[package]]
name = "bitflags"
version = "2.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8"
[[package]]
name = "block-buffer"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
dependencies = [
"generic-array",
]
[[package]]
name = "bytes"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593"
[[package]]
name = "cfg-if"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
[[package]]
name = "const-oid"
version = "0.9.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8"
[[package]]
name = "cpufeatures"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
dependencies = [
"libc",
]
[[package]]
name = "crossbeam-queue"
version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
[[package]]
name = "crypto-common"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
dependencies = [
"generic-array",
"typenum",
]
[[package]]
name = "curve25519-dalek"
version = "4.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be"
dependencies = [
"cfg-if",
"cpufeatures",
"curve25519-dalek-derive",
"digest",
"fiat-crypto",
"rustc_version",
"subtle",
"zeroize",
]
[[package]]
name = "curve25519-dalek-derive"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "dashboard"
version = "0.1.0"
dependencies = [
"axum",
"datastream",
"parking_lot",
"serde",
"serde_json",
"swactor",
"tokio",
"tokio-stream",
]
[[package]]
name = "datastream"
version = "0.1.0"
dependencies = [
"libc",
"serde",
"serde_json",
"swactor",
"swactor-transport",
]
[[package]]
name = "der"
version = "0.7.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb"
dependencies = [
"const-oid",
"zeroize",
]
[[package]]
name = "digest"
version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer",
"crypto-common",
]
[[package]]
name = "ed25519"
version = "2.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53"
dependencies = [
"pkcs8",
"serde",
"signature",
]
[[package]]
name = "ed25519-dalek"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9"
dependencies = [
"curve25519-dalek",
"ed25519",
"rand_core",
"serde",
"sha2",
"subtle",
"zeroize",
]
[[package]]
name = "fiat-crypto"
version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d"
[[package]]
name = "form_urlencoded"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf"
dependencies = [
"percent-encoding",
]
[[package]]
name = "futures-channel"
version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d"
dependencies = [
"futures-core",
]
[[package]]
name = "futures-core"
version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
[[package]]
name = "futures-task"
version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393"
[[package]]
name = "futures-util"
version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
dependencies = [
"futures-core",
"futures-task",
"pin-project-lite",
"slab",
]
[[package]]
name = "generic-array"
version = "0.14.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
dependencies = [
"typenum",
"version_check",
]
[[package]]
name = "getrandom"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
dependencies = [
"cfg-if",
"libc",
"wasi",
]
[[package]]
name = "http"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425"
dependencies = [
"bytes",
"itoa",
]
[[package]]
name = "http-body"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
dependencies = [
"bytes",
"http",
]
[[package]]
name = "http-body-util"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a"
dependencies = [
"bytes",
"futures-core",
"http",
"http-body",
"pin-project-lite",
]
[[package]]
name = "httparse"
version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
[[package]]
name = "httpdate"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
[[package]]
name = "hyper"
version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498"
dependencies = [
"atomic-waker",
"bytes",
"futures-channel",
"futures-core",
"http",
"http-body",
"httparse",
"httpdate",
"itoa",
"pin-project-lite",
"smallvec",
"tokio",
]
[[package]]
name = "hyper-util"
version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
dependencies = [
"bytes",
"http",
"http-body",
"hyper",
"pin-project-lite",
"tokio",
"tower-service",
]
[[package]]
name = "itoa"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
[[package]]
name = "libc"
version = "0.2.186"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
[[package]]
name = "lock_api"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
dependencies = [
"scopeguard",
]
[[package]]
name = "log"
version = "0.4.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
[[package]]
name = "matchit"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
[[package]]
name = "memchr"
version = "2.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
[[package]]
name = "mime"
version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "mio"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda"
dependencies = [
"libc",
"wasi",
"windows-sys",
]
[[package]]
name = "once_cell"
version = "1.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
[[package]]
name = "parking_lot"
version = "0.12.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
dependencies = [
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.9.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"smallvec",
"windows-link",
]
[[package]]
name = "percent-encoding"
version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
[[package]]
name = "pin-project-lite"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
[[package]]
name = "pkcs8"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7"
dependencies = [
"der",
"spki",
]
[[package]]
name = "proc-macro2"
version = "1.0.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368"
dependencies = [
"proc-macro2",
]
[[package]]
name = "rand_core"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
"getrandom",
]
[[package]]
name = "redox_syscall"
version = "0.5.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
dependencies = [
"bitflags",
]
[[package]]
name = "rustc_version"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
dependencies = [
"semver",
]
[[package]]
name = "ryu"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
[[package]]
name = "scopeguard"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "semver"
version = "1.0.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"
[[package]]
name = "serde"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
dependencies = [
"serde_core",
"serde_derive",
]
[[package]]
name = "serde_core"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "serde_json"
version = "1.0.150"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
dependencies = [
"itoa",
"memchr",
"serde",
"serde_core",
"zmij",
]
[[package]]
name = "serde_path_to_error"
version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457"
dependencies = [
"itoa",
"serde",
"serde_core",
]
[[package]]
name = "serde_urlencoded"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
dependencies = [
"form_urlencoded",
"itoa",
"ryu",
"serde",
]
[[package]]
name = "sha2"
version = "0.10.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
dependencies = [
"cfg-if",
"cpufeatures",
"digest",
]
[[package]]
name = "signature"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de"
dependencies = [
"rand_core",
]
[[package]]
name = "slab"
version = "0.4.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
[[package]]
name = "smallvec"
version = "1.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
[[package]]
name = "socket2"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51"
dependencies = [
"libc",
"windows-sys",
]
[[package]]
name = "spki"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d"
dependencies = [
"base64ct",
"der",
]
[[package]]
name = "subtle"
version = "2.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "swactor"
version = "0.1.0"
dependencies = [
"crossbeam-queue",
"crossbeam-utils",
"getrandom",
"parking_lot",
"serde",
]
[[package]]
name = "swactor-transport"
version = "0.1.0"
dependencies = [
"ed25519-dalek",
"rand_core",
"serde",
"serde_json",
"swactor",
]
[[package]]
name = "syn"
version = "2.0.118"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "sync_wrapper"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
[[package]]
name = "tokio"
version = "1.52.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"
dependencies = [
"libc",
"mio",
"pin-project-lite",
"socket2",
"tokio-macros",
"windows-sys",
]
[[package]]
name = "tokio-macros"
version = "2.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "tokio-stream"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70"
dependencies = [
"futures-core",
"pin-project-lite",
"tokio",
]
[[package]]
name = "tower"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4"
dependencies = [
"futures-core",
"futures-util",
"pin-project-lite",
"sync_wrapper",
"tokio",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "tower-layer"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
[[package]]
name = "tower-service"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
[[package]]
name = "tracing"
version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"
dependencies = [
"log",
"pin-project-lite",
"tracing-core",
]
[[package]]
name = "tracing-core"
version = "0.1.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a"
dependencies = [
"once_cell",
]
[[package]]
name = "typenum"
version = "1.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
[[package]]
name = "unicode-ident"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
[[package]]
name = "version_check"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "wasi"
version = "0.11.1+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
[[package]]
name = "windows-link"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
[[package]]
name = "windows-sys"
version = "0.61.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
dependencies = [
"windows-link",
]
[[package]]
name = "zeroize"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e"
[[package]]
name = "zmij"
version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"

15
crates/dashboard/Cargo.toml
View file

@ -1,19 +1,16 @@
[workspace]
[package]
name = "dashboard"
version = "0.1.0"
edition = "2024"
[dependencies]
axum = "0.8"
datastream = { path = "../datastream" }
swactor = { path = "../..", features = ["serde"] }
parking_lot = "0.12"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
axum = "0.8"
tokio = { version = "1", features = ["net", "rt-multi-thread", "sync", "time"] }
tokio = { version = "1", features = ["net", "rt-multi-thread", "sync"] }
tokio-stream = "0.1"
crossbeam-queue = "0.3.12"
datastream = { path = "../datastream" }
distribution = { path = "../distribution" }
[target.'cfg(target_os = "linux")'.dependencies]
libc = "0.2"

50
crates/dashboard/README.md
View file

@ -1,46 +1,16 @@
# dashboard
Datastream-only HTTP dashboard for visualizing swactor-derived telemetry in a browser. The dashboard consumes folded datastream records and serves live pages over HTTP/SSE.
Read-only HTML/SSE dashboard over incoming datastream frames.
## Features
The crate owns the Axum server, bounded raw frame window, and view registry. Component crates can keep their own view implementations beside their code and register them through `DashboardHandle::register_view`. The built-in swactor worker page is hosted here because worker/actor/message processing is universal to swactor programs.
| Feature | Default | Description |
|---------|---------|-------------|
| `distribution` | yes | `/distribution` page with SWIM membership, gossip directory routes, peer auth, and location cache data derived from datastream frames |
| Fleet view | yes | `/vastai` page showing nodes folded by `datastream_source::FleetView` |
## Routes
## HTTP Dashboard
- `GET /` — dashboard index
- `GET /events` — raw incoming frames as SSE
- `GET /api/frames` — recent raw frame window
- `GET /api/views` — registered view metadata
- `GET /view/swactor/workers` — built-in worker page
- `GET /api/view/swactor/workers` — worker page JSON snapshot
The dashboard is embedded by an application that owns a datastream sink. The
sink folds delivered frames through `datastream_source::FleetView`, then pushes
the resulting stats, activity lines, and cache-backed plugin JSON into the
dashboard handle.
Pages:
- `http://localhost:9090/` — live overview from the selected datastream node
- `http://localhost:9090/actors` — actor table reconstructed from actor telemetry records
- `http://localhost:9090/plugin/distribution` — SWIM membership, gossip directory routes, peer auth, and cache entries
- `http://localhost:9090/plugin/vastai` — fleet/node view fed by the shared fleet cache
The dashboard model is folded by `datastream_source::FleetView`. Producers
publish telemetry records to datastream channels; the dashboard sink folds those
records into cached JSON, pushes activity messages, and updates the HTTP/SSE
views.
## Public API
Embed the dashboard by constructing `DashboardConfig` and calling
`start_dashboard(config)`. The returned handle owns the HTTP server state and
supports externally pushed stats, activity messages, history access, plugin
registration, landing page overrides, extra routers, and shutdown.
Plugins are the extension boundary. New dashboard surfaces should register a
`DashboardPlugin` or use a cache-backed plugin such as
`fleet_cache_plugin(cache)` / `distribution_cache_plugin(cache)`, then feed it
from datastream-derived JSON caches.
## Pipeline app
`apps/pipeline-parallel-inference` enables the dashboard with `PP_DASHBOARD=1`.
Its orchestrator hosts the HTTP server, spawns the `datastream-sink` actor, and
feeds every dashboard view from `FleetView` updates.
All state is derived from observed frames. The dashboard sends no control signals back to producers.

400
crates/dashboard/src/bin/swactor_dummy_node.rs Normal file
View file

@ -0,0 +1,400 @@
use std::collections::HashMap;
use std::sync::Arc;
use std::thread;
use std::time::Duration;
use dashboard::swactor::{RUNTIME_ACTORS, RUNTIME_STATS, RUNTIME_WORKERS};
use dashboard::{DashboardConfig, DashboardHandle, start_dashboard};
use datastream::frame::{ChannelId, Frame, Lifetime, NodeId, Position, StreamId};
use parking_lot::Mutex;
use serde::Serialize;
use serde_json::{Value, json};
use swactor::actor::{ActorAddress, ActorInterface, Ctx};
use swactor::config::RuntimeConfig;
use swactor::runtime::Runtime;
use swactor::stats::{ActorSnapshot, StatsHook};
const NODE_ID: &str = "dashboard-swactor-dummy";
const WORKER_ACTORS: usize = 12;
const PUBLISH_INTERVAL: Duration = Duration::from_millis(250);
const PULSE_INTERVAL: Duration = Duration::from_millis(25);
const WORK_ITEM_DELAY: Duration = Duration::from_micros(200);
#[derive(Clone)]
struct PulseTick {
seq: u64,
}
#[derive(Clone)]
struct WorkItem {
seq: u64,
route: u32,
hops_left: u8,
}
#[derive(Clone)]
enum RouterMsg {
Configure { workers: Vec<ActorAddress> },
Beat { seq: u64 },
Complete { worker: u32, seq: u64, route: u32 },
}
struct PulseActor {
router: ActorAddress,
}
impl ActorInterface for PulseActor {
type Incoming = PulseTick;
type Response = ();
fn handle(&mut self, ctx: &Ctx, msg: PulseTick) {
let _ = ctx.send(self.router, RouterMsg::Beat { seq: msg.seq });
}
}
struct RouterActor {
workers: Vec<ActorAddress>,
next: usize,
completed: u64,
}
impl RouterActor {
fn new() -> Self {
Self {
workers: Vec::new(),
next: 0,
completed: 0,
}
}
}
impl ActorInterface for RouterActor {
type Incoming = RouterMsg;
type Response = ();
fn handle(&mut self, ctx: &Ctx, msg: RouterMsg) {
match msg {
RouterMsg::Configure { workers } => {
self.workers = workers;
self.next = 0;
}
RouterMsg::Beat { seq } => {
if self.workers.is_empty() {
return;
}
let burst = 48 + (seq as usize % 32);
for route in 0..burst {
let target = self.workers[self.next % self.workers.len()];
self.next = self.next.wrapping_add(1);
let _ = ctx.send(
target,
WorkItem {
seq,
route: route as u32,
hops_left: 1 + ((seq + route as u64) % 3) as u8,
},
);
}
}
RouterMsg::Complete { worker, seq, route } => {
self.completed = self.completed.wrapping_add(1);
if self.completed % 7 == 0 && !self.workers.is_empty() {
let target =
self.workers[(worker as usize + route as usize) % self.workers.len()];
let _ = ctx.send(
target,
WorkItem {
seq,
route: route.wrapping_add(1000),
hops_left: 1,
},
);
}
}
}
}
}
struct WorkerActor {
id: u32,
router: ActorAddress,
}
impl ActorInterface for WorkerActor {
type Incoming = WorkItem;
type Response = ();
fn handle(&mut self, ctx: &Ctx, msg: WorkItem) {
if msg.hops_left > 0 {
let _ = ctx.send(
ctx.self_addr(),
WorkItem {
seq: msg.seq,
route: msg.route,
hops_left: msg.hops_left - 1,
},
);
return;
}
thread::sleep(WORK_ITEM_DELAY);
let _ = ctx.send(
self.router,
RouterMsg::Complete {
worker: self.id,
seq: msg.seq,
route: msg.route,
},
);
}
}
struct QueuedSinkActor;
impl ActorInterface for QueuedSinkActor {
type Incoming = WorkItem;
type Response = ();
fn handle(&mut self, ctx: &Ctx, _msg: WorkItem) {
ctx.suspend_self();
}
}
#[derive(Default)]
struct DashboardStatsHook {
actors: Mutex<HashMap<ActorAddress, ActorDetail>>,
}
#[derive(Clone)]
struct ActorDetail {
worker_id: usize,
mailbox_depth: usize,
last_msg_type: Option<String>,
messages_processed: u64,
poisoned: bool,
message_type_counts: Vec<(String, u64)>,
}
impl StatsHook for DashboardStatsHook {
fn on_tick(&self, worker_id: usize, snapshots: &[ActorSnapshot]) {
let mut actors = self.actors.lock();
for snapshot in snapshots {
actors.insert(
snapshot.address,
ActorDetail {
worker_id,
mailbox_depth: snapshot.mailbox_depth,
last_msg_type: snapshot.last_msg_type.map(str::to_owned),
messages_processed: snapshot.messages_processed,
poisoned: snapshot.poisoned,
message_type_counts: snapshot
.message_type_counts
.iter()
.map(|(name, count)| ((*name).to_owned(), *count))
.collect(),
},
);
}
}
}
impl DashboardStatsHook {
fn snapshot(
&self,
live_workers: &[(ActorAddress, usize)],
names: &HashMap<ActorAddress, String>,
) -> Vec<ActorDetailFrame> {
let actors = self.actors.lock();
live_workers
.iter()
.map(|(address, worker_id)| {
let detail = actors.get(address);
ActorDetailFrame {
address: address.to_string(),
name: names.get(address).cloned(),
worker_id: detail.map_or(*worker_id, |detail| detail.worker_id),
mailbox_depth: detail.map_or(0, |detail| detail.mailbox_depth),
last_msg_type: detail.and_then(|detail| detail.last_msg_type.clone()),
messages_processed: detail.map_or(0, |detail| detail.messages_processed),
poisoned: detail.is_some_and(|detail| detail.poisoned),
message_type_counts: detail
.map(|detail| detail.message_type_counts.clone())
.unwrap_or_default(),
}
})
.collect()
}
}
#[derive(Serialize)]
struct ActorDetailFrame {
address: String,
name: Option<String>,
worker_id: usize,
mailbox_depth: usize,
last_msg_type: Option<String>,
messages_processed: u64,
poisoned: bool,
message_type_counts: Vec<(String, u64)>,
}
fn main() {
let dashboard = start_dashboard(DashboardConfig::default());
dashboard.start_http_standalone();
let mut runtime = Runtime::new(RuntimeConfig {
num_threads: 4,
max_actors: 128,
channel_buffer_size: 4096,
actor_message_budget: 8,
});
let stats_hook = Arc::new(DashboardStatsHook::default());
runtime.set_stats_hook(stats_hook.clone());
let router = runtime.spawn(RouterActor::new()).expect("spawn router");
let mut names = HashMap::new();
names.insert(router, "router".to_owned());
let mut workers = Vec::with_capacity(WORKER_ACTORS);
for id in 0..WORKER_ACTORS {
let address = runtime
.spawn(WorkerActor {
id: id as u32,
router,
})
.expect("spawn worker actor");
names.insert(address, format!("worker-{id}"));
workers.push(address);
}
let pulse = runtime.spawn(PulseActor { router }).expect("spawn pulse");
names.insert(pulse, "pulse".to_owned());
let queued_sink = runtime.spawn(QueuedSinkActor).expect("spawn queued sink");
names.insert(queued_sink, "queued-sink".to_owned());
runtime
.send_to(
router,
RouterMsg::Configure {
workers: workers.clone(),
},
)
.expect("configure router");
let runtime = runtime.run().expect("start swactor runtime");
let stream = StreamId::new(NodeId::new(NODE_ID), Lifetime(1));
let mut position = 0_u64;
let mut seq = 0_u64;
let mut ticks_until_publish = 0_u8;
println!(
"dashboard listening at http://127.0.0.1:{}/view/swactor/workers",
DashboardConfig::default().port
);
println!(
"dummy node {NODE_ID} running {} swactor actors",
names.len()
);
loop {
let _ = runtime.runtime.send_to(pulse, PulseTick { seq });
if seq % 2 == 0 {
let _ = runtime.runtime.send_to(
queued_sink,
WorkItem {
seq,
route: u32::MAX,
hops_left: 0,
},
);
}
seq = seq.wrapping_add(1);
if ticks_until_publish == 0 {
publish_runtime_snapshot(
&dashboard,
&stream,
&mut position,
&runtime.runtime,
&stats_hook,
&names,
);
ticks_until_publish = (PUBLISH_INTERVAL.as_millis() / PULSE_INTERVAL.as_millis()) as u8;
}
ticks_until_publish = ticks_until_publish.saturating_sub(1);
thread::sleep(PULSE_INTERVAL);
}
}
fn publish_runtime_snapshot(
dashboard: &DashboardHandle,
stream: &StreamId,
position: &mut u64,
runtime: &Runtime,
stats_hook: &DashboardStatsHook,
names: &HashMap<ActorAddress, String>,
) {
let stats = runtime.stats();
let actor_details = stats_hook.snapshot(&stats.actors, names);
let actors: Vec<Value> = stats
.actors
.iter()
.map(|(address, worker_id)| json!([address.to_string(), worker_id]))
.collect();
let workers = serde_json::to_value(&stats.workers).expect("serialize worker stats");
let actor_details = serde_json::to_value(actor_details).expect("serialize actor stats");
let tick_timings = serde_json::to_value(&stats.tick_timings).expect("serialize tick timings");
let total_mailbox_depth: usize = stats
.workers
.iter()
.map(|worker| worker.mailbox_depth)
.sum();
publish_json(
dashboard,
stream,
position,
RUNTIME_STATS,
json!({
"num_workers": stats.num_workers,
"uptime_ms": stats.uptime_ms,
"actors_live": stats.actors.len(),
"mailbox_depth": total_mailbox_depth,
"actors": actors,
"workers": workers,
"actor_details": actor_details,
"tick_timings": tick_timings,
}),
);
publish_json(
dashboard,
stream,
position,
RUNTIME_WORKERS,
json!({ "workers": stats.workers }),
);
publish_json(
dashboard,
stream,
position,
RUNTIME_ACTORS,
json!({ "actors": actor_details }),
);
}
fn publish_json(
dashboard: &DashboardHandle,
stream: &StreamId,
position: &mut u64,
channel: &str,
value: Value,
) {
let payload = serde_json::to_vec(&value).expect("serialize dashboard frame");
let frame = Frame::new(ChannelId::new(channel), Position(*position), payload);
dashboard.ingest(stream, &frame);
*position = position.wrapping_add(1);
}

1008
crates/dashboard/src/datastream_source.rs
View file

File diff suppressed because it is too large Load diff

1285
crates/dashboard/src/distribution_page.html
View file

File diff suppressed because it is too large Load diff

354
crates/dashboard/src/history.rs
View file

@ -1,354 +0,0 @@
//! In-process time-series history for dashboard sparklines and trend detection.
//!
//! Stores bounded ring buffers of per-worker and per-actor stats, sampled at
//! a configurable interval. All data is kept in memory with automatic eviction
//! of the oldest samples when capacity is reached.
use std::collections::{HashMap, VecDeque};
use std::sync::RwLock;
use swactor::actor::ActorAddress;
use swactor::stats::{ActorInfo, RuntimeStats};
/// Configuration for history collection.
#[derive(Debug, Clone)]
pub struct HistoryConfig {
/// Maximum samples per worker (default: 300 = 5 min at 1/sec).
pub max_worker_samples: usize,
/// Maximum samples per actor (default: 300).
pub max_actor_samples: usize,
/// Maximum number of actors tracked (LRU eviction). Default: 1000.
pub max_tracked_actors: usize,
}
impl Default for HistoryConfig {
fn default() -> Self {
Self {
max_worker_samples: 300,
max_actor_samples: 300,
max_tracked_actors: 1000,
}
}
}
/// Time-series data for a single worker.
#[derive(Debug, Clone)]
pub struct WorkerHistory {
pub message_rates: VecDeque<f64>,
pub mailbox_depths: VecDeque<u64>,
pub actor_counts: VecDeque<u32>,
prev_messages: u64,
}
impl WorkerHistory {
fn new() -> Self {
Self {
message_rates: VecDeque::new(),
mailbox_depths: VecDeque::new(),
actor_counts: VecDeque::new(),
prev_messages: 0,
}
}
fn push(
&mut self,
messages_processed: u64,
mailbox_depth: usize,
num_actors: usize,
cap: usize,
) {
let rate = messages_processed.saturating_sub(self.prev_messages) as f64;
self.prev_messages = messages_processed;
push_bounded(&mut self.message_rates, rate, cap);
push_bounded(&mut self.mailbox_depths, mailbox_depth as u64, cap);
push_bounded(&mut self.actor_counts, num_actors as u32, cap);
}
}
/// Time-series data for a single actor.
#[derive(Debug, Clone)]
pub struct ActorHistory {
pub mailbox_depths: VecDeque<u64>,
pub message_rates: VecDeque<f64>,
prev_messages: u64,
last_seen_sample: u64,
}
impl ActorHistory {
fn new(sample_counter: u64) -> Self {
Self {
mailbox_depths: VecDeque::new(),
message_rates: VecDeque::new(),
prev_messages: 0,
last_seen_sample: sample_counter,
}
}
fn push(&mut self, info: &ActorInfo, cap: usize, sample_counter: u64) {
let rate = info.messages_processed.saturating_sub(self.prev_messages) as f64;
self.prev_messages = info.messages_processed;
self.last_seen_sample = sample_counter;
push_bounded(&mut self.mailbox_depths, info.mailbox_depth as u64, cap);
push_bounded(&mut self.message_rates, rate, cap);
}
}
fn push_bounded<T>(buf: &mut VecDeque<T>, val: T, cap: usize) {
if buf.len() >= cap {
buf.pop_front();
}
buf.push_back(val);
}
/// Thread-safe history store. Written from datastream-pushed stats, read by SSE.
pub struct DashboardHistory {
inner: RwLock<HistoryInner>,
config: HistoryConfig,
}
struct HistoryInner {
workers: Vec<WorkerHistory>,
actors: HashMap<ActorAddress, ActorHistory>,
sample_counter: u64,
}
impl DashboardHistory {
pub fn new(config: HistoryConfig) -> Self {
Self {
inner: RwLock::new(HistoryInner {
workers: Vec::new(),
actors: HashMap::new(),
sample_counter: 0,
}),
config,
}
}
/// Record a stats snapshot. Called by the sampler thread.
pub fn record(&self, stats: &RuntimeStats) {
let mut inner = self.inner.write().unwrap();
inner.sample_counter += 1;
let counter = inner.sample_counter;
// Resize workers vec if needed
while inner.workers.len() < stats.workers.len() {
inner.workers.push(WorkerHistory::new());
}
// Record per-worker data
for w in &stats.workers {
if let Some(wh) = inner.workers.get_mut(w.id) {
wh.push(
w.messages_processed,
w.mailbox_depth,
w.num_actors,
self.config.max_worker_samples,
);
}
}
// Record per-actor data
for a in &stats.actor_details {
let ah = inner
.actors
.entry(a.address)
.or_insert_with(|| ActorHistory::new(counter));
ah.push(a, self.config.max_actor_samples, counter);
}
// LRU eviction: remove actors not seen recently if over capacity
if inner.actors.len() > self.config.max_tracked_actors {
let mut entries: Vec<(ActorAddress, u64)> = inner
.actors
.iter()
.map(|(addr, ah)| (*addr, ah.last_seen_sample))
.collect();
entries.sort_by_key(|&(_, seen)| seen);
let to_remove = inner.actors.len() - self.config.max_tracked_actors;
for (addr, _) in entries.into_iter().take(to_remove) {
inner.actors.remove(&addr);
}
}
}
/// Get a snapshot of worker history for rendering sparklines.
/// Returns Vec indexed by worker_id, each containing recent message rates.
pub fn worker_sparklines(&self) -> Vec<Vec<u64>> {
let inner = self.inner.read().unwrap();
inner
.workers
.iter()
.map(|wh| wh.message_rates.iter().map(|r| *r as u64).collect())
.collect()
}
/// Get worker mailbox depth history.
pub fn worker_mailbox_sparklines(&self) -> Vec<Vec<u64>> {
let inner = self.inner.read().unwrap();
inner
.workers
.iter()
.map(|wh| wh.mailbox_depths.iter().copied().collect())
.collect()
}
/// Get sparkline data for a specific actor.
pub fn actor_sparkline(&self, addr: &ActorAddress) -> Option<(Vec<u64>, Vec<u64>)> {
let inner = self.inner.read().unwrap();
inner.actors.get(addr).map(|ah| {
let mailbox: Vec<u64> = ah.mailbox_depths.iter().copied().collect();
let rates: Vec<u64> = ah.message_rates.iter().map(|r| *r as u64).collect();
(mailbox, rates)
})
}
/// Get total sample count (useful for knowing if history is available).
pub fn sample_count(&self) -> u64 {
self.inner.read().unwrap().sample_counter
}
/// Serialize worker history as JSON for the SSE initial payload.
pub fn worker_history_json(&self) -> String {
let sparklines = self.worker_sparklines();
let mailbox = self.worker_mailbox_sparklines();
serde_json::json!({
"workers": sparklines.iter().enumerate().map(|(i, rates)| {
serde_json::json!({
"id": i,
"message_rates": rates,
"mailbox_depths": mailbox.get(i).unwrap_or(&Vec::new()),
})
}).collect::<Vec<_>>(),
})
.to_string()
}
}
#[cfg(test)]
mod tests {
use super::*;
use swactor::stats::{ActorInfo, WorkerInfo};
fn make_stats(workers: Vec<(u64, usize, usize)>, actors: Vec<ActorInfo>) -> RuntimeStats {
RuntimeStats {
num_workers: workers.len(),
uptime_ms: 0,
actors: actors.iter().map(|a| (a.address, a.worker_id)).collect(),
workers: workers
.into_iter()
.enumerate()
.map(|(id, (msgs, depth, n_actors))| WorkerInfo {
id,
num_actors: n_actors,
mailbox_depth: depth,
messages_processed: msgs,
local_sends: 0,
cross_sends: 0,
inbox_sends: 0,
type_mismatches: 0,
panics: 0,
messages_dropped: 0,
restarts: 0,
stops: 0,
})
.collect(),
actor_details: actors,
tick_timings: Vec::new(),
}
}
fn make_actor(id: u8, worker: usize, depth: usize, msgs: u64) -> ActorInfo {
ActorInfo {
address: ActorAddress([id; 32]),
name: None,
worker_id: worker,
mailbox_depth: depth,
last_msg_type: None,
messages_processed: msgs,
poisoned: false,
message_type_counts: Vec::new(),
}
}
#[test]
fn worker_rates_accumulate_over_samples() {
let history = DashboardHistory::new(HistoryConfig::default());
// First sample: establishes baseline (rate will be the raw value since prev=0)
let stats1 = make_stats(vec![(100, 5, 2)], vec![]);
history.record(&stats1);
// Second sample: delta = 150 - 100 = 50
let stats2 = make_stats(vec![(150, 3, 2)], vec![]);
history.record(&stats2);
let sparklines = history.worker_sparklines();
assert_eq!(sparklines.len(), 1);
assert_eq!(sparklines[0].len(), 2);
assert_eq!(sparklines[0][0], 100); // first sample: 100 - 0
assert_eq!(sparklines[0][1], 50); // second sample: 150 - 100
}
#[test]
fn bounded_eviction_drops_oldest() {
let config = HistoryConfig {
max_worker_samples: 3,
..Default::default()
};
let history = DashboardHistory::new(config);
for i in 0..5u64 {
let stats = make_stats(vec![(i * 10, 0, 0)], vec![]);
history.record(&stats);
}
let sparklines = history.worker_sparklines();
assert_eq!(sparklines[0].len(), 3); // capped at 3
}
#[test]
fn actor_lru_eviction_keeps_most_recent() {
let config = HistoryConfig {
max_tracked_actors: 2,
..Default::default()
};
let history = DashboardHistory::new(config);
// Sample 1: actors A and B
let stats1 = make_stats(
vec![(0, 0, 2)],
vec![make_actor(1, 0, 0, 0), make_actor(2, 0, 0, 0)],
);
history.record(&stats1);
// Sample 2: actors B and C (A not seen)
let stats2 = make_stats(
vec![(0, 0, 2)],
vec![make_actor(2, 0, 0, 0), make_actor(3, 0, 0, 0)],
);
history.record(&stats2);
// A should be evicted (LRU), B and C kept
assert!(history.actor_sparkline(&ActorAddress([1; 32])).is_none());
assert!(history.actor_sparkline(&ActorAddress([2; 32])).is_some());
assert!(history.actor_sparkline(&ActorAddress([3; 32])).is_some());
}
#[test]
fn actor_rates_track_deltas() {
let history = DashboardHistory::new(HistoryConfig::default());
let stats1 = make_stats(vec![(0, 0, 1)], vec![make_actor(1, 0, 5, 100)]);
history.record(&stats1);
let stats2 = make_stats(vec![(0, 0, 1)], vec![make_actor(1, 0, 3, 175)]);
history.record(&stats2);
let (mailbox, rates) = history.actor_sparkline(&ActorAddress([1; 32])).unwrap();
assert_eq!(mailbox, vec![5, 3]);
assert_eq!(rates[0], 100); // first: 100 - 0
assert_eq!(rates[1], 75); // second: 175 - 100
}
}

2047
crates/dashboard/src/html.rs
View file

File diff suppressed because it is too large Load diff

123
crates/dashboard/src/layer.rs
View file

@ -1,123 +0,0 @@
use std::collections::VecDeque;
use std::sync::Mutex;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{SystemTime, UNIX_EPOCH};
use crossbeam_queue::ArrayQueue;
use serde::{Deserialize, Serialize};
/// A single dashboard activity event.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DashboardEvent {
pub seq: u64,
pub timestamp_ms: u64,
pub level: String,
pub message: String,
pub worker_id: Option<usize>,
pub actor_addr: Option<String>,
pub fields: serde_json::Map<String, serde_json::Value>,
}
/// Thread-safe ring buffer for dashboard events, with optional full-log recording.
pub struct EventStore {
events: Mutex<VecDeque<DashboardEvent>>,
capacity: usize,
next_seq: AtomicU64,
/// When recording is enabled, events are kept in a lock-free bounded ring buffer.
full_log: Option<ArrayQueue<DashboardEvent>>,
}
impl EventStore {
pub fn new(capacity: usize, record: bool, record_event_capacity: usize) -> Self {
Self {
events: Mutex::new(VecDeque::with_capacity(capacity)),
capacity,
next_seq: AtomicU64::new(0),
full_log: if record {
Some(ArrayQueue::new(record_event_capacity.max(1)))
} else {
None
},
}
}
pub fn push(&self, mut event: DashboardEvent) {
event.seq = self.next_seq.fetch_add(1, Ordering::Relaxed);
if let Some(ref log) = self.full_log {
let _ = log.force_push(event.clone());
}
let mut events = self.events.lock().unwrap();
if events.len() >= self.capacity {
events.pop_front();
}
events.push_back(event);
}
/// Read events starting from `cursor`. Returns the new events and the updated cursor.
pub fn read_from(&self, cursor: u64) -> (Vec<DashboardEvent>, u64) {
let events = self.events.lock().unwrap();
if events.is_empty() {
return (Vec::new(), cursor);
}
let first_seq = events.front().unwrap().seq;
let last_seq = events.back().unwrap().seq;
if cursor > last_seq {
return (Vec::new(), cursor);
}
let start = if cursor <= first_seq {
0
} else {
(cursor - first_seq) as usize
};
let batch: Vec<DashboardEvent> = events.iter().skip(start).cloned().collect();
let new_cursor = last_seq + 1;
(batch, new_cursor)
}
/// Read recent events for a specific actor address (hex prefix match).
/// Returns up to `limit` most recent matching events.
pub fn read_for_actor(&self, actor_hex: &str, limit: usize) -> Vec<DashboardEvent> {
let events = self.events.lock().unwrap();
let lower = actor_hex.to_lowercase();
events
.iter()
.rev()
.filter(|e| {
e.actor_addr
.as_ref()
.map(|a| {
a.to_lowercase().starts_with(&lower) || a.to_lowercase().contains(&lower)
})
.unwrap_or(false)
})
.take(limit)
.cloned()
.collect::<Vec<_>>()
.into_iter()
.rev()
.collect()
}
/// Drains the full recording log. Only available when recording is enabled.
/// This is destructive — events are consumed by callers that export logs.
pub fn all_events(&self) -> Option<Vec<DashboardEvent>> {
self.full_log.as_ref().map(|log| {
let mut out = Vec::new();
while let Some(ev) = log.pop() {
out.push(ev);
}
out
})
}
}
pub(crate) fn now_ms() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis() as u64
}

229
crates/dashboard/src/lib.rs
View file

@ -1,132 +1,131 @@
pub mod datastream_source;
pub mod history;
mod html;
mod layer;
pub mod plugin;
mod root_page;
mod server;
pub mod telemetry;
pub mod topology;
pub mod warnings;
pub use crate::layer::{DashboardEvent, EventStore};
mod store;
pub mod swactor;
pub mod view;
/// The canonical Distribution page (the SWIM connection-graph view). Owned by
/// the dashboard crate so every front-end that serves it — a live node's
/// `DistributionPlugin` and the datastream dashboard — renders the exact same
/// page and chrome, fed by the distribution-page JSON the datastream consumer
/// reconstructs (see [`crate::datastream_source`]).
pub const DISTRIBUTION_PAGE_HTML: &str = include_str!("distribution_page.html");
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use datastream::frame::{ChannelId, Frame, Lifetime, NodeId, Position, StreamId};
use parking_lot::Mutex;
use serde::Serialize;
use tokio::sync::broadcast;
use swactor::stats::RuntimeStats;
use crate::store::DashboardStore;
use crate::view::{DashboardView, ViewRegistry};
use crate::history::{DashboardHistory, HistoryConfig};
use crate::layer::now_ms;
use crate::plugin::PluginRegistry;
/// Configuration for the datastream-backed dashboard.
/// Configuration for the datastream dashboard server.
#[derive(Debug, Clone)]
pub struct DashboardConfig {
pub port: u16,
pub event_capacity: usize,
/// Enable full activity-log retention inside the event store.
pub record: bool,
/// Maximum events retained in the recording log. Only used when `record = true`.
pub record_event_capacity: usize,
/// Number of raw frame events retained by the SSE channel for slow clients.
pub frame_buffer: usize,
/// Number of recent raw frames retained for `/api/frames`.
pub raw_frame_history: usize,
}
impl Default for DashboardConfig {
fn default() -> Self {
Self {
port: 9090,
event_capacity: 10_000,
record: false,
record_event_capacity: 100_000,
frame_buffer: 1024,
raw_frame_history: 1024,
}
}
}
/// Handle to a running datastream-backed dashboard.
/// JSON shape emitted for each incoming datastream frame.
#[derive(Debug, Clone, Serialize)]
pub struct FrameEvent {
pub stream: StreamEvent,
pub channel: String,
pub position: u64,
pub payload: Vec<u8>,
}
#[derive(Debug, Clone, Serialize)]
pub struct StreamEvent {
pub node: String,
pub life: u64,
}
impl FrameEvent {
pub fn new(stream: &StreamId, frame: &Frame) -> Self {
Self {
stream: StreamEvent {
node: stream.node.as_str().to_string(),
life: stream.life.0,
},
channel: frame.channel.as_str().to_string(),
position: frame.position.0,
payload: frame.payload.clone(),
}
}
pub(crate) fn to_datastream_parts(&self) -> Option<(StreamId, Frame)> {
let stream = StreamId::new(NodeId::new(&self.stream.node), Lifetime(self.stream.life));
let frame = Frame::new(
ChannelId::new(&self.channel),
Position(self.position),
self.payload.clone(),
);
Some((stream, frame))
}
}
/// Handle to the read-only dashboard server.
///
/// The handle's data path publishes observed datastream frames to HTTP clients
/// and registered views. It does not send signals back to producers or mutate
/// runtime state.
pub struct DashboardHandle {
store: Arc<EventStore>,
/// Externally pushed stats from the datastream fold (latest wins).
pushed_stats: Arc<Mutex<Option<RuntimeStats>>>,
shutdown: Arc<AtomicBool>,
shutdown_notify: Arc<tokio::sync::Notify>,
history: Arc<DashboardHistory>,
port: u16,
plugin_registry: Arc<PluginRegistry>,
frames: broadcast::Sender<FrameEvent>,
store: Arc<DashboardStore>,
views: Arc<ViewRegistry>,
shutdown_notify: Arc<tokio::sync::Notify>,
standalone_rt: Mutex<Option<tokio::runtime::Runtime>>,
/// Optional override for the `/` landing page (e.g. a host serving a fleet
/// board instead of the single-node actor dashboard).
landing_html: Mutex<Option<Arc<str>>>,
/// Optional extra axum router merged into the live server, for hosts that
/// add disjoint routes of their own.
extra_router: Mutex<Option<axum::Router>>,
}
impl DashboardHandle {
/// Push a stats snapshot from the datastream fold (latest wins).
pub fn set_stats(&self, stats: RuntimeStats) {
*self.pushed_stats.lock().unwrap() = Some(stats);
/// Register a read-only view. External crates can keep their interpretation
/// code beside their component and plug it into this registry.
pub fn register_view(&self, view: Arc<dyn DashboardView>) {
self.views.register(view);
}
/// Push a dashboard activity line into the SSE event stream.
pub fn push_activity(&self, is_warn: bool, message: impl Into<String>) {
self.store.push(DashboardEvent {
seq: 0,
timestamp_ms: now_ms(),
level: if is_warn { "WARN" } else { "INFO" }.to_string(),
message: message.into(),
worker_id: None,
actor_addr: None,
fields: serde_json::Map::new(),
});
/// Publish one incoming datastream frame to raw clients and all matching views.
pub fn ingest(&self, stream: &StreamId, frame: &Frame) {
let event = self.store.ingest(stream, frame);
let _ = self.frames.send(event);
}
/// Register a plugin with the dashboard.
pub fn register_plugin(&self, plugin: Arc<dyn plugin::DashboardPlugin>) {
self.plugin_registry.register(plugin);
/// Publish an already-serialized frame event to raw clients and views.
pub fn publish(&self, event: FrameEvent) {
self.store.publish(event.clone());
let _ = self.frames.send(event);
}
/// Override the `/` landing page with custom HTML. Used when the dashboard
/// shows a fleet board rather than the single-node actor dashboard.
pub fn set_landing_html(&self, html: impl Into<Arc<str>>) {
*self.landing_html.lock().unwrap() = Some(html.into());
}
/// Merge an extra axum router into the live HTTP server. The routes must be
/// disjoint from the dashboard's own routes. Must be called before
/// `start_http`/`start_http_standalone`.
pub fn set_extra_router(&self, router: axum::Router) {
*self.extra_router.lock().unwrap() = Some(router);
}
/// Access the time-series history store.
pub fn history(&self) -> &Arc<DashboardHistory> {
&self.history
}
/// Signal the dashboard to shut down (SSE clients receive "done").
/// Stop the HTTP server.
pub fn shutdown(&self) {
self.shutdown.store(true, Ordering::Release);
self.shutdown_notify.notify_waiters();
}
/// Start the HTTP server on the provided tokio handle.
/// Use this when a tokio runtime already exists (e.g. IrohDriver's runtime).
/// Start the HTTP server on an existing Tokio runtime.
pub fn start_http(&self, handle: tokio::runtime::Handle) {
let state = self.build_app_state();
let extra = self.extra_router.lock().unwrap().take();
let state = server::AppState {
frames: self.frames.clone(),
store: Arc::clone(&self.store),
views: Arc::clone(&self.views),
shutdown_notify: Arc::clone(&self.shutdown_notify),
};
let port = self.port;
handle.spawn(async move {
server::run_server(state, port, extra).await;
server::run_server(state, port).await;
});
}
/// Start the HTTP server on a standalone tokio runtime (1 worker thread).
/// Use this when no external tokio runtime is available (e.g. non-async transport).
/// Start the HTTP server on a standalone Tokio runtime.
pub fn start_http_standalone(&self) {
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(1)
@ -134,53 +133,29 @@ impl DashboardHandle {
.build()
.expect("failed to create tokio runtime for dashboard HTTP");
let handle = rt.handle().clone();
*self.standalone_rt.lock().unwrap() = Some(rt);
*self.standalone_rt.lock() = Some(rt);
self.start_http(handle);
}
fn build_app_state(&self) -> server::AppState {
server::AppState {
store: Arc::clone(&self.store),
pushed_stats: Arc::clone(&self.pushed_stats),
shutdown: Arc::clone(&self.shutdown),
shutdown_notify: Arc::clone(&self.shutdown_notify),
history: Arc::clone(&self.history),
plugins: Arc::clone(&self.plugin_registry),
landing: self.landing_html.lock().unwrap().clone(),
}
}
}
/// Start a dashboard and return a handle.
/// Create the datastream dashboard state.
///
/// The dashboard state is created immediately but the HTTP server is NOT started.
/// Call `start_http()` or `start_http_standalone()` to begin serving.
/// Push datastream-folded stats with `set_stats()`; HTTP/SSE reads only those
/// snapshots and registered plugins.
/// The HTTP server is not started until `start_http` or `start_http_standalone`
/// is called.
pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle {
let store = Arc::new(EventStore::new(
config.event_capacity,
config.record,
config.record_event_capacity,
let views = Arc::new(ViewRegistry::new());
views.register(swactor::worker_view());
let store = Arc::new(DashboardStore::new(
config.raw_frame_history,
Arc::clone(&views),
));
let pushed_stats: Arc<Mutex<Option<RuntimeStats>>> = Arc::new(Mutex::new(None));
let shutdown = Arc::new(AtomicBool::new(false));
let shutdown_notify = Arc::new(tokio::sync::Notify::new());
let history = Arc::new(DashboardHistory::new(HistoryConfig::default()));
let port = config.port;
let plugin_registry = Arc::new(PluginRegistry::new());
let (frames, _) = broadcast::channel(config.frame_buffer.max(1));
DashboardHandle {
port: config.port,
frames,
store,
pushed_stats,
shutdown,
shutdown_notify,
history,
port,
plugin_registry,
views,
shutdown_notify: Arc::new(tokio::sync::Notify::new()),
standalone_rt: Mutex::new(None),
landing_html: Mutex::new(None),
extra_router: Mutex::new(None),
}
}

87
crates/dashboard/src/plugin.rs
View file

@ -1,87 +0,0 @@
//! Dashboard plugin system.
//!
//! Plugins provide subsystem-specific metrics, API endpoints, and UI pages
//! to the dashboard without the dashboard knowing about the subsystem.
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
/// Response from a plugin's request handler.
pub enum PluginResponse {
Json(String),
Binary { content_type: String, data: Vec<u8> },
Error { status: u16, message: String },
NotFound,
}
impl PluginResponse {
pub fn not_found() -> Self {
Self::NotFound
}
pub fn json(s: String) -> Self {
Self::Json(s)
}
pub fn error(status: u16, msg: impl Into<String>) -> Self {
Self::Error {
status,
message: msg.into(),
}
}
}
/// A composable dashboard plugin.
///
/// Plugins provide subsystem-specific metrics, API endpoints, and UI pages
/// to the dashboard without the dashboard knowing about the subsystem.
pub trait DashboardPlugin: Send + Sync {
/// Unique name — used as SSE event type and API route prefix `/api/plugin/{name}/...`
fn name(&self) -> &str;
/// JSON snapshot polled every ~200ms via SSE. Return None if no data available.
fn snapshot_json(&self) -> Option<String>;
/// Handle an API request to `/api/plugin/{name}/{path}`.
fn handle_request(
&self,
_method: &str,
_path: &str,
_query: &HashMap<String, String>,
_body: &[u8],
) -> PluginResponse {
PluginResponse::not_found()
}
/// Optional HTML page content. Dashboard will serve at `/plugin/{name}`.
fn html_page(&self) -> Option<&str> {
None
}
}
/// Thread-safe registry of plugins.
pub struct PluginRegistry {
plugins: Mutex<Vec<Arc<dyn DashboardPlugin>>>,
}
impl Default for PluginRegistry {
fn default() -> Self {
Self::new()
}
}
impl PluginRegistry {
pub fn new() -> Self {
Self {
plugins: Mutex::new(Vec::new()),
}
}
pub fn register(&self, plugin: Arc<dyn DashboardPlugin>) {
self.plugins.lock().unwrap().push(plugin);
}
pub fn snapshot(&self) -> Vec<Arc<dyn DashboardPlugin>> {
self.plugins.lock().unwrap().clone()
}
}

28
crates/dashboard/src/root_page.rs Normal file
View file

@ -0,0 +1,28 @@
pub const ROOT_HTML: &str = r#"<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>swactor dashboard</title>
<style>
:root { color-scheme: dark; font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; background: #111827; color: #e5e7eb; }
body { margin: 0; padding: 32px; }
a { color: #93c5fd; }
code { color: #fbbf24; }
.card { max-width: 760px; background: #1f2937; border: 1px solid #374151; border-radius: 16px; padding: 24px; }
li { margin: 10px 0; }
</style>
</head>
<body>
<main class="card">
<h1>swactor dashboard</h1>
<p>Read-only views over live datastream frames.</p>
<ul>
<li><a href="/view/swactor/workers">Swactor workers</a></li>
<li><a href="/api/views">Registered views JSON</a></li>
<li><code>/events</code> streams raw incoming frames as SSE.</li>
<li><code>/api/frames</code> returns the bounded recent raw frame window.</li>
</ul>
</main>
</body>
</html>"#;

387
crates/dashboard/src/server.rs
View file

@ -1,358 +1,115 @@
use std::collections::HashMap;
use std::convert::Infallible;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use std::sync::Arc;
use axum::Router;
use axum::body::Bytes;
use axum::extract::{Path, Query, State};
use axum::http::{StatusCode, header};
use axum::extract::{Path, State};
use axum::http::StatusCode;
use axum::response::sse::{Event, KeepAlive, Sse};
use axum::response::{IntoResponse, Response};
use axum::response::{Html, IntoResponse};
use axum::routing::get;
use axum::{Json, Router};
use tokio::sync::{broadcast, mpsc};
use tokio_stream::StreamExt;
use tokio_stream::wrappers::ReceiverStream;
use crate::history::DashboardHistory;
use crate::html::{ACTOR_DETAIL_HTML, ACTORS_HTML, DASHBOARD_HTML, TOPOLOGY_HTML};
use crate::layer::EventStore;
use crate::topology;
use crate::warnings::{WarningConfig, WarningDetector};
use crate::plugin::PluginRegistry;
/// Format a server-sent event.
fn format_sse(event: &str, data: &str) -> Event {
Event::default().event(event).data(data)
}
// ── Shared application state ────────────────────────────────────────────
use crate::FrameEvent;
use crate::root_page::ROOT_HTML;
use crate::store::DashboardStore;
use crate::view::ViewRegistry;
#[derive(Clone)]
pub(crate) struct AppState {
pub store: Arc<EventStore>,
/// Datastream-folded stats snapshot (latest wins).
pub pushed_stats: Arc<Mutex<Option<swactor::stats::RuntimeStats>>>,
pub shutdown: Arc<AtomicBool>,
pub frames: broadcast::Sender<FrameEvent>,
pub store: Arc<DashboardStore>,
pub views: Arc<ViewRegistry>,
pub shutdown_notify: Arc<tokio::sync::Notify>,
pub history: Arc<DashboardHistory>,
pub plugins: Arc<PluginRegistry>,
/// Optional HTML served at `/` instead of the actor dashboard.
pub landing: Option<Arc<str>>,
}
// ── Router builders ─────────────────────────────────────────────────────
pub(crate) fn build_live_router(state: AppState) -> Router {
let router = Router::new()
.route("/", get(page_dashboard))
.route("/actors", get(page_actors))
.route("/topology", get(page_topology))
.route("/events", get(handle_live_sse))
.route("/api/stats", get(handle_stats_api))
.route("/api/history", get(handle_history_api))
.route("/api/topology", get(handle_topology_api))
.route("/api/logs", get(handle_logs_api))
.route("/actor/{hex}", get(handle_actor_detail))
// Plugin routes. The bare form must be registered separately: a
// `{*rest}` wildcard never matches an empty remainder, and plugins
// answer their model snapshot on the bare path.
.route(
"/api/plugin/{name}",
get(handle_plugin_get_bare).post(handle_plugin_post_bare),
)
.route(
"/api/plugin/{name}/{*rest}",
get(handle_plugin_get).post(handle_plugin_post),
)
.route("/plugin/{name}", get(handle_plugin_page));
router.with_state(state)
}
// ── Server startup ──────────────────────────────────────────────────────
pub(crate) async fn run_server(state: AppState, port: u16, extra: Option<Router>) {
pub(crate) async fn run_server(state: AppState, port: u16) {
let listener = tokio::net::TcpListener::bind(format!("0.0.0.0:{port}"))
.await
.expect("failed to bind HTTP server");
let shutdown = state.shutdown_notify.clone();
let mut app = build_live_router(state);
if let Some(extra) = extra {
// Disjoint route sets (dashboard UI/API vs collector `/diag/*`) compose
// cleanly onto one listener; both are `Router<()>` after `with_state`.
app = app.merge(extra);
}
axum::serve(listener, app)
let shutdown = Arc::clone(&state.shutdown_notify);
axum::serve(listener, router(state))
.with_graceful_shutdown(async move { shutdown.notified().await })
.await
.expect("HTTP server error");
}
// ── HTML page handlers ──────────────────────────────────────────────────
fn html_response(template: &str, mode: &str) -> Response {
let html = template.replace("__DASHBOARD_MODE__", mode);
([(header::CONTENT_TYPE, "text/html; charset=utf-8")], html).into_response()
fn router(state: AppState) -> Router {
Router::new()
.route("/", get(root_page))
.route("/events", get(frame_stream))
.route("/api/frames", get(recent_frames))
.route("/api/views", get(views_json))
.route("/api/view/{*path}", get(view_snapshot))
.route("/view/{*path}", get(view_page))
.with_state(state)
}
async fn page_dashboard(State(state): State<AppState>) -> Response {
if let Some(html) = &state.landing {
return (
[(header::CONTENT_TYPE, "text/html; charset=utf-8")],
html.to_string(),
)
.into_response();
async fn root_page() -> Html<&'static str> {
Html(ROOT_HTML)
}
async fn recent_frames(State(state): State<AppState>) -> Json<Vec<FrameEvent>> {
Json(state.store.recent_frames())
}
async fn views_json(State(state): State<AppState>) -> Json<serde_json::Value> {
Json(serde_json::json!({ "views": state.views.descriptors() }))
}
async fn view_snapshot(
Path(path): Path<String>,
State(state): State<AppState>,
) -> impl IntoResponse {
match state.views.snapshot(&path) {
Some(snapshot) => Json(snapshot).into_response(),
None => (StatusCode::NOT_FOUND, "unknown dashboard view").into_response(),
}
html_response(DASHBOARD_HTML, "live")
}
async fn page_actors() -> Response {
html_response(ACTORS_HTML, "live")
async fn view_page(Path(path): Path<String>, State(state): State<AppState>) -> impl IntoResponse {
match state.views.html(&path) {
Some(html) => Html(html).into_response(),
None => (StatusCode::NOT_FOUND, "unknown dashboard view").into_response(),
}
}
async fn page_topology() -> Response {
html_response(TOPOLOGY_HTML, "live")
}
async fn handle_actor_detail(Path(hex_addr): Path<String>) -> Response {
let html = ACTOR_DETAIL_HTML
.replace("__DASHBOARD_MODE__", "live")
.replace("__ACTOR_ADDR__", &hex_addr);
([(header::CONTENT_TYPE, "text/html; charset=utf-8")], html).into_response()
}
// ── Live SSE handler ────────────────────────────────────────────────────
async fn handle_live_sse(
async fn frame_stream(
State(state): State<AppState>,
) -> Sse<impl tokio_stream::Stream<Item = Result<Event, Infallible>>> {
let (tx, rx) = tokio::sync::mpsc::channel::<Event>(32);
let mut rx = state.frames.subscribe();
let (tx, out) = mpsc::channel::<Event>(32);
tokio::spawn(async move {
let mut cursor: u64 = 0;
let mut warning_detector = WarningDetector::new(WarningConfig::default());
let mut tick_count: u64 = 0;
// Send initial history snapshot so sparklines render immediately
if state.history.sample_count() > 0 {
let json = state.history.worker_history_json();
if tx.send(format_sse("history", &json)).await.is_err() {
return;
}
}
loop {
// Send the latest datastream-folded stats snapshot, if any.
let maybe_stats = { state.pushed_stats.lock().unwrap().clone() };
if let Some(stats) = maybe_stats {
state.history.record(&stats);
let warnings = warning_detector.check(&stats);
if !warnings.is_empty()
&& let Ok(wjson) = serde_json::to_string(&warnings)
&& tx.send(format_sse("warnings", &wjson)).await.is_err()
{
return;
}
let json = serde_json::to_string(&stats).unwrap();
if tx.send(format_sse("stats", &json)).await.is_err() {
return;
}
tick_count += 1;
if tick_count.is_multiple_of(5) {
let topo = topology::worker_topology(&stats);
if let Ok(tjson) = serde_json::to_string(&topo)
&& tx.send(format_sse("topology", &tjson)).await.is_err()
match rx.recv().await {
Ok(frame) => {
let Ok(json) = serde_json::to_string(&frame) else {
continue;
};
if tx
.send(Event::default().event("frame").data(json))
.await
.is_err()
{
return;
}
}
}
// Poll all registered plugins
for plugin in state.plugins.snapshot() {
if let Some(json) = plugin.snapshot_json()
&& tx.send(format_sse(plugin.name(), &json)).await.is_err()
{
return;
Err(broadcast::error::RecvError::Lagged(skipped)) => {
let json = serde_json::json!({ "skipped": skipped }).to_string();
if tx
.send(Event::default().event("lagged").data(json))
.await
.is_err()
{
return;
}
}
Err(broadcast::error::RecvError::Closed) => return,
}
// Send new activity events
let (batch, new_cursor) = state.store.read_from(cursor);
if !batch.is_empty() {
let json = serde_json::to_string(&batch).unwrap();
if tx.send(format_sse("activity", &json)).await.is_err() {
return;
}
cursor = new_cursor;
}
if state.shutdown.load(Ordering::Relaxed) {
let _ = tx.send(format_sse("done", "{}")).await;
return;
}
tokio::time::sleep(Duration::from_millis(200)).await;
}
});
Sse::new(ReceiverStream::new(rx).map(Ok)).keep_alive(KeepAlive::default())
}
// ── JSON API handlers ───────────────────────────────────────────────────
fn json_response(json: String) -> Response {
([(header::CONTENT_TYPE, "application/json")], json).into_response()
}
fn json_error(status: StatusCode, msg: &str) -> Response {
let json = serde_json::json!({ "error": msg }).to_string();
(status, [(header::CONTENT_TYPE, "application/json")], json).into_response()
}
async fn handle_stats_api(State(state): State<AppState>) -> Response {
let json = state
.pushed_stats
.lock()
.unwrap()
.as_ref()
.map(|stats| serde_json::to_string(stats).unwrap())
.unwrap_or_else(|| "{}".to_string());
json_response(json)
}
async fn handle_topology_api(State(state): State<AppState>) -> Response {
let json = state
.pushed_stats
.lock()
.unwrap()
.as_ref()
.map(|stats| {
let topo = topology::worker_topology(stats);
serde_json::to_string(&topo).unwrap_or_else(|_| "{}".into())
})
.unwrap_or_else(|| "{}".to_string());
json_response(json)
}
async fn handle_logs_api(
State(state): State<AppState>,
Query(params): Query<HashMap<String, String>>,
) -> Response {
let actor = params.get("actor").cloned().unwrap_or_default();
let limit: usize = params
.get("limit")
.and_then(|s| s.parse().ok())
.unwrap_or(200);
let level = params.get("level").cloned();
let mut events = state.store.read_for_actor(&actor, limit);
// Filter by level if specified
if let Some(ref lvl) = level {
let lvl_upper = lvl.to_uppercase();
events.retain(|e| e.level == lvl_upper);
}
let json = serde_json::to_string(&events).unwrap_or_else(|_| "[]".into());
json_response(json)
}
async fn handle_history_api(State(state): State<AppState>) -> Response {
let json = state.history.worker_history_json();
json_response(json)
}
// ── Plugin handlers ─────────────────────────────────────────────────────
/// Dispatch one plugin API request and convert its [`PluginResponse`] to HTTP.
/// `rest` is the path after `/api/plugin/{name}/` — empty for the bare
/// `/api/plugin/{name}` form.
fn dispatch_plugin(
state: &AppState,
method: &str,
name: &str,
rest: &str,
params: &HashMap<String, String>,
body: &[u8],
) -> Response {
let plugins = state.plugins.snapshot();
let plugin = match plugins.iter().find(|p| p.name() == name) {
Some(p) => p,
None => return json_error(StatusCode::NOT_FOUND, &format!("plugin '{name}' not found")),
};
match plugin.handle_request(method, rest, params, body) {
crate::plugin::PluginResponse::Json(json) => json_response(json),
crate::plugin::PluginResponse::Binary { content_type, data } => {
([(header::CONTENT_TYPE, content_type)], data).into_response()
}
crate::plugin::PluginResponse::Error { status, message } => {
let code = StatusCode::from_u16(status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
json_error(code, &message)
}
crate::plugin::PluginResponse::NotFound => json_error(StatusCode::NOT_FOUND, "not found"),
}
}
async fn handle_plugin_get(
State(state): State<AppState>,
Path((name, rest)): Path<(String, String)>,
Query(params): Query<HashMap<String, String>>,
) -> Response {
dispatch_plugin(&state, "GET", &name, &rest, &params, &[])
}
async fn handle_plugin_post(
State(state): State<AppState>,
Path((name, rest)): Path<(String, String)>,
Query(params): Query<HashMap<String, String>>,
body: Bytes,
) -> Response {
dispatch_plugin(&state, "POST", &name, &rest, &params, &body)
}
/// `/api/plugin/{name}` with no trailing path. The `{*rest}` route cannot
/// match an empty remainder, so without this route the bare form — what the
/// distribution/netmap pages fetch for their first paint — would 404.
async fn handle_plugin_get_bare(
State(state): State<AppState>,
Path(name): Path<String>,
Query(params): Query<HashMap<String, String>>,
) -> Response {
dispatch_plugin(&state, "GET", &name, "", &params, &[])
}
async fn handle_plugin_post_bare(
State(state): State<AppState>,
Path(name): Path<String>,
Query(params): Query<HashMap<String, String>>,
body: Bytes,
) -> Response {
dispatch_plugin(&state, "POST", &name, "", &params, &body)
}
async fn handle_plugin_page(State(state): State<AppState>, Path(name): Path<String>) -> Response {
let plugins = state.plugins.snapshot();
let plugin = match plugins.iter().find(|p| p.name() == name) {
Some(p) => p,
None => {
return (StatusCode::NOT_FOUND, "plugin not found").into_response();
}
};
match plugin.html_page() {
Some(html) => {
let rendered = html.replace("__DASHBOARD_MODE__", "live");
(
[(header::CONTENT_TYPE, "text/html; charset=utf-8")],
rendered,
)
.into_response()
}
None => (StatusCode::NOT_FOUND, "no page for this plugin").into_response(),
}
Sse::new(ReceiverStream::new(out).map(Ok)).keep_alive(KeepAlive::default())
}

50
crates/dashboard/src/store.rs Normal file
View file

@ -0,0 +1,50 @@
use std::collections::VecDeque;
use std::sync::Arc;
use datastream::frame::{Frame, StreamId};
use parking_lot::Mutex;
use crate::FrameEvent;
use crate::view::ViewRegistry;
pub(crate) struct DashboardStore {
recent: Mutex<VecDeque<FrameEvent>>,
recent_cap: usize,
views: Arc<ViewRegistry>,
}
impl DashboardStore {
pub fn new(recent_cap: usize, views: Arc<ViewRegistry>) -> Self {
Self {
recent: Mutex::new(VecDeque::with_capacity(recent_cap.min(4096))),
recent_cap: recent_cap.max(1),
views,
}
}
pub fn ingest(&self, stream: &StreamId, frame: &Frame) -> FrameEvent {
let event = FrameEvent::new(stream, frame);
self.record(event.clone());
self.views.dispatch(stream, frame, &event);
event
}
pub fn publish(&self, event: FrameEvent) {
if let Some((stream, frame)) = event.to_datastream_parts() {
self.views.dispatch(&stream, &frame, &event);
}
self.record(event);
}
pub fn recent_frames(&self) -> Vec<FrameEvent> {
self.recent.lock().iter().cloned().collect()
}
fn record(&self, event: FrameEvent) {
let mut recent = self.recent.lock();
if recent.len() == self.recent_cap {
recent.pop_front();
}
recent.push_back(event);
}
}

16
crates/dashboard/src/swactor/mod.rs Normal file
View file

@ -0,0 +1,16 @@
use std::sync::Arc;
use crate::view::DashboardView;
mod worker_page;
mod worker_view;
pub use worker_view::SwactorWorkerView;
pub const RUNTIME_STATS: &str = "runtime.stats";
pub const RUNTIME_WORKERS: &str = "runtime.workers";
pub const RUNTIME_ACTORS: &str = "runtime.actors";
pub fn worker_view() -> Arc<dyn DashboardView> {
Arc::new(SwactorWorkerView::default())
}

187
crates/dashboard/src/swactor/worker_page.rs Normal file
View file

@ -0,0 +1,187 @@
pub const WORKER_HTML: &str = r#"<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>swactor workers</title>
<style>
:root { color-scheme: dark; font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; background: #0f172a; color: #e2e8f0; }
body { margin: 0; padding: 20px; }
header { display: flex; align-items: baseline; gap: 16px; margin-bottom: 18px; }
h1 { margin: 0; font-size: 28px; }
select, button { background: #1e293b; color: #e2e8f0; border: 1px solid #334155; border-radius: 8px; padding: 8px 10px; }
.muted { color: #94a3b8; }
.grid { display: grid; grid-template-columns: repeat(4, minmax(150px, 1fr)); gap: 12px; margin-bottom: 16px; }
.card { background: #1e293b; border: 1px solid #334155; border-radius: 14px; padding: 14px; }
.label { color: #94a3b8; font-size: 12px; text-transform: uppercase; letter-spacing: .08em; }
.value { font-size: 24px; margin-top: 6px; font-variant-numeric: tabular-nums; }
table { width: 100%; border-collapse: collapse; background: #111827; border: 1px solid #334155; border-radius: 12px; overflow: hidden; margin-bottom: 16px; }
th, td { padding: 9px 10px; border-bottom: 1px solid #1f2937; text-align: left; font-variant-numeric: tabular-nums; }
th { color: #93c5fd; background: #1e293b; cursor: pointer; }
tr[data-selected="true"] { background: #1d4ed833; }
.bad { color: #f87171; font-weight: 700; }
.warn { color: #fbbf24; }
.ok { color: #34d399; }
.section { margin-top: 20px; }
.spark-card { display: grid; gap: 8px; margin-bottom: 16px; }
.spark-head { display: flex; justify-content: space-between; gap: 12px; align-items: baseline; }
.legend { display: flex; gap: 14px; color: #94a3b8; font-size: 12px; }
.legend i { display: inline-block; width: 20px; height: 3px; margin-right: 6px; vertical-align: middle; border-radius: 99px; }
.spark { width: 100%; height: 72px; background: #111827; border: 1px solid #334155; border-radius: 12px; }
.empty { padding: 28px; border: 1px dashed #475569; border-radius: 12px; color: #94a3b8; }
@media (max-width: 900px) { .grid { grid-template-columns: repeat(2, minmax(150px, 1fr)); } body { padding: 12px; } }
</style>
</head>
<body>
<header>
<h1>Swactor workers</h1>
<span class="muted" id="status">loading…</span>
<select id="runtime"></select>
</header>
<section class="grid" id="summary"></section>
<section class="card spark-card">
<div class="spark-head">
<div><div class="label">activity</div><div class="muted">message rate and queued depth</div></div>
<div class="legend"><span><i style="background:#60a5fa"></i>msg/s</span><span><i style="background:#fbbf24"></i>queued</span></div>
</div>
<svg class="spark" id="spark" viewBox="0 0 800 120" preserveAspectRatio="none"></svg>
</section>
<section class="section">
<h2>Workers</h2>
<table>
<thead><tr id="workers-head"></tr></thead>
<tbody id="workers"></tbody>
</table>
</section>
<section class="section">
<h2>Actors <span class="muted" id="actor-filter"></span></h2>
<table>
<thead><tr><th>actor</th><th>name</th><th>worker</th><th>queued</th><th>msg/s</th><th>processed</th><th>last message</th><th>state</th></tr></thead>
<tbody id="actors"></tbody>
</table>
</section>
<script>
const runtimeSelect = document.getElementById('runtime');
const statusEl = document.getElementById('status');
const summaryEl = document.getElementById('summary');
const workersEl = document.getElementById('workers');
const workersHeadEl = document.getElementById('workers-head');
const actorsEl = document.getElementById('actors');
const actorFilterEl = document.getElementById('actor-filter');
const sparkEl = document.getElementById('spark');
let selectedRuntime = '';
let selectedWorker = null;
function fmt(n, digits = 0) {
if (n === null || n === undefined) return '—';
if (typeof n === 'number') return n.toLocaleString(undefined, { maximumFractionDigits: digits });
return String(n);
}
function workerLabel(id) { return id === null || id === undefined ? 'aggregate' : id; }
function setSummary(rt) {
const totals = rt.totals || {};
const cards = [
['workers', totals.workers, true],
['actors', totals.actors, true],
['queued', totals.mailbox_depth, true],
['msg/s', totals.msg_per_sec, true],
['local/s', totals.local_per_sec, true],
['cross/s', totals.cross_per_sec, true],
['tick p50 μs', totals.tick_p50_us, true],
['inbox/s', totals.inbox_per_sec, totals.inbox_per_sec > 0],
['drops', totals.messages_dropped, totals.messages_dropped > 0],
['panics', totals.panics, totals.panics > 0],
].filter(([, , show]) => show);
summaryEl.innerHTML = cards.map(([k, v]) => `<div class="card"><div class="label">${k}</div><div class="value">${fmt(v, 1)}</div></div>`).join('');
}
function drawSpark(history) {
const points = (history || []).slice(-80);
if (points.length < 2) { sparkEl.innerHTML = `<text x="400" y="64" text-anchor="middle" fill="currentColor">waiting for activity</text>`; return; }
const maxMsg = Math.max(1, ...points.map(p => p.msg_per_sec || 0));
const maxDepth = Math.max(1, ...points.map(p => p.mailbox_depth || 0));
const line = (field, max, color) => {
const coords = points.map((p, i) => {
const x = points.length === 1 ? 0 : i * 800 / (points.length - 1);
const y = 110 - ((p[field] || 0) / max) * 100;
return `${x.toFixed(1)},${y.toFixed(1)}`;
}).join(' ');
return `<polyline fill="none" stroke="${color}" stroke-width="3" points="${coords}" />`;
};
sparkEl.innerHTML = line('msg_per_sec', maxMsg, '#60a5fa') + line('mailbox_depth', maxDepth, '#fbbf24');
}
function renderWorkers(rt) {
const rows = [...(rt.workers || [])].sort((a, b) => (b.mailbox_depth || 0) - (a.mailbox_depth || 0));
const showInbox = rows.some(w => (w.inbox_per_sec || 0) > 0);
const showDrops = rows.some(w => (w.messages_dropped || 0) > 0);
const showPanics = rows.some(w => (w.panics || 0) > 0);
const columns = [
['worker', w => workerLabel(w.id)],
['actors', w => fmt(w.actor_count)],
['queued', w => `<span class="${w.mailbox_depth ? 'warn' : ''}">${fmt(w.mailbox_depth)}</span>`],
['msg/s', w => fmt(w.msg_per_sec, 1)],
['local/s', w => fmt(w.local_per_sec, 1)],
['cross/s', w => fmt(w.cross_per_sec, 1)],
['tick p50 μs', w => fmt(w.tick_p50_us)],
['inbox/s', w => fmt(w.inbox_per_sec, 1), showInbox],
['drops', w => `<span class="bad">${fmt(w.messages_dropped)}</span>`, showDrops],
['panics', w => `<span class="bad">${fmt(w.panics)}</span>`, showPanics],
].filter(([, , show = true]) => show);
workersHeadEl.innerHTML = columns.map(([label]) => `<th>${label}</th>`).join('');
workersEl.innerHTML = rows.map(w => {
const sel = String(workerLabel(w.id)) === String(selectedWorker);
return `<tr data-selected="${sel}" data-worker="${workerLabel(w.id)}">${columns.map(([, cell]) => `<td>${cell(w)}</td>`).join('')}</tr>`;
}).join('') || `<tr><td colspan="${columns.length}" class="empty">No worker frames received yet.</td></tr>`;
workersEl.querySelectorAll('tr[data-worker]').forEach(row => row.onclick = () => {
selectedWorker = row.dataset.worker;
render(currentSnapshot);
});
}
function renderActors(rt) {
const rows = [...(rt.actors || [])]
.filter(a => selectedWorker === null || String(workerLabel(a.worker_id)) === String(selectedWorker))
.sort((a, b) => (b.mailbox_depth || 0) - (a.mailbox_depth || 0));
actorFilterEl.textContent = selectedWorker === null ? '' : `(worker ${selectedWorker})`;
actorsEl.innerHTML = rows.map(a => `<tr>
<td>${a.address}</td><td>${a.name || ''}</td><td>${workerLabel(a.worker_id)}</td>
<td class="${a.mailbox_depth ? 'warn' : ''}">${fmt(a.mailbox_depth)}</td><td>${fmt(a.msg_per_sec, 1)}</td><td>${fmt(a.messages_processed)}</td>
<td>${a.last_msg_type || ''}</td><td class="${a.poisoned ? 'bad' : 'ok'}">${a.poisoned ? 'poisoned' : 'ok'}</td>
</tr>`).join('') || '<tr><td colspan="8" class="empty">No actor detail frames received yet.</td></tr>';
}
let currentSnapshot = null;
function render(snapshot) {
currentSnapshot = snapshot;
const runtimes = snapshot.runtimes || [];
const keys = runtimes.map(r => r.stream.key);
if (!selectedRuntime && keys.length) selectedRuntime = keys[0];
runtimeSelect.innerHTML = keys.map(k => `<option value="${k}" ${k === selectedRuntime ? 'selected' : ''}>${k}</option>`).join('');
const rt = runtimes.find(r => r.stream.key === selectedRuntime) || runtimes[0];
if (!rt) {
statusEl.textContent = 'waiting for runtime frames';
summaryEl.innerHTML = '<div class="empty">No swactor runtime frames received yet.</div>';
workersEl.innerHTML = '';
actorsEl.innerHTML = '';
sparkEl.innerHTML = '';
return;
}
statusEl.textContent = `${rt.live ? 'live' : 'stale'} · last seen ${fmt(rt.last_seen_ms_ago)}ms ago`;
setSummary(rt);
drawSpark(rt.history);
renderWorkers(rt);
renderActors(rt);
}
runtimeSelect.onchange = () => { selectedRuntime = runtimeSelect.value; selectedWorker = null; if (currentSnapshot) render(currentSnapshot); };
async function refresh() {
const res = await fetch('/api/view/swactor/workers', { cache: 'no-store' });
if (!res.ok) throw new Error(`HTTP ${res.status}`);
render(await res.json());
}
setInterval(() => refresh().catch(err => statusEl.textContent = err.message), 750);
refresh().catch(err => statusEl.textContent = err.message);
</script>
</body>
</html>"#;

783
crates/dashboard/src/swactor/worker_view.rs Normal file
View file

@ -0,0 +1,783 @@
use std::collections::{BTreeMap, VecDeque};
use std::time::{Duration, Instant};
use datastream::frame::{Frame, StreamId};
use parking_lot::RwLock;
use serde::Serialize;
use serde_json::{Value, json};
use crate::swactor::worker_page::WORKER_HTML;
use crate::swactor::{RUNTIME_ACTORS, RUNTIME_STATS, RUNTIME_WORKERS};
use crate::view::DashboardView;
use crate::{FrameEvent, StreamEvent};
const CHANNELS: &[&str] = &[RUNTIME_STATS, RUNTIME_WORKERS, RUNTIME_ACTORS];
const HISTORY_CAP: usize = 512;
const HISTORY_MIN_INTERVAL: Duration = Duration::from_millis(250);
const LIVE_TTL: Duration = Duration::from_secs(8);
#[derive(Default)]
pub struct SwactorWorkerView {
state: RwLock<WorkerViewState>,
}
#[derive(Default)]
struct WorkerViewState {
runtimes: BTreeMap<String, RuntimeState>,
}
struct RuntimeState {
stream: StreamEvent,
last_seen: Instant,
summary: RuntimeSummary,
aggregate: Option<WorkerState>,
workers: BTreeMap<u32, WorkerState>,
actors: BTreeMap<String, ActorState>,
history: VecDeque<HistorySample>,
}
#[derive(Default)]
struct RuntimeSummary {
num_workers: Option<u32>,
uptime_ms: Option<u64>,
actors_live: Option<u32>,
mailbox_depth: Option<u32>,
scheduled_tasks: Option<u32>,
}
#[derive(Clone)]
struct WorkerState {
id: Option<u32>,
actor_count: u32,
mailbox_depth: u32,
messages_processed: u64,
local_sends: u64,
cross_sends: u64,
inbox_sends: u64,
type_mismatches: u64,
panics: u64,
messages_dropped: u64,
restarts: u64,
stops: u64,
tick_p50_us: u64,
msg_per_sec: f64,
local_per_sec: f64,
cross_per_sec: f64,
inbox_per_sec: f64,
last_update: Option<Instant>,
}
impl WorkerState {
fn new(id: Option<u32>) -> Self {
Self {
id,
actor_count: 0,
mailbox_depth: 0,
messages_processed: 0,
local_sends: 0,
cross_sends: 0,
inbox_sends: 0,
type_mismatches: 0,
panics: 0,
messages_dropped: 0,
restarts: 0,
stops: 0,
tick_p50_us: 0,
msg_per_sec: 0.0,
local_per_sec: 0.0,
cross_per_sec: 0.0,
inbox_per_sec: 0.0,
last_update: None,
}
}
fn apply_json(&mut self, value: &Value, now: Instant) {
let elapsed = self
.last_update
.map(|then| now.duration_since(then).as_secs_f64())
.unwrap_or(0.0);
if let Some(id) = u32_field(value, &["id", "worker_id"]) {
self.id = Some(id);
}
assign_u32(
&mut self.actor_count,
value,
&["num_actors", "actor_count", "actors_live"],
);
assign_u32(
&mut self.mailbox_depth,
value,
&["mailbox_depth", "total_mailbox_depth"],
);
assign_u64_rate(
&mut self.messages_processed,
&mut self.msg_per_sec,
value,
&["messages_processed", "processed"],
elapsed,
);
assign_u64_rate(
&mut self.local_sends,
&mut self.local_per_sec,
value,
&["local_sends"],
elapsed,
);
assign_u64_rate(
&mut self.cross_sends,
&mut self.cross_per_sec,
value,
&["cross_sends"],
elapsed,
);
assign_u64_rate(
&mut self.inbox_sends,
&mut self.inbox_per_sec,
value,
&["inbox_sends"],
elapsed,
);
assign_u64(&mut self.type_mismatches, value, &["type_mismatches"]);
assign_u64(&mut self.panics, value, &["panics"]);
assign_u64(
&mut self.messages_dropped,
value,
&["messages_dropped", "dropped"],
);
assign_u64(&mut self.restarts, value, &["restarts"]);
assign_u64(&mut self.stops, value, &["stops"]);
assign_u64(&mut self.tick_p50_us, value, &["tick_p50_us", "tick_p50"]);
self.last_update = Some(now);
}
}
#[derive(Clone)]
struct ActorState {
address: String,
name: Option<String>,
worker_id: Option<u32>,
mailbox_depth: u32,
messages_processed: u64,
msg_per_sec: f64,
last_msg_type: Option<String>,
poisoned: bool,
message_type_counts: Vec<(String, u64)>,
last_update: Option<Instant>,
}
impl ActorState {
fn new(address: String) -> Self {
Self {
address,
name: None,
worker_id: None,
mailbox_depth: 0,
messages_processed: 0,
msg_per_sec: 0.0,
last_msg_type: None,
poisoned: false,
message_type_counts: Vec::new(),
last_update: None,
}
}
fn apply_json(&mut self, value: &Value, now: Instant) {
let elapsed = self
.last_update
.map(|then| now.duration_since(then).as_secs_f64())
.unwrap_or(0.0);
if let Some(name) = string_field(value, &["name"]).filter(|name| !name.is_empty()) {
self.name = Some(name);
}
if let Some(worker_id) = u32_field(value, &["worker_id", "worker"]) {
self.worker_id = Some(worker_id);
}
assign_u32(&mut self.mailbox_depth, value, &["mailbox_depth", "queued"]);
assign_u64_rate(
&mut self.messages_processed,
&mut self.msg_per_sec,
value,
&["messages_processed", "processed"],
elapsed,
);
if let Some(last) = string_field(
value,
&["last_msg_type", "last_message", "last_message_type"],
)
.filter(|last| !last.is_empty())
{
self.last_msg_type = Some(last);
}
if let Some(poisoned) = value.get("poisoned").and_then(Value::as_bool) {
self.poisoned = poisoned;
}
if let Some(counts) = parse_message_type_counts(value.get("message_type_counts")) {
self.message_type_counts = counts;
}
self.last_update = Some(now);
}
}
struct HistorySample {
at: Instant,
mailbox_depth: u32,
msg_per_sec: f64,
tick_p50_us: u64,
}
impl RuntimeState {
fn new(stream: StreamEvent, now: Instant) -> Self {
Self {
stream,
last_seen: now,
summary: RuntimeSummary::default(),
aggregate: None,
workers: BTreeMap::new(),
actors: BTreeMap::new(),
history: VecDeque::with_capacity(HISTORY_CAP),
}
}
fn update(&mut self, channel: &str, payload: &[u8], now: Instant) {
self.last_seen = now;
let Ok(value) = serde_json::from_slice::<Value>(payload) else {
return;
};
match channel {
RUNTIME_STATS => self.apply_runtime_stats(&value, now),
RUNTIME_WORKERS => self.apply_workers(&value, now),
RUNTIME_ACTORS => self.apply_actors(&value, now),
_ => {}
}
self.push_history(now);
}
fn apply_runtime_stats(&mut self, value: &Value, now: Instant) {
if let Some(num_workers) = u32_field(value, &["num_workers", "workers_live"]) {
self.summary.num_workers = Some(num_workers);
}
if let Some(uptime_ms) = u64_field(value, &["uptime_ms"]) {
self.summary.uptime_ms = Some(uptime_ms);
}
if let Some(actors_live) = u32_field(value, &["actors_live", "num_actors", "actor_count"]) {
self.summary.actors_live = Some(actors_live);
}
if let Some(mailbox_depth) = u32_field(value, &["mailbox_depth", "total_mailbox_depth"]) {
self.summary.mailbox_depth = Some(mailbox_depth);
}
if let Some(scheduled_tasks) = u32_field(value, &["scheduled_tasks"]) {
self.summary.scheduled_tasks = Some(scheduled_tasks);
}
if let Some(workers) = value.get("workers").and_then(Value::as_array) {
for worker in workers {
self.apply_worker(worker, now);
}
}
if let Some(actors) = value.get("actors").and_then(Value::as_array) {
self.summary.actors_live = Some(actors.len().min(u32::MAX as usize) as u32);
for actor in actors {
self.apply_actor_mapping(actor);
}
}
if let Some(actors) = value.get("actor_details").and_then(Value::as_array) {
for actor in actors {
self.apply_actor(actor, now);
}
}
if let Some(ticks) = value.get("tick_timings").and_then(Value::as_array) {
self.apply_tick_timings(ticks);
}
}
fn apply_workers(&mut self, value: &Value, now: Instant) {
if let Some(workers) = value.get("workers").and_then(Value::as_array) {
for worker in workers {
self.apply_worker(worker, now);
}
return;
}
if value.get("id").is_some() || value.get("worker_id").is_some() {
self.apply_worker(value, now);
return;
}
let worker = self.aggregate.get_or_insert_with(|| WorkerState::new(None));
worker.apply_json(value, now);
if let Some(num_workers) = u32_field(value, &["num_workers"]) {
self.summary.num_workers = Some(num_workers);
}
if let Some(scheduled_tasks) = u32_field(value, &["scheduled_tasks"]) {
self.summary.scheduled_tasks = Some(scheduled_tasks);
}
}
fn apply_actors(&mut self, value: &Value, now: Instant) {
let Some(actors) = value.get("actors").and_then(Value::as_array) else {
self.apply_actor(value, now);
return;
};
self.summary.actors_live = Some(actors.len().min(u32::MAX as usize) as u32);
for actor in actors {
self.apply_actor(actor, now);
}
}
fn apply_worker(&mut self, value: &Value, now: Instant) {
let Some(id) = u32_field(value, &["id", "worker_id"]) else {
let worker = self.aggregate.get_or_insert_with(|| WorkerState::new(None));
worker.apply_json(value, now);
return;
};
self.workers
.entry(id)
.or_insert_with(|| WorkerState::new(Some(id)))
.apply_json(value, now);
}
fn apply_actor(&mut self, value: &Value, now: Instant) {
let Some(address) = string_field(value, &["address", "addr", "actor_addr"]) else {
return;
};
self.actors
.entry(address.clone())
.or_insert_with(|| ActorState::new(address))
.apply_json(value, now);
}
fn apply_actor_mapping(&mut self, value: &Value) {
if let Some(items) = value.as_array()
&& items.len() >= 2
{
let Some(address) = value_to_string(&items[0]) else {
return;
};
let Some(worker_id) = value_to_u32(&items[1]) else {
return;
};
self.actors
.entry(address.clone())
.or_insert_with(|| ActorState::new(address))
.worker_id = Some(worker_id);
return;
}
let Some(address) = string_field(value, &["address", "addr", "actor_addr"]) else {
return;
};
if let Some(worker_id) = u32_field(value, &["worker_id", "worker"]) {
self.actors
.entry(address.clone())
.or_insert_with(|| ActorState::new(address))
.worker_id = Some(worker_id);
}
}
fn apply_tick_timings(&mut self, ticks: &[Value]) {
for (worker_id, worker_ticks) in ticks.iter().enumerate() {
let Some(series) = worker_ticks.as_array() else {
continue;
};
let mut totals: Vec<u64> = series
.iter()
.filter_map(|tick| tick.get("phase_us").and_then(Value::as_array))
.map(|phases| phases.iter().filter_map(Value::as_u64).sum())
.collect();
if totals.is_empty() {
continue;
}
totals.sort_unstable();
let p50 = totals[totals.len() / 2];
self.workers
.entry(worker_id as u32)
.or_insert_with(|| WorkerState::new(Some(worker_id as u32)))
.tick_p50_us = p50;
}
}
fn push_history(&mut self, now: Instant) {
let totals = self.totals();
if let Some(last) = self.history.back_mut()
&& now.duration_since(last.at) < HISTORY_MIN_INTERVAL
{
last.mailbox_depth = totals.mailbox_depth;
last.msg_per_sec = totals.msg_per_sec;
last.tick_p50_us = totals.tick_p50_us;
return;
}
if self.history.len() == HISTORY_CAP {
self.history.pop_front();
}
self.history.push_back(HistorySample {
at: now,
mailbox_depth: totals.mailbox_depth,
msg_per_sec: totals.msg_per_sec,
tick_p50_us: totals.tick_p50_us,
});
}
fn totals(&self) -> Totals {
let mut totals = Totals::default();
totals.workers = self
.summary
.num_workers
.unwrap_or_else(|| self.workers.len().min(u32::MAX as usize) as u32);
totals.actors = self
.summary
.actors_live
.unwrap_or_else(|| self.actors.len().min(u32::MAX as usize) as u32);
if let Some(aggregate) = &self.aggregate {
totals.mailbox_depth = aggregate.mailbox_depth;
totals.msg_per_sec = aggregate.msg_per_sec;
totals.local_per_sec = aggregate.local_per_sec;
totals.cross_per_sec = aggregate.cross_per_sec;
totals.inbox_per_sec = aggregate.inbox_per_sec;
totals.messages_dropped = aggregate.messages_dropped;
totals.panics = aggregate.panics;
totals.tick_p50_us = aggregate.tick_p50_us;
return totals;
}
for worker in self.workers.values() {
totals.mailbox_depth = totals.mailbox_depth.saturating_add(worker.mailbox_depth);
totals.msg_per_sec += worker.msg_per_sec;
totals.local_per_sec += worker.local_per_sec;
totals.cross_per_sec += worker.cross_per_sec;
totals.inbox_per_sec += worker.inbox_per_sec;
totals.messages_dropped = totals
.messages_dropped
.saturating_add(worker.messages_dropped);
totals.panics = totals.panics.saturating_add(worker.panics);
totals.tick_p50_us = totals.tick_p50_us.max(worker.tick_p50_us);
}
if totals.mailbox_depth == 0 {
totals.mailbox_depth = self.summary.mailbox_depth.unwrap_or(0);
}
totals
}
}
#[derive(Default, Serialize)]
struct Totals {
workers: u32,
actors: u32,
mailbox_depth: u32,
msg_per_sec: f64,
local_per_sec: f64,
cross_per_sec: f64,
inbox_per_sec: f64,
messages_dropped: u64,
panics: u64,
tick_p50_us: u64,
}
#[derive(Serialize)]
struct WorkerViewSnapshot {
runtimes: Vec<RuntimeSnapshot>,
}
#[derive(Serialize)]
struct RuntimeSnapshot {
stream: StreamSnapshot,
live: bool,
last_seen_ms_ago: u64,
summary: SummarySnapshot,
totals: Totals,
workers: Vec<WorkerSnapshot>,
actors: Vec<ActorSnapshot>,
history: Vec<HistorySnapshot>,
}
#[derive(Serialize)]
struct StreamSnapshot {
key: String,
node: String,
life: u64,
}
#[derive(Serialize)]
struct SummarySnapshot {
num_workers: Option<u32>,
uptime_ms: Option<u64>,
actors_live: Option<u32>,
mailbox_depth: Option<u32>,
scheduled_tasks: Option<u32>,
}
#[derive(Serialize)]
struct WorkerSnapshot {
id: Option<u32>,
actor_count: u32,
mailbox_depth: u32,
messages_processed: u64,
local_sends: u64,
cross_sends: u64,
inbox_sends: u64,
type_mismatches: u64,
panics: u64,
messages_dropped: u64,
restarts: u64,
stops: u64,
tick_p50_us: u64,
msg_per_sec: f64,
local_per_sec: f64,
cross_per_sec: f64,
inbox_per_sec: f64,
}
impl WorkerSnapshot {
fn from_state(state: &WorkerState, actor_count: u32) -> Self {
Self {
id: state.id,
actor_count: state.actor_count.max(actor_count),
mailbox_depth: state.mailbox_depth,
messages_processed: state.messages_processed,
local_sends: state.local_sends,
cross_sends: state.cross_sends,
inbox_sends: state.inbox_sends,
type_mismatches: state.type_mismatches,
panics: state.panics,
messages_dropped: state.messages_dropped,
restarts: state.restarts,
stops: state.stops,
tick_p50_us: state.tick_p50_us,
msg_per_sec: state.msg_per_sec,
local_per_sec: state.local_per_sec,
cross_per_sec: state.cross_per_sec,
inbox_per_sec: state.inbox_per_sec,
}
}
}
#[derive(Serialize)]
struct ActorSnapshot {
address: String,
name: Option<String>,
worker_id: Option<u32>,
mailbox_depth: u32,
messages_processed: u64,
msg_per_sec: f64,
last_msg_type: Option<String>,
poisoned: bool,
message_type_counts: Vec<(String, u64)>,
}
#[derive(Serialize)]
struct HistorySnapshot {
ms_ago: u64,
mailbox_depth: u32,
msg_per_sec: f64,
tick_p50_us: u64,
}
impl DashboardView for SwactorWorkerView {
fn id(&self) -> &'static str {
"swactor-workers"
}
fn title(&self) -> &'static str {
"Swactor workers"
}
fn path(&self) -> &'static str {
"swactor/workers"
}
fn channels(&self) -> &'static [&'static str] {
CHANNELS
}
fn ingest(&self, _stream: &StreamId, frame: &Frame, event: &FrameEvent) {
let now = Instant::now();
let mut state = self.state.write();
let key = stream_key(&event.stream);
state
.runtimes
.entry(key)
.or_insert_with(|| RuntimeState::new(event.stream.clone(), now))
.update(frame.channel.as_str(), &frame.payload, now);
}
fn snapshot_json(&self) -> Value {
let now = Instant::now();
let snapshot = WorkerViewSnapshot {
runtimes: self
.state
.read()
.runtimes
.values()
.map(|runtime| runtime_snapshot(runtime, now))
.collect(),
};
serde_json::to_value(snapshot).unwrap_or_else(|_| json!({ "runtimes": [] }))
}
fn html(&self) -> Option<&'static str> {
Some(WORKER_HTML)
}
}
fn runtime_snapshot(runtime: &RuntimeState, now: Instant) -> RuntimeSnapshot {
let mut worker_actor_counts: BTreeMap<Option<u32>, u32> = BTreeMap::new();
for actor in runtime.actors.values() {
*worker_actor_counts.entry(actor.worker_id).or_default() += 1;
}
let mut workers = Vec::new();
if let Some(aggregate) = &runtime.aggregate {
workers.push(WorkerSnapshot::from_state(
aggregate,
worker_actor_counts.get(&None).copied().unwrap_or_default(),
));
}
workers.extend(runtime.workers.iter().map(|(id, worker)| {
WorkerSnapshot::from_state(
worker,
worker_actor_counts
.get(&Some(*id))
.copied()
.unwrap_or_default(),
)
}));
for id in worker_actor_counts.keys().flatten() {
if !runtime.workers.contains_key(id) {
let mut worker = WorkerState::new(Some(*id));
worker.actor_count = worker_actor_counts
.get(&Some(*id))
.copied()
.unwrap_or_default();
workers.push(WorkerSnapshot::from_state(&worker, worker.actor_count));
}
}
RuntimeSnapshot {
stream: StreamSnapshot {
key: stream_key(&runtime.stream),
node: runtime.stream.node.clone(),
life: runtime.stream.life,
},
live: now.duration_since(runtime.last_seen) <= LIVE_TTL,
last_seen_ms_ago: now.duration_since(runtime.last_seen).as_millis() as u64,
summary: SummarySnapshot {
num_workers: runtime.summary.num_workers,
uptime_ms: runtime.summary.uptime_ms,
actors_live: runtime.summary.actors_live,
mailbox_depth: runtime.summary.mailbox_depth,
scheduled_tasks: runtime.summary.scheduled_tasks,
},
totals: runtime.totals(),
workers,
actors: runtime
.actors
.values()
.map(|actor| ActorSnapshot {
address: actor.address.clone(),
name: actor.name.clone(),
worker_id: actor.worker_id,
mailbox_depth: actor.mailbox_depth,
messages_processed: actor.messages_processed,
msg_per_sec: actor.msg_per_sec,
last_msg_type: actor.last_msg_type.clone(),
poisoned: actor.poisoned,
message_type_counts: actor.message_type_counts.clone(),
})
.collect(),
history: runtime
.history
.iter()
.map(|sample| HistorySnapshot {
ms_ago: now.duration_since(sample.at).as_millis() as u64,
mailbox_depth: sample.mailbox_depth,
msg_per_sec: sample.msg_per_sec,
tick_p50_us: sample.tick_p50_us,
})
.collect(),
}
}
fn assign_u32(slot: &mut u32, value: &Value, names: &[&str]) {
if let Some(v) = u32_field(value, names) {
*slot = v;
}
}
fn assign_u64(slot: &mut u64, value: &Value, names: &[&str]) {
if let Some(v) = u64_field(value, names) {
*slot = v;
}
}
fn assign_u64_rate(slot: &mut u64, rate: &mut f64, value: &Value, names: &[&str], elapsed: f64) {
if let Some(next) = u64_field(value, names) {
if elapsed > 0.0 && next > *slot {
*rate = (next - *slot) as f64 / elapsed;
} else if next < *slot {
*rate = 0.0;
}
*slot = next;
}
}
fn u32_field(value: &Value, names: &[&str]) -> Option<u32> {
u64_field(value, names).and_then(|v| u32::try_from(v).ok())
}
fn u64_field(value: &Value, names: &[&str]) -> Option<u64> {
names
.iter()
.find_map(|name| value.get(*name).and_then(value_to_u64))
}
fn string_field(value: &Value, names: &[&str]) -> Option<String> {
names
.iter()
.find_map(|name| value.get(*name).and_then(value_to_string))
}
fn value_to_u32(value: &Value) -> Option<u32> {
value_to_u64(value).and_then(|v| u32::try_from(v).ok())
}
fn value_to_u64(value: &Value) -> Option<u64> {
value
.as_u64()
.or_else(|| value.as_str().and_then(|s| s.parse::<u64>().ok()))
}
fn value_to_string(value: &Value) -> Option<String> {
value.as_str().map(ToOwned::to_owned).or_else(|| {
if value.is_null() {
None
} else {
Some(value.to_string())
}
})
}
fn parse_message_type_counts(value: Option<&Value>) -> Option<Vec<(String, u64)>> {
let value = value?;
if let Some(items) = value.as_array() {
let mut out = Vec::new();
for item in items {
if let Some(pair) = item.as_array()
&& pair.len() >= 2
&& let (Some(name), Some(count)) =
(value_to_string(&pair[0]), value_to_u64(&pair[1]))
{
out.push((name, count));
}
}
return Some(out);
}
if let Some(map) = value.as_object() {
let mut out: Vec<(String, u64)> = map
.iter()
.filter_map(|(name, count)| value_to_u64(count).map(|count| (name.clone(), count)))
.collect();
out.sort_by(|a, b| b.1.cmp(&a.1));
return Some(out);
}
None
}
fn stream_key(stream: &StreamEvent) -> String {
format!("{}#{}", stream.node, stream.life)
}

425
crates/dashboard/src/telemetry.rs
View file

@ -1,425 +0,0 @@
//! Dashboard-owned datastream channel records and local samplers.
//!
//! These records describe what the dashboard/fleet views consume. They are not a
//! datastream catalog: applications opt into them by importing this module and
//! submitting the records through `datastream`'s generic APIs.
use std::time::Instant;
use datastream::{ChannelId, Record};
use serde::{Deserialize, Serialize};
/// Identity / boot — emitted first in a stream; identifies the node and context.
pub const IDENTITY: &str = "identity";
/// Host / resource samples — periodic snapshots of machine resources.
pub const HOST_RESOURCE: &str = "host.resource";
/// Runtime stats — aggregate actor-runtime metrics.
pub const RUNTIME_STATS: &str = "runtime.stats";
/// Per-actor runtime detail — one row per live actor.
pub const RUNTIME_ACTORS: &str = "runtime.actors";
/// Worker-runtime counters — routing/error tallies and tick timing.
pub const RUNTIME_WORKERS: &str = "runtime.workers";
/// Which standard stream a span of process output came from.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ProcStream {
Stdout,
Stderr,
}
impl ProcStream {
pub fn as_str(self) -> &'static str {
match self {
ProcStream::Stdout => "stdout",
ProcStream::Stderr => "stderr",
}
}
}
/// A raw-text process-output channel: `proc.<label>.stdout|stderr`.
pub fn process_output(label: &str, stream: ProcStream) -> ChannelId {
ChannelId::new(format!("proc.{label}.{}", stream.as_str()))
}
/// Adapter for `DatastreamEmitter::process_observer_with`.
pub fn process_output_for_observer(label: &str, is_stderr: bool) -> ChannelId {
let stream = if is_stderr {
ProcStream::Stderr
} else {
ProcStream::Stdout
};
process_output(label, stream)
}
/// Identity / boot record.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct IdentityRecord {
pub node: String,
#[serde(default)]
pub life: u64,
#[serde(default)]
pub node_name: String,
#[serde(default)]
pub listen_addr: String,
#[serde(default)]
pub relay_url: String,
#[serde(default)]
pub version: String,
}
/// Host / resource sample.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ResourceSample {
#[serde(default)]
pub cpu_pct: f32,
#[serde(default)]
pub mem_used_mb: u32,
#[serde(default)]
pub mem_total_mb: u32,
#[serde(default)]
pub gpu_pct: f32,
#[serde(default)]
pub disk_used_gb: u32,
#[serde(default)]
pub net_rx_kbps: u32,
#[serde(default)]
pub net_tx_kbps: u32,
}
/// Aggregate runtime stats.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct RuntimeStats {
#[serde(default)]
pub actors_live: u32,
#[serde(default)]
pub mailbox_depth: u32,
#[serde(default)]
pub scheduled_tasks: u32,
}
/// Per-actor runtime detail.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct ActorRuntimeDetail {
#[serde(default)]
pub actors: Vec<ActorRec>,
}
/// One live actor's stats.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct ActorRec {
#[serde(default)]
pub address: String,
#[serde(default)]
pub name: String,
#[serde(default)]
pub mailbox_depth: u32,
#[serde(default)]
pub messages_processed: u64,
#[serde(default)]
pub last_msg_type: String,
#[serde(default)]
pub poisoned: bool,
#[serde(default)]
pub message_type_counts: Vec<(String, u64)>,
}
/// Worker-runtime counters.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct WorkerCounters {
#[serde(default)]
pub num_workers: u32,
#[serde(default)]
pub scheduled_tasks: u32,
#[serde(default)]
pub local_sends: u64,
#[serde(default)]
pub cross_sends: u64,
#[serde(default)]
pub inbox_sends: u64,
#[serde(default)]
pub type_mismatches: u64,
#[serde(default)]
pub panics: u64,
#[serde(default)]
pub messages_dropped: u64,
#[serde(default)]
pub restarts: u64,
#[serde(default)]
pub stops: u64,
#[serde(default)]
pub messages_processed: u64,
#[serde(default)]
pub tick_p50_us: u64,
}
impl Record for IdentityRecord {
const CHANNEL: &'static str = IDENTITY;
}
impl Record for ResourceSample {
const CHANNEL: &'static str = HOST_RESOURCE;
}
impl Record for RuntimeStats {
const CHANNEL: &'static str = RUNTIME_STATS;
}
impl Record for ActorRuntimeDetail {
const CHANNEL: &'static str = RUNTIME_ACTORS;
}
impl Record for WorkerCounters {
const CHANNEL: &'static str = RUNTIME_WORKERS;
}
/// Build the identity record a node emits first on its stream.
pub fn identity_record(node: &str, life: u64) -> IdentityRecord {
IdentityRecord {
node: node.to_string(),
life,
..Default::default()
}
}
/// Samples host CPU usage across calls.
#[derive(Default)]
pub struct CpuSampler {
prev: Option<(u64, Instant)>,
}
impl CpuSampler {
pub fn sample(&mut self) -> f32 {
let Some(now_busy) = read_proc_stat_busy_jiffies() else {
return 0.0;
};
let now = Instant::now();
let out = if let Some((prev_busy, prev_at)) = self.prev {
let busy = now_busy.saturating_sub(prev_busy) as f64;
let secs = now.duration_since(prev_at).as_secs_f64().max(0.001);
let cpus = num_cpus().max(1) as f64;
((busy / clock_ticks_per_sec()) / secs / cpus * 100.0).clamp(0.0, 100.0) as f32
} else {
0.0
};
self.prev = Some((now_busy, now));
out
}
}
/// Samples host network throughput across calls.
#[derive(Default)]
pub struct NetSampler {
prev: Option<((u64, u64), Instant)>,
}
impl NetSampler {
pub fn sample(&mut self) -> (u32, u32) {
let Some(now_bytes) = read_net_bytes() else {
return (0, 0);
};
let now = Instant::now();
let out = if let Some(((prev_rx, prev_tx), prev_at)) = self.prev {
let secs = now.duration_since(prev_at).as_secs_f64().max(0.001);
let rx = now_bytes.0.saturating_sub(prev_rx) as f64 / secs / 1024.0;
let tx = now_bytes.1.saturating_sub(prev_tx) as f64 / secs / 1024.0;
(
rx.min(u32::MAX as f64) as u32,
tx.min(u32::MAX as f64) as u32,
)
} else {
(0, 0)
};
self.prev = Some((now_bytes, now));
out
}
}
/// Host-resource sampler state carried between ticks.
pub struct HostSampler {
cpu: CpuSampler,
net: NetSampler,
}
impl HostSampler {
pub fn new() -> Self {
Self::default()
}
}
impl Default for HostSampler {
fn default() -> Self {
Self {
cpu: CpuSampler::default(),
net: NetSampler::default(),
}
}
}
/// Read a host-resource sample from real host sources where available.
pub fn read_host_resource(sampler: &mut HostSampler) -> ResourceSample {
let cpu_pct = sampler.cpu.sample();
let (net_rx_kbps, net_tx_kbps) = sampler.net.sample();
let (mem_total_mb, mem_used_mb) = read_meminfo_mb().unwrap_or((0, 0));
ResourceSample {
cpu_pct,
mem_used_mb,
mem_total_mb,
gpu_pct: read_gpu_pct(),
disk_used_gb: read_disk_used_gb(),
net_rx_kbps,
net_tx_kbps,
}
}
fn clock_ticks_per_sec() -> f64 {
#[cfg(target_os = "linux")]
unsafe {
let ticks = libc::sysconf(libc::_SC_CLK_TCK);
if ticks > 0 {
return ticks as f64;
}
}
100.0
}
fn num_cpus() -> u64 {
#[cfg(target_os = "linux")]
{
if let Ok(s) = std::fs::read_to_string("/proc/stat") {
let n = s
.lines()
.filter(|line| {
let rest = line.strip_prefix("cpu");
rest.is_some_and(|r| r.chars().next().is_some_and(|c| c.is_ascii_digit()))
})
.count() as u64;
if n > 0 {
return n;
}
}
}
std::thread::available_parallelism()
.map(|n| n.get() as u64)
.unwrap_or(1)
}
fn read_proc_stat_busy_jiffies() -> Option<u64> {
#[cfg(target_os = "linux")]
{
let s = std::fs::read_to_string("/proc/stat").ok()?;
let line = s.lines().find(|l| l.starts_with("cpu "))?;
let mut vals = line
.split_whitespace()
.skip(1)
.filter_map(|v| v.parse::<u64>().ok());
let user = vals.next()?;
let nice = vals.next()?;
let system = vals.next()?;
let _idle = vals.next()?;
let _iowait = vals.next().unwrap_or(0);
let irq = vals.next().unwrap_or(0);
let softirq = vals.next().unwrap_or(0);
let steal = vals.next().unwrap_or(0);
Some(user + nice + system + irq + softirq + steal)
}
#[cfg(not(target_os = "linux"))]
{
None
}
}
fn read_disk_used_gb() -> u32 {
#[cfg(target_os = "linux")]
unsafe {
let path = std::ffi::CString::new("/").expect("static path");
let mut st: libc::statvfs = std::mem::zeroed();
if libc::statvfs(path.as_ptr(), &mut st) == 0 {
let total = st.f_blocks as u128 * st.f_frsize as u128;
let free = st.f_bfree as u128 * st.f_frsize as u128;
return ((total.saturating_sub(free)) / 1_000_000_000).min(u32::MAX as u128) as u32;
}
}
0
}
fn read_net_bytes() -> Option<(u64, u64)> {
#[cfg(target_os = "linux")]
{
let s = std::fs::read_to_string("/proc/net/dev").ok()?;
let mut rx = 0u64;
let mut tx = 0u64;
for line in s.lines().skip(2) {
let (iface, rest) = line.split_once(':')?;
if iface.trim() == "lo" {
continue;
}
let vals: Vec<&str> = rest.split_whitespace().collect();
if vals.len() >= 16 {
rx = rx.saturating_add(vals[0].parse::<u64>().unwrap_or(0));
tx = tx.saturating_add(vals[8].parse::<u64>().unwrap_or(0));
}
}
Some((rx, tx))
}
#[cfg(not(target_os = "linux"))]
{
None
}
}
fn read_gpu_pct() -> f32 {
#[cfg(target_os = "linux")]
{
if std::path::Path::new("/proc/driver/nvidia/gpus")
.read_dir()
.map(|mut it| it.next().is_some())
.unwrap_or(false)
{
return query_nvidia_smi_util().unwrap_or(0.0);
}
}
0.0
}
#[cfg(target_os = "linux")]
fn query_nvidia_smi_util() -> Option<f32> {
let out = std::process::Command::new("nvidia-smi")
.args([
"--query-gpu=utilization.gpu",
"--format=csv,noheader,nounits",
])
.output()
.ok()?;
if !out.status.success() {
return None;
}
let s = String::from_utf8(out.stdout).ok()?;
let vals: Vec<f32> = s
.lines()
.filter_map(|l| l.trim().parse::<f32>().ok())
.collect();
if vals.is_empty() {
None
} else {
Some(vals.iter().sum::<f32>() / vals.len() as f32)
}
}
fn read_meminfo_mb() -> Option<(u32, u32)> {
#[cfg(target_os = "linux")]
{
let s = std::fs::read_to_string("/proc/meminfo").ok()?;
let mut total = None;
let mut avail = None;
for line in s.lines() {
if let Some(rest) = line.strip_prefix("MemTotal:") {
total = rest.split_whitespace().next()?.parse::<u64>().ok();
} else if let Some(rest) = line.strip_prefix("MemAvailable:") {
avail = rest.split_whitespace().next()?.parse::<u64>().ok();
}
}
let total = total? / 1024;
let avail = avail? / 1024;
Some((total as u32, total.saturating_sub(avail) as u32))
}
#[cfg(not(target_os = "linux"))]
{
None
}
}

82
crates/dashboard/src/topology.rs
View file

@ -1,82 +0,0 @@
//! Actor-to-actor (and worker-to-worker) message flow topology.
//!
//! Currently derives topology from per-worker cross_sends/local_sends stats.
//! Future: sample-based per-actor source→destination tracking with core instrumentation.
use swactor::stats::RuntimeStats;
/// An edge in the topology graph.
#[derive(Debug, Clone, serde::Serialize)]
pub struct TopologyEdge {
pub source: String,
pub target: String,
pub weight: u64,
pub label: String,
}
/// A node in the topology graph.
#[derive(Debug, Clone, serde::Serialize)]
pub struct TopologyNode {
pub id: String,
pub label: String,
pub actor_count: usize,
pub group: usize,
}
/// A snapshot of the current topology.
#[derive(Debug, Clone, serde::Serialize)]
pub struct TopologySnapshot {
pub nodes: Vec<TopologyNode>,
pub edges: Vec<TopologyEdge>,
}
/// Build a worker-level topology from RuntimeStats.
///
/// Workers are nodes, edges represent message flow:
/// - Self-loops for local_sends
/// - Cross-edges distributed proportionally (until per-destination tracking exists)
pub fn worker_topology(stats: &RuntimeStats) -> TopologySnapshot {
let mut nodes = Vec::new();
let mut edges = Vec::new();
for w in &stats.workers {
nodes.push(TopologyNode {
id: format!("w{}", w.id),
label: format!("W{}", w.id),
actor_count: w.num_actors,
group: w.id,
});
// Local sends = self-loop
if w.local_sends > 0 {
edges.push(TopologyEdge {
source: format!("w{}", w.id),
target: format!("w{}", w.id),
weight: w.local_sends,
label: format!("{} local", w.local_sends),
});
}
// Cross sends — without per-destination data, distribute evenly to other workers
if w.cross_sends > 0 && stats.workers.len() > 1 {
let others: Vec<&swactor::stats::WorkerInfo> =
stats.workers.iter().filter(|o| o.id != w.id).collect();
let per_worker = w.cross_sends / others.len() as u64;
let remainder = w.cross_sends % others.len() as u64;
for (i, other) in others.iter().enumerate() {
let count = per_worker + if (i as u64) < remainder { 1 } else { 0 };
if count > 0 {
edges.push(TopologyEdge {
source: format!("w{}", w.id),
target: format!("w{}", other.id),
weight: count,
label: format!("{} cross", count),
});
}
}
}
}
TopologySnapshot { nodes, edges }
}

94
crates/dashboard/src/view.rs Normal file
View file

@ -0,0 +1,94 @@
use std::sync::Arc;
use datastream::frame::{Frame, StreamId};
use parking_lot::RwLock;
use serde::Serialize;
use serde_json::Value;
use crate::FrameEvent;
/// Read-only interpretation of one or more datastream channels.
///
/// Views are observation-only: they fold incoming frames into local state and
/// expose JSON/HTML. They do not send control messages back to the runtime.
pub trait DashboardView: Send + Sync {
fn id(&self) -> &'static str;
fn title(&self) -> &'static str;
fn path(&self) -> &'static str {
self.id()
}
fn channels(&self) -> &'static [&'static str];
fn ingest(&self, stream: &StreamId, frame: &Frame, event: &FrameEvent);
fn snapshot_json(&self) -> Value;
fn html(&self) -> Option<&'static str> {
None
}
}
#[derive(Debug, Clone, Serialize)]
pub struct ViewDescriptor {
pub id: &'static str,
pub title: &'static str,
pub path: &'static str,
pub page: String,
pub api: String,
pub channels: &'static [&'static str],
}
#[derive(Default)]
pub(crate) struct ViewRegistry {
views: RwLock<Vec<Arc<dyn DashboardView>>>,
}
impl ViewRegistry {
pub fn new() -> Self {
Self::default()
}
pub fn register(&self, view: Arc<dyn DashboardView>) {
let mut views = self.views.write();
views.retain(|existing| existing.id() != view.id() && existing.path() != view.path());
views.push(view);
}
pub fn dispatch(&self, stream: &StreamId, frame: &Frame, event: &FrameEvent) {
let channel = frame.channel.as_str();
for view in self.views.read().iter() {
let channels = view.channels();
if channels.is_empty() || channels.contains(&channel) {
view.ingest(stream, frame, event);
}
}
}
pub fn descriptors(&self) -> Vec<ViewDescriptor> {
self.views
.read()
.iter()
.map(|view| ViewDescriptor {
id: view.id(),
title: view.title(),
path: view.path(),
page: format!("/view/{}", view.path()),
api: format!("/api/view/{}", view.path()),
channels: view.channels(),
})
.collect()
}
pub fn snapshot(&self, path: &str) -> Option<Value> {
self.views
.read()
.iter()
.find(|view| view.path() == path || view.id() == path)
.map(|view| view.snapshot_json())
}
pub fn html(&self, path: &str) -> Option<&'static str> {
self.views
.read()
.iter()
.find(|view| view.path() == path || view.id() == path)
.and_then(|view| view.html())
}
}

422
crates/dashboard/src/warnings.rs
View file

@ -1,422 +0,0 @@
//! Automated anomaly detection for the runtime dashboard.
//!
//! Runs on each stats sample, comparing consecutive snapshots to detect
//! growing mailboxes, stalled actors, worker imbalance, and other conditions.
use std::collections::HashMap;
use swactor::actor::ActorAddress;
use swactor::stats::RuntimeStats;
/// Types of warnings the detector can produce.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize)]
#[serde(rename_all = "snake_case")]
pub enum WarningType {
GrowingMailbox,
StalledActor,
PoisonedActor,
WorkerImbalance,
EmptyWorker,
MailboxOverflow,
}
/// Severity levels for warnings.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Serialize)]
#[serde(rename_all = "lowercase")]
pub enum Severity {
Low,
Medium,
High,
Critical,
}
/// An active warning.
#[derive(Debug, Clone, serde::Serialize)]
pub struct Warning {
pub warning_type: WarningType,
pub severity: Severity,
pub entity: String,
pub description: String,
}
/// Configuration for warning thresholds.
#[derive(Debug, Clone)]
pub struct WarningConfig {
/// Consecutive samples with increasing mailbox depth before warning.
pub growing_mailbox_threshold: usize,
/// Consecutive ticks with no message processing while mailbox > 0.
pub stalled_actor_threshold: usize,
/// A worker is "imbalanced" if it has > this ratio times the average load.
pub worker_imbalance_ratio: f64,
}
impl Default for WarningConfig {
fn default() -> Self {
Self {
growing_mailbox_threshold: 5,
stalled_actor_threshold: 10,
worker_imbalance_ratio: 2.0,
}
}
}
/// Per-actor tracking state.
struct ActorState {
prev_mailbox: usize,
prev_messages: u64,
growing_streak: usize,
stalled_streak: usize,
}
/// Warning detection engine. Call `check()` on each stats sample.
pub struct WarningDetector {
config: WarningConfig,
actors: HashMap<ActorAddress, ActorState>,
}
impl WarningDetector {
pub fn new(config: WarningConfig) -> Self {
Self {
config,
actors: HashMap::new(),
}
}
/// Analyze a stats snapshot and return all active warnings.
pub fn check(&mut self, stats: &RuntimeStats) -> Vec<Warning> {
let mut warnings = Vec::new();
// Track which actors are still alive
let mut live_addrs: std::collections::HashSet<ActorAddress> =
std::collections::HashSet::new();
for actor in &stats.actor_details {
live_addrs.insert(actor.address);
// Poisoned actor — immediate critical warning
if actor.poisoned {
warnings.push(Warning {
warning_type: WarningType::PoisonedActor,
severity: Severity::Critical,
entity: format!("{}", actor.address),
description: "Actor is poisoned (panicked)".to_string(),
});
}
let state = self.actors.entry(actor.address).or_insert(ActorState {
prev_mailbox: actor.mailbox_depth,
prev_messages: actor.messages_processed,
growing_streak: 0,
stalled_streak: 0,
});
// Growing mailbox detection
if actor.mailbox_depth > state.prev_mailbox && actor.mailbox_depth > 0 {
state.growing_streak += 1;
} else {
state.growing_streak = 0;
}
if state.growing_streak >= self.config.growing_mailbox_threshold {
warnings.push(Warning {
warning_type: WarningType::GrowingMailbox,
severity: Severity::Medium,
entity: format!("{}", actor.address),
description: format!(
"Mailbox growing for {} consecutive samples (depth: {})",
state.growing_streak, actor.mailbox_depth,
),
});
}
// Stalled actor detection
if actor.messages_processed == state.prev_messages && actor.mailbox_depth > 0 {
state.stalled_streak += 1;
} else {
state.stalled_streak = 0;
}
if state.stalled_streak >= self.config.stalled_actor_threshold {
warnings.push(Warning {
warning_type: WarningType::StalledActor,
severity: Severity::High,
entity: format!("{}", actor.address),
description: format!(
"No messages processed for {} ticks with {} pending",
state.stalled_streak, actor.mailbox_depth,
),
});
}
state.prev_mailbox = actor.mailbox_depth;
state.prev_messages = actor.messages_processed;
}
// Clean up dead actors
self.actors.retain(|addr, _| live_addrs.contains(addr));
// Mailbox overflow detection
for w in &stats.workers {
if w.messages_dropped > 0 {
warnings.push(Warning {
warning_type: WarningType::MailboxOverflow,
severity: Severity::Medium,
entity: format!("Worker {}", w.id),
description: format!("{} messages dropped", w.messages_dropped),
});
}
}
// Worker imbalance and empty worker detection
if stats.workers.len() > 1 {
let total_actors: usize = stats.workers.iter().map(|w| w.num_actors).sum();
let avg = total_actors as f64 / stats.workers.len() as f64;
for w in &stats.workers {
if avg > 0.0 && w.num_actors as f64 > avg * self.config.worker_imbalance_ratio {
warnings.push(Warning {
warning_type: WarningType::WorkerImbalance,
severity: Severity::Low,
entity: format!("Worker {}", w.id),
description: format!(
"{} actors vs {:.0} average ({:.1}x)",
w.num_actors,
avg,
w.num_actors as f64 / avg,
),
});
}
if w.num_actors == 0 && total_actors > 0 {
warnings.push(Warning {
warning_type: WarningType::EmptyWorker,
severity: Severity::Low,
entity: format!("Worker {}", w.id),
description: "Worker has no actors while others do".to_string(),
});
}
}
}
// Sort by severity (critical first)
warnings.sort_by(|a, b| b.severity.cmp(&a.severity));
warnings
}
}
#[cfg(test)]
mod tests {
use super::*;
use swactor::stats::{ActorInfo, WorkerInfo};
fn make_worker(id: usize, actors: usize, dropped: u64) -> WorkerInfo {
WorkerInfo {
id,
num_actors: actors,
mailbox_depth: 0,
messages_processed: 0,
local_sends: 0,
cross_sends: 0,
inbox_sends: 0,
type_mismatches: 0,
panics: 0,
messages_dropped: dropped,
restarts: 0,
stops: 0,
}
}
fn make_actor(id: u8, depth: usize, msgs: u64, poisoned: bool) -> ActorInfo {
ActorInfo {
address: ActorAddress([id; 32]),
name: None,
worker_id: 0,
mailbox_depth: depth,
last_msg_type: None,
messages_processed: msgs,
poisoned,
message_type_counts: Vec::new(),
}
}
fn make_stats(workers: Vec<WorkerInfo>, actors: Vec<ActorInfo>) -> RuntimeStats {
RuntimeStats {
num_workers: workers.len(),
uptime_ms: 0,
actors: actors.iter().map(|a| (a.address, a.worker_id)).collect(),
workers,
actor_details: actors,
tick_timings: Vec::new(),
}
}
#[test]
fn poisoned_actor_triggers_critical_warning() {
let mut detector = WarningDetector::new(WarningConfig::default());
let stats = make_stats(vec![make_worker(0, 1, 0)], vec![make_actor(1, 0, 10, true)]);
let warnings = detector.check(&stats);
assert!(
warnings
.iter()
.any(|w| w.warning_type == WarningType::PoisonedActor)
);
assert!(warnings.iter().any(|w| w.severity == Severity::Critical));
}
#[test]
fn growing_mailbox_triggers_after_threshold() {
let config = WarningConfig {
growing_mailbox_threshold: 3,
..Default::default()
};
let mut detector = WarningDetector::new(config);
// 4 samples with increasing mailbox: should trigger at sample 4
for depth in 1..=4 {
let stats = make_stats(
vec![make_worker(0, 1, 0)],
vec![make_actor(1, depth, 0, false)],
);
let warnings = detector.check(&stats);
if depth < 4 {
assert!(
!warnings
.iter()
.any(|w| w.warning_type == WarningType::GrowingMailbox),
"should not trigger at depth {}",
depth
);
} else {
assert!(
warnings
.iter()
.any(|w| w.warning_type == WarningType::GrowingMailbox),
"should trigger at depth {}",
depth
);
}
}
}
#[test]
fn growing_mailbox_resets_on_decrease() {
let config = WarningConfig {
growing_mailbox_threshold: 3,
..Default::default()
};
let mut detector = WarningDetector::new(config);
// Grow for 2 samples, then decrease, then grow again
for depth in [1, 2, 1, 2, 3, 4] {
let stats = make_stats(
vec![make_worker(0, 1, 0)],
vec![make_actor(1, depth, 0, false)],
);
detector.check(&stats);
}
// After 1,2 → streak=2; then 1 → streak=0; then 2,3,4 → streak=3 → triggers
let stats = make_stats(vec![make_worker(0, 1, 0)], vec![make_actor(1, 5, 0, false)]);
let warnings = detector.check(&stats);
assert!(
warnings
.iter()
.any(|w| w.warning_type == WarningType::GrowingMailbox)
);
}
#[test]
fn stalled_actor_triggers_when_not_processing() {
let config = WarningConfig {
stalled_actor_threshold: 3,
..Default::default()
};
let mut detector = WarningDetector::new(config);
// Same messages_processed, nonzero mailbox for 4 ticks
for _ in 0..4 {
let stats = make_stats(
vec![make_worker(0, 1, 0)],
vec![make_actor(1, 5, 100, false)],
);
let warnings = detector.check(&stats);
// Last one should trigger
if warnings
.iter()
.any(|w| w.warning_type == WarningType::StalledActor)
{
return; // test passed
}
}
panic!("expected StalledActor warning");
}
#[test]
fn worker_imbalance_detected() {
let mut detector = WarningDetector::new(WarningConfig::default());
// Worker 0: 10 actors, Worker 1: 1 actor. Avg=5.5, ratio=10/5.5=1.8
// With ratio threshold 2.0, this should NOT trigger
let stats = make_stats(vec![make_worker(0, 10, 0), make_worker(1, 1, 0)], vec![]);
let warnings = detector.check(&stats);
assert!(
!warnings
.iter()
.any(|w| w.warning_type == WarningType::WorkerImbalance)
);
// Worker 0: 20 actors, Worker 1: 1 actor. Avg=10.5, ratio=20/10.5=1.9 — still no
// Worker 0: 30 actors, Worker 1: 1 actor. Avg=15.5, ratio=30/15.5=1.9 — still no
// Worker 0: 100 actors, Worker 1: 1 actor. Avg=50.5, ratio=100/50.5=1.98 — almost
// Worker 0: 100 actors, Worker 1: 0 actor. Avg=50, ratio=100/50=2.0 — at threshold
let stats2 = make_stats(vec![make_worker(0, 100, 0), make_worker(1, 1, 0)], vec![]);
let warnings2 = detector.check(&stats2);
// 100 / 50.5 = 1.98 — not > 2.0
assert!(
!warnings2
.iter()
.any(|w| w.warning_type == WarningType::WorkerImbalance)
);
// Now 200 vs 1: 200/100.5 = ~1.99 — still not. Let's do 300 vs 1: 300/150.5 = ~2.0
// Actually need > 2x. Let's do 50 vs 1: avg=25.5, ratio=50/25.5=1.96. Nope.
// 10 vs 1 vs 1: avg=4, ratio=10/4=2.5 — triggers!
let stats3 = make_stats(
vec![
make_worker(0, 10, 0),
make_worker(1, 1, 0),
make_worker(2, 1, 0),
],
vec![],
);
let warnings3 = detector.check(&stats3);
assert!(
warnings3
.iter()
.any(|w| w.warning_type == WarningType::WorkerImbalance)
);
}
#[test]
fn empty_worker_detected() {
let mut detector = WarningDetector::new(WarningConfig::default());
let stats = make_stats(vec![make_worker(0, 5, 0), make_worker(1, 0, 0)], vec![]);
let warnings = detector.check(&stats);
assert!(
warnings
.iter()
.any(|w| w.warning_type == WarningType::EmptyWorker)
);
}
#[test]
fn mailbox_overflow_detected() {
let mut detector = WarningDetector::new(WarningConfig::default());
let stats = make_stats(vec![make_worker(0, 1, 42)], vec![]);
let warnings = detector.check(&stats);
assert!(
warnings
.iter()
.any(|w| w.warning_type == WarningType::MailboxOverflow)
);
}
}

153
crates/dashboard/tests/dashboard_core.rs
View file

@ -1,153 +0,0 @@
use std::io::{Read, Write};
use std::net::{SocketAddr, TcpListener, TcpStream};
use std::thread;
use std::time::{Duration, Instant};
use dashboard::{DashboardConfig, start_dashboard};
fn free_port() -> u16 {
TcpListener::bind("127.0.0.1:0")
.expect("bind ephemeral port")
.local_addr()
.expect("local addr")
.port()
}
fn wait_for_http(port: u16) {
let addr: SocketAddr = format!("127.0.0.1:{port}").parse().unwrap();
let deadline = Instant::now() + Duration::from_secs(5);
while Instant::now() < deadline {
if TcpStream::connect_timeout(&addr, Duration::from_millis(100)).is_ok() {
return;
}
thread::sleep(Duration::from_millis(20));
}
panic!("dashboard HTTP server did not start on port {port}");
}
fn read_sse_until(port: u16, needle: &str) -> String {
let mut stream = TcpStream::connect(("127.0.0.1", port)).expect("connect dashboard");
stream
.set_read_timeout(Some(Duration::from_millis(250)))
.expect("set read timeout");
write!(
stream,
"GET /events HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAccept: text/event-stream\r\nConnection: close\r\n\r\n"
)
.expect("write request");
let deadline = Instant::now() + Duration::from_secs(5);
let mut out = String::new();
let mut buf = [0_u8; 4096];
while Instant::now() < deadline {
match stream.read(&mut buf) {
Ok(0) => break,
Ok(n) => {
out.push_str(&String::from_utf8_lossy(&buf[..n]));
if out.contains(needle) {
return out;
}
}
Err(e)
if e.kind() == std::io::ErrorKind::WouldBlock
|| e.kind() == std::io::ErrorKind::TimedOut => {}
Err(e) => panic!("read SSE response: {e}"),
}
}
panic!("SSE response did not contain {needle:?}; response was: {out}");
}
fn activity_json_from_sse(response: &str) -> serde_json::Value {
let mut in_activity = false;
for line in response.lines() {
if line == "event: activity" {
in_activity = true;
continue;
}
if in_activity && let Some(json) = line.strip_prefix("data: ") {
return serde_json::from_str(json).expect("activity JSON");
}
}
panic!("no activity event in SSE response: {response}");
}
#[test]
fn activity_events_are_streamed_over_public_http_api() {
let port = free_port();
let dashboard = start_dashboard(DashboardConfig {
port,
event_capacity: 10,
..DashboardConfig::default()
});
dashboard.start_http_standalone();
wait_for_http(port);
dashboard.push_activity(false, "datastream connected");
dashboard.push_activity(true, "mailbox pressure rising");
let response = read_sse_until(port, "mailbox pressure rising");
dashboard.shutdown();
let events = activity_json_from_sse(&response);
let rows = events.as_array().expect("activity array");
assert_eq!(rows.len(), 2);
assert_eq!(rows[0]["message"], "datastream connected");
assert_eq!(rows[0]["level"], "INFO");
assert_eq!(rows[1]["message"], "mailbox pressure rising");
assert_eq!(rows[1]["level"], "WARN");
}
#[test]
fn activity_stream_honors_event_capacity_for_late_clients() {
let port = free_port();
let dashboard = start_dashboard(DashboardConfig {
port,
event_capacity: 2,
..DashboardConfig::default()
});
dashboard.start_http_standalone();
wait_for_http(port);
dashboard.push_activity(false, "oldest");
dashboard.push_activity(false, "middle");
dashboard.push_activity(false, "newest");
let response = read_sse_until(port, "newest");
dashboard.shutdown();
let events = activity_json_from_sse(&response);
let rows = events.as_array().expect("activity array");
assert_eq!(rows.len(), 2);
assert_eq!(rows[0]["message"], "middle");
assert_eq!(rows[1]["message"], "newest");
assert!(
!response.contains("oldest"),
"late client should only receive the retained activity window"
);
}
#[test]
fn activity_sequences_are_gap_free_for_retained_window() {
let port = free_port();
let dashboard = start_dashboard(DashboardConfig {
port,
event_capacity: 10,
..DashboardConfig::default()
});
dashboard.start_http_standalone();
wait_for_http(port);
for i in 0..5 {
dashboard.push_activity(false, format!("source-{}-event", i % 2));
}
let response = read_sse_until(port, "source-0-event");
dashboard.shutdown();
let events = activity_json_from_sse(&response);
let rows = events.as_array().expect("activity array");
assert_eq!(rows.len(), 5);
for (idx, row) in rows.iter().enumerate() {
assert_eq!(row["seq"], idx as u64);
}
}

155
crates/dashboard/tests/t_datastream_render.rs
View file

@ -1,155 +0,0 @@
//! Dashboard render contract: source records → frames → reconstructed views.
//!
//! Feeds the real `FleetView` consumer a scripted set of one node's frames and
//! checks the `FleetUpdate` it produces carries the migrated metrics: the
//! distribution panels show real cache / registry / directory / peer-auth
//! values, and the Actors table shows the real per-actor rows. Every expectation
//! is derived from the input records, never read back from the consumer, and the
//! test asserts on reconstructed JSON / stats — not on rendered HTML — so it
//! survives a UI refactor.
use dashboard::datastream_source::{FleetUpdate, FleetView};
use dashboard::telemetry::{ActorRec, ActorRuntimeDetail, IdentityRecord};
use datastream::Record;
use datastream::frame::{Frame, Lifetime, NodeId, Position, StreamId};
use distribution::telemetry::DistributionState;
fn node_stream() -> StreamId {
StreamId::new(NodeId::new(&"ab".repeat(32)), Lifetime(1))
}
fn frame<R: Record>(record: &R, pos: u64) -> Frame {
Frame::new(R::channel(), Position(pos), record.encode())
}
fn identity(stream: &StreamId, node_name: &str) -> IdentityRecord {
IdentityRecord {
node: stream.node.as_str().to_string(),
life: stream.life.0,
node_name: node_name.to_string(),
listen_addr: "host:4242".to_string(),
relay_url: String::new(),
version: "v1".to_string(),
}
}
/// Fold a script of frames for one node and return the final `FleetUpdate`.
fn ingest_all(view: &mut FleetView, stream: &StreamId, frames: &[Frame]) -> FleetUpdate {
let mut last = None;
for f in frames {
last = Some(view.ingest(stream, f));
}
last.expect("at least one frame")
}
#[test]
fn distribution_panels_render_real_state_from_frames() {
let stream = node_stream();
let mut view = FleetView::new(None);
let dist = DistributionState {
cache_size: 4,
directory_route_count: 9,
registry_size: 6,
registry_tombstones: 1,
peer_auth_mode: "allow-list".into(),
authorized_peer_count: 3,
..Default::default()
};
let frames = vec![
frame(&identity(&stream, "swift-falcon"), 0),
frame(&dist, 1),
];
let update = ingest_all(&mut view, &stream, &frames);
let dj: serde_json::Value = serde_json::from_str(
update
.dist_json
.as_ref()
.expect("dist json for selected node"),
)
.unwrap();
// The migrated distribution metrics arrive as real values (not the zeros the
// pre-datastream reconstruction used to fill).
assert_eq!(dj["cache_size"], 4);
assert_eq!(dj["directory_route_count"], 9);
assert_eq!(dj["registry_size"], 6);
assert_eq!(dj["registry_tombstones"], 1);
assert_eq!(dj["peer_auth_mode"], "allow-list");
assert_eq!(dj["authorized_peer_count"], 3);
// Identity extras ride the identity channel.
assert_eq!(dj["node_name"], "swift-falcon");
assert_eq!(dj["listen_addr"], "host:4242");
assert_eq!(dj["version"], "v1");
}
#[test]
fn actors_table_renders_real_per_actor_rows() {
let stream = node_stream();
let mut view = FleetView::new(None);
let detail = ActorRuntimeDetail {
actors: vec![
ActorRec {
address: "ab".repeat(32),
name: "SwimActor".into(),
mailbox_depth: 2,
messages_processed: 42,
last_msg_type: "Ping".into(),
poisoned: false,
message_type_counts: vec![("Ping".into(), 42)],
},
ActorRec {
address: "cd".repeat(32),
name: "RegistryActor".into(),
mailbox_depth: 0,
messages_processed: 7,
last_msg_type: "Tick".into(),
poisoned: true,
message_type_counts: vec![],
},
],
};
let frames = vec![frame(&identity(&stream, "node"), 0), frame(&detail, 1)];
let update = ingest_all(&mut view, &stream, &frames);
let stats = update.stats.expect("stats for the selected node");
// The real per-actor rows are rendered (not synthetic per-channel rows): the
// names, message tallies, and poisoned flag carry through unchanged.
let swim = stats
.actor_details
.iter()
.find(|a| a.name.as_deref() == Some("SwimActor"))
.expect("SwimActor row");
assert_eq!(swim.messages_processed, 42);
assert_eq!(swim.mailbox_depth, 2);
assert!(!swim.poisoned);
let registry = stats
.actor_details
.iter()
.find(|a| a.name.as_deref() == Some("RegistryActor"))
.expect("RegistryActor row");
assert!(registry.poisoned, "poisoned flag carried through");
}
#[test]
fn dashboard_ignores_unknown_plugin_channel_without_corrupting_stream_state() {
let stream = node_stream();
let mut view = FleetView::new(None);
let frames = vec![
frame(&identity(&stream, "node"), 0),
Frame::new(
"external.plugin.sample",
Position(1),
br#"{"value":42}"#.to_vec(),
),
];
let update = ingest_all(&mut view, &stream, &frames);
let fleet: serde_json::Value = serde_json::from_str(&update.fleet_json).unwrap();
let rows = fleet["nodes"].as_array().expect("fleet rows");
assert_eq!(rows.len(), 1);
assert_eq!(rows[0]["id"], stream.node.as_str());
}

6
crates/datastream/DATASTREAM_SPEC.md
View file

@ -308,8 +308,8 @@ pub trait ChannelClassifier {
```
Producers and consumers own the records and channel constants for their domains.
A dashboard, plugin, or application may compose a registry of the channels it
understands, but the pipe never consults that registry. Adding a channel or
A plugin or application may compose a registry of the channels it understands,
but the pipe never consults that registry. Adding a channel or
teaching a view a new codec changes nothing in the mux, transport, ingest, or
store.
@ -433,7 +433,7 @@ them changes anything above or below.
### 6.2 Fan-out: drain once, distribute
A node may feed more than one consumer (a local dashboard render **and** a
A node may feed more than one consumer (for example, a local renderer and a
remote collector). Draining is destructive — the buffer can be drained once —
so fan-out **cannot** be "each sink drains."

9
crates/datastream/src/sink_actor.rs
View file

@ -6,11 +6,10 @@
//! on the collector (e.g. the orchestrator) — receives them, decodes each back
//! into a `(StreamId, Frame)` delivery, and hands it to a caller-supplied fold.
//!
//! It deliberately knows nothing about the dashboard: folding a delivery into a
//! `FleetView` lives in the `dashboard` crate, which `distribution` must not
//! depend on. The actor therefore owns an opaque callback, and the binary that
//! has both crates in scope wires a `FleetView` into it. Malformed payloads are
//! dropped silently — the same best-effort tolerance the UDP ingest had.
//! It deliberately knows nothing about any view layer. The actor owns an opaque
//! callback so binaries can wire the decoded deliveries into whichever fold they
//! need. Malformed payloads are dropped silently — the same best-effort tolerance
//! the UDP ingest had.
use swactor::actor::ActorInterface;
use swactor::runtime::Ctx;

2
crates/datastream/tests/t_datastream.rs
View file

@ -197,7 +197,7 @@ fn typed_codec_tolerates_version_skew() {
// A consolidated record evolves the same way: an unknown panel is ignored,
// and a record predating the per-entry vectors decodes them as empty — so a
// newer fleet dashboard and an older node stay compatible (spec §6.3).
// newer fleet view and an older node stay compatible (spec §6.3).
let dist_skew = br#"{"cache_size":4,"unknown_panel":[1,2,3]}"#;
let decoded = DistributionState::decode(dist_skew).expect("dist.state tolerates skew");
assert_eq!(decoded.cache_size, 4);

2
crates/distribution/src/registry.rs
View file

@ -333,7 +333,7 @@ impl ClusterRegistry {
tombstone: e.tombstone,
})
.collect();
// Stable order so the dashboard table doesn't reshuffle each tick.
// Stable order so observers don't reshuffle each tick.
entries.sort_by(|a, b| a.name.cmp(&b.name));
let tombstones = entries.iter().filter(|e| e.tombstone).count();
RegistrySnapshot {

14
crates/distribution/src/snapshot.rs
View file

@ -1,13 +1,11 @@
//! Serializable shape of a node's observable distribution state.
//!
//! This is the JSON contract the Distribution page renders. The node no longer
//! *collects* it by polling — per-node telemetry now flows over the datastream,
//! and the dashboard's datastream consumer reconstructs this exact field shape
//! from the `membership` / `identity` / `dist.state` channels. The type is
//! retained as the shared wire shape so external consumers (the example
//! clusters, the docker integration tests) can deserialize a node's
//! `/api/distribution` response, and so a producer that builds the shape
//! directly (the example apps) has one definition to target.
//! This is the JSON contract distribution observers render. The node no longer
//! *collects* it by polling — per-node telemetry now flows over the datastream.
//! The type is retained as the shared wire shape so external consumers (the
//! example clusters, the docker integration tests) can deserialize a node's
//! `/api/distribution` response, and so producers that build the shape directly
//! have one definition to target.
use serde::{Deserialize, Serialize};

7
crates/iroh-driver/src/iroh_driver.rs
View file

@ -551,9 +551,8 @@ impl IrohDriver {
/// Capture the driver-owned slice of the node's observable state: identity,
/// listen address, and the directory route-view extent. The core node no
/// longer polls this (its telemetry flows over the datastream), but the
/// example apps still build their dashboards from this shape, so it is kept
/// as a convenience over [`listen_addr`](Self::listen_addr) and
/// longer polls this (its telemetry flows over the datastream), so this is
/// kept as a convenience over [`listen_addr`](Self::listen_addr) and
/// [`directory_route_count`](Self::directory_route_count).
pub fn snapshot(&self) -> DistributionNodeSnapshot {
let mut snap = DistributionNodeSnapshot::empty(self.node_id());
@ -596,7 +595,7 @@ impl IrohDriver {
.filter(|(_, host)| **host != self_id)
.map(|(actor, host)| (*actor, *host))
.collect();
// Stable order so the dashboard table doesn't reshuffle each tick.
// Stable order so observers don't reshuffle each tick.
entries.sort_by(|a, b| a.0.0.cmp(&b.0.0));
entries
})

5
crates/iroh-driver/tests/common/iroh.rs
View file

@ -14,8 +14,7 @@
//!
//! Membership is observed through the harness `membership_mirror` (a
//! `MemberList` filled by the [`MembershipFanout`] from SWIM's
//! `MembershipChanged` stream) — the same derivation production uses for its
//! dashboard snapshot. The driver snapshot no longer carries members.
//! `MembershipChanged` stream). The driver snapshot no longer carries members.
#![allow(dead_code)]
use std::collections::HashMap;
@ -393,7 +392,7 @@ where
/// Whether `node` sees `peer_key` in membership `state`. Membership comes from
/// the [`MembershipFanout`]-filled mirror, not the (now memberless) driver
/// snapshot — exactly as production derives its dashboard members.
/// snapshot.
pub fn sees_state(node: &IrohNode, peer_key: &PublicKey, state: &str) -> bool {
let want = match state {
"alive" => MemberState::Alive,

12
crates/mvp-system/Cargo.toml
View file

@ -13,7 +13,6 @@ local-e2e = [
"dep:iroh-driver",
"dep:iroh",
"dep:tokio",
"dep:dashboard",
"dep:parking_lot",
]
@ -27,7 +26,6 @@ swactor-transport = { path = "../transport", optional = true }
distribution = { path = "../distribution", optional = true }
iroh-driver = { path = "../iroh-driver", optional = true }
iroh = { version = "0.98", optional = true }
dashboard = { path = "../dashboard", optional = true }
tokio = { version = "1", features = ["rt-multi-thread", "macros", "process", "io-util", "sync", "time"], optional = true }
[target.'cfg(target_os = "linux")'.dependencies]
@ -38,6 +36,11 @@ name = "mvp-local-e2e"
path = "src/bin/local_e2e.rs"
required-features = ["local-e2e"]
[[bin]]
name = "mvp-local-e2e-cluster"
path = "src/bin/local_e2e_cluster.rs"
required-features = ["local-e2e"]
[[bin]]
name = "mvp-dumb-worker"
path = "src/bin/dumb_worker.rs"
@ -52,3 +55,8 @@ required-features = ["local-e2e"]
name = "gpu_worker_node_e2e"
path = "tests/gpu_worker_node_e2e.rs"
required-features = ["local-e2e"]
[[test]]
name = "local-e2e-cluster"
path = "tests/local_e2e_cluster.rs"
required-features = ["local-e2e"]

64
crates/mvp-system/src/arena_manager.rs
View file

@ -184,7 +184,7 @@ impl ArenaManager {
})
}
fn request(&mut self, request: ArenaRequest) -> Vec<ArenaEvent> {
pub fn request(&mut self, request: ArenaRequest) -> Vec<ArenaEvent> {
match request {
ArenaRequest::LeaseRing(request) => self.lease_ring(request),
ArenaRequest::CancelLease { request_id } => {
@ -196,16 +196,31 @@ impl ArenaManager {
}
}
fn live_leases(&self) -> &[RingLease] {
pub fn live_leases(&self) -> &[RingLease] {
&self.live_order
}
fn lookup_lease(&self, ring_id: RingId) -> Option<&RingLease> {
pub fn lookup_lease(&self, ring_id: RingId) -> Option<&RingLease> {
self.live_index
.get(&ring_id)
.and_then(|index| self.live_order.get(*index))
}
#[cfg(target_os = "linux")]
pub fn arena_fd(&self) -> std::os::fd::RawFd {
self._backing.fd()
}
#[cfg(target_os = "linux")]
pub fn write_arena(&self, offset: u64, bytes: &[u8]) -> Result<(), std::io::Error> {
self._backing.write_at(offset, bytes)
}
#[cfg(target_os = "linux")]
pub fn read_arena(&self, offset: u64, len: usize) -> Result<Vec<u8>, std::io::Error> {
self._backing.read_at(offset, len)
}
fn lease_ring(&mut self, request: LeaseRing) -> Vec<ArenaEvent> {
if self.state == ArenaState::ShuttingDown {
return vec![ArenaEvent::RingLeaseRejected {
@ -479,7 +494,7 @@ fn gcd(mut left: u64, mut right: u64) -> u64 {
struct ArenaBacking {
len: usize,
ptr: *mut libc::c_void,
_fd: std::os::fd::OwnedFd,
fd: std::os::fd::OwnedFd,
}
#[cfg(target_os = "linux")]
@ -497,7 +512,7 @@ impl ArenaBacking {
let fd = unsafe {
let name = b"mvp-system-arena\0";
libc::memfd_create(name.as_ptr().cast(), libc::MFD_CLOEXEC)
libc::memfd_create(name.as_ptr().cast(), 0)
};
if fd < 0 {
return Err(ArenaFault::BackingUnavailable);
@ -524,7 +539,44 @@ impl ArenaBacking {
return Err(ArenaFault::BackingUnavailable);
}
Ok(Self { len, ptr, _fd: fd })
Ok(Self { len, ptr, fd })
}
fn fd(&self) -> std::os::fd::RawFd {
std::os::fd::AsRawFd::as_raw_fd(&self.fd)
}
fn write_at(&self, offset: u64, bytes: &[u8]) -> Result<(), std::io::Error> {
let written = unsafe {
libc::pwrite(
self.fd(),
bytes.as_ptr().cast(),
bytes.len(),
offset as libc::off_t,
)
};
if written == bytes.len() as isize {
Ok(())
} else {
Err(std::io::Error::last_os_error())
}
}
fn read_at(&self, offset: u64, len: usize) -> Result<Vec<u8>, std::io::Error> {
let mut bytes = vec![0u8; len];
let read = unsafe {
libc::pread(
self.fd(),
bytes.as_mut_ptr().cast(),
bytes.len(),
offset as libc::off_t,
)
};
if read == len as isize {
Ok(bytes)
} else {
Err(std::io::Error::last_os_error())
}
}
}

218
crates/mvp-system/src/bin/local_e2e.rs
View file

@ -18,14 +18,13 @@ use mvp_system::actors::orchestrator::{
StageRefWire,
};
use mvp_system::actors::register_mvp_actor_codecs;
use mvp_system::dashboard::MvpDashboard;
use mvp_system::distribution_stack::DistributionRuntimeStack;
use mvp_system::driver_pumps;
use mvp_system::engine_builder as engine;
use mvp_system::observability_surface as obs;
use mvp_system::orchestrator_run_fsm as fsm;
use mvp_system::run_plan as plan;
use mvp_system::stage_controller as stage;
use mvp_system::tx_rx_edge_actor as edge_actor;
use serde::{Deserialize, Serialize};
use serde_json::json;
use swactor::actor::ActorAddress;
@ -41,10 +40,8 @@ fn main() -> ExitCode {
let args = std::env::args().collect::<Vec<_>>();
let result = if args.iter().any(|arg| arg == "--role=node") {
run_node_role(&args)
} else if std::env::var_os("MVP_DASHBOARD").is_some() {
run_supervisor_dashboard_loop()
} else {
run_supervisor_once(RUN_ID, None, true)
run_supervisor_once(RUN_ID, true)
};
match result {
@ -81,86 +78,10 @@ struct NodeReady {
struct NodeStdoutLine {
#[serde(rename = "type")]
kind: String,
stage_index: Option<u32>,
event: Option<String>,
}
fn record_dashboard_event(dashboard: &mut Option<&mut MvpDashboard>, event: obs::Event) {
if let Some(dashboard) = dashboard.as_deref_mut() {
dashboard.record_event(event);
}
}
fn run_event(run_id: u64, kind: obs::EventKind) -> obs::Event {
obs::Event::RunScoped {
kind,
run_id: obs::RunId(run_id),
reason: None,
component: obs::Component::Orchestrator,
}
}
fn run_fault_event(run_id: u64) -> obs::Event {
obs::Event::RunScoped {
kind: obs::EventKind::RunFaulted,
run_id: obs::RunId(run_id),
reason: Some(obs::FaultReason::WorkerCrashed),
component: obs::Component::Orchestrator,
}
}
fn node_event(node_id: u64, kind: obs::EventKind) -> obs::Event {
obs::Event::NodeScoped {
kind,
node_id: obs::NodeId(node_id),
component: obs::Component::NodeBoot,
}
}
fn stage_event(run_id: u64, stage_index: u32, kind: obs::EventKind) -> obs::Event {
obs::Event::StageScoped {
kind,
run_id: obs::RunId(run_id),
stage_index: obs::StageIndex(stage_index),
reason: if kind == obs::EventKind::StageFaulted {
Some(obs::FaultReason::WorkerCrashed)
} else {
None
},
component: obs::Component::StageController,
}
}
fn object_event(run_id: u64, object_id: u64, sequence: u64, kind: obs::EventKind) -> obs::Event {
let _ = run_id;
obs::Event::ObjectScoped {
kind,
object_id: obs::ObjectId(object_id),
sequence: obs::Sequence(sequence),
component: obs::Component::TokenEndpoint,
}
}
fn run_supervisor_dashboard_loop() -> Result<(), String> {
let mut dashboard = MvpDashboard::start_from_env()?;
eprintln!("mvp-local-e2e: dashboard {}", dashboard.url());
eprintln!("mvp-local-e2e: MVP_DASHBOARD=1, repeating local scenario until Ctrl+C");
let mut run_id = RUN_ID;
loop {
match run_supervisor_once(run_id, Some(&mut dashboard), false) {
Ok(()) => eprintln!("mvp-local-e2e: run {run_id} ok"),
Err(error) => eprintln!("mvp-local-e2e: run {run_id} failed: {error}"),
}
run_id = run_id.saturating_add(1);
thread::sleep(Duration::from_secs(1));
}
}
fn run_supervisor_once(
run_id: u64,
mut dashboard: Option<&mut MvpDashboard>,
print_summary: bool,
) -> Result<(), String> {
fn run_supervisor_once(run_id: u64, print_summary: bool) -> Result<(), String> {
let _tokio = tokio::runtime::Runtime::new().map_err(|e| format!("tokio runtime: {e}"))?;
let mut driver = new_driver(_tokio.handle().clone())?;
let stack = DistributionRuntimeStack::new_with_codecs(
@ -193,10 +114,6 @@ fn run_supervisor_once(
))
.map_err(|e| format!("spawn orchestrator actor: {e}"))?;
stack.register_local_actor(driver.register_actor(orchestrator_addr, 1));
record_dashboard_event(
&mut dashboard,
node_event(ORCHESTRATOR_LOGICAL_NODE_ID, obs::EventKind::NodeStarted),
);
let token_out_listener = TcpListener::bind("127.0.0.1:0")
.map_err(|e| format!("bind orchestrator token-out listener: {e}"))?;
@ -245,16 +162,6 @@ fn run_supervisor_once(
&self_endpoint_json,
&orchestrator_actor_json,
)?;
for stage in [&stage1, &stage0] {
record_dashboard_event(
&mut dashboard,
node_event(stage.node_id.0, obs::EventKind::NodeStarted),
);
record_dashboard_event(
&mut dashboard,
node_event(stage.node_id.0, obs::EventKind::NodeAvailable),
);
}
let started_at = Instant::now();
wait_for_routes(
@ -277,7 +184,7 @@ fn run_supervisor_once(
},
)
.map_err(|e| format!("observe pool ready: {e}"))?;
record_dashboard_event(&mut dashboard, run_event(run_id, obs::EventKind::PoolReady));
stack
.runtime
.send_to(
@ -295,10 +202,7 @@ fn run_supervisor_once(
},
)
.map_err(|e| format!("observe plan: {e}"))?;
record_dashboard_event(
&mut dashboard,
run_event(run_id, obs::EventKind::RunPlanned),
);
stack
.runtime
.send_to(
@ -313,10 +217,6 @@ fn run_supervisor_once(
OrchestratorMsg::ObserveTokenOutEndpointReady,
)
.map_err(|e| format!("observe token-out endpoint: {e}"))?;
record_dashboard_event(
&mut dashboard,
run_event(run_id, obs::EventKind::ReadinessBarrierPassed),
);
let mut injected = false;
let mut completed = false;
@ -326,10 +226,12 @@ fn run_supervisor_once(
let mut sent_stop_to_node0 = false;
let mut sent_stop_to_node1 = false;
let mut token_in_object_allocator =
edge_actor::ObjectIdAllocator::new(edge_actor::EdgeId(stage0.inbound_edge.0));
while started_at.elapsed() < Duration::from_secs(30) {
pump_network(&mut driver, &stack);
stage_ready_count += drain_node_stdout(&node0.stdout_rx, run_id, &mut dashboard);
stage_ready_count += drain_node_stdout(&node1.stdout_rx, run_id, &mut dashboard);
stage_ready_count += drain_node_stdout(&node0.stdout_rx);
stage_ready_count += drain_node_stdout(&node1.stdout_rx);
while let Some(report) = orchestrator_report.try_recv() {
match report {
@ -345,29 +247,18 @@ fn run_supervisor_once(
.runtime
.send_to(target, NodeAgentMsg::ProvisionStage(provision))
.map_err(|e| format!("send provision to stage {stage_index}: {e}"))?;
record_dashboard_event(
&mut dashboard,
stage_event(run_id, stage_index, obs::EventKind::StageProvisionStarted),
);
}
RunCommandWire::InjectPrompt {
sequence, prompt, ..
} => {
injected = true;
record_dashboard_event(
&mut dashboard,
object_event(
run_id,
9_000 + sequence,
sequence,
obs::EventKind::PromptInjected,
),
);
let prompt_object = token_in_object_allocator.alloc();
write_edge_frame(
node0.ready.token_in_addr,
EdgeFrame {
edge_id: 7000,
object_id: 9_000 + sequence,
edge_id: stage0.inbound_edge.0,
object_id: prompt_object.object_id.0,
sequence,
kind: "token".to_owned(),
token_id: prompt.first().copied(),
@ -390,10 +281,6 @@ fn run_supervisor_once(
.runtime
.send_to(target, NodeAgentMsg::StopRun { run_id })
.map_err(|e| format!("send stop to stage {stage_index}: {e}"))?;
record_dashboard_event(
&mut dashboard,
stage_event(run_id, stage_index, obs::EventKind::StopRunSent),
);
}
RunCommandWire::TearDownTokenEndpoints { .. } => {
stack
@ -409,28 +296,15 @@ fn run_supervisor_once(
| RunCommandWire::BroadcastStart { .. } => {}
},
OrchestratorReport::Lifecycle(event) => match event {
LifecycleEventWire::RunCompleted { run_id } => {
LifecycleEventWire::RunCompleted { .. } => {
token_received = true;
record_dashboard_event(
&mut dashboard,
object_event(run_id, 9_000, 0, obs::EventKind::TokenReceived),
);
record_dashboard_event(
&mut dashboard,
run_event(run_id, obs::EventKind::RunCompleted),
);
completed = true;
}
LifecycleEventWire::RunTornDown { run_id } => {
record_dashboard_event(
&mut dashboard,
run_event(run_id, obs::EventKind::RunTornDown),
);
LifecycleEventWire::RunTornDown { .. } => {
torn_down = true;
}
LifecycleEventWire::RunRejected { run_id }
| LifecycleEventWire::RunFaulted { run_id } => {
record_dashboard_event(&mut dashboard, run_fault_event(run_id));
LifecycleEventWire::RunRejected { .. }
| LifecycleEventWire::RunFaulted { .. } => {
return Err(format!("run failed: {event:?}"));
}
},
@ -495,7 +369,6 @@ fn run_supervisor_once(
thread::sleep(Duration::from_millis(10));
}
record_dashboard_event(&mut dashboard, run_fault_event(run_id));
shutdown_node(&mut node0);
shutdown_node(&mut node1);
Err(format!(
@ -516,8 +389,9 @@ fn run_node_role(args: &[String]) -> Result<(), String> {
let outbound_addr = parse_arg(args, "--outbound-addr")?
.parse::<SocketAddr>()
.map_err(|e| format!("outbound addr: {e}"))?;
let seed: EndpointAddr = serde_json::from_str(parse_arg(args, "--seed-endpoint")?)
.map_err(|e| format!("seed endpoint json: {e}"))?;
let coordinator: EndpointAddr =
serde_json::from_str(parse_arg(args, "--coordinator-endpoint")?)
.map_err(|e| format!("coordinator endpoint json: {e}"))?;
let orchestrator_addr: ActorAddress =
serde_json::from_str(parse_arg(args, "--orchestrator-actor")?)
.map_err(|e| format!("orchestrator actor json: {e}"))?;
@ -537,7 +411,7 @@ fn run_node_role(args: &[String]) -> Result<(), String> {
stack.relay_mirror.clone(),
stack.route_view.clone(),
);
driver.join(&[seed]);
driver.join(&[coordinator]);
let node_report = stack
.runtime
@ -594,6 +468,7 @@ fn run_node_role(args: &[String]) -> Result<(), String> {
let mut worker = WorkerProc::spawn()?;
let mut pending_commands = VecDeque::new();
let mut outbound_stream: Option<TcpStream> = None;
let mut outbound_object_allocator: Option<edge_actor::ObjectIdAllocator> = None;
let started_at = Instant::now();
loop {
@ -642,6 +517,8 @@ fn run_node_role(args: &[String]) -> Result<(), String> {
&mut worker,
outbound_addr,
&mut outbound_stream,
&mut outbound_object_allocator,
stage_index,
)?;
}
pending_commands = deferred;
@ -881,6 +758,8 @@ fn handle_node_command(
worker: &mut WorkerProc,
outbound_addr: SocketAddr,
outbound_stream: &mut Option<TcpStream>,
outbound_object_allocator: &mut Option<edge_actor::ObjectIdAllocator>,
local_stage_index: u32,
) -> Result<(), String> {
match command {
StageCommandWire::EstablishInboundEdge { edge_id } => runtime
@ -895,6 +774,9 @@ fn handle_node_command(
)))
.map_err(|e| format!("write outbound preamble: {e}"))?;
*outbound_stream = Some(stream);
*outbound_object_allocator = Some(edge_actor::ObjectIdAllocator::new(
edge_actor::EdgeId(edge_id),
));
runtime
.send_to(node_actor, NodeAgentMsg::MarkOutboundEdgeReady { edge_id })
.map_err(|e| format!("mark outbound ready: {e}"))
@ -919,22 +801,27 @@ fn handle_node_command(
.send_to(node_actor, NodeAgentMsg::StepCompleted { step_id })
.map_err(|e| format!("mark step completed: {e}"))?;
let output_edge_id = output_edge_ids.first().copied().unwrap_or(0);
let output_key = outbound_object_allocator
.as_mut()
.ok_or_else(|| "outbound object allocator missing for ExecuteStep".to_owned())?
.alloc();
let stream = outbound_stream
.as_mut()
.ok_or_else(|| "outbound stream missing for ExecuteStep".to_owned())?;
let final_stage = local_stage_index == 1;
write_json_frame(
stream,
&EdgeFrame {
edge_id: output_edge_id,
object_id: 10_000 + step_id,
object_id: output_key.object_id.0,
sequence,
kind: if output_edge_id == 7002 {
kind: if final_stage {
"token".to_owned()
} else {
"activation".to_owned()
},
token_id: (output_edge_id == 7002).then_some(99),
eos: output_edge_id == 7002,
token_id: final_stage.then_some(99),
eos: final_stage,
},
)
}
@ -1039,7 +926,7 @@ fn spawn_node_process(
stage_index: u32,
inbound_edge_id: u64,
outbound_addr: SocketAddr,
seed_endpoint_json: &str,
coordinator_endpoint_json: &str,
orchestrator_actor_json: &str,
) -> Result<NodeChild, String> {
let mut child = Command::new(std::env::current_exe().map_err(|e| format!("current exe: {e}"))?)
@ -1052,8 +939,8 @@ fn spawn_node_process(
.arg(inbound_edge_id.to_string())
.arg("--outbound-addr")
.arg(outbound_addr.to_string())
.arg("--seed-endpoint")
.arg(seed_endpoint_json)
.arg("--coordinator-endpoint")
.arg(coordinator_endpoint_json)
.arg("--orchestrator-actor")
.arg(orchestrator_actor_json)
.stdin(Stdio::piped())
@ -1112,38 +999,17 @@ fn shutdown_node(node: &mut NodeChild) {
let _ = node.child.wait();
}
fn drain_node_stdout(
rx: &Receiver<NodeStdoutLine>,
run_id: u64,
dashboard: &mut Option<&mut MvpDashboard>,
) -> usize {
fn drain_node_stdout(rx: &Receiver<NodeStdoutLine>) -> usize {
let mut stage_ready_count = 0;
while let Ok(line) = rx.try_recv() {
if line.kind != "node_lifecycle" {
continue;
}
let Some(stage_index) = line.stage_index else {
continue;
};
let Some(event) = line.event.as_deref() else {
continue;
};
if event.contains("StageReady") {
stage_ready_count += 1;
record_dashboard_event(
dashboard,
stage_event(run_id, stage_index, obs::EventKind::StageReady),
);
} else if event.contains("StageStopped") {
record_dashboard_event(
dashboard,
stage_event(run_id, stage_index, obs::EventKind::StageStopped),
);
} else if event.contains("StageFault") {
record_dashboard_event(
dashboard,
stage_event(run_id, stage_index, obs::EventKind::StageFaulted),
);
}
}
stage_ready_count

2252
crates/mvp-system/src/bin/local_e2e_cluster.rs Normal file
View file

File diff suppressed because it is too large Load diff

478
crates/mvp-system/src/dashboard.rs
View file

@ -1,478 +0,0 @@
//! MVP-system dashboard adapter.
//!
//! This module owns the MVP-specific view over `mvp.lifecycle` datastream
//! records. The generic `dashboard` crate remains only an HTTP/SSE/plugin host;
//! MVP semantics live here.
use std::collections::{BTreeMap, VecDeque};
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use ::dashboard as dash;
use datastream::Record;
use datastream::frame::{Frame, Lifetime, NodeId, Position, StreamId};
use parking_lot::Mutex;
use serde_json::json;
use swactor::actor::ActorAddress;
use swactor::stats::{ActorInfo, RuntimeStats, WorkerInfo};
use crate::observability_surface as obs;
use crate::telemetry::{MVP_LIFECYCLE, MvpLifecycleRecord};
const RECENT_LIMIT: usize = 64;
const DEFAULT_PORT: u16 = 9090;
const MVP_PAGE: &str = r#"<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>MVP Dashboard</title>
<style>
body { margin: 0; font-family: Menlo, Consolas, monospace; background: #0f1117; color: #e0e0e0; font-size: 13px; }
.header { display: flex; align-items: center; justify-content: space-between; padding: 12px 20px; background: #161822; border-bottom: 1px solid #2a2d3e; }
a { color: #9ab; text-decoration: none; margin-right: 12px; }
.content { padding: 16px 20px; }
.cards { display: grid; grid-template-columns: repeat(5, minmax(120px, 1fr)); gap: 10px; margin-bottom: 16px; }
.card, .panel { background: #161822; border: 1px solid #2a2d3e; border-radius: 6px; padding: 12px; }
.value { font-size: 22px; font-weight: 700; color: #fff; }
.label { color: #888; font-size: 10px; text-transform: uppercase; margin-top: 3px; }
table { width: 100%; border-collapse: collapse; }
th, td { text-align: left; border-bottom: 1px solid #25283a; padding: 6px 8px; }
th { color: #888; font-size: 10px; text-transform: uppercase; }
.ok { color: #4caf50; } .warn { color: #ff9800; } .bad { color: #f44336; }
pre { white-space: pre-wrap; line-height: 1.5; margin: 0; max-height: 420px; overflow: auto; }
</style>
</head>
<body>
<div class="header"><div><strong>MVP Dashboard</strong> <a href="/">Overview</a><a href="/actors">Actors</a><a href="/plugin/mvp">MVP</a></div><span id="status">connecting</span></div>
<div class="content">
<div class="cards">
<div class="card"><div class="value" id="run">-</div><div class="label">run</div></div>
<div class="card"><div class="value" id="state">-</div><div class="label">state</div></div>
<div class="card"><div class="value" id="events">0</div><div class="label">events</div></div>
<div class="card"><div class="value" id="completed">0</div><div class="label">completed</div></div>
<div class="card"><div class="value" id="faulted">0</div><div class="label">faulted</div></div>
</div>
<div class="panel"><h3>Stages</h3><table><thead><tr><th>Stage</th><th>Status</th><th>Events</th><th>Last event</th></tr></thead><tbody id="stages"></tbody></table></div>
<div class="panel" style="margin-top:16px"><h3>Recent lifecycle</h3><pre id="recent"></pre></div>
</div>
<script>
function cls(s) { return s === 'faulted' ? 'bad' : (s === 'ready' || s === 'completed' || s === 'torn_down' ? 'ok' : 'warn'); }
function render(m) {
document.getElementById('status').textContent = 'live';
document.getElementById('run').textContent = m.run_id ?? '-';
document.getElementById('state').textContent = m.state || '-';
document.getElementById('state').className = 'value ' + cls(m.state);
document.getElementById('events').textContent = m.event_count || 0;
document.getElementById('completed').textContent = m.completed_runs || 0;
document.getElementById('faulted').textContent = m.faulted_runs || 0;
const tbody = document.getElementById('stages'); tbody.innerHTML = '';
Object.entries(m.stages || {}).forEach(([stage, s]) => {
const tr = document.createElement('tr');
tr.innerHTML = '<td>' + stage + '</td><td class="' + cls(s.status) + '">' + s.status + '</td><td>' + s.event_count + '</td><td>' + (s.last_event || '') + '</td>';
tbody.appendChild(tr);
});
document.getElementById('recent').textContent = (m.recent || []).join('\n');
}
fetch('/api/plugin/mvp').then(r => r.json()).then(render).catch(() => {});
const es = new EventSource('/events');
es.addEventListener('mvp', e => { try { render(JSON.parse(e.data)); } catch (_) {} });
es.addEventListener('done', () => { document.getElementById('status').textContent = 'done'; es.close(); });
</script>
</body>
</html>"#;
/// Running MVP dashboard handle used by the local E2E observation mode.
pub struct MvpDashboard {
handle: Arc<dash::DashboardHandle>,
cache: Arc<Mutex<Option<String>>>,
view: MvpView,
stream: StreamId,
next_position: u64,
port: u16,
}
impl MvpDashboard {
/// Start the dashboard using `MVP_DASHBOARD_PORT` or 9090.
pub fn start_from_env() -> Result<Self, String> {
let port = match std::env::var("MVP_DASHBOARD_PORT") {
Ok(raw) => raw
.trim()
.parse::<u16>()
.map_err(|e| format!("MVP_DASHBOARD_PORT: {e}"))?,
Err(_) => DEFAULT_PORT,
};
let handle = Arc::new(dash::start_dashboard(dash::DashboardConfig {
port,
..Default::default()
}));
let cache = Arc::new(Mutex::new(None));
handle.register_plugin(Arc::new(MvpPlugin {
cache: Arc::clone(&cache),
}));
handle.start_http_standalone();
Ok(Self {
handle,
cache,
view: MvpView::default(),
stream: StreamId::new(NodeId::new("mvp-local-e2e"), Lifetime(now_secs())),
next_position: 0,
port,
})
}
pub fn port(&self) -> u16 {
self.port
}
pub fn url(&self) -> String {
format!("http://127.0.0.1:{}/", self.port)
}
/// Convenience path for local producers: encode the event as the MVP-owned
/// datastream record, then ingest the frame through the same consumer path.
pub fn record_event(&mut self, event: obs::Event) {
let record = MvpLifecycleRecord::new(event);
let frame = Frame::new(
MvpLifecycleRecord::CHANNEL,
Position(self.next_position),
record.encode(),
);
self.next_position = self.next_position.saturating_add(1);
let stream = self.stream.clone();
self.ingest(&stream, &frame);
}
/// Fold a delivered datastream frame into the MVP dashboard view.
pub fn ingest(&mut self, _stream: &StreamId, frame: &Frame) {
if frame.channel.as_str() != MVP_LIFECYCLE {
return;
}
let Ok(record) = MvpLifecycleRecord::decode(&frame.payload) else {
return;
};
let is_warn = fault_event(&record.event);
let line = format_event(&record.event);
self.view.observe(&record.event);
let json = self.view.json();
*self.cache.lock() = Some(json);
self.handle.push_activity(is_warn, line);
self.handle.set_stats(self.view.runtime_stats());
}
pub fn shutdown(&self) {
self.handle.shutdown();
}
}
#[derive(Default)]
struct MvpView {
run_id: Option<u64>,
state: &'static str,
event_count: u64,
completed_runs: u64,
faulted_runs: u64,
torn_down_runs: u64,
nodes: BTreeMap<u64, NodeView>,
stages: BTreeMap<u32, StageView>,
recent: VecDeque<String>,
}
#[derive(Default)]
struct NodeView {
status: &'static str,
event_count: u64,
last_event: String,
}
#[derive(Default)]
struct StageView {
status: &'static str,
event_count: u64,
last_event: String,
}
impl MvpView {
fn observe(&mut self, event: &obs::Event) {
self.event_count = self.event_count.saturating_add(1);
if let Some(run_id) = run_id(event) {
self.run_id = Some(run_id);
}
let kind = event.kind();
match event {
obs::Event::RunScoped { kind, .. } => self.observe_run(*kind),
obs::Event::NodeScoped { node_id, kind, .. } => {
let node = self.nodes.entry(node_id.0).or_default();
node.event_count = node.event_count.saturating_add(1);
node.last_event = format!("{kind:?}");
node.status = match kind {
obs::EventKind::NodeStarted => "started",
obs::EventKind::NodeAvailable => "available",
_ => node.status,
};
}
obs::Event::StageScoped {
stage_index, kind, ..
} => {
let stage = self.stages.entry(stage_index.0).or_default();
stage.event_count = stage.event_count.saturating_add(1);
stage.last_event = format!("{kind:?}");
stage.status = match kind {
obs::EventKind::StageProvisionStarted => "provisioning",
obs::EventKind::StageReady => "ready",
obs::EventKind::StopRunSent => "stopping",
obs::EventKind::StageStopped => "stopped",
obs::EventKind::StageFaulted => "faulted",
_ => stage.status,
};
}
_ => {}
}
push_recent(&mut self.recent, format_event(event));
if matches!(kind, obs::EventKind::RunCompleted) {
self.completed_runs = self.completed_runs.saturating_add(1);
}
if matches!(kind, obs::EventKind::RunFaulted) {
self.faulted_runs = self.faulted_runs.saturating_add(1);
}
if matches!(kind, obs::EventKind::RunTornDown) {
self.torn_down_runs = self.torn_down_runs.saturating_add(1);
}
}
fn observe_run(&mut self, kind: obs::EventKind) {
self.state = match kind {
obs::EventKind::PoolReady => "pool_ready",
obs::EventKind::RunPlanned => "planned",
obs::EventKind::ReadinessBarrierPassed => "ready",
obs::EventKind::PromptInjected => "running",
obs::EventKind::RunCompleted => "completed",
obs::EventKind::RunFaulted => "faulted",
obs::EventKind::RunTornDown => "torn_down",
_ => self.state,
};
}
fn json(&self) -> String {
let nodes = self
.nodes
.iter()
.map(|(id, node)| {
(
id.to_string(),
json!({
"status": node.status,
"event_count": node.event_count,
"last_event": node.last_event,
}),
)
})
.collect::<serde_json::Map<_, _>>();
let stages = self
.stages
.iter()
.map(|(idx, stage)| {
(
idx.to_string(),
json!({
"status": stage.status,
"event_count": stage.event_count,
"last_event": stage.last_event,
}),
)
})
.collect::<serde_json::Map<_, _>>();
json!({
"run_id": self.run_id,
"state": self.state,
"event_count": self.event_count,
"completed_runs": self.completed_runs,
"faulted_runs": self.faulted_runs,
"torn_down_runs": self.torn_down_runs,
"nodes": nodes,
"stages": stages,
"recent": self.recent.iter().cloned().collect::<Vec<_>>(),
})
.to_string()
}
fn runtime_stats(&self) -> RuntimeStats {
let mut actor_details = Vec::new();
actor_details.push(actor_info(
0,
"mvp-orchestrator",
self.state,
self.event_count,
));
for (node_id, node) in &self.nodes {
actor_details.push(actor_info(
1 + (*node_id as usize % 64),
&format!("mvp-node-{node_id}"),
node.status,
node.event_count,
));
}
for (stage_index, stage) in &self.stages {
actor_details.push(actor_info(
128 + *stage_index as usize,
&format!("mvp-stage-{stage_index}"),
stage.status,
stage.event_count,
));
}
let actors = actor_details
.iter()
.map(|actor| (actor.address, actor.worker_id))
.collect::<Vec<_>>();
RuntimeStats {
num_workers: 1,
uptime_ms: self.event_count.saturating_mul(100),
actors,
workers: vec![WorkerInfo {
id: 0,
num_actors: actor_details.len(),
mailbox_depth: 0,
messages_processed: self.event_count,
local_sends: self.event_count,
cross_sends: 0,
inbox_sends: 0,
type_mismatches: 0,
panics: self.faulted_runs,
messages_dropped: 0,
restarts: self.completed_runs.saturating_sub(1),
stops: self.torn_down_runs,
}],
actor_details,
tick_timings: Vec::new(),
}
}
}
struct MvpPlugin {
cache: Arc<Mutex<Option<String>>>,
}
impl dash::plugin::DashboardPlugin for MvpPlugin {
fn name(&self) -> &str {
"mvp"
}
fn snapshot_json(&self) -> Option<String> {
self.cache.lock().clone()
}
fn handle_request(
&self,
method: &str,
path: &str,
_query: &std::collections::HashMap<String, String>,
_body: &[u8],
) -> dash::plugin::PluginResponse {
match (method, path) {
("GET", "" | "model" | "snapshot") => dash::plugin::PluginResponse::json(
self.cache.lock().clone().unwrap_or_else(|| "{}".into()),
),
_ => dash::plugin::PluginResponse::not_found(),
}
}
fn html_page(&self) -> Option<&str> {
Some(MVP_PAGE)
}
}
fn actor_info(slot: usize, name: &str, status: &str, messages_processed: u64) -> ActorInfo {
ActorInfo {
address: actor_address(slot),
worker_id: 0,
mailbox_depth: 0,
last_msg_type: Some(status.to_owned()),
messages_processed,
poisoned: status == "faulted",
name: Some(name.to_owned()),
message_type_counts: vec![(status.to_owned(), messages_processed)],
}
}
fn actor_address(slot: usize) -> ActorAddress {
let mut bytes = [0_u8; 32];
bytes[..8].copy_from_slice(&(slot as u64).to_be_bytes());
bytes[8..11].copy_from_slice(b"mvp");
ActorAddress(bytes)
}
fn push_recent(recent: &mut VecDeque<String>, line: String) {
if recent.len() >= RECENT_LIMIT {
recent.pop_front();
}
recent.push_back(line);
}
fn run_id(event: &obs::Event) -> Option<u64> {
match event {
obs::Event::RunScoped { run_id, .. } | obs::Event::StageScoped { run_id, .. } => {
Some(run_id.0)
}
_ => None,
}
}
fn fault_event(event: &obs::Event) -> bool {
matches!(
event.kind(),
obs::EventKind::RunFaulted | obs::EventKind::StageFaulted
)
}
fn format_event(event: &obs::Event) -> String {
match event {
obs::Event::RunScoped {
kind,
run_id,
reason,
..
} => format!("run {} {kind:?}{}", run_id.0, reason_text(*reason)),
obs::Event::NodeScoped { kind, node_id, .. } => format!("node {} {kind:?}", node_id.0),
obs::Event::StageScoped {
kind,
run_id,
stage_index,
reason,
..
} => format!(
"run {} stage {} {kind:?}{}",
run_id.0,
stage_index.0,
reason_text(*reason)
),
obs::Event::EdgeScoped { kind, edge_id, .. } => format!("edge {} {kind:?}", edge_id.0),
obs::Event::RingScoped { kind, ring_id, .. } => format!("ring {} {kind:?}", ring_id.0),
obs::Event::ObjectScoped {
kind,
object_id,
sequence,
..
} => format!("object {} seq {} {kind:?}", object_id.0, sequence.0),
obs::Event::StepScoped { kind, step_id, .. } => format!("step {} {kind:?}", step_id.0),
obs::Event::WorkerScoped {
kind,
worker_generation,
..
} => format!("worker generation {} {kind:?}", worker_generation.0),
}
}
fn reason_text(reason: Option<obs::FaultReason>) -> String {
reason
.map(|reason| format!(" ({reason:?})"))
.unwrap_or_default()
}
fn now_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}

3
crates/mvp-system/src/driver_pumps.rs
View file

@ -151,9 +151,6 @@ pub enum DriverEventOut {
edge_id: EdgeId,
ring_id: RingId,
},
ObjectHeaderParsed {
edge_id: EdgeId,
},
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]

4
crates/mvp-system/src/edge_establisher.rs
View file

@ -228,10 +228,6 @@ pub enum EdgeCommand {
ring_id: RingId,
proof: QuiescenceProof,
},
CopyHotPathBytes {
edge_id: EdgeId,
bytes: Vec<u8>,
},
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]

42
crates/mvp-system/src/engine_builder/engine.rs
View file

@ -6,7 +6,7 @@ use crate::run_plan::RunId;
use super::error::EngineBuildError;
use super::events::EngineEvent;
use super::launcher::{
LaunchedNode, NodeControl, NodeFacts, NodeLaunchSpec, NodeLauncher, SeedSpec,
CoordinatorJoinSpec, LaunchedNode, NodeControl, NodeFacts, NodeLaunchSpec, NodeLauncher,
};
use super::model::ModelSpec;
use super::node_image::NodeImageSpec;
@ -117,33 +117,31 @@ impl ClusterBuilder {
let mut nodes = Vec::with_capacity(leases.len());
let mut iter = leases.into_iter();
let seed_lease = iter.next().ok_or(EngineBuildError::EmptyPool)?;
let mut seed = launcher.launch_node(
&seed_lease,
let coordinator_lease = iter.next().ok_or(EngineBuildError::EmptyPool)?;
let mut coordinator = launcher.launch_node(
&coordinator_lease,
NodeLaunchSpec {
cluster_id: self.cluster_id.clone(),
image: image.clone(),
seed: None,
is_seed: true,
coordinator: None,
is_coordinator: true,
env: BTreeMap::new(),
},
)?;
events.push(EngineEvent::NodeLaunched {
node_id: seed.lease.logical_node_id,
seed: true,
node_id: coordinator.lease.logical_node_id,
coordinator: true,
});
let seed_facts = seed.control.wait_boot_ready(self.boot_timeout)?;
let coordinator_facts = coordinator.control.wait_boot_ready(self.boot_timeout)?;
events.push(EngineEvent::NodeBootReady {
node_id: seed_facts.node_id,
node_id: coordinator_facts.node_id,
});
let seed_endpoint =
seed_facts
.seed_endpoint
.clone()
.ok_or(EngineBuildError::SeedEndpointMissing {
node_id: seed_facts.node_id.0,
})?;
nodes.push(EngineNode::new(seed, seed_facts));
let coordinator_endpoint = coordinator_facts.coordinator_endpoint.clone().ok_or(
EngineBuildError::CoordinatorEndpointMissing {
node_id: coordinator_facts.node_id.0,
},
)?;
nodes.push(EngineNode::new(coordinator, coordinator_facts));
for lease in iter {
let mut node = launcher.launch_node(
@ -151,16 +149,16 @@ impl ClusterBuilder {
NodeLaunchSpec {
cluster_id: self.cluster_id.clone(),
image: image.clone(),
seed: Some(SeedSpec {
endpoint: seed_endpoint.clone(),
coordinator: Some(CoordinatorJoinSpec {
endpoint: coordinator_endpoint.clone(),
}),
is_seed: false,
is_coordinator: false,
env: BTreeMap::new(),
},
)?;
events.push(EngineEvent::NodeLaunched {
node_id: node.lease.logical_node_id,
seed: false,
coordinator: false,
});
let facts = node.control.wait_boot_ready(self.boot_timeout)?;
events.push(EngineEvent::NodeBootReady {

9
crates/mvp-system/src/engine_builder/error.rs
View file

@ -7,7 +7,7 @@ use crate::run_plan;
pub enum EngineBuildError {
MissingComponent(&'static str),
EmptyPool,
SeedEndpointMissing { node_id: u64 },
CoordinatorEndpointMissing { node_id: u64 },
RoleTargetMissing { node_id: u64 },
Pool(PoolError),
Launch(LaunchError),
@ -20,8 +20,11 @@ impl fmt::Display for EngineBuildError {
match self {
Self::MissingComponent(name) => write!(f, "missing engine builder component: {name}"),
Self::EmptyPool => write!(f, "pool provider returned no nodes"),
Self::SeedEndpointMissing { node_id } => {
write!(f, "seed node {node_id} did not report a seed endpoint")
Self::CoordinatorEndpointMissing { node_id } => {
write!(
f,
"coordinator node {node_id} did not report a coordinator endpoint"
)
}
Self::RoleTargetMissing { node_id } => {
write!(f, "role assignment targeted unknown node {node_id}")

2
crates/mvp-system/src/engine_builder/events.rs
View file

@ -5,7 +5,7 @@ use super::roles::RoleKind;
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum EngineEvent {
PoolAcquired { node_count: usize },
NodeLaunched { node_id: NodeId, seed: bool },
NodeLaunched { node_id: NodeId, coordinator: bool },
NodeBootReady { node_id: NodeId },
ClusterConverged { node_count: usize },
RolesPlanned { stage_count: usize },

12
crates/mvp-system/src/engine_builder/launcher.rs
View file

@ -31,29 +31,29 @@ pub trait NodeControl: Send {
pub struct NodeLaunchSpec {
pub cluster_id: String,
pub image: NodeImageSpec,
pub seed: Option<SeedSpec>,
pub is_seed: bool,
pub coordinator: Option<CoordinatorJoinSpec>,
pub is_coordinator: bool,
pub env: BTreeMap<String, String>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct SeedSpec {
pub struct CoordinatorJoinSpec {
pub endpoint: String,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct NodeFacts {
pub node_id: NodeId,
pub seed_endpoint: Option<String>,
pub coordinator_endpoint: Option<String>,
pub resources: ResourceFacts,
pub capabilities: BTreeSet<NodeCapability>,
}
impl NodeFacts {
pub fn from_lease(lease: &NodeLease, seed_endpoint: Option<String>) -> Self {
pub fn from_lease(lease: &NodeLease, coordinator_endpoint: Option<String>) -> Self {
Self {
node_id: lease.logical_node_id,
seed_endpoint,
coordinator_endpoint,
resources: lease.expected_resources.clone(),
capabilities: lease.capabilities.clone(),
}

2
crates/mvp-system/src/engine_builder/mod.rs
View file

@ -23,7 +23,7 @@ pub use engine::{ClusterBuilder, ClusterHandle, NodeSummary};
pub use error::{EngineBuildError, LaunchError, NodeControlError, PlanningError, PoolError};
pub use events::EngineEvent;
pub use launcher::{
LaunchedNode, NodeControl, NodeFacts, NodeLaunchSpec, NodeLauncher, SeedSpec,
CoordinatorJoinSpec, LaunchedNode, NodeControl, NodeFacts, NodeLaunchSpec, NodeLauncher,
StaticNodeLauncher,
};
pub use model::{DTypeFamily, ModelArchitecture, ModelArtifact, ModelSpec};

4
crates/mvp-system/src/engine_builder/runtime_stack.rs
View file

@ -87,8 +87,8 @@ impl RuntimeNode {
self.driver.endpoint_addr()
}
pub fn join(&mut self, seeds: &[EndpointAddr]) {
self.driver.join(seeds);
pub fn join(&mut self, coordinators: &[EndpointAddr]) {
self.driver.join(coordinators);
}
pub fn register_actor_route(&mut self, actor_addr: ActorAddress, generation: u64) {

11
crates/mvp-system/src/gpu_worker_ctl.rs
View file

@ -78,9 +78,6 @@ pub enum WorkerCommand {
handle: DeviceHandle,
},
ShutdownWorker,
PayloadBytes {
bytes: Vec<u8>,
},
}
#[derive(Clone, Debug, PartialEq, Eq)]
@ -174,8 +171,7 @@ enum CtlState {
Stopped,
}
#[cfg(test)]
pub struct GpuWorkerCtlHarness {
pub struct GpuWorkerCtl {
config: WorkerConfig,
state: CtlState,
current_generation: WorkerGeneration,
@ -188,8 +184,7 @@ pub struct GpuWorkerCtlHarness {
installed_rings: std::collections::BTreeSet<RingId>,
}
#[cfg(test)]
impl GpuWorkerCtlHarness {
impl GpuWorkerCtl {
pub fn new(config: WorkerConfig) -> Self {
Self {
config,
@ -376,3 +371,5 @@ impl GpuWorkerCtlHarness {
}
}
}
pub type GpuWorkerCtlHarness = GpuWorkerCtl;

32
crates/mvp-system/src/gpu_worker_egress_producer.rs
View file

@ -9,6 +9,17 @@ pub struct PortId(pub String);
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct ObjectId(pub u64);
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct ObjectKey {
pub edge_id: EdgeId,
pub object_id: ObjectId,
}
impl ObjectKey {
pub fn new(edge_id: EdgeId, object_id: ObjectId) -> Self {
Self { edge_id, object_id }
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct StepId(pub u64);
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@ -151,6 +162,7 @@ pub struct HeaderDecodeError;
const HEADER_LEN: usize = 48;
#[cfg(test)]
#[derive(Clone, Debug, PartialEq, Eq)]
struct PendingStep {
step_id: StepId,
@ -167,7 +179,7 @@ pub struct EgressProducerHarness {
committed: std::collections::BTreeMap<RingId, Vec<u8>>,
payload_committed: std::collections::BTreeMap<RingId, u64>,
full_rings: std::collections::BTreeSet<RingId>,
produced: std::collections::BTreeSet<ObjectId>,
produced: std::collections::BTreeSet<ObjectKey>,
wake_hints: Vec<WakeHint>,
events: Vec<WorkerEgressOut>,
}
@ -323,7 +335,10 @@ impl EgressProducerHarness {
.or_default()
.extend(std::iter::repeat(0).take(byte_count as usize));
*self.payload_committed.entry(output.ring_id).or_insert(0) += byte_count;
if self.produced.insert(object_id) {
let Some(object_key) = self.output_key(output) else {
return;
};
if self.produced.insert(object_key) {
self.events.push(WorkerEgressOut::ObjectProduced {
ring_id: output.ring_id,
object_id,
@ -362,6 +377,12 @@ impl EgressProducerHarness {
});
}
fn output_key(&self, output: OutputBinding) -> Option<ObjectKey> {
self.rings
.get(&output.ring_id)
.map(|ring| ObjectKey::new(ring.edge_id, output.object_id))
}
fn maybe_step_completed(&mut self, step_id: StepId) {
let Some(step) = self.steps.iter().find(|step| step.step_id == step_id) else {
return;
@ -369,14 +390,17 @@ impl EgressProducerHarness {
if !step.role_state_updated {
return;
}
if step.outputs.iter().all(|output| self.produced.contains(&output.object_id))
&& !self.events.iter().any(|event| matches!(event, WorkerEgressOut::StepCompleted { step_id: seen } if *seen == step_id))
if step.outputs.iter().all(|output| {
self.output_key(*output)
.is_some_and(|key| self.produced.contains(&key))
}) && !self.events.iter().any(|event| matches!(event, WorkerEgressOut::StepCompleted { step_id: seen } if *seen == step_id))
{
self.events.push(WorkerEgressOut::StepCompleted { step_id });
}
}
}
#[cfg(test)]
fn encode_header(output: OutputBinding) -> Vec<u8> {
let mut out = Vec::with_capacity(HEADER_LEN);
out.extend_from_slice(b"MO01");

100
crates/mvp-system/src/gpu_worker_ingress_parser.rs
View file

@ -9,6 +9,18 @@ pub struct PortId(pub String);
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct ObjectId(pub u64);
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct ObjectKey {
pub edge_id: EdgeId,
pub object_id: ObjectId,
}
impl ObjectKey {
pub fn new(edge_id: EdgeId, object_id: ObjectId) -> Self {
Self { edge_id, object_id }
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct DeviceHandle {
pub generation: WorkerGeneration,
@ -195,19 +207,32 @@ impl ObjectRecordBuilder {
}
}
const HEADER_LEN: usize = 48;
pub const HEADER_LEN: usize = 48;
#[derive(Clone, Debug, PartialEq, Eq)]
struct ParsedRecord {
object_id: ObjectId,
sequence: u64,
extent: u64,
total_len: usize,
pub struct ObjectRecord {
pub object_id: ObjectId,
pub sequence: u64,
pub extent: u64,
pub total_len: usize,
}
impl ObjectRecord {
pub fn payload<'a>(&self, bytes: &'a [u8]) -> Option<&'a [u8]> {
bytes.get(HEADER_LEN..self.total_len)
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum ObjectRecordRead {
Incomplete,
Complete(ObjectRecord),
}
#[cfg(test)]
#[derive(Clone, Debug, PartialEq, Eq)]
struct PendingObject {
record: ParsedRecord,
record: ObjectRecord,
copy_done: bool,
handle: Option<DeviceHandle>,
}
@ -221,7 +246,7 @@ pub struct IngressParserHarness {
cursor_reload: std::collections::BTreeMap<RingId, u64>,
faulted_rings: std::collections::BTreeSet<RingId>,
expected_sequence: u64,
pending: std::collections::BTreeMap<ObjectId, PendingObject>,
pending: std::collections::BTreeMap<ObjectKey, PendingObject>,
copy_log: Vec<DeviceCopyLog>,
events: Vec<WorkerIngressOut>,
}
@ -263,22 +288,26 @@ impl IngressParserHarness {
object_id,
byte_count,
});
if let Some(pending) = self.pending.get_mut(&object_id) {
pending.copy_done = true;
if byte_count == pending.record.extent {
if let Some(install) = &self.install {
*self.consume.entry(install.ring_id).or_insert(0) +=
pending.record.total_len as u64;
if let Some(key) = self.object_key(object_id) {
if let Some(pending) = self.pending.get_mut(&key) {
pending.copy_done = true;
if byte_count == pending.record.extent {
if let Some(install) = &self.install {
*self.consume.entry(install.ring_id).or_insert(0) +=
pending.record.total_len as u64;
}
}
}
self.maybe_loaded(key);
}
self.maybe_loaded(object_id);
}
WorkerIngressEvent::DeviceHandleCreated { object_id, handle } => {
if let Some(pending) = self.pending.get_mut(&object_id) {
pending.handle = Some(handle);
if let Some(key) = self.object_key(object_id) {
if let Some(pending) = self.pending.get_mut(&key) {
pending.handle = Some(handle);
}
self.maybe_loaded(key);
}
self.maybe_loaded(object_id);
}
WorkerIngressEvent::RingFault { ring_id } => {
self.faulted_rings.insert(ring_id);
@ -321,8 +350,8 @@ impl IngressParserHarness {
if buffer.is_empty() {
return;
}
match decode_record(&buffer, install.object_spec, eof) {
Ok(record) => {
match read_object_record(&buffer, install.object_spec, eof) {
Ok(ObjectRecordRead::Complete(record)) => {
if record.sequence != self.expected_sequence {
self.events.push(WorkerIngressOut::ObjectFailed {
ring_id,
@ -333,7 +362,7 @@ impl IngressParserHarness {
}
self.expected_sequence += 1;
self.pending.insert(
record.object_id,
ObjectKey::new(install.edge_id, record.object_id),
PendingObject {
record: record.clone(),
copy_done: false,
@ -345,6 +374,7 @@ impl IngressParserHarness {
byte_count: record.extent,
});
}
Ok(ObjectRecordRead::Incomplete) => {}
Err(reason) => self.events.push(WorkerIngressOut::ObjectFailed {
ring_id,
object_id: None,
@ -353,8 +383,14 @@ impl IngressParserHarness {
}
}
fn maybe_loaded(&mut self, object_id: ObjectId) {
let Some(pending) = self.pending.get(&object_id).cloned() else {
fn object_key(&self, object_id: ObjectId) -> Option<ObjectKey> {
self.install
.as_ref()
.map(|install| ObjectKey::new(install.edge_id, object_id))
}
fn maybe_loaded(&mut self, key: ObjectKey) {
let Some(pending) = self.pending.get(&key).cloned() else {
return;
};
let Some(handle) = pending.handle else {
@ -364,12 +400,12 @@ impl IngressParserHarness {
return;
}
let install = self.install.as_ref().unwrap();
if !self.events.iter().any(|event| matches!(event, WorkerIngressOut::ObjectLoaded { object_id: seen, .. } if *seen == object_id)) {
if !self.events.iter().any(|event| matches!(event, WorkerIngressOut::ObjectLoaded { edge_id, object_id, .. } if *edge_id == key.edge_id && *object_id == key.object_id)) {
self.events.push(WorkerIngressOut::ObjectLoaded {
ring_id: install.ring_id,
edge_id: install.edge_id,
port_id: install.port_id.clone(),
object_id,
object_id: key.object_id,
sequence: pending.record.sequence,
extent: pending.record.extent,
handle,
@ -378,16 +414,16 @@ impl IngressParserHarness {
}
}
fn decode_record(
pub fn read_object_record(
bytes: &[u8],
spec: ObjectSpec,
eof: bool,
) -> Result<ParsedRecord, ObjectFailureReason> {
) -> Result<ObjectRecordRead, ObjectFailureReason> {
if bytes.len() < HEADER_LEN {
return if eof {
return if eof && !bytes.is_empty() {
Err(ObjectFailureReason::EofBeforeFullPayload)
} else {
Err(ObjectFailureReason::MalformedHeaderLength)
Ok(ObjectRecordRead::Incomplete)
};
}
if &bytes[0..4] != b"MO01" {
@ -413,13 +449,13 @@ fn decode_record(
return if eof {
Err(ObjectFailureReason::EofBeforeFullPayload)
} else {
Err(ObjectFailureReason::MalformedHeaderLength)
Ok(ObjectRecordRead::Incomplete)
};
}
Ok(ParsedRecord {
Ok(ObjectRecordRead::Complete(ObjectRecord {
object_id,
sequence,
extent,
total_len,
})
}))
}

2
crates/mvp-system/src/lib.rs
View file

@ -4,8 +4,6 @@ extern crate self as mvp_system;
#[cfg(feature = "local-e2e")]
pub mod actors;
pub mod arena_manager;
#[cfg(feature = "local-e2e")]
pub mod dashboard;
pub mod device_bridge;
#[cfg(feature = "local-e2e")]
pub mod distribution_stack;

51
crates/mvp-system/src/run_plan.rs
View file

@ -13,6 +13,29 @@ pub struct NodeId(pub u64);
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct EdgeId(pub u64);
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct EdgeAllocator {
next: u64,
}
impl EdgeAllocator {
pub fn new() -> Self {
Self { next: 1 }
}
pub fn alloc(&mut self) -> EdgeId {
let edge_id = EdgeId(self.next);
self.next += 1;
edge_id
}
}
impl Default for EdgeAllocator {
fn default() -> Self {
Self::new()
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum DTypeFamily {
BFloat,
@ -227,10 +250,18 @@ pub fn plan_run(input: PlannerInput) -> Result<RunPlan, PlanRejection> {
max_extent: activation_extent,
};
let mut edge_allocator = EdgeAllocator::new();
let token_in_edge = edge_allocator.alloc();
let mut activation_edges = Vec::with_capacity(input.stage_count.saturating_sub(1) as usize);
for _ in 0..input.stage_count.saturating_sub(1) {
activation_edges.push(edge_allocator.alloc());
}
let token_out_edge = edge_allocator.alloc();
let mut edges = Vec::with_capacity(input.stage_count as usize + 1);
edges.push(EdgePlan {
run_id: input.run_id,
edge_id: EdgeId(7000),
edge_id: token_in_edge,
kind: EdgeKind::TokenIn,
producer: EdgeEndpoint::Orchestrator {
node_id: input.orchestrator_node_id,
@ -246,7 +277,7 @@ pub fn plan_run(input: PlannerInput) -> Result<RunPlan, PlanRejection> {
for stage_index in 0..input.stage_count.saturating_sub(1) {
edges.push(EdgePlan {
run_id: input.run_id,
edge_id: EdgeId(7001 + u64::from(stage_index)),
edge_id: activation_edges[stage_index as usize],
kind: EdgeKind::Activation,
producer: EdgeEndpoint::Stage {
node_id: placements[stage_index as usize].node_id,
@ -263,7 +294,7 @@ pub fn plan_run(input: PlannerInput) -> Result<RunPlan, PlanRejection> {
edges.push(EdgePlan {
run_id: input.run_id,
edge_id: EdgeId(7000 + u64::from(input.stage_count)),
edge_id: token_out_edge,
kind: EdgeKind::TokenOut,
producer: EdgeEndpoint::Stage {
node_id: placements[input.stage_count as usize - 1].node_id,
@ -280,6 +311,16 @@ pub fn plan_run(input: PlannerInput) -> Result<RunPlan, PlanRejection> {
for placement in &placements {
let stage_index = placement.stage_index;
let (start, end) = layer_range(input.model.num_layers, input.stage_count, stage_index);
let inbound_edge = if stage_index == 0 {
token_in_edge
} else {
activation_edges[stage_index as usize - 1]
};
let outbound_edge = if stage_index + 1 == input.stage_count {
token_out_edge
} else {
activation_edges[stage_index as usize]
};
stages.push(StagePlan {
run_id: input.run_id,
stage_index,
@ -287,8 +328,8 @@ pub fn plan_run(input: PlannerInput) -> Result<RunPlan, PlanRejection> {
node_id: placement.node_id,
layer_start: start,
layer_end_exclusive: end,
inbound_edge: EdgeId(7000 + u64::from(stage_index)),
outbound_edge: EdgeId(7001 + u64::from(stage_index)),
inbound_edge,
outbound_edge,
});
}

8
crates/mvp-system/src/tests/driver_pumps_guarantees.rs
View file

@ -190,12 +190,8 @@ fn recv_pump_copies_bytes_without_parsing_and_respects_backpressure() {
edge_id: driver::EdgeId(7001),
bytes: driver::fake_object_header_bytes(),
});
assert!(
!harness
.events()
.iter()
.any(|event| { matches!(event, driver::DriverEventOut::ObjectHeaderParsed { .. }) })
);
// The public driver event enum has no object-header event; object parsing
// belongs to the worker ingress parser, not the recv pump.
assert!(harness.ring_commit(driver::EdgeId(7001)) > 0);
assert!(harness.wake_hints().iter().any(|wake| {
matches!(

7
crates/mvp-system/src/tests/edge_establisher_guarantees.rs
View file

@ -246,12 +246,7 @@ fn driver_ready_marks_local_edge_actor_ready() {
}));
// After readiness, stream and pump behavior belongs to the driver; the
// establisher should not emit hot-path byte commands.
assert!(
!tx.commands()
.iter()
.any(|command| { matches!(command, edge::EdgeCommand::CopyHotPathBytes { .. }) })
);
// public command enum has no hot-path byte variant.
}
// This proves StopEdge cancels queued leases, stops pumps, uninstalls worker

3
crates/mvp-system/src/tests/gpu_worker_ctl_guarantees.rs
View file

@ -142,9 +142,6 @@ fn command_routing_requires_running_current_generation_and_is_payload_free() {
| ctl::WorkerCommand::ExecuteStep { .. }
| ctl::WorkerCommand::ReleaseDeviceObject { .. }
| ctl::WorkerCommand::ShutdownWorker { .. } => {}
ctl::WorkerCommand::PayloadBytes { .. } => {
panic!("worker control command carried payload bytes: {command:?}")
}
}
}

10
crates/mvp-system/src/tests/local_mock/assertions.rs
View file

@ -113,11 +113,13 @@ pub fn assert_topology_surface(outcome: &LocalMockOutcome) {
);
assert_eq!(deliveries[0].kind, MockObjectKind::Token);
assert_eq!(deliveries.last().unwrap().kind, MockObjectKind::Token);
assert!(
assert_eq!(
deliveries
.windows(2)
.all(|pair| pair[0].edge_id.0 + 1 == pair[1].edge_id.0),
"sequence {sequence} must follow the linear edge chain"
.iter()
.map(|delivery| delivery.edge_id)
.collect::<Vec<_>>(),
outcome.edge_chain,
"sequence {sequence} must follow the planned linear edge chain"
);
assert_eq!(
deliveries

49
crates/mvp-system/src/tests/local_mock/environment.rs
View file

@ -5,6 +5,7 @@ use mvp_system::observability_surface as obs;
use mvp_system::orchestrator_run_fsm as fsm;
use mvp_system::run_plan as plan;
use mvp_system::stage_controller as stage;
use mvp_system::tx_rx_edge_actor as edge_actor;
use super::mock_node::MockNode;
use super::mock_transport::{Delivery, MockObject, MockObjectKind, MockTransport};
@ -40,6 +41,7 @@ pub struct LocalMockCluster {
transport: MockTransport,
resources: ResourceTracker,
observed_edges: BTreeSet<plan::EdgeId>,
object_allocators: BTreeMap<plan::EdgeId, edge_actor::ObjectIdAllocator>,
scenario: LocalMockScenario,
}
@ -54,6 +56,7 @@ pub struct LocalMockOutcome {
pub live_stage_runs: usize,
pub transport_delivery_count: usize,
pub transport_deliveries: Vec<Delivery>,
pub edge_chain: Vec<plan::EdgeId>,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
@ -203,6 +206,7 @@ impl LocalMockCluster {
transport: MockTransport::default(),
resources: ResourceTracker::default(),
observed_edges: BTreeSet::new(),
object_allocators: BTreeMap::new(),
scenario: LocalMockScenario::Happy,
}
}
@ -305,6 +309,7 @@ impl LocalMockCluster {
self.transport = MockTransport::default();
self.resources = ResourceTracker::default();
self.observed_edges.clear();
self.object_allocators.clear();
self.orchestrator_command_cursor = 0;
self.orchestrator_event_cursor = 0;
self.scenario = scenario;
@ -508,16 +513,17 @@ impl LocalMockCluster {
obs::Component::Orchestrator,
);
}
let token_in_edge = self.edge_by_kind(plan::EdgeKind::TokenIn).edge_id;
let object_id = self.allocate_object_id(token_in_edge);
self.push_object(
obs::EventKind::PromptInjected,
9_000 + sequence,
object_id,
sequence,
obs::Component::TokenEndpoint,
);
let token_in_edge = self.edge_by_kind(plan::EdgeKind::TokenIn).edge_id;
let object = self.transport.deliver(MockObject {
edge_id: token_in_edge,
object_id: 9_000 + sequence,
object_id,
sequence,
kind: MockObjectKind::Token,
token_id: None,
@ -535,7 +541,6 @@ impl LocalMockCluster {
.map(|stage| stage.stage_index)
.collect();
for stage_index in stage_indices {
let edge_id = current.edge_id;
let mut stage_object = current;
if self.scenario
== (LocalMockScenario::SequenceViolation {
@ -596,7 +601,14 @@ impl LocalMockCluster {
);
self.push_step(obs::EventKind::StepCompleted, execution.step_id);
let delivered = self.transport.deliver(execution.produced);
debug_assert_eq!(delivered.edge_id.0, edge_id.0 + 1);
let expected_edge = self
.plan
.stages
.iter()
.find(|stage| stage.stage_index == stage_index)
.expect("mock stage must exist")
.outbound_edge;
debug_assert_eq!(delivered.edge_id, expected_edge);
current = delivered;
}
self.push_object(
@ -621,9 +633,11 @@ impl LocalMockCluster {
.iter()
.find(|stage| stage.stage_index == stage_index)
.expect("mock stage must exist");
let wrong_edge = stage.outbound_edge;
let object_id = self.allocate_object_id(wrong_edge);
let object = MockObject {
edge_id: plan::EdgeId(stage.inbound_edge.0 + 10_000),
object_id: 30_000 + u64::from(stage_index),
edge_id: wrong_edge,
object_id,
sequence: 0,
kind: MockObjectKind::Token,
token_id: None,
@ -786,6 +800,26 @@ impl LocalMockCluster {
.expect("mock plan must contain requested edge")
}
fn edge_chain(&self) -> Vec<plan::EdgeId> {
let mut stages = self.plan.stages.iter().collect::<Vec<_>>();
stages.sort_by_key(|stage| stage.stage_index);
let mut chain = Vec::with_capacity(stages.len() + 1);
if let Some(first) = stages.first() {
chain.push(first.inbound_edge);
}
chain.extend(stages.into_iter().map(|stage| stage.outbound_edge));
chain
}
fn allocate_object_id(&mut self, edge_id: plan::EdgeId) -> u64 {
self.object_allocators
.entry(edge_id)
.or_insert_with(|| edge_actor::ObjectIdAllocator::new(edge_actor::EdgeId(edge_id.0)))
.alloc()
.object_id
.0
}
fn finish_outcome(&self) -> LocalMockOutcome {
LocalMockOutcome {
engine_events: self.engine_events.clone(),
@ -801,6 +835,7 @@ impl LocalMockCluster {
live_stage_runs: self.resources.live_stage_runs.len(),
transport_delivery_count: self.transport.deliveries().len(),
transport_deliveries: self.transport.deliveries().to_vec(),
edge_chain: self.edge_chain(),
}
}

11
crates/mvp-system/src/tests/local_mock/mock_node.rs
View file

@ -1,5 +1,6 @@
use mvp_system::run_plan as plan;
use mvp_system::stage_controller as stage;
use mvp_system::tx_rx_edge_actor as edge_actor;
use super::mock_transport::MockObject;
use super::mock_worker::MockWorker;
@ -14,6 +15,7 @@ pub struct MockNode {
stage_count: u32,
inbound_edge: Option<plan::EdgeId>,
outbound_edge: Option<plan::EdgeId>,
outbound_object_allocator: Option<edge_actor::ObjectIdAllocator>,
controller: stage::StageControllerHarness,
worker: MockWorker,
event_cursor: usize,
@ -31,6 +33,7 @@ impl MockNode {
stage_count,
inbound_edge: None,
outbound_edge: None,
outbound_object_allocator: None,
controller: stage::StageControllerHarness::new(stage::NodeId(node_id.0)),
worker: MockWorker::new(stage_index, eos_after_sequence),
event_cursor: 0,
@ -40,6 +43,9 @@ impl MockNode {
pub fn provision(&mut self, from: stage::NodeId, provision: stage::ProvisionStage) {
self.inbound_edge = Some(plan::EdgeId(provision.inbound.edge_id.0));
self.outbound_edge = Some(plan::EdgeId(provision.outbound.edge_id.0));
self.outbound_object_allocator = Some(edge_actor::ObjectIdAllocator::new(
edge_actor::EdgeId(provision.outbound.edge_id.0),
));
self.controller
.observe(stage::StageEvent::ProvisionStage { from, provision });
}
@ -47,6 +53,9 @@ impl MockNode {
pub fn provision_from_wrong_orchestrator(&mut self, provision: stage::ProvisionStage) {
self.inbound_edge = Some(plan::EdgeId(provision.inbound.edge_id.0));
self.outbound_edge = Some(plan::EdgeId(provision.outbound.edge_id.0));
self.outbound_object_allocator = Some(edge_actor::ObjectIdAllocator::new(
edge_actor::EdgeId(provision.outbound.edge_id.0),
));
self.controller.observe(stage::StageEvent::ProvisionStage {
from: stage::NodeId(provision.authorized_orchestrator.0 + 1),
provision,
@ -87,9 +96,11 @@ impl MockNode {
stage::StageCommand::ExecuteStep(step) => Some(step.clone()),
_ => None,
})?;
let output_object_id = self.outbound_object_allocator.as_mut()?.alloc().object_id.0;
let produced = self.worker.execute(
&step,
outbound_edge,
output_object_id,
self.stage_index + 1 == self.stage_count,
);
self.controller.observe(stage::StageEvent::StepCompleted {

10
crates/mvp-system/src/tests/local_mock/mock_worker.rs
View file

@ -4,26 +4,22 @@ use mvp_system::stage_controller as stage;
use super::mock_transport::{MockObject, MockObjectKind};
pub struct MockWorker {
stage_index: u32,
eos_after_sequence: u64,
}
impl MockWorker {
pub fn new(stage_index: u32, eos_after_sequence: u64) -> Self {
Self {
stage_index,
eos_after_sequence,
}
pub fn new(_stage_index: u32, eos_after_sequence: u64) -> Self {
Self { eos_after_sequence }
}
pub fn execute(
&self,
step: &stage::ExecuteStep,
outbound_edge: plan::EdgeId,
object_id: u64,
is_last_stage: bool,
) -> MockObject {
let sequence = step.input.sequence;
let object_id = 10_000 + u64::from(self.stage_index) * 1_000 + sequence;
if is_last_stage {
MockObject {
edge_id: outbound_edge,

25
crates/mvp-system/src/tests/tx_rx_edge_actor_guarantees.rs
View file

@ -16,6 +16,25 @@ fn edge_id() -> edge_actor::EdgeId {
edge_actor::EdgeId(7001)
}
#[test]
fn object_id_allocator_is_scoped_to_one_producer_edge() {
let mut stage0_output = edge_actor::ObjectIdAllocator::new(edge_actor::EdgeId(7001));
let mut stage1_output = edge_actor::ObjectIdAllocator::new(edge_actor::EdgeId(7002));
assert_eq!(
stage0_output.alloc(),
edge_actor::ObjectKey::new(edge_actor::EdgeId(7001), edge_actor::ObjectId(1))
);
assert_eq!(
stage1_output.alloc(),
edge_actor::ObjectKey::new(edge_actor::EdgeId(7002), edge_actor::ObjectId(1))
);
assert_eq!(
stage0_output.alloc(),
edge_actor::ObjectKey::new(edge_actor::EdgeId(7001), edge_actor::ObjectId(2))
);
}
// Tx starts in provisioning and represents the producer side of one edge. The
// harness records only actor messages, not bytes or flow-control details.
fn new_tx() -> edge_actor::TxActorHarness {
@ -43,12 +62,6 @@ fn assert_actor_message_is_payload_free(message: &edge_actor::ActorMessage) {
| edge_actor::ActorMessage::ObjectIdentity { .. }
| edge_actor::ActorMessage::OpaqueHandle { .. }
| edge_actor::ActorMessage::CoarseFault { .. } => {}
edge_actor::ActorMessage::PayloadBytes { .. }
| edge_actor::ActorMessage::HostPointer { .. }
| edge_actor::ActorMessage::ByteRange { .. }
| edge_actor::ActorMessage::CreditCount { .. } => {
panic!("edge actor message carried payload or flow-control detail: {message:?}")
}
}
}

68
crates/mvp-system/src/tx_rx_edge_actor.rs
View file

@ -7,6 +7,36 @@ pub struct PortId(pub String);
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct ObjectId(pub u64);
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct ObjectKey {
pub edge_id: EdgeId,
pub object_id: ObjectId,
}
impl ObjectKey {
pub fn new(edge_id: EdgeId, object_id: ObjectId) -> Self {
Self { edge_id, object_id }
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ObjectIdAllocator {
edge_id: EdgeId,
next: u64,
}
impl ObjectIdAllocator {
pub fn new(edge_id: EdgeId) -> Self {
Self { edge_id, next: 1 }
}
pub fn alloc(&mut self) -> ObjectKey {
let object_key = ObjectKey::new(self.edge_id, ObjectId(self.next));
self.next += 1;
object_key
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct OpaqueHandle(pub u64);
@ -56,23 +86,6 @@ pub enum ActorMessage {
edge_id: EdgeId,
reason: ActorFaultReason,
},
PayloadBytes {
edge_id: EdgeId,
bytes: Vec<u8>,
},
HostPointer {
edge_id: EdgeId,
address: usize,
},
ByteRange {
edge_id: EdgeId,
start: u64,
len: u64,
},
CreditCount {
edge_id: EdgeId,
credits: u64,
},
}
impl ActorMessage {
@ -81,11 +94,7 @@ impl ActorMessage {
ActorMessage::Lifecycle { edge_id, .. }
| ActorMessage::ObjectIdentity { edge_id, .. }
| ActorMessage::OpaqueHandle { edge_id, .. }
| ActorMessage::CoarseFault { edge_id, .. }
| ActorMessage::PayloadBytes { edge_id, .. }
| ActorMessage::HostPointer { edge_id, .. }
| ActorMessage::ByteRange { edge_id, .. }
| ActorMessage::CreditCount { edge_id, .. } => *edge_id,
| ActorMessage::CoarseFault { edge_id, .. } => *edge_id,
}
}
}
@ -141,15 +150,13 @@ enum ActorState {
Stopped,
}
#[cfg(test)]
pub struct TxActorHarness {
pub struct TxEdgeActor {
config: TxConfig,
state: ActorState,
messages: Vec<ActorMessage>,
}
#[cfg(test)]
impl TxActorHarness {
impl TxEdgeActor {
pub fn new(config: TxConfig) -> Self {
Self {
config,
@ -231,15 +238,13 @@ impl TxActorHarness {
}
}
#[cfg(test)]
pub struct RxActorHarness {
pub struct RxEdgeActor {
config: RxConfig,
state: ActorState,
messages: Vec<ActorMessage>,
}
#[cfg(test)]
impl RxActorHarness {
impl RxEdgeActor {
pub fn new(config: RxConfig) -> Self {
Self {
config,
@ -322,3 +327,6 @@ impl RxActorHarness {
});
}
}
pub type TxActorHarness = TxEdgeActor;
pub type RxActorHarness = RxEdgeActor;

141
crates/mvp-system/tests/local_e2e_cluster.rs Normal file
View file

@ -0,0 +1,141 @@
use std::path::Path;
use std::process::Command;
const IMAGE: &str = "swactor-mvp-local-e2e-cluster:latest";
#[test]
fn local_e2e_cluster_docker_cpu_pipeline_prompt() {
if std::env::var_os("MVP_SYSTEM_LOCAL_E2E_CLUSTER").is_none() {
eprintln!("skipping; set MVP_SYSTEM_LOCAL_E2E_CLUSTER=1 to run Docker CPU cluster e2e");
return;
}
build_docker_fixture();
let output = Command::new(env!("CARGO_BIN_EXE_mvp-local-e2e-cluster"))
.arg("--prompt")
.arg("ping")
.env("MVP_LOCAL_E2E_CLUSTER_IMAGE", IMAGE)
.output()
.expect("run mvp-local-e2e-cluster");
assert!(
output.status.success(),
"mvp-local-e2e-cluster failed\nstdout:\n{}\nstderr:\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
let value: serde_json::Value = serde_json::from_slice(&output.stdout).expect("json stdout");
assert_eq!(value["ok"], true);
assert_eq!(value["actor_plane"], "iroh-swactor");
assert_eq!(value["data_plane"], "iroh-quic-persistent-edge-streams");
assert_eq!(
value["edge_protocol"],
"edge-id-preamble-mo01-object-records"
);
assert_eq!(
value["node_local_data_plane"],
"arena-backed-rings-json-metadata-only"
);
assert_eq!(
value["worker_processes"],
"docker-tinygrad-cpu-worker-per-node"
);
assert_eq!(value["tinygrad_device"], "CPU");
assert_eq!(value["prompt_text"], "ping");
assert_eq!(value["response_text"], "pong");
assert_eq!(value["response_tokens"].as_array().map(Vec::len), Some(1));
assert_eq!(
value["engine_builder_pattern"],
"host-coordinator-static-topology-docker-workers"
);
assert_eq!(value["engine_builder_node_count"], 3);
assert_eq!(value["engine_builder_stage_assignments"], 2);
assert_eq!(value["injected_prompt_observed"], true);
assert_eq!(value["token_received_observed"], true);
assert_eq!(value["run_completed_observed"], true);
assert_eq!(value["run_torn_down_observed"], true);
assert_eq!(value["stop_sent_to_all_nodes"], true);
assert_eq!(value["stage_ready_stdout_count"], 2);
assert!(
value["node0_endpoint"]["addrs"]
.as_array()
.is_some_and(|addrs| !addrs.is_empty()),
"{value}"
);
assert!(
value["node1_endpoint"]["addrs"]
.as_array()
.is_some_and(|addrs| !addrs.is_empty()),
"{value}"
);
}
fn build_docker_fixture() {
let crate_dir = Path::new(env!("CARGO_MANIFEST_DIR"));
let workspace = crate_dir
.parent()
.and_then(Path::parent)
.expect("workspace root")
.canonicalize()
.expect("canonical workspace root");
let context = std::env::temp_dir().join(format!(
"mvp-system-local-e2e-cluster-context-{}",
std::process::id()
));
let _ = std::fs::remove_dir_all(&context);
copy_workspace_context(&workspace, &context);
let dockerfile = context.join("crates/mvp-system/tests/local_e2e_cluster/Dockerfile");
phase("building Docker CPU cluster fixture image");
let build = Command::new("docker")
.args(["build", "-f"])
.arg(&dockerfile)
.args(["-t", IMAGE])
.arg(&context)
.status()
.expect("run docker build");
assert!(build.success(), "docker build failed with status {build}");
}
fn copy_workspace_context(source: &Path, dest: &Path) {
std::fs::create_dir_all(dest).expect("create docker context");
for entry in std::fs::read_dir(source).expect("read workspace") {
let entry = entry.expect("read workspace entry");
let name = entry.file_name();
let name = name.to_string_lossy();
if matches!(name.as_ref(), ".git" | "target" | ".dockerignore") {
continue;
}
copy_context_entry(&entry.path(), &dest.join(name.as_ref()));
}
}
fn copy_context_entry(source: &Path, dest: &Path) {
let metadata = std::fs::symlink_metadata(source).expect("context metadata");
if metadata.file_type().is_symlink() {
return;
}
if metadata.is_dir() {
std::fs::create_dir_all(dest).expect("create context dir");
for entry in std::fs::read_dir(source).expect("read context dir") {
let entry = entry.expect("read context entry");
let name = entry.file_name();
let name = name.to_string_lossy();
if matches!(name.as_ref(), ".git" | "target" | ".dockerignore") {
continue;
}
copy_context_entry(&entry.path(), &dest.join(name.as_ref()));
}
} else if metadata.is_file() {
if let Some(parent) = dest.parent() {
std::fs::create_dir_all(parent).expect("create context parent");
}
std::fs::copy(source, dest).expect("copy context file");
}
}
fn phase(message: &str) {
eprintln!("local-e2e-cluster: {message}");
}

20
crates/mvp-system/tests/local_e2e_cluster/Dockerfile Normal file
View file

@ -0,0 +1,20 @@
FROM rust:1-bookworm
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
ca-certificates \
pkg-config \
python3 \
python3-pip && \
python3 -m pip install --no-cache-dir --break-system-packages tinygrad==0.12.0 numpy && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
ENV DEV=CPU
ENV PYTHONDONTWRITEBYTECODE=1
ENV CARGO_TARGET_DIR=/workspace/target
COPY . /workspace
WORKDIR /workspace
RUN cargo build -p mvp-system --features local-e2e --bin mvp-local-e2e-cluster
ENTRYPOINT ["/workspace/target/debug/mvp-local-e2e-cluster"]

254
crates/mvp-system/tests/local_e2e_cluster/tinygrad_cpu_worker.py Executable file
View file

@ -0,0 +1,254 @@
#!/usr/bin/env python3
from __future__ import annotations
import json
import mmap
import os
import struct
import sys
from typing import Any
HEADER_LEN = 48
worker_generation = 0
arena: mmap.mmap | None = None
rings: dict[int, dict[str, Any]] = {}
objects: dict[int, dict[str, Any]] = {}
role: dict[str, Any] = {}
Tensor: Any = None
dtypes: Any = None
next_handle = 42
def control(**event: Any) -> None:
print(json.dumps(event, separators=(",", ":")), flush=True)
def fatal(reason: str, **fields: Any) -> None:
control(type="WorkerFatal", reason=reason, **fields)
raise SystemExit(1)
def require_arena() -> mmap.mmap:
if arena is None:
fatal("ArenaNotMapped")
return arena
def require_tinygrad() -> tuple[Any, Any]:
if Tensor is None or dtypes is None:
fatal("BackendNotInitialized")
return Tensor, dtypes
def initialize(cmd: dict[str, Any]) -> None:
global arena, Tensor, dtypes, worker_generation
if int(cmd["required_ring_helper_abi"]) != 1:
fatal("UnsupportedHelperAbi", required_ring_helper_abi=cmd["required_ring_helper_abi"])
worker_generation = int(cmd["worker_generation"])
fd = int(os.environ["SWACTOR_ARENA_FD"])
size = int(cmd.get("arena_ceiling", os.environ["SWACTOR_ARENA_BYTES"]))
arena = mmap.mmap(fd, size)
os.environ.setdefault("DEV", cmd.get("backend", {}).get("device", "CPU"))
from tinygrad import Tensor as TinyTensor, dtypes as tiny_dtypes
Tensor = TinyTensor
dtypes = tiny_dtypes
Tensor([1], dtype=dtypes.int32).realize().numpy().tolist()
control(
type="WorkerReady",
pid=os.getpid(),
worker_generation=worker_generation,
ring_helper_abi=1,
backend={"device": os.environ.get("DEV", "CPU")},
)
def install_ring(cmd: dict[str, Any]) -> None:
ring_id = int(cmd["ring_id"])
layout = cmd["layout"]
spec = cmd["object_spec"]
rings[ring_id] = {
"ring_id": ring_id,
"edge_id": int(cmd["edge_id"]),
"port_id": cmd["port_id"],
"direction": cmd["direction"],
"data_offset": int(layout["data_offset"]),
"data_capacity": int(layout["data_capacity"]),
"max_extent": int(spec["max_extent"]),
"alignment": int(spec["alignment"]),
"next_sequence": 0,
}
control(type="RingInstalled", ring_id=ring_id, edge_id=rings[ring_id]["edge_id"], port_id=cmd["port_id"])
def configure_role(cmd: dict[str, Any]) -> None:
config = cmd["config"]
role.clear()
role.update(
role_id=int(cmd["role_id"]),
run_id=int(config["run_id"]),
stage_index=int(config["stage_index"]),
layer_start=int(config["layer_start"]),
layer_end_exclusive=int(config["layer_end_exclusive"]),
)
control(type="RoleConfigured", role_id=role["role_id"])
def parse_record(ring: dict[str, Any]) -> tuple[int, int, int, bytes]:
view = require_arena()
base = ring["data_offset"]
header = view[base : base + HEADER_LEN]
if header[0:4] != b"MO01" or header[4] != 1 or header[5] != HEADER_LEN:
fatal("InvalidObjectHeader", ring_id=ring["ring_id"])
object_id = struct.unpack_from("<Q", header, 8)[0]
sequence = struct.unpack_from("<Q", header, 16)[0]
extent = struct.unpack_from("<Q", header, 24)[0]
flags = struct.unpack_from("<I", header, 32)[0]
reserved = struct.unpack_from("<I", header, 36)[0]
del flags, reserved
if extent > ring["max_extent"] or (ring["alignment"] and extent % ring["alignment"]):
fatal("ObjectExtentInvalid", ring_id=ring["ring_id"], object_id=object_id, extent=extent)
if sequence != ring["next_sequence"]:
fatal("SequenceViolation", ring_id=ring["ring_id"], expected=ring["next_sequence"], actual=sequence)
payload = bytes(view[base + HEADER_LEN : base + HEADER_LEN + extent])
ring["next_sequence"] += 1
return object_id, sequence, extent, payload
def ring_readable(cmd: dict[str, Any]) -> None:
global next_handle
tensor, dtype_mod = require_tinygrad()
ring_id = int(cmd["ring_id"])
ring = rings[ring_id]
if ring["direction"] != "ingress":
fatal("WrongRingDirection", ring_id=ring_id)
object_id, sequence, extent, payload = parse_record(ring)
values = list(struct.unpack(f"<{extent // 4}i", payload))
loaded = tensor(values, dtype=dtype_mod.int32).realize()
handle = next_handle
next_handle += 1
objects[handle] = {
"object_id": object_id,
"sequence": sequence,
"tensor": loaded,
"extent": extent,
}
control(
type="ObjectLoaded",
ring_id=ring_id,
edge_id=ring["edge_id"],
port_id=ring["port_id"],
object_id=object_id,
sequence=sequence,
extent=extent,
device_handle={"worker_generation": worker_generation, "id": handle},
)
def write_record(ring: dict[str, Any], object_id: int, sequence: int, words: list[int], flags: int) -> int:
payload = b"".join(struct.pack("<i", word) for word in words)
extent = len(payload)
if extent > ring["max_extent"]:
fatal("OutputExtentInvalid", ring_id=ring["ring_id"], extent=extent)
header = bytearray(HEADER_LEN)
header[0:4] = b"MO01"
header[4] = 1
header[5] = HEADER_LEN
struct.pack_into("<Q", header, 8, object_id)
struct.pack_into("<Q", header, 16, sequence)
struct.pack_into("<Q", header, 24, extent)
struct.pack_into("<I", header, 32, flags)
struct.pack_into("<I", header, 36, 0)
view = require_arena()
base = ring["data_offset"]
view[base : base + HEADER_LEN] = header
view[base + HEADER_LEN : base + HEADER_LEN + extent] = payload
return HEADER_LEN + extent
def execute_step(cmd: dict[str, Any]) -> None:
if not role:
fatal("RoleNotConfigured")
tensor, _ = require_tinygrad()
del tensor
role_id = int(cmd["role_id"])
if role_id != role["role_id"]:
fatal("RoleMismatch", expected=role["role_id"], actual=role_id)
step_id = int(cmd["step_id"])
input_binding = cmd["inputs"][0]
output_binding = cmd["outputs"][0]
device_handle = input_binding["device_handle"]
if int(device_handle["worker_generation"]) != worker_generation:
fatal("OldGenerationHandle", handle=device_handle)
handle = int(device_handle["id"])
obj = objects[handle]
if int(input_binding["object_id"]) != obj["object_id"] or int(input_binding["sequence"]) != obj["sequence"]:
fatal("InputBindingMismatch", step_id=step_id)
transformed = int(obj["tensor"].sum().item()) + role["layer_start"] + role["layer_end_exclusive"] + role["stage_index"]
if bool(cmd.get("runtime", {}).get("final_stage")):
words = [6 if transformed % 2 == 1 else 8]
else:
words = [transformed if transformed > 0 else 1]
ring = rings[int(output_binding["ring_id"])]
if ring["direction"] != "egress":
fatal("WrongRingDirection", ring_id=ring["ring_id"])
committed = write_record(
ring,
int(output_binding["object_id"]),
int(output_binding["sequence"]),
words,
int(output_binding.get("flags", 0)),
)
control(
type="ObjectProduced",
ring_id=ring["ring_id"],
edge_id=ring["edge_id"],
port_id=ring["port_id"],
object_id=int(output_binding["object_id"]),
sequence=int(output_binding["sequence"]),
committed_bytes=committed,
)
if bool(cmd.get("release_inputs_after")):
objects.pop(handle, None)
control(type="StepCompleted", role_id=role_id, step_id=step_id)
def release_device_object(cmd: dict[str, Any]) -> None:
handle = int(cmd["device_handle"]["id"])
objects.pop(handle, None)
control(type="DeviceObjectReleased", device_handle=cmd["device_handle"])
def uninstall_ring(cmd: dict[str, Any]) -> None:
ring_id = int(cmd["ring_id"])
rings.pop(ring_id, None)
control(type="RingQuiesced", ring_id=ring_id)
def shutdown_worker(_: dict[str, Any]) -> None:
control(type="WorkerStopped", reason="Graceful")
raise SystemExit(0)
HANDLERS = {
"InitializeWorker": initialize,
"InstallRing": install_ring,
"ConfigureRole": configure_role,
"RingReadable": ring_readable,
"ExecuteStep": execute_step,
"ReleaseDeviceObject": release_device_object,
"UninstallRing": uninstall_ring,
"ShutdownWorker": shutdown_worker,
}
for raw in sys.stdin:
if not raw.strip():
continue
try:
command = json.loads(raw)
except json.JSONDecodeError as exc:
fatal("InvalidJson", error=str(exc))
handler = HANDLERS.get(command.get("type"))
if handler is None:
fatal("UnknownCommand", command=command.get("type"))
handler(command)

2
src/stats.rs
View file

@ -142,7 +142,7 @@ pub trait StatsHook: Send + Sync {
fn on_tick(&self, worker_id: usize, snapshots: &[ActorSnapshot]);
}
/// Per-actor info for the dashboard.
/// Per-actor runtime stats.
#[derive(Debug, Clone)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct ActorInfo {

8
src/std/extension.rs
View file

@ -48,11 +48,9 @@ impl RuntimeExtension for StdExtension {
let mut notifications = Vec::new();
for (addr, reason, exit_value) in dead {
let watch_notifications = self.watch_registry.notify_death(
*addr,
stop_to_exit(*reason),
exit_value.clone(),
);
let watch_notifications =
self.watch_registry
.notify_death(*addr, stop_to_exit(*reason), exit_value.clone());
for (watcher, exited) in watch_notifications {
notifications.push((watcher, Box::new(exited) as Box<dyn Any + Send>));
}

6
src/std/group_registry.rs
View file

@ -35,7 +35,11 @@ impl GroupRegistry {
.entry(group.clone())
.or_insert_with(|| HashSet::with_hasher(AddrBuildHasher))
.insert(addr);
self.memberships.write().entry(addr).or_default().insert(group);
self.memberships
.write()
.entry(addr)
.or_default()
.insert(group);
}
/// Remove an actor from a named group. Empty groups are auto-deleted.

1
src/worker.rs
View file

@ -17,7 +17,6 @@ use crate::stats::{ActorSnapshot, TickTiming, WorkerStats};
use crate::extension::WorkerExtension;
/// Whether an actor should be skipped during `tick_all`.
pub(crate) fn should_skip_actor(poisoned: bool, stopping: bool, suspended: bool) -> bool {
poisoned || stopping || suspended

28
tests/core_extension_seams.rs
View file

@ -3,7 +3,9 @@ use std::sync::Arc;
use parking_lot::Mutex;
use swactor::actor::{ActorAddress, ActorInterface, Ctx, Environment, EnvironmentBuilder, ExitValue, StopReason};
use swactor::actor::{
ActorAddress, ActorInterface, Ctx, Environment, EnvironmentBuilder, ExitValue, StopReason,
};
use swactor::config::RuntimeConfig;
use swactor::extension::{RuntimeExtension, WorkerExtension};
use swactor::runtime::Runtime;
@ -62,18 +64,16 @@ impl RuntimeExtension for SeamExtension {
&self,
dead: &[(ActorAddress, StopReason, Option<ExitValue>)],
) -> Vec<(ActorAddress, Box<dyn Any + Send>)> {
let _ = dead.iter().map(|(_, reason, value)| (reason, value)).count();
let _ = dead
.iter()
.map(|(_, reason, value)| (reason, value))
.count();
let Some(report_to) = *self.state.death_report_to.lock() else {
return Vec::new();
};
dead.iter()
.map(|(addr, _, _)| {
(
report_to,
Box::new(DeathSeen(*addr)) as Box<dyn Any + Send>,
)
})
.map(|(addr, _, _)| (report_to, Box::new(DeathSeen(*addr)) as Box<dyn Any + Send>))
.collect()
}
@ -124,12 +124,7 @@ impl WorkerExtension for SeamWorkerExtension {
.worker_pending
.lock()
.pop()
.map(|target| {
(
target,
Box::new(WorkerExtFired) as Box<dyn Any + Send>,
)
})
.map(|target| (target, Box::new(WorkerExtFired) as Box<dyn Any + Send>))
.into_iter()
.collect()
}
@ -238,8 +233,9 @@ impl ActorInterface for WorkerRequestActor {
type Response = ();
fn handle(&mut self, ctx: &Ctx, _msg: ()) {
ctx.raw_inner()
.post_worker_request(Box::new(WorkerRequest { target: self.target }));
ctx.raw_inner().post_worker_request(Box::new(WorkerRequest {
target: self.target,
}));
}
}

1
tests/message_delivery.rs
View file

@ -398,4 +398,3 @@ fn fairness_budget_prevents_starvation() {
"unlimited budget drains all"
);
}

5
tests/std_extension.rs
View file

@ -82,7 +82,10 @@ fn runtime_naming_lifecycle() {
tick_n(&rt, 2);
assert_eq!(inbox.try_recv(), Some(Pong));
assert!(rt.register_name("worker", rt.spawn(PingPongActor).unwrap()).is_err());
assert!(
rt.register_name("worker", rt.spawn(PingPongActor).unwrap())
.is_err()
);
let mut names = rt.registered_names();
names.sort();