feat: wire stage roster and binary-swap verify
Orchestrator resolves per-stage SWIM roster + emits pp_stage_roster; pp_gpu_node verifies fetched binary sha-256 digest (sha2 dep, strip release). Collector gains /diag/runs + /diag/stream SSE; vastai/dockerfile redeploy hardening. Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
This commit is contained in:
parent
00fb3b2d91
commit
1af7e48201
23 changed files with 3504 additions and 2105 deletions
1
Cargo.lock
generated
1
Cargo.lock
generated
|
|
@ -1344,6 +1344,7 @@ dependencies = [
|
|||
"swactor-transport",
|
||||
"tar",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ relay = [
|
|||
collector = [
|
||||
"dep:axum",
|
||||
"dep:tokio",
|
||||
"dep:tokio-stream",
|
||||
"tokio/macros",
|
||||
"tokio/net",
|
||||
"tokio/sync",
|
||||
|
|
@ -53,6 +54,7 @@ iroh-relay = { version = "0.98", features = ["test-utils"], optional = true }
|
|||
iroh-metrics = { version = "0.38", optional = true }
|
||||
tokio = { version = "1", features = ["rt-multi-thread"], optional = true }
|
||||
axum = { version = "0.8", optional = true }
|
||||
tokio-stream = { version = "0.1", features = ["sync"], optional = true }
|
||||
tar = { version = "0.4", optional = true }
|
||||
flate2 = { version = "1", optional = true }
|
||||
|
||||
|
|
|
|||
|
|
@ -13,18 +13,25 @@
|
|||
//! pending hints from [`CollectorState::take_pending_hints`], and (for
|
||||
//! finalize) drive the T1.7 snapshot-then-tar sequence.
|
||||
|
||||
use std::convert::Infallible;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use axum::body::Body;
|
||||
use axum::extract::{Path, State};
|
||||
use axum::http::{HeaderMap, StatusCode, header};
|
||||
use axum::response::sse::{Event, KeepAlive, Sse};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::routing::{get, post};
|
||||
use axum::{Json, Router};
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
use tokio_stream::StreamExt;
|
||||
use tokio_stream::wrappers::BroadcastStream;
|
||||
use tokio_stream::wrappers::errors::BroadcastStreamRecvError;
|
||||
|
||||
use super::protocol::{ClockEcho, Hints, PostAck, RecordKind};
|
||||
use super::state::{CollectorState, MAX_BODY_BYTES};
|
||||
use super::state::{CollectorState, MAX_BODY_BYTES, NodeStats, RunStats};
|
||||
use super::{bundle, wall_ms_now};
|
||||
|
||||
/// Build the axum router. Wire this into `axum::serve` from the
|
||||
|
|
@ -33,9 +40,101 @@ pub fn router(state: Arc<CollectorState>) -> Router {
|
|||
Router::new()
|
||||
.route("/diag/{kind}", post(ingest))
|
||||
.route("/diag/bundle/{run_id}", get(download_bundle))
|
||||
.route("/diag/runs", get(list_runs))
|
||||
.route("/diag/stream/{run_id}", get(stream_run))
|
||||
.with_state(state)
|
||||
}
|
||||
|
||||
/// SSE event for an in-memory record. The `event` field carries the
|
||||
/// `RecordKind` so clients can `addEventListener("snapshot", …)`.
|
||||
fn format_sse(event: &str, data: &str) -> Event {
|
||||
Event::default().event(event).data(data)
|
||||
}
|
||||
|
||||
/// Per-node accounting shape returned from `/diag/runs`. Mirrors
|
||||
/// `NodeStats` minus the cached `identity` blob (potentially large;
|
||||
/// belongs in the bundle, not a directory listing).
|
||||
#[derive(Serialize)]
|
||||
struct NodeSummary {
|
||||
node_id: String,
|
||||
boot_recorded: bool,
|
||||
event_batches: u64,
|
||||
snapshots: u64,
|
||||
finalize_recorded: bool,
|
||||
}
|
||||
|
||||
/// Per-run accounting shape returned from `/diag/runs`.
|
||||
#[derive(Serialize)]
|
||||
struct RunSummary {
|
||||
run_id: String,
|
||||
run_start_collector_ms: Option<u64>,
|
||||
run_end_collector_ms: Option<u64>,
|
||||
finalize_received: bool,
|
||||
nodes: Vec<NodeSummary>,
|
||||
}
|
||||
|
||||
impl RunSummary {
|
||||
fn from_stats(run_id: String, stats: RunStats) -> Self {
|
||||
let mut nodes: Vec<NodeSummary> = stats
|
||||
.nodes
|
||||
.into_iter()
|
||||
.map(|(node_id, n): (String, NodeStats)| NodeSummary {
|
||||
node_id,
|
||||
boot_recorded: n.boot_recorded,
|
||||
event_batches: n.event_batches,
|
||||
snapshots: n.snapshots,
|
||||
finalize_recorded: n.finalize_recorded,
|
||||
})
|
||||
.collect();
|
||||
nodes.sort_by(|a, b| a.node_id.cmp(&b.node_id));
|
||||
RunSummary {
|
||||
run_id,
|
||||
run_start_collector_ms: stats.run_start_collector_ms,
|
||||
run_end_collector_ms: stats.run_end_collector_ms,
|
||||
finalize_received: stats.finalize_received,
|
||||
nodes,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn list_runs(State(state): State<Arc<CollectorState>>) -> Json<Vec<RunSummary>> {
|
||||
let summaries = state
|
||||
.run_summaries()
|
||||
.into_iter()
|
||||
.map(|(run_id, stats)| RunSummary::from_stats(run_id, stats))
|
||||
.collect();
|
||||
Json(summaries)
|
||||
}
|
||||
|
||||
/// Subscribe to live persisted records for a single run as
|
||||
/// Server-Sent Events. Each event's name is the record `kind`
|
||||
/// (`boot`, `events`, `snapshot`, `finalize`); the data payload is
|
||||
/// the JSON-serialized `LiveRecord`.
|
||||
///
|
||||
/// Unknown `run_id`s are accepted — the connection stays open and
|
||||
/// the client will see records once they arrive. Slow subscribers
|
||||
/// that fall behind the broadcast capacity silently skip the gap
|
||||
/// (`/diag/bundle/{run_id}` is the catch-up path).
|
||||
async fn stream_run(
|
||||
Path(run_id): Path<String>,
|
||||
State(state): State<Arc<CollectorState>>,
|
||||
) -> Sse<impl tokio_stream::Stream<Item = Result<Event, Infallible>>> {
|
||||
let rx = state.subscribe();
|
||||
let stream = BroadcastStream::new(rx).filter_map(move |item| match item {
|
||||
Ok(rec) if rec.run_id == run_id => {
|
||||
let json = serde_json::to_string(&*rec).ok()?;
|
||||
Some(Ok(format_sse(rec.kind.as_str(), &json)))
|
||||
}
|
||||
Ok(_) => None,
|
||||
Err(BroadcastStreamRecvError::Lagged(_)) => None,
|
||||
});
|
||||
Sse::new(stream).keep_alive(
|
||||
KeepAlive::new()
|
||||
.interval(Duration::from_secs(15))
|
||||
.text("ping"),
|
||||
)
|
||||
}
|
||||
|
||||
async fn ingest(
|
||||
State(state): State<Arc<CollectorState>>,
|
||||
Path(kind): Path<String>,
|
||||
|
|
|
|||
|
|
@ -96,6 +96,23 @@ impl RecordKind {
|
|||
}
|
||||
}
|
||||
|
||||
/// A persisted record, fanned out to live SSE subscribers via
|
||||
/// `GET /diag/stream/{run_id}`.
|
||||
///
|
||||
/// Ordering guarantee: per-(run_id, node_id, kind) monotonic by
|
||||
/// `seq`. Across kinds or nodes, ordering is best-effort — the
|
||||
/// broadcast preserves send order, but `Lagged` receivers see gaps
|
||||
/// and must reconcile via `GET /diag/bundle/{run_id}`.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct LiveRecord {
|
||||
pub run_id: String,
|
||||
pub node_id: String,
|
||||
pub kind: RecordKind,
|
||||
pub recv_ms: u64,
|
||||
pub seq: u64,
|
||||
pub body: serde_json::Value,
|
||||
}
|
||||
|
||||
/// Per-node summary written into `MANIFEST.json` at the root of a
|
||||
/// finalized bundle.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
|
|
|
|||
|
|
@ -7,12 +7,19 @@
|
|||
use std::collections::HashMap;
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use serde_json::Value;
|
||||
use tokio::sync::broadcast;
|
||||
|
||||
use super::protocol::{Hints, RecordKind};
|
||||
use super::protocol::{Hints, LiveRecord, RecordKind};
|
||||
|
||||
/// Default fan-out capacity for the live SSE broadcast — overridable
|
||||
/// via `SWACTOR_DIAG_STREAM_CAPACITY`. 1024 is ~70s of buffer at the
|
||||
/// ~14 rec/s typical of an 11-stage cluster; slow subscribers see
|
||||
/// `Lagged` rather than backpressuring the ingest path.
|
||||
pub const DEFAULT_STREAM_CAPACITY: usize = 1024;
|
||||
|
||||
/// How long `/diag/finalize` waits between marking nodes for
|
||||
/// snapshot_now and assembling the tarball, by default.
|
||||
|
|
@ -51,6 +58,11 @@ pub struct CollectorState {
|
|||
/// handler rebuilds from current staging. This is the
|
||||
/// "node-count heuristic" the spec names.
|
||||
canonical_node_counts: Mutex<HashMap<String, usize>>,
|
||||
/// Fan-out of every persisted record to live SSE subscribers.
|
||||
/// Lossy: when a subscriber falls behind the channel's capacity
|
||||
/// it observes `Lagged` and resumes from the next send. The
|
||||
/// catch-up path is `GET /diag/bundle/{run_id}`.
|
||||
live_tx: broadcast::Sender<Arc<LiveRecord>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
|
|
@ -88,6 +100,12 @@ pub struct NodeStats {
|
|||
|
||||
impl CollectorState {
|
||||
pub fn new(root: impl Into<PathBuf>) -> Self {
|
||||
let cap = std::env::var("SWACTOR_DIAG_STREAM_CAPACITY")
|
||||
.ok()
|
||||
.and_then(|s| s.parse::<usize>().ok())
|
||||
.filter(|n| *n > 0)
|
||||
.unwrap_or(DEFAULT_STREAM_CAPACITY);
|
||||
let (live_tx, _) = broadcast::channel(cap);
|
||||
Self {
|
||||
root: root.into(),
|
||||
finalize_wait: DEFAULT_FINALIZE_WAIT,
|
||||
|
|
@ -95,6 +113,7 @@ impl CollectorState {
|
|||
runs: Mutex::new(HashMap::new()),
|
||||
pending_hints: Mutex::new(HashMap::new()),
|
||||
canonical_node_counts: Mutex::new(HashMap::new()),
|
||||
live_tx,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -130,6 +149,17 @@ impl CollectorState {
|
|||
self
|
||||
}
|
||||
|
||||
/// Override the live-broadcast capacity. Production reads
|
||||
/// `SWACTOR_DIAG_STREAM_CAPACITY` in [`Self::new`]; tests use
|
||||
/// this builder to exercise the lossy-lagged path without
|
||||
/// racing other tests on a shared env var.
|
||||
pub fn with_stream_capacity(mut self, cap: usize) -> Self {
|
||||
let cap = cap.max(1);
|
||||
let (tx, _) = broadcast::channel(cap);
|
||||
self.live_tx = tx;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn finalize_wait(&self) -> Duration {
|
||||
self.finalize_wait
|
||||
}
|
||||
|
|
@ -174,9 +204,37 @@ impl CollectorState {
|
|||
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
|
||||
std::fs::write(&path, bytes)?;
|
||||
self.update_stats(run_id, node_id, kind, recv_ms, body);
|
||||
// Fan out to live SSE subscribers. SendError (zero receivers)
|
||||
// is steady state; ignore.
|
||||
let _ = self.live_tx.send(Arc::new(LiveRecord {
|
||||
run_id: run_id.to_string(),
|
||||
node_id: node_id.to_string(),
|
||||
kind,
|
||||
recv_ms,
|
||||
seq,
|
||||
body: body.clone(),
|
||||
}));
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
/// Subscribe to the live fan-out of persisted records. Each
|
||||
/// receiver gets every record sent after subscription; if the
|
||||
/// receiver falls behind the channel capacity it observes
|
||||
/// `RecvError::Lagged(n)` and continues from the next send.
|
||||
pub fn subscribe(&self) -> broadcast::Receiver<Arc<LiveRecord>> {
|
||||
self.live_tx.subscribe()
|
||||
}
|
||||
|
||||
/// Snapshot of all known runs and their accounting, sorted by
|
||||
/// `run_id`. Used by `GET /diag/runs`.
|
||||
pub fn run_summaries(&self) -> Vec<(String, RunStats)> {
|
||||
let runs = self.runs.lock().expect("collector runs mutex poisoned");
|
||||
let mut out: Vec<(String, RunStats)> =
|
||||
runs.iter().map(|(k, v)| (k.clone(), v.clone())).collect();
|
||||
out.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
out
|
||||
}
|
||||
|
||||
fn next_seq(&self, run_id: &str, node_id: &str, kind: RecordKind) -> u64 {
|
||||
let mut seqs = self.seqs.lock().expect("collector seq mutex poisoned");
|
||||
let key = SeqKey {
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@
|
|||
|
||||
#![cfg(feature = "collector")]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::Read;
|
||||
use std::net::SocketAddr;
|
||||
use std::path::PathBuf;
|
||||
|
|
@ -39,33 +40,50 @@ use tokio::net::TcpStream;
|
|||
struct Fixture {
|
||||
addr: SocketAddr,
|
||||
root: PathBuf,
|
||||
state: Arc<CollectorState>,
|
||||
_tmpdir: TempDir,
|
||||
_server: tokio::task::JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl Fixture {
|
||||
async fn start() -> Self {
|
||||
Self::start_with(|s| s).await
|
||||
}
|
||||
|
||||
/// Like `start`, but allows the caller to layer additional
|
||||
/// builder calls onto the [`CollectorState`] before it's wrapped
|
||||
/// in an `Arc` and handed to the server. Used by the stream
|
||||
/// tests that need a smaller broadcast capacity to exercise the
|
||||
/// lagged path.
|
||||
async fn start_with(configure: impl FnOnce(CollectorState) -> CollectorState) -> Self {
|
||||
let tmpdir = TempDir::new();
|
||||
let root = tmpdir.path().to_path_buf();
|
||||
// Tests don't have aggregator clients chasing hints, so the
|
||||
// finalize wait would just stall every assertion. Collapse it.
|
||||
let state = Arc::new(
|
||||
let state = configure(
|
||||
CollectorState::new(&root).with_finalize_wait(Duration::from_millis(0)),
|
||||
);
|
||||
let state = Arc::new(state);
|
||||
let listener = bind("127.0.0.1:0".parse().unwrap()).await.expect("bind");
|
||||
let addr = listener.local_addr().expect("local_addr");
|
||||
let serve_state = Arc::clone(&state);
|
||||
let handle = tokio::spawn(async move {
|
||||
let _ = serve(listener, state).await;
|
||||
let _ = serve(listener, serve_state).await;
|
||||
});
|
||||
// Tiny pause so the spawned task gets to accept().
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
Fixture {
|
||||
addr,
|
||||
root,
|
||||
state,
|
||||
_tmpdir: tmpdir,
|
||||
_server: handle,
|
||||
}
|
||||
}
|
||||
|
||||
fn state(&self) -> &Arc<CollectorState> {
|
||||
&self.state
|
||||
}
|
||||
}
|
||||
|
||||
fn now_ms() -> u64 {
|
||||
|
|
@ -345,6 +363,335 @@ async fn header_values_are_sanitized_against_path_traversal() {
|
|||
);
|
||||
}
|
||||
|
||||
// ── Live stream / runs endpoint tests ────────────────────────────────────
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn live_subscriber_sees_persisted_records_in_post_order() {
|
||||
// A live subscriber receives every persisted record, in the order
|
||||
// it was POSTed, with per-(run, node, kind) monotonic seq.
|
||||
let fx = Fixture::start().await;
|
||||
let mut rx = fx.state().subscribe();
|
||||
let run_id = "run-live-1";
|
||||
let node_id = "n".repeat(64);
|
||||
|
||||
for i in 0..3 {
|
||||
let resp = post_json(
|
||||
&fx,
|
||||
"/diag/events",
|
||||
run_id,
|
||||
&node_id,
|
||||
now_ms(),
|
||||
&json!([{"i": i}]),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(resp.status, 200);
|
||||
}
|
||||
|
||||
let mut got_seqs = Vec::new();
|
||||
for _ in 0..3 {
|
||||
let rec = tokio::time::timeout(Duration::from_secs(1), rx.recv())
|
||||
.await
|
||||
.expect("recv within 1s")
|
||||
.expect("recv ok");
|
||||
assert_eq!(rec.run_id, run_id);
|
||||
assert_eq!(rec.node_id, node_id);
|
||||
// We POSTed only events — so every fan-out record is events.
|
||||
// If a future change accidentally mis-tags records, this will
|
||||
// catch it without echoing the persist() shape.
|
||||
assert_eq!(
|
||||
serde_json::to_string(&rec.kind).unwrap(),
|
||||
"\"events\"",
|
||||
"every record from /diag/events must be tagged kind=events"
|
||||
);
|
||||
got_seqs.push(rec.seq);
|
||||
}
|
||||
assert_eq!(got_seqs, vec![1, 2, 3], "events seq must be monotonic from 1");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn live_subscriber_records_carry_origin_run_and_node_ids() {
|
||||
// Two concurrent runs share one subscriber. Each fanned-out
|
||||
// record carries the run_id and node_id it was POSTed under, so
|
||||
// downstream filters (the SSE handler's run_id filter, or a
|
||||
// future per-node consumer) can split the stream correctly.
|
||||
let fx = Fixture::start().await;
|
||||
let mut rx = fx.state().subscribe();
|
||||
let run_a = "run-a";
|
||||
let run_b = "run-b";
|
||||
let node_a = "a".repeat(64);
|
||||
let node_b = "b".repeat(64);
|
||||
|
||||
let resp = post_json(
|
||||
&fx,
|
||||
"/diag/boot",
|
||||
run_a,
|
||||
&node_a,
|
||||
now_ms(),
|
||||
&json!({"role": "stage"}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(resp.status, 200);
|
||||
let resp = post_json(
|
||||
&fx,
|
||||
"/diag/boot",
|
||||
run_b,
|
||||
&node_b,
|
||||
now_ms(),
|
||||
&json!({"role": "orchestrator"}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(resp.status, 200);
|
||||
|
||||
let mut seen: HashMap<String, String> = HashMap::new();
|
||||
for _ in 0..2 {
|
||||
let rec = tokio::time::timeout(Duration::from_secs(1), rx.recv())
|
||||
.await
|
||||
.expect("recv within 1s")
|
||||
.expect("recv ok");
|
||||
seen.insert(rec.run_id.clone(), rec.node_id.clone());
|
||||
}
|
||||
assert_eq!(seen.get(run_a), Some(&node_a));
|
||||
assert_eq!(seen.get(run_b), Some(&node_b));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn live_subscriber_observes_lag_and_keeps_receiving() {
|
||||
// Lossy-but-live contract: when the broadcast buffer overflows,
|
||||
// the subscriber sees a Lagged error and the next live record
|
||||
// still reaches it. The collector ingest path must not be
|
||||
// backpressured by a slow subscriber.
|
||||
use tokio::sync::broadcast::error::RecvError;
|
||||
|
||||
let fx = Fixture::start_with(|s| s.with_stream_capacity(4)).await;
|
||||
let mut rx = fx.state().subscribe();
|
||||
let run_id = "run-lag";
|
||||
let node_id = "n".repeat(64);
|
||||
|
||||
// Fire 10 sends without draining — capacity is 4, so the receiver
|
||||
// is at least 6 behind and is guaranteed to observe Lagged.
|
||||
for i in 0..10 {
|
||||
let resp = post_json(
|
||||
&fx,
|
||||
"/diag/events",
|
||||
run_id,
|
||||
&node_id,
|
||||
now_ms(),
|
||||
&json!([{"i": i}]),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(resp.status, 200);
|
||||
}
|
||||
|
||||
let mut saw_lag = false;
|
||||
let mut drained = 0;
|
||||
loop {
|
||||
match tokio::time::timeout(Duration::from_millis(200), rx.recv()).await {
|
||||
Ok(Ok(_)) => drained += 1,
|
||||
Ok(Err(RecvError::Lagged(n))) => {
|
||||
saw_lag = true;
|
||||
assert!(n > 0, "Lagged must report a non-zero gap");
|
||||
}
|
||||
Ok(Err(other)) => panic!("unexpected recv error during drain: {other:?}"),
|
||||
Err(_) => break, // drained
|
||||
}
|
||||
}
|
||||
assert!(saw_lag, "lagged subscriber must observe Lagged at least once");
|
||||
assert!(drained >= 1, "lagged subscriber must still get buffered records");
|
||||
|
||||
// After the gap, a fresh send still lands at this same receiver.
|
||||
let resp = post_json(
|
||||
&fx,
|
||||
"/diag/events",
|
||||
run_id,
|
||||
&node_id,
|
||||
now_ms(),
|
||||
&json!([{"fresh": true}]),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(resp.status, 200);
|
||||
let rec = tokio::time::timeout(Duration::from_secs(1), rx.recv())
|
||||
.await
|
||||
.expect("recv within 1s after lag")
|
||||
.expect("recv ok after lag");
|
||||
assert_eq!(rec.run_id, run_id);
|
||||
assert_eq!(rec.body, json!([{"fresh": true}]));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn runs_endpoint_lists_active_runs() {
|
||||
// /diag/runs answers "what's there to stream?" — every run that
|
||||
// has had at least one POST appears with its per-node accounting.
|
||||
let fx = Fixture::start().await;
|
||||
let run_a = "run-list-a";
|
||||
let run_b = "run-list-b";
|
||||
let node = "n".repeat(64);
|
||||
|
||||
let resp = post_json(
|
||||
&fx,
|
||||
"/diag/boot",
|
||||
run_a,
|
||||
&node,
|
||||
now_ms(),
|
||||
&json!({"role": "stage"}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(resp.status, 200);
|
||||
let resp = post_json(
|
||||
&fx,
|
||||
"/diag/boot",
|
||||
run_b,
|
||||
&node,
|
||||
now_ms(),
|
||||
&json!({"role": "orchestrator"}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(resp.status, 200);
|
||||
|
||||
let resp = get(&fx, "/diag/runs").await;
|
||||
assert_eq!(resp.status, 200, "body: {}", body_str(&resp));
|
||||
let runs: Value = serde_json::from_slice(&resp.body).expect("runs json");
|
||||
let arr = runs.as_array().expect("runs is array");
|
||||
let ids: Vec<&str> = arr
|
||||
.iter()
|
||||
.filter_map(|v| v.get("run_id").and_then(|r| r.as_str()))
|
||||
.collect();
|
||||
assert!(ids.contains(&run_a), "ids={ids:?}");
|
||||
assert!(ids.contains(&run_b), "ids={ids:?}");
|
||||
|
||||
// Boot landed but finalize did not; surface honestly.
|
||||
for run_id in [run_a, run_b] {
|
||||
let entry = arr
|
||||
.iter()
|
||||
.find(|v| v.get("run_id").and_then(|r| r.as_str()) == Some(run_id))
|
||||
.expect("entry present");
|
||||
assert_eq!(entry.get("finalize_received"), Some(&Value::Bool(false)));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn sse_stream_delivers_a_record_over_http() {
|
||||
// End-to-end SSE: subscribe via HTTP, POST a boot, parse the
|
||||
// first SSE frame off the wire. Covers the HTTP/SSE framing
|
||||
// path that the broadcast-level tests above intentionally skip.
|
||||
let fx = Fixture::start().await;
|
||||
let run_id = "run-sse-1";
|
||||
let node_id = "c".repeat(64);
|
||||
|
||||
let mut stream = open_sse_stream(fx.addr, &format!("/diag/stream/{run_id}")).await;
|
||||
|
||||
// Now that the subscribe has happened (handler runs before the
|
||||
// response head is flushed), drive one record into the channel.
|
||||
let resp = post_json(
|
||||
&fx,
|
||||
"/diag/boot",
|
||||
run_id,
|
||||
&node_id,
|
||||
now_ms(),
|
||||
&json!({"role": "stage", "stage_index": 0}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(resp.status, 200);
|
||||
|
||||
let chunk = tokio::time::timeout(Duration::from_secs(2), read_chunk(&mut stream))
|
||||
.await
|
||||
.expect("first chunk within 2s")
|
||||
.expect("non-empty chunk");
|
||||
let frame = std::str::from_utf8(&chunk).expect("chunk utf8");
|
||||
|
||||
let (event_name, data_json) = parse_sse_frame(frame).expect("well-formed SSE frame");
|
||||
assert_eq!(event_name, "boot");
|
||||
let body: Value = serde_json::from_str(&data_json).expect("data is json");
|
||||
assert_eq!(body.get("run_id").and_then(|v| v.as_str()), Some(run_id));
|
||||
assert_eq!(body.get("node_id").and_then(|v| v.as_str()), Some(node_id.as_str()));
|
||||
assert_eq!(body.get("kind").and_then(|v| v.as_str()), Some("boot"));
|
||||
}
|
||||
|
||||
// ── SSE / chunked-transfer test helpers ─────────────────────────────────
|
||||
|
||||
/// Open a TCP stream, send a GET, consume HTTP headers. Returns the
|
||||
/// stream positioned at the first body byte (first chunk header).
|
||||
/// Verifies the response advertised chunked transfer-encoding so
|
||||
/// `read_chunk` can rely on the framing.
|
||||
async fn open_sse_stream(addr: SocketAddr, path: &str) -> TcpStream {
|
||||
let mut stream = TcpStream::connect(addr).await.expect("connect");
|
||||
let req = format!(
|
||||
"GET {path} HTTP/1.1\r\nhost: 127.0.0.1\r\naccept: text/event-stream\r\n\r\n"
|
||||
);
|
||||
stream.write_all(req.as_bytes()).await.expect("write");
|
||||
stream.flush().await.ok();
|
||||
|
||||
// Read until end-of-headers.
|
||||
let mut buf = Vec::with_capacity(1024);
|
||||
let mut tmp = [0u8; 512];
|
||||
loop {
|
||||
let n = stream.read(&mut tmp).await.expect("read headers");
|
||||
if n == 0 {
|
||||
panic!("EOF before SSE headers");
|
||||
}
|
||||
buf.extend_from_slice(&tmp[..n]);
|
||||
if let Some(idx) = find_double_crlf(&buf) {
|
||||
assert_eq!(
|
||||
&buf[idx + 4..],
|
||||
b"",
|
||||
"test reads must not span past the headers terminator"
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
let head = std::str::from_utf8(&buf).expect("header utf8");
|
||||
assert!(
|
||||
head.to_ascii_lowercase().contains("transfer-encoding: chunked"),
|
||||
"expected chunked SSE response, got:\n{head}"
|
||||
);
|
||||
stream
|
||||
}
|
||||
|
||||
/// Read one HTTP/1.1 transfer-encoding chunk's payload. Returns
|
||||
/// `None` on the terminator chunk (size 0).
|
||||
async fn read_chunk(stream: &mut TcpStream) -> Option<Vec<u8>> {
|
||||
let mut size_line = Vec::with_capacity(8);
|
||||
loop {
|
||||
let mut b = [0u8; 1];
|
||||
stream.read_exact(&mut b).await.expect("read chunk size byte");
|
||||
size_line.push(b[0]);
|
||||
if size_line.ends_with(b"\r\n") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let s = std::str::from_utf8(&size_line[..size_line.len() - 2]).expect("size utf8");
|
||||
let s = s.split(';').next().unwrap().trim();
|
||||
let size = usize::from_str_radix(s, 16).expect("hex chunk size");
|
||||
if size == 0 {
|
||||
return None;
|
||||
}
|
||||
let mut data = vec![0u8; size];
|
||||
stream.read_exact(&mut data).await.expect("read chunk data");
|
||||
let mut trailer = [0u8; 2];
|
||||
stream.read_exact(&mut trailer).await.expect("read chunk trailer");
|
||||
assert_eq!(&trailer, b"\r\n", "chunk trailer must be CRLF");
|
||||
Some(data)
|
||||
}
|
||||
|
||||
/// Parse one SSE frame of the form `event: NAME\ndata: PAYLOAD\n\n`
|
||||
/// (or with a trailing single `\n`). Returns `(event_name, data)`.
|
||||
fn parse_sse_frame(frame: &str) -> Option<(String, String)> {
|
||||
let trimmed = frame.trim_end_matches('\n');
|
||||
let mut event = None;
|
||||
let mut data: Option<String> = None;
|
||||
for line in trimmed.split('\n') {
|
||||
if let Some(rest) = line.strip_prefix("event: ") {
|
||||
event = Some(rest.to_string());
|
||||
} else if let Some(rest) = line.strip_prefix("data: ") {
|
||||
// SSE allows multiple `data:` lines, joined by '\n'.
|
||||
data = Some(match data {
|
||||
Some(prev) => format!("{prev}\n{rest}"),
|
||||
None => rest.to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
Some((event?, data?))
|
||||
}
|
||||
|
||||
// ---------- HTTP helpers ----------
|
||||
|
||||
struct HttpResponse {
|
||||
|
|
|
|||
2
examples/pipeline-parallel-inference/Cargo.lock
generated
2
examples/pipeline-parallel-inference/Cargo.lock
generated
|
|
@ -810,6 +810,7 @@ dependencies = [
|
|||
"swactor-transport",
|
||||
"tar",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
|
|
@ -2619,6 +2620,7 @@ dependencies = [
|
|||
"reqwest 0.12.28",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"swactor",
|
||||
"swactor-process",
|
||||
"tokio",
|
||||
|
|
|
|||
|
|
@ -18,6 +18,12 @@ iroh = "0.98"
|
|||
urlencoding = "2"
|
||||
base64 = "0.22"
|
||||
libc = "0.2"
|
||||
# PROTOTYPE_BINARY_SWAP (spec §5.1): pp-gpu-node verifies an
|
||||
# operator-supplied SHA-256 digest of the fetched replacement binary.
|
||||
# Already a transitive dep of the iroh stack; pinned here as a direct
|
||||
# dep so a future §5.1 removal can drop this line cleanly along with
|
||||
# the swap module.
|
||||
sha2 = "0.10"
|
||||
|
||||
[[bin]]
|
||||
name = "pp-gpu-node"
|
||||
|
|
@ -30,3 +36,9 @@ path = "src/bin/pp_smoke_run.rs"
|
|||
[dev-dependencies]
|
||||
wiremock = "0.6"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
|
||||
# These binaries are shipped to rented GPU nodes — over the docker image on a
|
||||
# cold lease, and over scp through vast.ai's throttled SSH proxy on a redeploy.
|
||||
# Stripping debug symbols (~26MB -> ~18MB) trims both paths at no runtime cost.
|
||||
[profile.release]
|
||||
strip = true
|
||||
|
|
|
|||
|
|
@ -1,35 +1,111 @@
|
|||
FROM nvidia/cuda:12.6.3-runtime-ubuntu24.04
|
||||
# Pipeline-parallel runtime image.
|
||||
#
|
||||
# Spec §4.10 (permanent contracts):
|
||||
# * MUST be produced by a multi-stage build, with build artifacts
|
||||
# (compilers, headers, dev libraries) confined to the builder stage.
|
||||
# * MUST be ≤ 1 GB compressed (§4.10 / acceptance §6.11).
|
||||
# * Apt caches, pip caches, __pycache__, test data, and docs MUST NOT
|
||||
# be present in the runtime layer.
|
||||
# * MUST be self-sufficient — booting MUST NOT fetch any binary from
|
||||
# an external host (the §5.1 binary-swap path is opt-in only).
|
||||
# * SHOULD use a slim CUDA runtime image, not a -devel image.
|
||||
#
|
||||
# Build context must be the workspace root:
|
||||
# docker build -f examples/pipeline-parallel-inference/Dockerfile -t <tag> .
|
||||
|
||||
# ─── Builder stage ───────────────────────────────────────────────────
|
||||
# Confines the dev-only CUDA headers (cuda-cudart-dev) and the pip
|
||||
# install machinery here so the runtime layer keeps neither. The pip
|
||||
# install also produces __pycache__ + bundled tests; we strip both
|
||||
# inside this stage so they cannot ride along on any COPY out.
|
||||
FROM nvidia/cuda:12.6.3-runtime-ubuntu24.04 AS builder
|
||||
|
||||
# Runtime base, not devel: the devel base alone is ~5 GB and blew past the
|
||||
# vastai image-pull budget (the previous 7.55 GB build). NVRTC — the kernel
|
||||
# compiler tinygrad's CUDA backend uses — ships in the runtime image, but the
|
||||
# CUDA *toolkit headers* do not, and tinygrad's generated fp16 kernels
|
||||
# `#include <cuda_fp16.h>`. Pull in just the cudart dev headers (~7 MB) so
|
||||
# NVRTC's `-I/usr/local/cuda/include` resolves them — the minimal alternative
|
||||
# to the full devel base. Without this every real-model stage dies at
|
||||
# graph-realize with NVRTC_ERROR_COMPILATION ("cannot open cuda_fp16.h").
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
python3 \
|
||||
python3-venv \
|
||||
python3-pip \
|
||||
ca-certificates \
|
||||
cuda-cudart-dev-12-6 && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install tinygrad and numpy
|
||||
# Stage tinygrad + numpy into a self-contained directory we COPY into
|
||||
# the runtime stage. --target keeps them off the system path so the
|
||||
# runtime can drop them under PYTHONPATH without dragging /usr/lib.
|
||||
RUN python3 -m pip install --no-cache-dir --break-system-packages \
|
||||
tinygrad==0.12.0 \
|
||||
numpy
|
||||
--target=/opt/pp-pydeps \
|
||||
tinygrad==0.12.0 numpy && \
|
||||
find /opt/pp-pydeps -depth -type d \
|
||||
\( -name '__pycache__' -o -name 'tests' -o -name 'test' \) \
|
||||
-exec rm -rf {} + && \
|
||||
find /opt/pp-pydeps -name '*.pyc' -delete && \
|
||||
find /opt/pp-pydeps -type d -name '*.dist-info' -exec rm -rf {} +
|
||||
|
||||
# Copy the pipeline-parallel binaries and tinygrad worker
|
||||
# Build context should be the workspace root:
|
||||
# docker build -f examples/pipeline-parallel-inference/Dockerfile -t <tag> .
|
||||
# Stage the NVRTC headers tinygrad's generated fp16 / bf16 kernels
|
||||
# `#include`. They are pure source files (kilobytes); the runtime stage
|
||||
# picks them up without the full cuda-cudart-dev package.
|
||||
RUN mkdir -p /opt/pp-nvrtc-include && \
|
||||
cp /usr/local/cuda/include/cuda_fp16.h \
|
||||
/usr/local/cuda/include/cuda_fp16.hpp \
|
||||
/opt/pp-nvrtc-include/ && \
|
||||
if [ -f /usr/local/cuda/include/cuda_bf16.h ]; then \
|
||||
cp /usr/local/cuda/include/cuda_bf16.h \
|
||||
/usr/local/cuda/include/cuda_bf16.hpp \
|
||||
/opt/pp-nvrtc-include/; \
|
||||
fi
|
||||
|
||||
# ─── Runtime stage ───────────────────────────────────────────────────
|
||||
# Slim CUDA -base image (no math libs) plus exactly the libraries
|
||||
# tinygrad needs at JIT time: libcudart (CUDA runtime) and libnvrtc
|
||||
# (kernel compiler). The cuda_fp16.h header comes from the builder; no
|
||||
# -dev / -devel package lands here.
|
||||
FROM nvidia/cuda:12.6.3-base-ubuntu24.04 AS runtime
|
||||
|
||||
# Full python3 (not -minimal) for stdlib coverage tinygrad+numpy need
|
||||
# (numpy 2.x imports `contextvars` from stdlib; python3-minimal omits
|
||||
# it). `apt-get clean` + `rm -rf` keep apt's archive cache out of the
|
||||
# layer (spec §4.10 forbids it). The find calls scrub __pycache__ and
|
||||
# .pyc generated by post-install scripts of the packages we DO need.
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
python3 \
|
||||
cuda-cudart-12-6 \
|
||||
cuda-nvrtc-12-6 \
|
||||
ca-certificates \
|
||||
procps && \
|
||||
apt-get clean && \
|
||||
rm -rf /var/lib/apt/lists/* \
|
||||
/var/cache/apt/archives/* \
|
||||
/var/cache/apt/*.bin \
|
||||
/var/log/apt/* \
|
||||
/var/log/dpkg.log \
|
||||
/tmp/* \
|
||||
/var/tmp/* \
|
||||
/usr/share/doc/* \
|
||||
/usr/share/man/* \
|
||||
/usr/share/info/* && \
|
||||
find /usr -depth -type d -name '__pycache__' -exec rm -rf {} + 2>/dev/null || true && \
|
||||
find /usr -type f -name '*.pyc' -delete 2>/dev/null || true
|
||||
|
||||
# Headers tinygrad's NVRTC backend resolves via its default -I path.
|
||||
COPY --from=builder /opt/pp-nvrtc-include/ /usr/local/cuda/include/
|
||||
|
||||
# Pip-installed Python deps from the builder, already stripped of
|
||||
# __pycache__ + bundled tests (spec §4.10: caches / test data MUST NOT
|
||||
# be present in the runtime layer).
|
||||
COPY --from=builder /opt/pp-pydeps /opt/pp-pydeps
|
||||
ENV PYTHONPATH=/opt/pp-pydeps
|
||||
# Avoid regenerating bytecode at runtime — keeps the runtime FS clean
|
||||
# of fresh __pycache__ writes after first import.
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
|
||||
# Pipeline binaries + worker script. The binaries are statically linked
|
||||
# enough that the runtime stage's libc is all they need; the worker is
|
||||
# pure Python.
|
||||
COPY examples/pipeline-parallel-inference/target/release/pp-gpu-node /usr/local/bin/pp-gpu-node
|
||||
COPY examples/pipeline-parallel-inference/target/release/pp-smoke-run /usr/local/bin/pp-smoke-run
|
||||
COPY examples/pipeline-parallel-inference/pp_tinygrad_worker.py /usr/local/share/pp_tinygrad_worker.py
|
||||
|
||||
# Enable CUDA backend for tinygrad (override with -e DEV=CPU for CPU runs)
|
||||
# Enable CUDA backend for tinygrad (override with -e CUDA=0 for CPU runs).
|
||||
ENV CUDA=1
|
||||
ENV WORKER_SCRIPT=/usr/local/share/pp_tinygrad_worker.py
|
||||
|
||||
|
|
|
|||
|
|
@ -1,520 +0,0 @@
|
|||
# N=3 collection coverage extension — behavioral spec
|
||||
|
||||
Companion to `N3_POSTMORTEM_2026-05-25_1779733878.md`,
|
||||
`N3_SIM_TEST_BATTERY_SPEC.md`, and the simulator's `SIM_SPEC.md`.
|
||||
This document is the contract for a separate coding agent to extend
|
||||
diagnostic collection coverage along three layers — **production
|
||||
diagnostics**, **simulator emit/model**, and **simulator test
|
||||
verification** — for the gaps the `1779733878` run surfaced.
|
||||
|
||||
This is a *behavioral* spec. It names the gap, the contract the
|
||||
collected data must satisfy, and the layer(s) the contract threads
|
||||
through. It does not prescribe field names, file layout, or
|
||||
implementation choices.
|
||||
|
||||
---
|
||||
|
||||
## 0. Motivation
|
||||
|
||||
The `1779733878` run validated the prior observability upgrade —
|
||||
A+B+C tiers were load-bearing, the bundle attributed the failure to
|
||||
"dials to orchestrator fail by timeout 7/11 while every inter-stage
|
||||
dial succeeds 19/19" in one table — and surfaced five residual
|
||||
collection gaps the upgrade either left as carry-forward (gaps 1, 5,
|
||||
8 from the original scorecard) or that this run exposed for the
|
||||
first time (response-leg event absence, bundle-serve behavior under
|
||||
run-id reuse).
|
||||
|
||||
A gap whose collection landed only in production but not in the sim
|
||||
is a gap that the sim test battery can never guard — the next regression
|
||||
in that field will be caught only by another live deploy. A gap whose
|
||||
sim model exists but is not exercised by a test is dead code. The
|
||||
coverage in this spec is required to thread through every layer
|
||||
where it can — and the spec is explicit when a layer does not
|
||||
apply.
|
||||
|
||||
Five coverages, each threaded through up to three layers. Each
|
||||
coverage may close one of {Pass, Mixed, Fail} against the current
|
||||
source, and each names the close criterion.
|
||||
|
||||
---
|
||||
|
||||
## 1. Cross-cutting requirements
|
||||
|
||||
These hold for every coverage in §2.
|
||||
|
||||
### 1.1 Three-layer threading
|
||||
|
||||
For each coverage, the spec names which of the three layers it
|
||||
threads through:
|
||||
|
||||
- **D — Diagnostics**: the production bundle gains a field, event,
|
||||
or section that closes the gap the postmortem named.
|
||||
- **S — Sim**: the simulator's relevant component (host kind,
|
||||
network, relay vertex, bundle writer) emits the same field /
|
||||
event / section under the same conditions, with bundle-shape
|
||||
parity per `SIM_SPEC.md §5` (cross-cutting "Bundle-shape parity
|
||||
with prod") and §9 (bundle schema).
|
||||
- **T — Tests**: the sim test battery gains a scenario or property
|
||||
test asserting the bundle carries the new data when the
|
||||
triggering condition holds, and gains a discriminator assertion
|
||||
when absence is meaningful (per the honesty-under-absence pattern
|
||||
the prior upgrade established).
|
||||
|
||||
A coverage that threads through fewer than three layers is honest
|
||||
about which it skips and why. Skipping S because "the simulator
|
||||
does not model this surface" is acceptable; skipping it because
|
||||
"this is not interesting" is not.
|
||||
|
||||
### 1.2 Honesty-under-absence carries forward
|
||||
|
||||
The prior upgrade's `status_source` discriminator pattern (a status
|
||||
field always paired with a field naming how that status was derived
|
||||
— `"iroh"` for native, `"derived"` for inferred) is the model. Any
|
||||
new field whose value might be absent or derived must carry an
|
||||
adjacent discriminator. A bundle reader must never be left guessing
|
||||
"unknown means the thing is unknown" vs "we couldn't ask."
|
||||
|
||||
### 1.3 Additive evolution
|
||||
|
||||
Every new field on `SnapshotBody`, every new event variant, every
|
||||
new section in the post-processor output is additive. An old bundle
|
||||
reader on a new bundle still parses; a new bundle reader on an old
|
||||
bundle reports the new field absent rather than erroring. The
|
||||
prior upgrade established this contract; coverage 2.x preserves it.
|
||||
|
||||
### 1.4 Sim/prod schema parity
|
||||
|
||||
Per `SIM_SPEC.md §9.2`: the event payload schema is exactly the
|
||||
production diagnostics schema for that kind. The sim invents no new
|
||||
event kinds. A coverage that lands an event in prod and in sim
|
||||
**uses the same schema in both**, verified by the existing parity
|
||||
tests under `crates/simulation/tests/sim_cross_pollination.rs`. A
|
||||
schema added to sim ahead of prod is a deliberate amendment and
|
||||
declares so explicitly.
|
||||
|
||||
### 1.5 Verdict-first per layer
|
||||
|
||||
Every coverage in §2 declares, per layer, its expected status on
|
||||
the current source: **landed** (the layer satisfies the contract;
|
||||
the work is verification / regression-guard), **partial** (the
|
||||
layer has structure but not data flow), **absent** (the layer has
|
||||
nothing today). The implementing agent's work is to bring each
|
||||
layer to "landed" against this spec or to file a structural reason
|
||||
why a layer cannot land.
|
||||
|
||||
---
|
||||
|
||||
## 2. The coverages
|
||||
|
||||
Five, ordered by the postmortem's own ranking of residual gaps.
|
||||
|
||||
### 2.1 Orchestrator-side host provider metadata forwarding
|
||||
|
||||
**Source**: postmortem §"Observability upgrade scorecard" row "gap
|
||||
5 host metadata" (◐); postmortem §"Data-collection / deployment
|
||||
gaps surfaced by this run" item 1.
|
||||
|
||||
**Gap**: The orchestrator has each rental's public IP, datacenter,
|
||||
country, and contract id at `lease_chain` return time. The
|
||||
container can read these from `SWACTOR_DIAG_*` env vars. The
|
||||
container env is never set. The boot record's
|
||||
`host_ip_public`/`datacenter_id`/`host_country`/`vastai_contract_id`/
|
||||
`home_relay_url_at_boot` fields are still null in every bundle.
|
||||
The contract id arrives but in `container_id`, not
|
||||
`vastai_contract_id` — the naming is currently load-bearing-but-wrong.
|
||||
|
||||
**Contract — what closing the gap looks like**:
|
||||
|
||||
- D: the orchestrator's per-rental env payload, at the point it
|
||||
creates each container, carries every field the boot record can
|
||||
consume — public IP, datacenter id, host country, vast.ai
|
||||
contract id, the home relay URL the container will use. The boot
|
||||
record reflects every field as a concrete value, not `null`,
|
||||
whenever the orchestrator had the data. The fields that name
|
||||
cloud-provider state stay absent only on hosts where they
|
||||
genuinely do not apply (e.g., local development), and the
|
||||
bundle's `## Hosts` section renders `?` for absent fields
|
||||
(already implemented per `S-A2`).
|
||||
- S: scenarios declare per-peer host context as part of the peer's
|
||||
`kind_config`. The sim's stage host populates its boot record /
|
||||
`HostContext` from the scenario declaration the same way prod
|
||||
populates from env. A scenario without declared host context
|
||||
produces a bundle whose `## Hosts` section is all-`?` for that
|
||||
peer — same absence shape as a local-dev prod bundle.
|
||||
- T: a scenario declaring heterogeneous host context across three
|
||||
peers (e.g., two datacenters, two countries) produces a bundle
|
||||
whose `## Hosts` section renders the declared fields verbatim.
|
||||
A scenario that declares no context for one peer and full context
|
||||
for the others produces a bundle distinguishable from "no context
|
||||
declared for any peer" by the `?` placement.
|
||||
|
||||
**Expected status**:
|
||||
|
||||
- D: partial. The container reads the env vars (per `S-A2`); the
|
||||
orchestrator does not set them. The misnaming of contract id →
|
||||
`container_id` is a separate cleanup.
|
||||
- S: absent. The sim's stage host carries no host context in its
|
||||
current scenario schema.
|
||||
- T: absent. No test exercises this discriminator.
|
||||
|
||||
**Close criterion**: a deployed bundle's `## Hosts` section names
|
||||
the datacenter, country, public IP, and contract id of every
|
||||
vast.ai rental, and the docker `container_id` field carries the
|
||||
docker container id, not the vast.ai contract id. A sim bundle
|
||||
with declared host context produces the matching shape.
|
||||
|
||||
### 2.2 Relay-port reachability probe
|
||||
|
||||
**Source**: postmortem §"Observability upgrade scorecard" row "gap
|
||||
8 relay-port probe" (✗); postmortem §"Data-collection / deployment
|
||||
gaps surfaced by this run" item 3.
|
||||
|
||||
**Gap**: Stage probe arrays carry only `collector_udp_echo`
|
||||
(:9081). No probe targets the relay's actual port (:7843).
|
||||
Whether a stage retained transport-level reachability to the relay
|
||||
at the moment its peer-connection died is currently inferable only
|
||||
from a *different* port on the same host. The `S-E1` work is
|
||||
documented as landed (per the prior iteration log) but the
|
||||
`1779733878` bundle shows no relay-port probe records. The wiring
|
||||
is in place; the data is not.
|
||||
|
||||
**Contract — what closing the gap looks like**:
|
||||
|
||||
- D: every stage's snapshot carries a probe outcome for the
|
||||
relay's UDP listener (host + port resolved from the home relay
|
||||
URL). The outcome is one of the five-discriminator set the
|
||||
prior upgrade established: `ok` / `timeout` / `refused` /
|
||||
`unresolved` / `error`. A snapshot taken when the relay is
|
||||
reachable carries `ok` with an RTT; a snapshot taken when the
|
||||
relay is unreachable carries the appropriate failure
|
||||
discriminator with no silent fallback to "absent."
|
||||
- S: the sim's stage host emits the same probe record on every
|
||||
snapshot, sourced from a query the network answers about the
|
||||
stage→relay edge. The relay vertex's `RelayKill` /
|
||||
`RelayCapacityChange` mutations are reflected in the probe's
|
||||
outcome distribution.
|
||||
- T: a scenario that issues a `RelayKill` mutation mid-run
|
||||
produces a bundle whose every stage's relay-port probe outcome
|
||||
flips from `ok` to `unresolved` (or `timeout`, per the
|
||||
network's policy) at the mutation's `at_ns` and remains there
|
||||
through `RelayBoot`. The probe-outcome timeline is the test's
|
||||
discriminator between "tunnel down" and "tunnel up but peer
|
||||
conn down" — coverage 2.x.A from the battery spec consumes
|
||||
this signal.
|
||||
|
||||
**Expected status**:
|
||||
|
||||
- D: partial. Probe scheduler wires the target; emission to the
|
||||
bundle is unverified by this run's evidence.
|
||||
- S: absent. The sim's network has no probe-query surface today.
|
||||
- T: absent.
|
||||
|
||||
**Close criterion**: the next deployment's bundle has a
|
||||
relay-port probe outcome on every stage's snapshots. A sim
|
||||
scenario with `RelayKill` produces the probe-outcome flip in the
|
||||
bundle.
|
||||
|
||||
### 2.3 Relay session lifecycle on the relay side
|
||||
|
||||
**Source**: postmortem §"Observability upgrade scorecard" row "gap
|
||||
1 relay observability" (◐); postmortem §"Data-collection /
|
||||
deployment gaps surfaced by this run" item 4.
|
||||
|
||||
**Gap**: The relay reports identity and 186 snapshots into the
|
||||
bundle but cannot answer "who closed session X and why" — the
|
||||
per-session lifecycle hooks are the documented skeleton with
|
||||
`active=0 opens=0 closes=0`. `iroh_relay::server` exposes no
|
||||
session hooks. Until it does, a relay-side eviction is
|
||||
unanswerable from the relay's own data; the postmortem fell back
|
||||
to node-side dial outcomes.
|
||||
|
||||
**Contract — what closing the gap looks like**:
|
||||
|
||||
- D: the relay's bundle contribution names, per peer session, the
|
||||
open time, close time, close-initiator discriminator
|
||||
(`relay` / `peer` / `transport` / `unknown`), close reason
|
||||
string (relay-specific or transport-specific), bytes
|
||||
transferred per direction, and duration. The mechanism is
|
||||
free — middleware around the relay binary, kernel-layer
|
||||
observation, a forked relay, or upstream hooks when iroh
|
||||
exposes them. The contract is the *shape*, not the source.
|
||||
When the source is unavailable, the relay's bundle
|
||||
contribution still emits the gap-1 absence-line the prior
|
||||
upgrade introduced in `summary.md` (the post-processor's
|
||||
acceptance branch for "no relay-role node has session data").
|
||||
- S: the sim's relay vertex emits `RelaySessionOpened` /
|
||||
`RelaySessionClosed` records when it accepts and releases
|
||||
per-peer queues. The records carry the same shape D
|
||||
requires. A `RelayKill` mutation produces a
|
||||
`RelaySessionClosed { initiator: "relay", reason: "killed",
|
||||
... }` for every session active at the mutation time.
|
||||
- T: a scenario where the relay accepts three peer sessions, runs
|
||||
to steady state, then receives a `RelayKill` mutation,
|
||||
produces a bundle whose relay contribution names three
|
||||
`RelaySessionOpened` events at the convergence boundary and
|
||||
three `RelaySessionClosed { initiator: "relay" }` events at
|
||||
the mutation time. A scenario where a peer voluntarily
|
||||
disconnects produces a session-closed event with
|
||||
`initiator: "peer"`. The discriminator must hold.
|
||||
|
||||
**Expected status**:
|
||||
|
||||
- D: skeleton — wired call sites, no data flow. Whether the
|
||||
unblock path is upstream hooks, middleware, or kernel
|
||||
observation is implementer's call.
|
||||
- S: partial. `RelayObservability` exists on the host side per the
|
||||
prior upgrade (`S-B1`); the sim's relay vertex itself does not
|
||||
emit lifecycle events as engine-synthesized records.
|
||||
- T: absent.
|
||||
|
||||
**Close criterion**: a deployed bundle from a run that included a
|
||||
peer dial failure attributable to a relay-side close names the
|
||||
close-initiator and reason in the relay's bundle contribution. A
|
||||
sim `RelayKill` scenario produces the matching event stream.
|
||||
|
||||
### 2.4 Inference response-leg instrumentation
|
||||
|
||||
**Source**: postmortem §"Data-collection / deployment gaps
|
||||
surfaced by this run" item 5.
|
||||
|
||||
**Gap**: The `1779733878` postmortem's conclusion — "last stage
|
||||
could not deliver the response" — was inferred from dial timeouts
|
||||
plus the absence of an inbound `InferenceResponse`, not from a
|
||||
typed event on the last stage saying "I tried to send the response
|
||||
and the send outcome was X." The chain `stage-(N-1)
|
||||
→ InferenceResponse → orchestrator's inbox` has no event on the
|
||||
sending side. A typed event makes attribution a one-line read
|
||||
rather than a triangulation.
|
||||
|
||||
**Contract — what closing the gap looks like**:
|
||||
|
||||
- D: the production stage actor, on attempting to send an
|
||||
`InferenceResponse` upstream, emits a typed event naming the
|
||||
target peer, the request id the response corresponds to, the
|
||||
byte size, and the send outcome. The outcome discriminator is
|
||||
the iroh-level result the transport returns (succeed / timeout
|
||||
/ connection-closed / refused / unresolved / queued-but-not-
|
||||
acked-in-budget). The post-processor surfaces these in
|
||||
`summary.md` under a section that names which inference
|
||||
request was answered by which stage's send and how that send
|
||||
resolved.
|
||||
- S: the sim's stage host kind grows a minimal inference
|
||||
message surface (`InferenceRequest` inbound to stage-0,
|
||||
`InferenceResponse` outbound from stage-(N-1), forwarded
|
||||
between adjacent stages as opaque payload in the MVP). The
|
||||
stage host emits the same typed response-send event when it
|
||||
attempts the outbound to the orchestrator. The codec contract
|
||||
(`SIM_SPEC.md §3.3`) carries the inference messages with
|
||||
byte-equality between sim and prod encoding.
|
||||
- T: a scenario where the orchestrator's inbound path is broken
|
||||
via `RelayPeerConnDown` on the last leg (stage-(N-1) → orch)
|
||||
while every other leg works produces a bundle whose last
|
||||
stage emits exactly one `InferenceResponseSent` event with
|
||||
`send_outcome` in the failure-discriminator set. A scenario
|
||||
where every leg works produces an `InferenceResponseSent`
|
||||
with `send_outcome=success` and a matching
|
||||
`InferenceResponseReceived` (or equivalent) on the
|
||||
orchestrator's side.
|
||||
|
||||
**Expected status**:
|
||||
|
||||
- D: absent. The current stage actor's send call is not wrapped
|
||||
in a typed diagnostic event for the response leg.
|
||||
- S: absent. The sim's stage host kind today produces no `Send`
|
||||
actions during its lifecycle (`SIM_SPEC.md §6A.5` notes this
|
||||
explicitly and defers inter-stage traffic to a later revision).
|
||||
Closing this coverage moves that deferral forward.
|
||||
- T: absent.
|
||||
|
||||
**Close criterion**: a deployed bundle from any run where the
|
||||
response did not return names the send outcome of the last
|
||||
stage's response attempt in a single event. A sim scenario
|
||||
modeling the same failure produces the same shape.
|
||||
|
||||
### 2.5 Bundle serve hardening under run-id reuse
|
||||
|
||||
**Source**: postmortem §"Bundle recovery" caveat; postmortem
|
||||
§"Data-collection / deployment gaps surfaced by this run" item 2.
|
||||
|
||||
**Gap**: When a run id is reused across the failed-first-lease /
|
||||
successful-second-lease shape the `1779733878` run exhibited, a
|
||||
finalize record from the first phase pins a stale canonical
|
||||
bundle in the collector's cache. A subsequent `GET` serves the
|
||||
stale 5.3 KB bundle instead of synthesizing the rich 9.3 MB one
|
||||
from current staging. Two adjacent quirks: `finalize_received`
|
||||
stays `true` after the on-disk `finalize-*.json` is deleted, and
|
||||
the synthesized manifest still lists a removed node directory.
|
||||
|
||||
**Contract — what closing the gap looks like**:
|
||||
|
||||
- D: the collector's `download_bundle` handler prefers the
|
||||
*richer* of {canonical-cached, synthesized-from-current-staging}
|
||||
by a size or node-count heuristic, or rebuilds canonical when
|
||||
staging has grown past the cached bundle's manifest. Deleting a
|
||||
node directory from staging clears the corresponding finalize
|
||||
record from in-memory state. The synthesized manifest reflects
|
||||
the current on-disk state, never a stale in-memory record. The
|
||||
`finalize_received` boolean is sourced from the same place the
|
||||
serve decision is sourced from — a single source of truth, not
|
||||
two diverging caches.
|
||||
- S: not applicable. The sim writes bundles directly to a
|
||||
destination directory; there is no serve logic, no finalize
|
||||
cache, no run-id reuse semantics. The coverage threads through
|
||||
D only.
|
||||
- T: not applicable as a *sim test*. The discriminator (stale vs
|
||||
fresh serve on a finalize-then-staging-growth sequence) is a
|
||||
collector unit-test concern living under
|
||||
`crates/distribution/tests/`, not a scenario the sim engine
|
||||
can express. The implementing agent should land the collector
|
||||
test alongside the D-layer change; it is named here so that the
|
||||
coverage's verification surface is honest about where it lives.
|
||||
|
||||
**Expected status**:
|
||||
|
||||
- D: absent. Current serve logic prefers cached canonical
|
||||
unconditionally when `finalize_received` is true.
|
||||
- S: not applicable.
|
||||
- T: collector unit test absent.
|
||||
|
||||
**Close criterion**: a collector unit test writes two phases of
|
||||
staging with an intervening finalize, deletes the first-phase
|
||||
node, and verifies the second `GET` serves the richer bundle and
|
||||
that the cleared node does not appear in the manifest.
|
||||
|
||||
---
|
||||
|
||||
### 2.6 Per-SWIM-probe RTT and observed latency distribution
|
||||
|
||||
**Source**: postmortem §"SWIM churn and relay events" (1701
|
||||
SwimTransitions over ~7 min, all with `conn_type=Relay`); postmortem
|
||||
§"UDP echo probes" (tier-2 RTTs spread 181–405 ms; SWIM probes
|
||||
ride a relay-mediated path on top of these); `SWIM_TUNING_REPORT.md`
|
||||
§6 limit 3 ("SWIM host adapter does not emit `probe_sent` /
|
||||
`probe_received` / `probe_timed_out` events").
|
||||
|
||||
**Gap**: The bundle has tier-2 UDP-echo RTT to docean:9081 — a
|
||||
host-level surface that does not represent the latency SWIM
|
||||
actually sees. SWIM rides a relay-mediated peer connection whose
|
||||
RTT is at least one extra hop and is subject to relay-side HOL
|
||||
queueing under load. The bundle currently exposes:
|
||||
|
||||
- per-snapshot iroh counters (cumulative `MessageSent` /
|
||||
`MessageReceived`),
|
||||
- aggregate `SwimTransition` counts,
|
||||
- per-peer dial outcomes (`Timeout` / `Success` rollup),
|
||||
|
||||
but it does not expose per-probe RTT, per-peer RTT distribution
|
||||
over the run window, or correlation between
|
||||
`probe_timed_out`-class outcomes and observed RTT spikes. Without
|
||||
this surface, SWIM tuning is a guess against the deploy's actual
|
||||
latency distribution rather than a measurement.
|
||||
|
||||
This gap also mirrors the simulator's own limit per
|
||||
`SWIM_TUNING_REPORT.md` §6.3: the SWIM host adapter does not emit
|
||||
the probe lifecycle events, so the §10 evaluator's
|
||||
`no_flap_while_probes_ok` is structurally `Inconclusive`. Closing
|
||||
the gap on both sides closes the assertion's precondition.
|
||||
|
||||
**Contract — what closing the gap looks like**:
|
||||
|
||||
- D: each SWIM ping/ack pair emits a typed event naming the
|
||||
observer, target, virtual-or-wall send time, virtual-or-wall
|
||||
receive time, the resulting RTT, and the discriminator
|
||||
(`success` / `timeout` / `connection-closed` / etc.). The
|
||||
post-processor surfaces a `## Probe RTT distribution` section
|
||||
with median, p95, p99 per (observer, target) pair, plus per
|
||||
five-second bucket so degradation over time is visible. A
|
||||
`probe_timed_out` outcome carries the configured timeout
|
||||
budget alongside the observed RTT (where one exists) so a
|
||||
reader sees "probe missed a 3 s budget by 200 ms" vs "no
|
||||
response within 3 s, never arrived."
|
||||
- S: the simulator's SWIM host adapter emits the same probe
|
||||
lifecycle events. Per `SIM_SPEC.md §9.2` parity, the schema is
|
||||
identical to D's. This is the §6.3 limit from
|
||||
`SWIM_TUNING_REPORT.md` closing simultaneously with D — the
|
||||
bundle reader cannot tell a sim run from a prod run by this
|
||||
surface.
|
||||
- T: a scenario with a declared per-link latency distribution
|
||||
(heavy-tailed, peer-symmetric) produces a bundle whose
|
||||
postproc RTT section's median, p95, p99 fall within stated
|
||||
tolerance of the scenario's declared distribution. A scenario
|
||||
with a `LatencySpike` mutation produces a bundle whose RTT
|
||||
section shows the spike at the mutation time. The
|
||||
precondition for `no_flap_while_probes_ok` is now satisfied;
|
||||
the assertion moves off `Inconclusive` for every scenario
|
||||
using a SWIM-host kind.
|
||||
|
||||
**Expected status**:
|
||||
|
||||
- D: absent. No per-probe event today.
|
||||
- S: absent. `SWIM_TUNING_REPORT.md` §6.3 names this explicitly.
|
||||
- T: absent.
|
||||
|
||||
**Close criterion**: a deployed bundle's postproc summary names
|
||||
the median / p99 RTT per (observer, target) and a sim bundle
|
||||
produces the matching surface. `no_flap_while_probes_ok` resolves
|
||||
to `Pass` or `Fail` (not `Inconclusive`) on every SWIM scenario in
|
||||
the calibration library.
|
||||
|
||||
**Downstream**: this coverage is the data surface
|
||||
`N3_SWIM_TUNING_SPEC.md` consumes. SWIM tuning itself is
|
||||
downstream of collection and lives in that sibling document.
|
||||
|
||||
---
|
||||
|
||||
## 3. Out of scope
|
||||
|
||||
- **Inference protocol surface beyond the response leg.** Coverage
|
||||
2.4 instruments the response-send event. A full inference-
|
||||
protocol event stream (microbatch routing, KV cache, per-stage
|
||||
worker activity) is broader than what the `1779733878` postmortem
|
||||
could not answer; it belongs in a separate spec when a
|
||||
postmortem demands it.
|
||||
- **Post-processor summary enhancements.** SWIM transition
|
||||
distributions, per-(observer, target, reason) breakdowns,
|
||||
cross-node temporal alignment around the moment of failure —
|
||||
these are renderer concerns, not collection concerns. They
|
||||
presuppose the data is in the bundle; this spec is about the
|
||||
data.
|
||||
- **Orchestrator-topology fixes.** The `1779733878` postmortem's
|
||||
item 6 names the root cause as a NAT'd local orchestrator with
|
||||
no reachable port. That is a deployment-shape question for the
|
||||
runbook, not a collection-coverage question.
|
||||
- **Runbook fixes.** The `--gpu RTX_4090` vs `RTX 4090` line in
|
||||
`DEPLOYMENT_TEST.md` (postmortem item 7) is a runbook bug, not a
|
||||
collection gap.
|
||||
- **Sim coverage of upstream-blocked surfaces.** If
|
||||
`iroh_relay::server` continues to expose no session hooks, the
|
||||
sim's relay vertex can model the lifecycle events the contract
|
||||
requires, but the production D layer of coverage 2.3 may remain
|
||||
partial. That partiality is a structural blind spot to file per
|
||||
the established blind-spot discipline; this spec does not
|
||||
resolve it.
|
||||
|
||||
---
|
||||
|
||||
## 4. References
|
||||
|
||||
- `N3_POSTMORTEM_2026-05-25_1779733878.md` — the second
|
||||
2026-05-25 deployment's postmortem. §"Observability upgrade
|
||||
scorecard" is the source for coverages 2.1, 2.2, 2.3; §"Data-
|
||||
collection / deployment gaps surfaced by this run" items 1–5 map
|
||||
to coverages 2.1, 2.5, 2.2, 2.3, 2.4 respectively.
|
||||
- `N3_SIM_TEST_BATTERY_SPEC.md` — the sim-test battery spec. The
|
||||
battery's families A (relay peer-conn down) and the discriminator
|
||||
it builds against the relay-port probe (coverage 2.2) and the
|
||||
relay session lifecycle (coverage 2.3) consume the data this
|
||||
spec lands.
|
||||
- `crates/simulation/SIM_SPEC.md` — the simulator's behavioral
|
||||
surface. §3.3 codec contract, §5A relay vertex, §6A stage host
|
||||
kind, §9 bundle layout are the load-bearing references for the
|
||||
S-layer contracts.
|
||||
- `crates/simulation/SWIM_TUNING_REPORT.md` — the prior tuning
|
||||
pass against simulated 60 ms latency. §6 limits (especially
|
||||
§6.3 "SWIM host adapter does not emit `probe_sent` /
|
||||
`probe_received` / `probe_timed_out` events") are the source
|
||||
for the S-layer of coverage 2.6.
|
||||
- `N3_SWIM_TUNING_SPEC.md` — the downstream spec that consumes
|
||||
coverage 2.6's data surface to retune SWIM against the
|
||||
observed `1779733878` latency distribution. Sibling document.
|
||||
|
|
@ -1,324 +0,0 @@
|
|||
# N=3 vast.ai deployment post-mortem — 2026-05-25 (run `1779733878`)
|
||||
|
||||
Second N=3 deployment of 2026-05-25, and the **first run on the
|
||||
observability upgrade** (commit `e8be135`, the work specified in
|
||||
`N3_OBSERVABILITY_UPGRADE_SPEC.md` and motivated by
|
||||
`N3_DATA_GAPS.md`). Companion to the earlier post-mortem
|
||||
`N3_POSTMORTEM_2026-05-25.md` (run `1779720002`), whose failure this
|
||||
deployment was meant to (a) avoid and (b) make diagnosable.
|
||||
|
||||
One invocation of `pp-smoke-run --vastai --num-stages 3` (stub worker)
|
||||
on 2026-05-25 (`vastai-N3-1779733878`). The cluster came up cleanly,
|
||||
all three stage workers reached `ready`, the request was sent — and
|
||||
then SWIM membership flapped continuously and no `InferenceResponse`
|
||||
ever returned. The operator declared a deadstop at ~7 min and killed
|
||||
the orchestrator. Unlike last time, the bundle was recovered through
|
||||
the collector's own endpoint (gap 7), and the new diagnostics
|
||||
**attributed the failure to a specific edge**: the response path from
|
||||
the last stage back to the (NAT'd, locally-run) orchestrator.
|
||||
|
||||
## Outcome in one line
|
||||
|
||||
Not a worker bug and not the relay-session loss of run `1779720002`.
|
||||
The stage chain was healthy; the weak link was reaching the
|
||||
orchestrator over the relay. Per-peer dial data (gap 9) shows dials
|
||||
**to the orchestrator failing 7/11 with `Timeout`** while every
|
||||
inter-stage dial succeeded (19/19). This was indistinguishable in the
|
||||
previous bundle and is a one-table answer now.
|
||||
|
||||
## Cleanup note
|
||||
|
||||
The run was terminated with `SIGTERM` (operator kill on deadstop),
|
||||
which — like the previous `SIGKILL` — skips the orchestrator's
|
||||
destroy-on-exit handler. Three rentals (`37803546`, `37803550`,
|
||||
`37803555`) were destroyed via `DELETE /api/v0/instances/<id>/`
|
||||
(HTTP 200 each). Post-cleanup instance count = 0, verified. No leak
|
||||
from the earlier failed lease attempt either (see Sequence).
|
||||
|
||||
## Sequence
|
||||
|
||||
Orchestrator runs locally (behind home NAT, no direct port); three
|
||||
stages on vast.ai RTX 4090 hosts; relay + collector on docean
|
||||
(`146.190.110.128`).
|
||||
|
||||
```
|
||||
t=— first launch dies instantly: lease_chain failed,
|
||||
"no offers available (after geo/exclusion filter)".
|
||||
Root cause: --gpu RTX_4090 (underscore) matches 0 vast.ai
|
||||
offers; the API uses "RTX 4090" (space). 0 instances leased.
|
||||
t=0 relaunch with --gpu "RTX 4090": orchestrator node 146aef53,
|
||||
custom-relay banner emitted.
|
||||
t=~135s contracts 37803546/37803550/37803555 reach running,
|
||||
leased as stage-0 (aa0ef1c5), stage-1 (32abf9d6),
|
||||
stage-2 (a1ebaa08).
|
||||
"waiting for SWIM convergence (3 alive)" → converges.
|
||||
t=~150s registered pp-orchestrator; pp-entry resolved; one
|
||||
InferenceRequest sent. Enter await_response (600s budget).
|
||||
t=~165s+ SWIM begins flapping. Members oscillate, e.g.:
|
||||
+63s : 32abf9d6=suspect
|
||||
+79s : 32abf9d6=dead, a1ebaa08=dead
|
||||
+94s : 32abf9d6=dead (others alive)
|
||||
+194s: 32abf9d6=dead, a1ebaa08=suspect
|
||||
+216s: aa0ef1c5=suspect
|
||||
iroh keeps exchanging messages throughout; connect-timeout
|
||||
count to peers is 0 (contrast 1779720002).
|
||||
t=~419s await_response still open, members momentarily all-alive,
|
||||
still no InferenceResponse.
|
||||
t≈7min operator declares deadstop (a peer Dead across two
|
||||
consecutive 45s heartbeats with zero forward progress),
|
||||
kills orchestrator (SIGTERM). No finalize record written.
|
||||
```
|
||||
|
||||
## Bundle recovery (gap 7 — worked, with a caveat)
|
||||
|
||||
`GET /diag/bundle/vastai-N3-1779733878` returned **HTTP 200** with a
|
||||
usable tarball — no hand tar/scp/reshape, unlike last time. The
|
||||
gap-7 synthesis-from-staging path is the intended fix and it
|
||||
functioned.
|
||||
|
||||
Caveat surfaced by this run: the run id was **reused across the
|
||||
failed first lease attempt**. That attempt's orchestrator
|
||||
(`e8151ed8`) called `finalize("lease_chain_error")`, which made the
|
||||
collector build and cache a tiny (5.3 KB) canonical bundle from the
|
||||
staging that existed *at that moment* — orchestrator + relay only, no
|
||||
stages. Because `finalize_received` was then true, the first `GET`
|
||||
served that **stale canonical bundle** rather than synthesizing from
|
||||
current staging. Removing the cached bundle and the junk `e8151ed8`
|
||||
node forced re-synthesis → full **9.3 MB** bundle with all five real
|
||||
nodes. Two residual quirks observed even after removal:
|
||||
|
||||
- `finalize_received` stayed `true` (the collector retains an
|
||||
in-memory finalize record that outlives deletion of the on-disk
|
||||
`finalize-*.json`).
|
||||
- the synthesized `MANIFEST.json` still listed the deleted `e8151ed8`
|
||||
node (with `finalize_recorded: true`) although no such directory
|
||||
was in the tarball.
|
||||
|
||||
Neither blocked analysis, but both are worth hardening: gap-7 assumed
|
||||
finalize == end-of-run, and run-id reuse breaks that assumption.
|
||||
|
||||
## Bundle findings
|
||||
|
||||
### Volume
|
||||
|
||||
| node | role | snapshots | events |
|
||||
|----------------------------|--------------|-----------|--------|
|
||||
| `146aef53` orchestrator | orchestrator | 409 | 4385 |
|
||||
| `1c8357a1` relay (docean) | relay | 186 | 187 |
|
||||
| `aa0ef1c5` stage-0 | stage | 771 | 8638 |
|
||||
| `32abf9d6` stage-1 | stage | 305 | 3234 |
|
||||
| `a1ebaa08` stage-2 | stage | 535 | 5963 |
|
||||
|
||||
`run_start_ms=1779734098442`, `run_end_ms=1779735024439`
|
||||
(`duration_ms=925997`; the tail includes the relay's continued
|
||||
periodic reporting after the orchestrator died — the relay on docean
|
||||
is still pinned to this run id, see Infra state).
|
||||
|
||||
### Subprocess lifecycle (gap 4) — decisive
|
||||
|
||||
`SubprocessSpawned: 3`. `Custom(worker_starting): 3`,
|
||||
`Custom(worker_ready): 3`, `Custom(worker_heartbeat): 37`. No
|
||||
`SubprocessExited`.
|
||||
|
||||
**All three stage workers spawned and became ready and stayed up.**
|
||||
This is the single fact the `1779720002` bundle could not establish
|
||||
(there, stage-2 emitted neither `worker_starting` nor `worker_ready`,
|
||||
and we could not tell "never spawned" from "spawned and died"). The
|
||||
worker is conclusively ruled out as the cause this time.
|
||||
|
||||
### Per-peer dials (gap 9) — the attribution
|
||||
|
||||
Totals: `started=50, succeeded=42, failed=7, in-flight=1`.
|
||||
|
||||
| peer | started | ok | failed | in-flight | last_outcome | at_ms |
|
||||
|-----------------------|---------|----|--------|-----------|--------------|----------------|
|
||||
| orchestrator-146aef53 | 11 | 3 | **7** | 1 | **Timeout** | 1779734955567 |
|
||||
| stage-0 (aa0ef1c5) | 1 | 1 | 0 | 0 | Success | 1779734410397 |
|
||||
| stage-1 (32abf9d6) | 19 | 19 | 0 | 0 | Success | 1779734518356 |
|
||||
| stage-2 (a1ebaa08) | 19 | 19 | 0 | 0 | Success | 1779734522415 |
|
||||
|
||||
Every dial *between stages* succeeded. Only dials *to the
|
||||
orchestrator* failed, and they failed by timeout. The last stage's
|
||||
`InferenceResponse` is addressed to the orchestrator's inbox; if it
|
||||
cannot dial the orchestrator, the response never lands. This table is
|
||||
the proximate cause of the empty result.
|
||||
|
||||
### Relay-session field (gap 2) and conn type
|
||||
|
||||
stage-2's latest snapshot `body.iroh.relay_session`:
|
||||
|
||||
```
|
||||
relay_url: http://146.190.110.128:7843/
|
||||
status: connected
|
||||
status_changed_at_ms:1779734401024
|
||||
status_entered_at_ms:1779734401024
|
||||
status_source: derived
|
||||
```
|
||||
|
||||
The tunnel to the relay was **connected**, with `status_source:
|
||||
derived` honestly flagging that iroh does not expose this natively
|
||||
(per spec §2). So the failure is *not* "tunnel died" — it is "tunnel
|
||||
alive, peer-connection-through-tunnel to the orchestrator dead." That
|
||||
distinction was the explicit acceptance criterion for gap 2, and it
|
||||
holds here.
|
||||
|
||||
`First peer to go Dead`: stage-2 marked stage-1 (`32abf9d6`) Dead at
|
||||
`t=1779734428439`, reason `suspicion-timeout`. Both sides
|
||||
`conn_type=Relay` (no direct hole-punch anywhere in the run). Observer
|
||||
`probes_ok=yes`, peer `probes_ok=no`.
|
||||
|
||||
### SWIM churn and relay events (gap 3)
|
||||
|
||||
`SwimTransition: 1701` over a ~7-minute run — heavy flapping,
|
||||
consistent with the relay-mediated reachability of a NAT'd
|
||||
orchestrator and the §10.3 self-incarnation flap (see
|
||||
`SWIM_TUNING_REPORT`). `RelaySessionStateChanged: 8`, `RelayChanged:
|
||||
4`, `IrohConnTypeChanged: 8` — relay/transport flips are now on the
|
||||
event stream, not just counter deltas.
|
||||
|
||||
### Kernel network drops (gap 11)
|
||||
|
||||
`udp.no_ports` delta across the run:
|
||||
|
||||
| node | udp.no_ports |
|
||||
|-------------------|--------------|
|
||||
| orchestrator | +1 |
|
||||
| stage-0 | +139 |
|
||||
| stage-1 | +134 |
|
||||
| stage-2 | **+1424** |
|
||||
|
||||
stage-2 took ~10× the no-listening-port UDP drops of its siblings —
|
||||
an interface-level corroboration of localized relay/hole-punch path
|
||||
instability, surfaced automatically in the summary.
|
||||
|
||||
### Gossip receipts (gap 10)
|
||||
|
||||
| node | swim_piggyback | bytes | items |
|
||||
|--------------|----------------|--------|-------|
|
||||
| orchestrator | 806 | 263825 | 1607 |
|
||||
| stage-0 | 1589 | 522855 | 3178 |
|
||||
| stage-1 | 563 | 194236 | 1183 |
|
||||
| stage-2 | 1082 | 340115 | 2069 |
|
||||
|
||||
Every node received gossip. Gossip starvation is ruled out — the
|
||||
flap is not "a node never heard membership," it is "membership churned
|
||||
because the underlying relay path to a peer was unreliable."
|
||||
|
||||
### UDP echo probes (collector tier-2)
|
||||
|
||||
| node | result |
|
||||
|--------------|---------------------------------|
|
||||
| orchestrator | ok, rtt=293 ms, 34/35 |
|
||||
| stage-0 | ok, rtt=181 ms, 55/55 |
|
||||
| stage-1 | ok, rtt=405 ms, 27/28 |
|
||||
| stage-2 | ok, rtt=184 ms, 38/38 |
|
||||
|
||||
All nodes had clean tier-2 reachability to docean:9081 — i.e. the
|
||||
hosts themselves were on the network. The failure was at the iroh
|
||||
peer-connection layer, not raw host reachability.
|
||||
|
||||
### iroh version honesty (gap 6)
|
||||
|
||||
`iroh_version: "0.98.2"` on every node and in every
|
||||
`iroh_api_missing` event (4 total) — matches `Cargo.lock`. The
|
||||
hard-coded `"0.96"` literal from `1779720002` is gone. The API-gap
|
||||
list now also carries the `RelayTunnel.*` derived-field markers.
|
||||
|
||||
## Observability upgrade scorecard
|
||||
|
||||
What this run confirms the upgrade delivers, versus what it doesn't:
|
||||
|
||||
| gap | status | evidence |
|
||||
|-----|--------|----------|
|
||||
| 2 relay-session field | ✅ | `relay_session.status=connected, status_source=derived` |
|
||||
| 3 relay events | ✅ | `RelaySessionStateChanged: 8`, `RelayChanged: 4` |
|
||||
| 4 subprocess introspector | ✅ | `SubprocessSpawned: 3`; worker ruled out |
|
||||
| 6 iroh version honesty | ✅ | `0.98.2` everywhere, lockfile match |
|
||||
| 7 bundle without finalize | ✅¹ | `GET` returned a usable 9.3 MB bundle; ¹run-id reuse exposed stale-canonical serve + sticky finalize flag |
|
||||
| 9 per-peer dials | ✅ | the orchestrator-reachability table (the headline finding) |
|
||||
| 10 gossip receipts | ✅ | per-node `swim_piggyback` breakdown |
|
||||
| 11 kernel counters | ✅ | stage-2 `udp.no_ports +1424` surfaced |
|
||||
| 1 relay observability | ◐ | relay reports identity + 186 snapshots, but per-session lifecycle is the documented skeleton: `active=0 opens=0 closes=0` (iroh-relay exposes no session hooks) |
|
||||
| 5 host metadata | ◐ | `container_id`/`hostname`/`git_sha`/`iroh_version`/`binary_version` present; `host_ip_public`/`datacenter_id`/`host_country`/`vastai_contract_id` still **null** — provider fields not forwarded. (The contract id does arrive, but in `container_id`, not `vastai_contract_id`.) |
|
||||
| 8 relay-port probe | ✗ | not present: stage probe arrays carry only `collector_udp_echo` (:9081); no relay-port (:7843) probe |
|
||||
|
||||
A+B+C (the tiers the previous investigation needed) all landed and
|
||||
were load-bearing here. The polish/robustness tiers (5 ip/dc/country,
|
||||
7 finalize edge cases, 8 relay probe, 1 relay session lifecycle) have
|
||||
remaining work.
|
||||
|
||||
## Data-collection / deployment gaps surfaced by this run
|
||||
|
||||
1. **Host provider fields not forwarded (gap 5 incomplete).** The
|
||||
orchestrator has each rental's public IP, datacenter, country, and
|
||||
contract id at `lease_chain` return, but only the docker container
|
||||
id (landing in `container_id`) and hostname reach the boot record.
|
||||
`host_ip_public`/`datacenter_id`/`host_country`/`vastai_contract_id`
|
||||
/`home_relay_url_at_boot` are still null. "Which rental was
|
||||
stage-2" is answerable only via the `container_id`↔contract map,
|
||||
not the dedicated fields.
|
||||
|
||||
2. **gap-7 stale-canonical serve on run-id reuse.** A finalize from an
|
||||
earlier phase of the same run id pins a stale cached bundle, and
|
||||
`finalize_received` is sticky in collector memory across staging
|
||||
deletion. Serve logic should prefer the richer of {canonical,
|
||||
synthesized-from-current-staging} or rebuild canonical when staging
|
||||
has grown past the cached bundle.
|
||||
|
||||
3. **No relay-port reachability probe (gap 8 absent).** Whether
|
||||
stage-2 could reach docean:7843 (the relay) at moment T is still
|
||||
inferred from a different port (:9081 echo). The exact gap the
|
||||
previous post-mortem flagged remains open.
|
||||
|
||||
4. **Relay per-session lifecycle still skeleton (gap 1).** The relay
|
||||
reports into the bundle but cannot yet say "who closed session X
|
||||
and why" because `iroh_relay::server` exposes no session hooks.
|
||||
Until it does, "was this a relay-side eviction" is unanswerable
|
||||
from the relay side; we relied on node-side dial outcomes instead.
|
||||
|
||||
5. **No response-leg instrumentation.** The conclusion "last stage
|
||||
could not deliver the response" was inferred from dial timeouts +
|
||||
absence of an inbound response, not from a typed event on the last
|
||||
stage ("attempted to send InferenceResponse to orchestrator,
|
||||
outcome=…"). A response-send event would make this a direct read
|
||||
rather than an inference.
|
||||
|
||||
6. **Environmental: orchestrator topology is the root cause.** A
|
||||
locally-run, NAT'd orchestrator with no direct port is reachable
|
||||
only via the relay, and the relay path to it proved unreliable
|
||||
under load (7/11 inbound dials timed out, 1701 SWIM transitions).
|
||||
Running the orchestrator on a reachable host (e.g. docean) or
|
||||
giving it a direct/forwarded port is the likely fix to test next.
|
||||
|
||||
7. **`DEPLOYMENT_TEST.md` GPU flag is wrong.** Line 84 shows
|
||||
`--gpu RTX_4090`; vast.ai matches `gpu_name` literally and the
|
||||
underscore form returns 0 offers. The orchestrator's own default
|
||||
is the correct `"RTX 4090"`. Fix the runbook example.
|
||||
|
||||
## Artifacts
|
||||
|
||||
In `.vastai-logs/` (gitignored) after recovery:
|
||||
|
||||
```
|
||||
vastai-N3-1779733878.bundle.tar.gz full synthesized bundle (9.3 MB, 5 nodes)
|
||||
vastai-N3-1779733878.staging-full.tar.gz raw collector staging backup (~10 MB)
|
||||
vastai-N3-1779733878.tar.gz the stale 5.3 KB canonical bundle (kept for reference)
|
||||
vastai-N3-1779733878.log orchestrator stdout (both launch attempts)
|
||||
vastai-N3-1779733878.out/summary.md postproc summary
|
||||
vastai-N3-1779733878.out/reachability.tsv
|
||||
vastai-N3-1779733878.out/timeline-*.tsv per-link timelines
|
||||
```
|
||||
|
||||
Staging copy retained on docean at
|
||||
`/var/lib/swactor-diag/vastai-N3-1779733878/` (minus the removed
|
||||
`e8151ed8` junk node).
|
||||
|
||||
## Infrastructure state at end of session
|
||||
|
||||
- docean (`146.190.110.128`): collector and relay running (rebuilt
|
||||
static-musl binaries from `e8be135`). The relay is **still pinned to
|
||||
`SWACTOR_DIAG_RUN_ID=vastai-N3-1779733878`** and continues appending
|
||||
periodic snapshots to that run's staging; the next run's redeploy
|
||||
re-pins it. Re-pulling the bundle later will include those extra
|
||||
relay snapshots.
|
||||
- vast.ai instances under `$VAST_API_KEY`: **0** (verified).
|
||||
|
|
@ -1,579 +0,0 @@
|
|||
# N=3 sim-test battery — behavioral specification
|
||||
|
||||
Companion to `N3_POSTMORTEM_2026-05-25.md`, `N3_DATA_GAPS.md`,
|
||||
`N3_DEPLOYMENT_REPORT.md`, `SIM_HARDENING_SPEC.md`, and the simulator's
|
||||
`SIM_SPEC.md`. This document is the contract for a separate coding agent
|
||||
that will land a battery of simulator tests covering the general failure
|
||||
shapes the latest deployment exposed.
|
||||
|
||||
This is a *behavioral* spec. It names the failure shapes, the contracts
|
||||
each test must establish, and the verdicts each must produce. It does
|
||||
not prescribe file layout, TOML field values, or internal helper code.
|
||||
|
||||
---
|
||||
|
||||
## 0. Motivation and framing
|
||||
|
||||
The 2026-05-25 deployment surfaced one new failure shape (`stage-2`'s
|
||||
relay-mediated path died at ~5 s and never recovered, while its tunnel
|
||||
to the relay apparently survived) layered on top of failure shapes
|
||||
prior deploys also exhibited (silent-worker subprocess, gossip-only
|
||||
membership view, asymmetric host reachability, bundle-recovery only
|
||||
via staging-file scrape). Together these are the **general** failure
|
||||
cases the battery must cover — not one scenario per postmortem, but a
|
||||
*family* per shape, as `SIM_HARDENING_SPEC.md §5` requires.
|
||||
|
||||
The simulator has now landed every observability and sim-cross-
|
||||
pollination contract those postmortems demanded (`F1`–`F3`, `S-A1`
|
||||
through `S-E2`; see `.loop/verdict.md`). The pieces needed to express
|
||||
these scenarios all exist: `MutationKind::RelayPeerConnDown`, the
|
||||
`stage` host kind with `WorkerExit`, the `relay` vertex with policy
|
||||
mutations, and the §10.1 assertion catalog. **The battery is the
|
||||
exercise of those pieces against the latest deployment's known shapes,
|
||||
expressed end-to-end through scenario files and verdicts — not new
|
||||
sim machinery.**
|
||||
|
||||
Why a *battery* rather than one test per shape: the
|
||||
`SIM_HARDENING_SPEC §5` family rule. A fix that resolves the
|
||||
2026-05-25 incident's specific timing (relay-peer-down at +5 s) but
|
||||
regresses a sibling instance of the family (relay-peer-down at +30 s,
|
||||
or during partition heal, or on only the inbound leg) is a regression
|
||||
the battery must catch.
|
||||
|
||||
A diagnostic deployment is running concurrently to gather data we
|
||||
don't yet have for the silent-worker class. This spec is written
|
||||
against the evidence already in the bundle from 2026-05-25; the
|
||||
implementing agent should not block on that deploy's results. When
|
||||
results land they will sharpen the parameters of family **B**
|
||||
(silent-worker) but will not change the shape of the battery.
|
||||
|
||||
---
|
||||
|
||||
## 1. Cross-cutting requirements
|
||||
|
||||
These hold for every family in §3.
|
||||
|
||||
### 1.1 No white-box / structural tests
|
||||
|
||||
A test in the battery passes or fails based on the *bundle* the
|
||||
scenario produces and the verdicts the §10.1 assertion catalog
|
||||
returns against that bundle. No test reads simulator internals, no
|
||||
test inserts a value via one API path and reads it back via another,
|
||||
no test asserts that an internal Rust struct has a particular field
|
||||
shape. A test that would survive a refactor of the engine, the
|
||||
network, or any host kind, but fail when the *deployment-relevant
|
||||
behavior* drifts, is a test that belongs.
|
||||
|
||||
Litmus test: if removing the assertion would change the bundle's
|
||||
prose summary in a way a deployment investigator would notice, the
|
||||
assertion belongs. If removing it would not, the assertion is
|
||||
echoing internals and does not belong.
|
||||
|
||||
### 1.2 Test taxonomy and priority
|
||||
|
||||
Each family ships at least one **scenario test** (story-shape:
|
||||
declared scenario + declared assertion + declared expected verdict)
|
||||
and where the parameter space is large, at least one **property
|
||||
test** (a parameterized scenario whose `seed` ranges over the §1.3
|
||||
family axes). Scenario tests are mandatory; property tests are
|
||||
required only where §3 names them.
|
||||
|
||||
A small number of **contract tests** sit alongside the families: they
|
||||
assert that the bundle's event schema matches the production
|
||||
diagnostics schema for the event kinds the battery exercises (the
|
||||
`SubprocessSpawned`/`SubprocessExited`, `RelaySessionStateChanged`,
|
||||
`GossipReceived`, and `Tier2RelaySession` shapes the observability
|
||||
upgrade landed). The contract tests are not per-family; they live
|
||||
once and protect every family from sim/prod drift.
|
||||
|
||||
### 1.3 Family-based, not single-seed
|
||||
|
||||
Every family in §3 declares its **mutation axes** — the dimensions
|
||||
along which the postmortem's parameters are "plausibly variable in
|
||||
the wild" per `SIM_HARDENING_SPEC §5`. The family's scenario tests
|
||||
cover the central case (the specific incident's parameters) and the
|
||||
named extreme cases (e.g., "session closes at +1 s" and "session
|
||||
closes at +5 min" for family A). The family's property test ranges
|
||||
over the axes within their declared bounds.
|
||||
|
||||
### 1.4 Deterministic replay
|
||||
|
||||
Every scenario test's `(scenario, seed)` is recorded in the test
|
||||
itself; running the test produces a byte-identical bundle to any
|
||||
previous run on any supported architecture. A property-test failure
|
||||
prints the seed; running the scenario with that seed reproduces the
|
||||
failure. This is mechanical — the simulator already guarantees it
|
||||
(`SIM_SPEC.md §7`); the battery must not undo it. No test reads any
|
||||
wall-clock or system source of randomness.
|
||||
|
||||
### 1.5 Sub-second per scenario
|
||||
|
||||
A 3-node scenario test (including bundle assembly and verdict
|
||||
evaluation) completes in under one second on the developer's
|
||||
machine. The full battery completes in under thirty seconds locally
|
||||
and under three minutes in CI. A scenario that grows above this
|
||||
budget is a regression in the test, not in the simulator; the test
|
||||
author tightens the scenario rather than relaxing the budget.
|
||||
|
||||
### 1.6 Verdict-first
|
||||
|
||||
Every test in the battery declares its **expected verdict on the
|
||||
current source** before it lands: `Pass` (the simulator already
|
||||
satisfies the contract; the test guards against regression), `Fail`
|
||||
(the simulator currently violates the contract; landing the test
|
||||
makes the failure visible, and the test is expected to pass after a
|
||||
fix names in §4), or `Mixed` (some seeds pass, some fail — typical
|
||||
for property tests against a probabilistic shape).
|
||||
|
||||
A test landing as `Fail` is **not** a build break in the test
|
||||
binary; it is a verdict in the bundle's `verdicts.json` whose CI
|
||||
exposure is named in §1.7. A test landing as `Pass` runs with
|
||||
`#[test]` semantics — a regression in the simulator is a CI break.
|
||||
|
||||
### 1.7 CI exposure
|
||||
|
||||
Tests with expected verdict `Pass` run as standard `cargo test`
|
||||
binaries under `crates/simulation/tests/`. Tests with expected
|
||||
verdict `Fail` or `Mixed` run as a separate
|
||||
`cargo test --package simulation --test battery_expected_failures`
|
||||
binary that asserts the verdict matches expectation (`Fail` →
|
||||
`Fail`, `Mixed` → at least one `Fail` across the seed range, at
|
||||
least one `Pass`). Promoting a `Fail` test to `Pass` after a fix is
|
||||
a one-line move between binaries and a deletion from the expected-
|
||||
failures registry; the implementer should make this move trivial.
|
||||
|
||||
### 1.8 Library layout
|
||||
|
||||
The battery's scenarios live under
|
||||
`crates/simulation/scenarios/reproduction/n3_2026_05_25/`, one
|
||||
subdirectory per family. Each family directory contains:
|
||||
|
||||
- A `README.md` naming the family, pointing at the postmortem, and
|
||||
listing the family's mutation axes.
|
||||
- One scenario file per named central or extreme case
|
||||
(`central.toml`, `extreme_*.toml`).
|
||||
- A `property.toml` file declaring the property-test seed range and
|
||||
axis bounds where §3 requires a property test.
|
||||
|
||||
This layout is the existing `scenarios/reproduction/` convention
|
||||
extended one level. No new top-level directories.
|
||||
|
||||
---
|
||||
|
||||
## 2. The shared scenario shape
|
||||
|
||||
Every scenario in the battery has the following shape unless its
|
||||
family in §3 names a divergence:
|
||||
|
||||
- **Three peers**: one orchestrator-kind, two stage-kind. IDs
|
||||
`orch`, `stage-0`, `stage-2` (the latter named to match the
|
||||
postmortem's victim peer). The third stage from production is
|
||||
omitted only when its absence does not change the shape of the
|
||||
failure under test; families that require N=4 to manifest must say
|
||||
so explicitly. (`stage-1` may appear as a peer in families that
|
||||
need it; otherwise the simulator's N=3 minimum is the target.)
|
||||
- **One relay vertex** `R`, with policy seeded from the
|
||||
`vastai-N3-2` calibration scenario (own-relay shape — widened
|
||||
egress, modest queue depth). Per-family scenarios may tighten or
|
||||
loosen this; the central case for each family uses the calibration
|
||||
defaults.
|
||||
- **Routing**: all host-to-host edges declared `via = R`. The 2026-
|
||||
05-25 incident exercised the relay path exclusively; no direct
|
||||
edges in the battery's central cases. Extreme cases that need
|
||||
direct edges declare them per `SIM_SPEC.md §8.1`.
|
||||
- **Duration**: 10 simulated minutes (`duration_ns = 600_000_000_000`)
|
||||
matching the 2026-05-25 run's wall-clock budget. Scenarios may
|
||||
shorten but not lengthen — long scenarios violate the sub-second
|
||||
budget in §1.5.
|
||||
- **Snapshots**: at least one snapshot per peer per simulated
|
||||
minute, plus a snapshot one virtual nanosecond before and one
|
||||
after every named fault, so the bundle reader can see the state
|
||||
on each side of each transition. (This is a property of the
|
||||
scenario, not of the engine: the scenario's `[[snapshots]]` array
|
||||
declares these.)
|
||||
- **Assertions**: each family in §3 names its required assertions.
|
||||
Scenarios may add further assertions from §10.1 to tighten the
|
||||
contract; they may not remove or relax the named ones.
|
||||
|
||||
---
|
||||
|
||||
## 3. The families
|
||||
|
||||
Six families, each named for the failure shape it covers. Families
|
||||
A, B, and C are derived directly from the 2026-05-25 incident.
|
||||
Families D, E, and F are derived from the broader N≥3 deployment
|
||||
history that the latest run did not contradict and should not
|
||||
regress.
|
||||
|
||||
### Family A — Relay-mediated peer-connection drop with surviving tunnel
|
||||
|
||||
**Source**: `N3_POSTMORTEM_2026-05-25.md` "iroh state — orchestrator's
|
||||
view of stage-2"; `N3_DATA_GAPS.md` gaps 1, 2, 3.
|
||||
|
||||
**Shape**: A peer-to-peer path through a relay opens, succeeds for a
|
||||
short window, then dies. The relay's tunnel to the victim peer
|
||||
remains apparently healthy — the victim's `Tier2RelaySession.status`
|
||||
stays `connected` or is reported as such by the relay, while the
|
||||
orchestrator's `connection_cache[victim].last_failure_reason` shows
|
||||
the path closed. iroh does not re-establish.
|
||||
|
||||
**Central case** (`central.toml`): `RelayPeerConnDown { relay: R,
|
||||
from: orch, to: stage-2, at_ns: 5_000_000_000, duration_ns: 0 }`
|
||||
(permanent until run end), inserted shortly after SWIM convergence.
|
||||
No other faults.
|
||||
|
||||
**Mutation axes** (the family's parameter space):
|
||||
|
||||
1. `at_ns`: when the cut fires. Central +5 s; extremes +1 s, +30 s,
|
||||
+1 min, +5 min.
|
||||
2. `duration_ns`: how long the cut persists. Central permanent;
|
||||
extremes 100 ms, 5 s, 30 s.
|
||||
3. Direction: cut on `(orch → stage-2)` only, on `(stage-2 → orch)`
|
||||
only, or on both. The 2026-05-25 evidence is ambiguous about
|
||||
direction; the battery covers all three.
|
||||
4. Flap: a sequence of `RelayPeerConnDown` mutations interleaved with
|
||||
their natural recovery — close, reopen, close. Inter-flap durations
|
||||
100 ms, 1 s, 5 s.
|
||||
5. Phase: cut during SWIM convergence (before all peers Alive); cut
|
||||
during steady-state after convergence; cut during a
|
||||
`Partition`+`Heal` cycle's heal phase (per `SIM_HARDENING_SPEC §9`).
|
||||
|
||||
**Required assertions**:
|
||||
|
||||
- `no_flap_while_probes_ok { peer: stage-2, window_start_ns:
|
||||
at_ns, window_end_ns: duration_ns_end }` — the family asserts the
|
||||
*observability* contract that a relay-peer cut produces a typed
|
||||
event chain (`RelayPeerConnDown` mutation record →
|
||||
`RelaySessionStateChanged` or equivalent on the victim's view →
|
||||
`connection-closed` in the observer's cache). What it does *not*
|
||||
assert is that the simulator's SWIM tolerates the cut — the
|
||||
current simulator does not.
|
||||
- `event_count { kind: "RelaySessionStateChanged", min: 1 }` on
|
||||
the central case — a cut must produce at least one transition
|
||||
event for the bundle reader to see.
|
||||
- `dead_peer_resurrects_within { peer: stage-2, after_ns:
|
||||
heal_at_ns, within_ns: 30_000_000_000 }` on the finite-duration
|
||||
extreme cases — once the cut lifts, the cluster must reconverge.
|
||||
|
||||
**Property test**: `property.toml` ranges seeds 0..256 over axes 1,
|
||||
2, and 5. The seed search reports any seed whose run violates
|
||||
`no_flap_while_probes_ok` while the cut is *not* active (a
|
||||
false-flap during a healthy window — the bug class the family
|
||||
exists to catch).
|
||||
|
||||
**Expected verdict on current source**: `Mixed`. The central case
|
||||
is expected `Fail` against the current SWIM source (the
|
||||
deployment's actual failure mode); the flap extreme and the
|
||||
phase-during-heal extreme are also expected `Fail`. The finite-
|
||||
duration extremes with short cuts may pass.
|
||||
|
||||
**Family closes when**: a fix lands that lets the central case
|
||||
pass and at least the flap and phase-during-heal extremes pass,
|
||||
with no other family regressing.
|
||||
|
||||
### Family B — Silent stage subprocess (never spawned, spawned-and-stuck, spawned-and-exited)
|
||||
|
||||
**Source**: `N3_POSTMORTEM_2026-05-25.md` "Custom (worker) events"
|
||||
table (`stage-2` emitted zero `worker_starting`, zero `worker_ready`);
|
||||
`N3_DATA_GAPS.md` gap 4; `SIM_HARDENING_SPEC §5`.
|
||||
|
||||
**Shape**: A stage's worker subprocess fails to reach the
|
||||
`worker_ready` state. The stage actor itself is alive — snapshots
|
||||
still arrive, events still flow — but no work begins. The failure
|
||||
splits into three buckets per the §4 spec the observability upgrade
|
||||
already landed: never-spawned, spawned-and-stalled-before-ready,
|
||||
spawned-and-exited-before-ready.
|
||||
|
||||
**Central case** (`central.toml`): install a `SubprocessFakeSpec`
|
||||
on `stage-2` with `never_ready = true`, no `exit_after_ns`. The
|
||||
orchestrator's view: `SubprocessSpawned` arrives, no `worker_ready`
|
||||
Custom event ever does. The sim already supports this via `F1`.
|
||||
|
||||
**Mutation axes**:
|
||||
|
||||
1. Bucket: `never_spawned` (no `SubprocessFakeSpec` installed at
|
||||
all; stage actor never registers); `stalled` (spawned, never
|
||||
ready); `early_exit` (spawned, exits before ready with named
|
||||
exit code / signal).
|
||||
2. `exit_after_ns` for the `early_exit` bucket: 100 ms (faster than
|
||||
any plausible ready), 1 s, 10 s.
|
||||
3. Number of victim stages: one (central), two (whole stage layer
|
||||
silent), zero (control — all stages reach `worker_ready` —
|
||||
sanity).
|
||||
4. Whether SWIM convergence completes before or after the worker
|
||||
silence is observable.
|
||||
|
||||
**Required assertions**:
|
||||
|
||||
- The bundle must make the three buckets distinguishable at the
|
||||
verdict level. The discriminator is the joint state of
|
||||
`SubprocessSpawned`, `SubprocessExited`, and the `worker_ready`
|
||||
Custom event for the victim peer, with the buckets mapping as:
|
||||
- `never_spawned`: `SubprocessSpawned == 0`, `worker_ready == 0`.
|
||||
- `stalled`: `SubprocessSpawned == 1`, `worker_ready == 0`, no
|
||||
`SubprocessExited` for the run's duration.
|
||||
- `early_exit`: `SubprocessSpawned == 1`, `worker_ready == 0`,
|
||||
`SubprocessExited == 1` with the declared reason.
|
||||
- `name_resolves_within { name: "pp-entry", observers: [orch],
|
||||
within_ns: 300_000_000_000, from_ns: 0 }` — the orchestrator's
|
||||
resolution of the pipeline entry name must fail when any victim
|
||||
stage is silent. The contract: `Inconclusive` is **not**
|
||||
acceptable — the bundle must clearly say "the orchestrator looked
|
||||
and the name was absent," not "we don't know if the orchestrator
|
||||
looked."
|
||||
|
||||
**Property test**: not required for B. The bucket count is small
|
||||
enough that all combinations land as scenario tests.
|
||||
|
||||
**Expected verdict on current source**: per-bucket. `never_spawned`
|
||||
and `stalled` expected `Fail` on the `name_resolves_within`
|
||||
assertion (correct — the cluster cannot resolve `pp-entry` if a
|
||||
stage is silent). `early_exit` expected `Fail` on the same plus
|
||||
`event_count { kind: "SubprocessExited", min: 1 }` with the
|
||||
correct exit code observable in the bundle.
|
||||
|
||||
The battery's job here is to **prove the bucket is observable**, not
|
||||
to prove the cluster recovers. Recovery from a silent worker is a
|
||||
product question, not a sim contract.
|
||||
|
||||
**Family closes when**: the bundle's `summary.md` (rendered through
|
||||
`swactor-diag-postproc`) names which bucket the victim stage is in,
|
||||
in human-readable prose, for every scenario in the family.
|
||||
|
||||
### Family C — Gossip-arrival absence (control-plane vs data-plane discriminator)
|
||||
|
||||
**Source**: `N3_POSTMORTEM_2026-05-25.md` "iroh state — stage-2's
|
||||
view of itself" (`peers: [orchestrator only]`); `N3_DATA_GAPS.md`
|
||||
gap 10; `SIM_HARDENING_SPEC` §1 and §2.
|
||||
|
||||
**Shape**: A victim peer's local membership view contains only the
|
||||
orchestrator, never its siblings. Two possible causes are
|
||||
indistinguishable from the postmortem bundle: gossip about siblings
|
||||
never arrived (control-plane failure), or gossip arrived but the
|
||||
dials based on it never connected (data-plane failure). The battery
|
||||
must let a single scenario+verdict pair disambiguate these.
|
||||
|
||||
**Central case** (`central.toml`): a `Partition` mutation that
|
||||
isolates `stage-2` from `stage-0` and `stage-1` at the
|
||||
network-graph layer (no direct, no relayed route between them),
|
||||
while leaving each stage's path to `orch` intact. Stage-2 should
|
||||
never receive gossip naming stage-0 / stage-1.
|
||||
|
||||
**Mutation axes**:
|
||||
|
||||
1. Topology: full isolation (central); one-way isolation (stage-2
|
||||
receives gossip, dials silently dropped); periodic gossip drops
|
||||
modulated by `LossBurst`.
|
||||
2. Whether the orchestrator's gossip-piggyback ever names the
|
||||
siblings (which depends on its own membership view at the time
|
||||
stage-2 boots and receives its first ping).
|
||||
|
||||
**Required assertions**:
|
||||
|
||||
- `event_count { kind: "GossipReceived", peer: stage-2,
|
||||
payload_kind: "NameRegistry", min: N }` where `N` depends on the
|
||||
axis: for the central case, `N >= 1` (gossip must reach
|
||||
stage-2); the assertion lets us prove the discriminator. A
|
||||
scenario in which gossip *did* arrive but dials failed produces
|
||||
`GossipReceived >= 1` and `DialOutcome` with failure reasons for
|
||||
the siblings; a scenario in which gossip never arrived produces
|
||||
`GossipReceived == 0`. The two bundles are now distinguishable
|
||||
by the verdict.
|
||||
- `event_count { kind: "DialStarted", peer: stage-2, target: stage-0,
|
||||
min: 1 }` on the one-way-isolation axis: dials must be observable
|
||||
in the data-plane-failure case.
|
||||
|
||||
**Property test**: not required.
|
||||
|
||||
**Expected verdict on current source**: `Pass` for all cases — the
|
||||
observability upgrade landed `GossipReceived` (`S-E2`) and the
|
||||
per-peer dial rollup (`S-A3`), so the discriminator is already
|
||||
expressible. The battery's job is to *guard* this contract against
|
||||
regression in the simulator or in the post-processor.
|
||||
|
||||
**Family closes when**: a probe-by-grep against the bundle's
|
||||
`summary.md` confirms the discriminator is named in prose, not
|
||||
buried in raw event counts.
|
||||
|
||||
### Family D — Asymmetric host reachability (NAT / mapping pathology)
|
||||
|
||||
**Source**: `N3_POSTMORTEM_2026-05-25.md` "UDP echo probes" (stage-2
|
||||
1/12 timeout while others were clean); `N3_DATA_GAPS.md` gaps 8 and
|
||||
11; `SIM_HARDENING_SPEC §2` host-environment-level faults.
|
||||
|
||||
**Shape**: One peer's host network behaves correctly *most* of the
|
||||
time, but exhibits asymmetric loss, NAT-rebind, or kernel-UDP-buffer
|
||||
overflow in a pattern that downstream iroh layers cannot
|
||||
distinguish from a relay-side issue or a peer-software issue. The
|
||||
postmortem could not tell which.
|
||||
|
||||
**Central case** (`central.toml`): a `LossBurst` on
|
||||
`(stage-2 → R)` with `prob_ppm = 80_000` (8% loss) lasting 30 s
|
||||
during steady state. This is the smallest fault that produces the
|
||||
postmortem's "one peer flaky, others clean" symptom.
|
||||
|
||||
**Mutation axes**:
|
||||
|
||||
1. Symmetry: loss on outbound from victim, on inbound to victim,
|
||||
on both directions, none (control).
|
||||
2. Burst shape: continuous low-rate loss vs short high-rate burst.
|
||||
3. Co-occurrence: loss alone vs loss + clock skew on the same peer
|
||||
(compound — per `SIM_HARDENING_SPEC §7`).
|
||||
|
||||
**Required assertions**:
|
||||
|
||||
- The bundle's UDP echo probe records must show the victim's
|
||||
outcome distribution (`ok` / `timeout` / `refused` / `unresolved`
|
||||
/ `error`) differing from the other peers' by a margin evident
|
||||
to a human reader.
|
||||
- Across the run, the victim's
|
||||
`Tier3InterfaceCounters.rx_packets_dropped` or
|
||||
`Tier3UdpKernelStats.in_errors` is non-zero in the bundle, while
|
||||
the other peers' is zero. This is the "kernel saw the loss, not
|
||||
just iroh" contract gap 11 demanded.
|
||||
|
||||
**Property test**: required, seeds 0..128. Range over axes 1 and
|
||||
2. The property: for every seed in which the victim's UDP echo
|
||||
shows >5% loss, the bundle must surface a non-zero kernel-counter
|
||||
delta on the same peer. (This is the discriminator gap 11 asked
|
||||
for.)
|
||||
|
||||
**Expected verdict on current source**: `Mixed`. The observability
|
||||
upgrade landed kernel counters in the bundle (`S-A4`); the
|
||||
simulator's stage host needs to emit `Tier3InterfaceCounters` under
|
||||
the loss-burst mutation for the discriminator to hold. If it does
|
||||
not, that is a sim-coverage gap belonging in `SIM_BLIND_SPOTS.md`
|
||||
per `SIM_HARDENING_SPEC §10`, not a reason to relax the assertion.
|
||||
|
||||
**Family closes when**: the property test runs to 128 seeds with
|
||||
the loss-discriminator holding on every seed it sees loss; the
|
||||
sim-coverage gap, if it exists, is filed.
|
||||
|
||||
### Family E — Bundle integrity under operator SIGKILL
|
||||
|
||||
**Source**: `N3_POSTMORTEM_2026-05-25.md` "Bundle recovery";
|
||||
`N3_DATA_GAPS.md` gap 7; observability upgrade `S-D` (bundle
|
||||
without finalize).
|
||||
|
||||
**Shape**: The orchestrator is killed ungracefully (SIGKILL via
|
||||
TaskStop, not graceful shutdown). No finalize record is written.
|
||||
The diagnostic bundle must still be assemblable from staging files
|
||||
on disk, with `manifest.finalize_received: false`.
|
||||
|
||||
**Central case** (`central.toml`): a `PeerKill { peer: orch,
|
||||
at_ns: 60_000_000_000 }` mutation 60 s into the run. No
|
||||
`PeerResurrect`. The scenario's `duration_ns` extends 30 s past
|
||||
the kill so the collector has time to observe and the bundle has
|
||||
time to coalesce.
|
||||
|
||||
**Mutation axes**:
|
||||
|
||||
1. Timing of kill: during convergence, during steady state, during
|
||||
a partition heal.
|
||||
2. Which peer: orchestrator, a stage, the relay.
|
||||
|
||||
**Required assertions**:
|
||||
|
||||
- The bundle's `manifest.json` must exist and contain
|
||||
`finalize_received: false`.
|
||||
- Every peer's pre-kill events and snapshots must be present in
|
||||
the bundle (the kill must not erase prior records).
|
||||
- The `verdicts.json` must contain a verdict for every declared
|
||||
assertion, with `Inconclusive` for any assertion whose
|
||||
preconditions did not fire (e.g., a steady-state assertion when
|
||||
steady state was never reached).
|
||||
|
||||
**Property test**: not required.
|
||||
|
||||
**Expected verdict on current source**: `Pass`. The observability
|
||||
upgrade landed `S-D` (bundle assembly without finalize). This
|
||||
family guards that contract against regression.
|
||||
|
||||
**Family closes when**: every scenario in the family produces a
|
||||
parseable bundle whose `summary.md` renders cleanly through
|
||||
`swactor-diag-postproc`.
|
||||
|
||||
### Family F — Compound faults under recovery
|
||||
|
||||
**Source**: `SIM_HARDENING_SPEC §7` and §9.
|
||||
|
||||
**Shape**: Two or more faults active during a single recovery
|
||||
window — a partition heal during a relay-peer-down, a clock skew
|
||||
during a worker respawn, a kernel UDP overflow during SWIM gossip
|
||||
burst. The 2026-05-25 incident is consistent with at least two
|
||||
overlapping faults (relay-peer-down + silent-worker); the battery
|
||||
must cover the next overlap before it lands in prod.
|
||||
|
||||
**Central case** (`central.toml`): a `Partition` cutting `stage-2`
|
||||
from `stage-0` from t=10 s to t=30 s; a `RelayPeerConnDown { from:
|
||||
orch, to: stage-2, at_ns: 20_000_000_000, duration_ns:
|
||||
20_000_000_000 }` overlapping the partition's last 10 s and
|
||||
extending 10 s past its heal. The scenario tests whether SWIM
|
||||
behaves under the *overlap* and the *heal* sequence the postmortem
|
||||
mentions but did not isolate.
|
||||
|
||||
**Mutation axes**:
|
||||
|
||||
1. Which two faults overlap (cross product of the four single-fault
|
||||
families above, restricted to combinations that produce
|
||||
distinguishable bundles).
|
||||
2. Overlap geometry: full overlap, partial overlap, abutting (one
|
||||
ends as the other begins).
|
||||
3. Recovery phase: which recovery phase the second fault hits, per
|
||||
`SIM_HARDENING_SPEC §9`.
|
||||
|
||||
**Required assertions**: family-dependent — each compound test
|
||||
combines the assertions of its constituent families. The compound
|
||||
test passes only if every constituent assertion holds.
|
||||
|
||||
**Property test**: required, seeds 0..512. Range over all three
|
||||
axes. The property: any seed in which a compound bundle violates
|
||||
*more* assertions than the sum of the constituents' individual
|
||||
violations is a true compound bug, reported separately.
|
||||
|
||||
**Expected verdict on current source**: `Mixed`. Compound failures
|
||||
are the under-tested corner; the implementing agent should expect
|
||||
to find at least one new sim-coverage gap during this family's
|
||||
implementation and file it.
|
||||
|
||||
**Family closes when**: at least one compound bug is either fixed
|
||||
or filed as a sim-coverage gap with a structural reason.
|
||||
|
||||
---
|
||||
|
||||
## 4. Out of scope
|
||||
|
||||
- Tuning the simulator's existing scenarios under
|
||||
`scenarios/calibration/` or `scenarios/smoke/`.
|
||||
- Adding new failure shapes the 2026-05-25 deployment did not
|
||||
surface (the diagnostic deployment running in parallel may; if
|
||||
so, those land as a new spec, not as an amendment to this one).
|
||||
- Changes to the simulator's engine, network, host kinds, bundle
|
||||
writer, or post-processor. The battery exercises them; it does
|
||||
not modify them.
|
||||
- Changes to the production diagnostics code path. The
|
||||
observability upgrade landed; the battery consumes its output.
|
||||
- Documentation of the simulator beyond `SIM_BLIND_SPOTS.md`
|
||||
amendments. `SIM_HARDENING_SPEC.md` and `SIM_SPEC.md` already
|
||||
exist; this document is the only new prose required.
|
||||
|
||||
---
|
||||
|
||||
## 5. References
|
||||
|
||||
- `N3_POSTMORTEM_2026-05-25.md` — source for families A, B, C, D, E.
|
||||
- `N3_DATA_GAPS.md` — source for the gap-named contracts each
|
||||
family asserts the simulator's bundle must satisfy.
|
||||
- `N3_DEPLOYMENT_REPORT.md` — historical context: Layers A/B/C from
|
||||
the prior eight deploys.
|
||||
- `N3_OBSERVABILITY_UPGRADE_SPEC.md` — the contract the bundle
|
||||
*already* satisfies. The battery consumes that contract.
|
||||
- `SIM_HARDENING_SPEC.md` — the family / mutation-axis discipline
|
||||
that §1 and §3 above enforce.
|
||||
- `crates/simulation/SIM_SPEC.md` — the simulator's behavioral
|
||||
surface. §3.1 components, §5.5 mutations, §6A stage host, §10.1
|
||||
assertion catalog, §8 scenario format are the load-bearing
|
||||
references.
|
||||
- `.loop/notes.md`, `.loop/verdict.md` — the observability-upgrade
|
||||
iteration log and verdict, current as of 2026-05-25; STATUS:
|
||||
DONE, VERDICT: PASS.
|
||||
|
|
@ -1,366 +0,0 @@
|
|||
# N=3 SWIM retuning against deployed latency — behavioral spec
|
||||
|
||||
Companion to `N3_POSTMORTEM_2026-05-25_1779733878.md`,
|
||||
`N3_COVERAGE_EXTENSION_SPEC.md`, and the simulator's
|
||||
`SWIM_TUNING_REPORT.md`. This document is the contract for a SWIM
|
||||
retuning pass that uses the `1779733878` deployment's observed
|
||||
latency and churn data as the evidence the tune is calibrated
|
||||
against — rather than the simulated 60 ms latency the prior tune
|
||||
used.
|
||||
|
||||
This is a *behavioral* spec. It names the targets the retuning
|
||||
must hit, the evidence each target is calibrated against, and the
|
||||
prerequisites that must be in place before a retuning pass can be
|
||||
evidence-driven rather than guess-driven. It does not prescribe
|
||||
specific knob values.
|
||||
|
||||
---
|
||||
|
||||
## 0. Motivation
|
||||
|
||||
`SWIM_TUNING_REPORT.md` documented a prior tuning pass against the
|
||||
§10.3 gossip-flap property and the three N3 calibration scenarios.
|
||||
That pass collapsed `self_incarnation_peak` from 86–94 to 7–10 — a
|
||||
significant win — but its calibration latency was **60 ms RTT with
|
||||
15 ms jitter** (per §2 of that report). The simulator's calibration
|
||||
scenarios used these numbers because the live bundles available at
|
||||
the time did not surface per-SWIM-probe RTT.
|
||||
|
||||
The `1779733878` run exposes a different reality:
|
||||
|
||||
- **Tier-2 (host-level) UDP echo RTT**: orchestrator 293 ms,
|
||||
stage-0 181 ms, stage-1 405 ms, stage-2 184 ms. p99 spread is
|
||||
multi-hundred-millisecond and asymmetric across peers.
|
||||
- **Topology**: every peer connection is `conn_type=Relay`. No
|
||||
hole-punching succeeded. SWIM probes ride a relay-mediated path
|
||||
whose RTT is strictly higher than the tier-2 floor and is
|
||||
subject to relay-side HOL queueing.
|
||||
- **Observed churn**: 1701 `SwimTransition` events over a ~7-minute
|
||||
run while iroh continued to exchange messages (`connect-timeout
|
||||
count = 0`). The chosen `probe_timeout = 15 ticks = 3 s` was
|
||||
selected against 60 ms RTT; against a relay-mediated path with
|
||||
p99 multi-hundred-millisecond tier-2 floor and load-driven
|
||||
queueing on top, that budget may be marginal or worse.
|
||||
- **Configuration**: the prior tune's knobs landed at
|
||||
`probe_interval=10, probe_timeout=15, suspicion_timeout=75,
|
||||
indirect_probes=2, dead_reprobe_interval=50` ticks plus
|
||||
`max_piggyback=6`. These are committed defaults; the question
|
||||
this spec opens is whether they hold under the observed
|
||||
deployment shape, not whether the prior tuning method was
|
||||
correct.
|
||||
|
||||
The previous postmortem (run `1779720002`) could not have driven
|
||||
this retune: its bundle lacked the data the observability upgrade
|
||||
landed afterward. The `1779733878` bundle is the first one rich
|
||||
enough to retune against. This spec captures the contract that
|
||||
retuning must satisfy.
|
||||
|
||||
---
|
||||
|
||||
## 1. Prerequisites
|
||||
|
||||
Retuning is not evidence-driven until the data the tune calibrates
|
||||
against is in the bundle. Three prerequisites are explicit.
|
||||
|
||||
### 1.1 Coverage 2.6 from `N3_COVERAGE_EXTENSION_SPEC.md`
|
||||
|
||||
Per-SWIM-probe RTT events and the post-processor's `## Probe RTT
|
||||
distribution` section must land in production *and* in the sim
|
||||
adapter. Until this coverage is in place:
|
||||
|
||||
- The deployed evidence is tier-2 RTT (UDP echo to the collector),
|
||||
which understates the relay-mediated SWIM RTT by an unknown
|
||||
factor.
|
||||
- The simulator's `no_flap_while_probes_ok` assertion is
|
||||
`Inconclusive` on every SWIM scenario (per
|
||||
`SWIM_TUNING_REPORT.md` §6.3), so the assertion can neither
|
||||
pass nor fail the retune.
|
||||
|
||||
A retuning pass that lands without 2.6 is a guess against
|
||||
tier-2 latency — the same mistake the prior tune made against
|
||||
60 ms simulated latency, only with a different proxy for the real
|
||||
number.
|
||||
|
||||
### 1.2 Determinism fix from `SWIM_TUNING_REPORT.md` §6.7
|
||||
|
||||
`MemberList`'s `HashMap<NodeId, _>` randomises iteration order per
|
||||
process; the prior tune reports ±20 % run-to-run variance as a
|
||||
result. A retuning pass that has to average across five samples
|
||||
per grid point to estimate variance is exactly five times slower
|
||||
and five times noisier than one against deterministic substream
|
||||
selection. `HashMap` → `BTreeMap` is the one-line fix the prior
|
||||
report names; it must land before retuning, not after, so the
|
||||
retune's results have signal-to-noise high enough to read.
|
||||
|
||||
### 1.3 The Layer-B1 refute-on-stale-Suspect bug (`SWIM_TUNING_REPORT.md` §6.1)
|
||||
|
||||
`crates/distribution/src/swim/node.rs::apply_membership_update`
|
||||
refutes against `self_id()` whenever
|
||||
`update.state ∈ {Suspect, Dead}` regardless of whether
|
||||
`update.incarnation` is current. This creates a non-zero floor on
|
||||
`self_incarnation_peak` that no tuning can collapse. A retuning
|
||||
pass against the `1779733878` shape, where the relay-mediated path
|
||||
keeps stale Suspect entries in the dissemination queue for many
|
||||
probe cycles, will hit this floor and conclude — incorrectly —
|
||||
that further tuning gain is unavailable.
|
||||
|
||||
The one-condition gate the prior report names is the
|
||||
priority-1 follow-up the prior tune deferred. It is a
|
||||
prerequisite for evidence-driven retuning against this deployment,
|
||||
not a downstream cleanup.
|
||||
|
||||
---
|
||||
|
||||
## 2. Calibration data the retune is driven by
|
||||
|
||||
The retuning pass's evidence comes from the `1779733878` bundle
|
||||
(and any subsequent N=3 deploy bundles that land before the
|
||||
retune). Three numbers anchor the calibration:
|
||||
|
||||
### 2.1 Observed tier-2 RTT distribution
|
||||
|
||||
| node | RTT (ms) | echo success |
|
||||
|--------------|----------|--------------|
|
||||
| orchestrator | 293 | 34/35 |
|
||||
| stage-0 | 181 | 55/55 |
|
||||
| stage-1 | 405 | 27/28 |
|
||||
| stage-2 | 184 | 38/38 |
|
||||
|
||||
The tier-2 echo path is collector-bound, not peer-bound. It
|
||||
establishes the floor below which a relay-mediated SWIM probe
|
||||
cannot land.
|
||||
|
||||
### 2.2 Per-SWIM-probe RTT distribution (post coverage 2.6)
|
||||
|
||||
After coverage 2.6 lands, the bundle will carry per-probe RTT
|
||||
distributions per (observer, target) pair, plus per-bucket
|
||||
distributions over the run window. The retune calibrates
|
||||
`probe_timeout` such that the configured budget exceeds the
|
||||
observed p99 of legitimate (non-failure) probe RTT with a margin
|
||||
the retune explicitly justifies. Until 2.6 is collected against a
|
||||
live run, the retune uses §2.1 as a lower-bound proxy and is
|
||||
explicit about that.
|
||||
|
||||
### 2.3 SWIM churn and dial outcomes
|
||||
|
||||
`SwimTransition: 1701` across a ~7-minute run is the load-bearing
|
||||
churn signal. The retune is calibrated such that a scenario
|
||||
configured to mirror the `1779733878` shape produces a churn
|
||||
count within a stated factor (target: <300, an order-of-magnitude
|
||||
collapse comparable to the prior tune's `self_incarnation`
|
||||
collapse).
|
||||
|
||||
Per-peer dials from the postmortem (orchestrator 7/11 timeout,
|
||||
inter-stage 19/19 success) are the discriminator the retune must
|
||||
not undo: a retuned SWIM that makes inter-stage probes flap is a
|
||||
regression even if it makes orchestrator-bound probes more stable.
|
||||
|
||||
---
|
||||
|
||||
## 3. Retuning targets
|
||||
|
||||
Six, ordered by load-bearing impact.
|
||||
|
||||
### 3.1 `probe_timeout` against relay-mediated p99 RTT
|
||||
|
||||
**Target**: `probe_timeout` exceeds the bundle's observed p99 of
|
||||
legitimate probe RTT (post-2.6) by a margin the retune justifies
|
||||
in prose — the margin must account for relay-side HOL queueing
|
||||
peaks the steady-state distribution does not capture.
|
||||
|
||||
**Anti-target**: the budget cannot be set so high that suspicion
|
||||
takes longer than the operator's deadstop threshold. The
|
||||
postmortem named ~7 min as the operator's deadstop budget; SWIM's
|
||||
detection time (`probe_timeout + suspicion_timeout`) must remain
|
||||
well under that, with a documented headroom.
|
||||
|
||||
**Evidence**: per-probe RTT histogram from §2.2; SwimTransition
|
||||
churn count from §2.3.
|
||||
|
||||
### 3.2 `suspicion_timeout` under relay-mediated reachability
|
||||
|
||||
**Target**: a peer whose relay-mediated path is intermittently
|
||||
unreachable (the `1779733878` shape — repeated probe failures
|
||||
interleaved with successes) does not flap between Alive and
|
||||
Suspect more than the prior tune's bound on the gossip-flap
|
||||
property, when the scenario mirrors the deployment's latency
|
||||
distribution.
|
||||
|
||||
**Anti-target**: a peer whose path is genuinely dead is not
|
||||
falsely held Alive past the operator's deadstop window.
|
||||
|
||||
**Evidence**: §2.3 churn count; the `no_flap_while_probes_ok`
|
||||
assertion (now resolvable post-2.6) against the calibration
|
||||
scenario.
|
||||
|
||||
### 3.3 `indirect_probes` count against relay HOL behavior
|
||||
|
||||
**Target**: indirect probes still provide redundant coverage when
|
||||
the direct probe times out, but their cumulative bandwidth
|
||||
contribution to the relay's egress queue does not push the
|
||||
`relay_queue_depth_bounded` assertion to fail under own-relay
|
||||
policy.
|
||||
|
||||
**Anti-target**: dropping the count below the prior tune's 2
|
||||
collapses indirect coverage, which the prior tune's §5 already
|
||||
documents.
|
||||
|
||||
**Evidence**: `relay_queue_depth_bounded` under own-relay calibration;
|
||||
churn count from §2.3.
|
||||
|
||||
### 3.4 `probe_interval` against the dial-rate signal
|
||||
|
||||
**Target**: probe rate is set such that the orchestrator-bound
|
||||
dial failures the `1779733878` run exhibited (7/11 timeout) do
|
||||
not bottleneck convergence beyond a tolerance the spec names.
|
||||
|
||||
**Anti-target**: probe rate is not lifted so high that
|
||||
`message_size_bounded` regresses against own-relay policy.
|
||||
|
||||
**Evidence**: per-peer dial table from §2.3; piggyback byte
|
||||
totals from the bundle.
|
||||
|
||||
### 3.5 `max_piggyback` against observed gossip-receipt sizes
|
||||
|
||||
**Target**: piggyback gossip stays within the
|
||||
`message_size_bounded` envelope under own-relay policy, given
|
||||
the `1779733878` per-node piggyback byte totals (806–1589
|
||||
piggybacks per node, 194–522 KB total).
|
||||
|
||||
**Anti-target**: lowering `max_piggyback` below the prior tune's
|
||||
6 stops convergence within the property's window
|
||||
(`SWIM_TUNING_REPORT.md` §5).
|
||||
|
||||
**Evidence**: gossip-receipt totals from the postmortem's "Gossip
|
||||
receipts" section; `message_size_bounded` assertion under
|
||||
own-relay.
|
||||
|
||||
### 3.6 `LifeguardConfig` wiring (formerly out of scope)
|
||||
|
||||
**Target**: the dynamic suspicion-timeout formula in
|
||||
`crates/distribution/src/swim/lifeguard.rs` is wired into
|
||||
`SwimNode`'s suspicion state machine. Until wiring lands, the
|
||||
constants in `lifeguard.rs` have no observable effect — per
|
||||
`SWIM_TUNING_REPORT.md` §6.5, the prior tune could not sweep "the
|
||||
lifeguard band" because it was dead code.
|
||||
|
||||
This target is the only one that requires code beyond a knob
|
||||
change. It is included here because the prior tune named it as a
|
||||
priority follow-up and because the `1779733878` data motivates
|
||||
adaptive suspicion: a path whose RTT varies 2× under load benefits
|
||||
from adaptive timeouts more than a static budget can capture.
|
||||
|
||||
**Anti-target**: landing the wiring without sweeping its
|
||||
parameters reproduces the prior tune's dead-code condition for the
|
||||
new fields. The wiring must come with a sweep against the
|
||||
calibration scenarios.
|
||||
|
||||
**Evidence**: the new dynamic-suspicion code path is exercised by
|
||||
at least one scenario whose assertion verdict changes when the
|
||||
multiplier changes.
|
||||
|
||||
---
|
||||
|
||||
## 4. Calibration scenario updates
|
||||
|
||||
The three N3 calibration scenarios under
|
||||
`crates/simulation/scenarios/calibration/` were last updated to
|
||||
mirror the prior tune's defaults at the scenario's 200 ms tick
|
||||
(`SWIM_TUNING_REPORT.md` §3). The retune updates these scenarios
|
||||
along two axes:
|
||||
|
||||
- **Latency distribution**: per-link latency is set against the
|
||||
`1779733878` per-peer tier-2 RTT distribution, not the prior
|
||||
60 ms baseline. Heavy-tailed per `SIM_HARDENING_SPEC §8` (the
|
||||
prior battery spec's reference); the distribution's median, p95,
|
||||
and p99 fall within tolerance of the live bundle's after
|
||||
coverage 2.6 lands.
|
||||
- **Topology**: every host-to-host link is routed through the
|
||||
relay vertex (`via = R` in scenario syntax). The
|
||||
`1779733878` shape had `conn_type=Relay` everywhere; the
|
||||
calibration scenarios must reflect that to be evidence-faithful.
|
||||
|
||||
The scenarios' `kind_config` blocks are updated to the retune's
|
||||
chosen operating point. The current calibration block (per the
|
||||
prior report) gives probes a 333 ms budget against 60 ms RTT;
|
||||
against multi-hundred-millisecond relay-mediated RTT, the same
|
||||
budget under-budgets by an order of magnitude. The retune's new
|
||||
budget is the §3.1 target.
|
||||
|
||||
---
|
||||
|
||||
## 5. Acceptance
|
||||
|
||||
The retune is complete when:
|
||||
|
||||
1. Every prerequisite in §1 is in place (coverage 2.6, the
|
||||
determinism fix, the Layer-B1 gate).
|
||||
2. Each target in §3 has a chosen operating point and a one-line
|
||||
prose justification anchored to the §2 evidence.
|
||||
3. The `1779733878` calibration scenario, configured to mirror
|
||||
the deployment's latency and topology, produces fewer than
|
||||
300 `SwimTransition` events in a 7-minute virtual run (an
|
||||
order-of-magnitude reduction from 1701).
|
||||
4. Inter-stage dial outcomes in the calibration bundle remain at
|
||||
the `1779733878` shape (≥95 % success on inter-stage edges)
|
||||
— the retune does not improve orchestrator-bound stability at
|
||||
the cost of inter-stage flakiness.
|
||||
5. The §10.3 gossip-flap property's `self_incarnation_peak` does
|
||||
not regress from the prior tune's 7–10 band.
|
||||
6. A retuning report (a successor to `SWIM_TUNING_REPORT.md`)
|
||||
documents the new operating point, the evidence each knob
|
||||
choice was calibrated against, the before/after numbers across
|
||||
every calibration scenario, and the limits the retune could
|
||||
not move.
|
||||
|
||||
---
|
||||
|
||||
## 6. Out of scope
|
||||
|
||||
- **Adding new SWIM features.** The retune adjusts existing knobs
|
||||
and lands the Layer-B1 gate / Lifeguard wiring the prior report
|
||||
named. New algorithmic features (push-pull anti-entropy,
|
||||
alternative failure detectors) are not in scope.
|
||||
- **Relay-side fixes.** The `relay_queue_depth_bounded` failure
|
||||
on the canary topology is structurally out-of-reach for SWIM
|
||||
tuning (`SWIM_TUNING_REPORT.md` §6.2). The retune does not
|
||||
attempt to make canary pass; it does not regress own-relay.
|
||||
- **Orchestrator-topology changes.** Running the orchestrator on
|
||||
a reachable host (the `1779733878` postmortem's item 6) is a
|
||||
deployment-shape change, not a SWIM-tuning change. The retune
|
||||
is calibrated against the NAT'd-orchestrator shape because that
|
||||
is the deployment we have, but the conclusion may be "even
|
||||
optimally-tuned SWIM cannot stabilize this topology" — that
|
||||
conclusion is a valid retune outcome.
|
||||
- **The gossip-flap property's `self_incarnation_bounded`
|
||||
assertion.** The prior tune collapsed it from 86–94 to 7–10
|
||||
without removing the non-zero floor; the retune holds that
|
||||
result. Removing the floor is the Layer-B1 fix's job (a §1.3
|
||||
prerequisite, not a §3 target).
|
||||
- **Scenarios beyond the calibration corpus.** The reproduction
|
||||
and topology scenarios remain on their current SWIM config.
|
||||
Retuning them is a follow-up that should wait for the
|
||||
calibration retune to converge.
|
||||
|
||||
---
|
||||
|
||||
## 7. References
|
||||
|
||||
- `N3_POSTMORTEM_2026-05-25_1779733878.md` — source of the
|
||||
observed latency distribution (§"UDP echo probes"), the churn
|
||||
signal (§"SWIM churn and relay events"), the dial outcomes
|
||||
(§"Per-peer dials"), and the topology context
|
||||
(`conn_type=Relay` everywhere, NAT'd orchestrator).
|
||||
- `N3_COVERAGE_EXTENSION_SPEC.md §2.6` — the data surface this
|
||||
spec consumes. §1.1 of this spec is a hard prerequisite.
|
||||
- `crates/simulation/SWIM_TUNING_REPORT.md` — the prior tuning
|
||||
pass. §3 (configuration), §5 (tradeoff curve), §6 (limits) are
|
||||
the load-bearing prior art the retune does not re-derive. §6.1,
|
||||
§6.5, §6.7 limits are §1.3, §3.6, §1.2 prerequisites
|
||||
respectively in this spec.
|
||||
- `crates/simulation/SIM_SPEC.md` — the simulator's calibration
|
||||
contract (§11) and the assertion catalog (§10.1) the retune is
|
||||
scored against.
|
||||
- `N3_SIM_TEST_BATTERY_SPEC.md` — the sim-test battery. A
|
||||
retuned SWIM that regresses any battery family is a retune
|
||||
regression, not a battery regression.
|
||||
|
|
@ -70,14 +70,20 @@ from __future__ import annotations
|
|||
|
||||
import argparse
|
||||
import base64
|
||||
import functools
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import struct
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Sequence
|
||||
|
||||
# Stub-mode constants — small so test payloads stay tiny. Both values
|
||||
|
|
@ -113,6 +119,620 @@ def compute_layer_range(stage: int, num_stages: int, total_blocks: int) -> tuple
|
|||
return start, end
|
||||
|
||||
|
||||
# ggml type tables for the sharded loader: quantized types map to
|
||||
# (elements_per_block, bytes_per_block); native types map to byte width. These
|
||||
# mirror tinygrad 0.12.0's ggml_data_to_tensor and let us size each tensor's raw
|
||||
# byte slice so only the kept weights are copied off disk.
|
||||
_GGML_QUANT_BLOCK = {2: (32, 18), 3: (32, 20), 8: (32, 34), 12: (256, 144), 14: (256, 210), 39: (32, 17)}
|
||||
_GGML_NATIVE_ITEMSIZE = {0: 4, 1: 2, 16: 1, 17: 2, 18: 4}
|
||||
|
||||
|
||||
def _ggml_tensor_nbytes(n_elements: int, ggml_type: int) -> int:
|
||||
"""Raw byte size of an ``n_elements`` ggml tensor of ``ggml_type``."""
|
||||
if ggml_type in _GGML_NATIVE_ITEMSIZE:
|
||||
return _GGML_NATIVE_ITEMSIZE[ggml_type] * n_elements
|
||||
if ggml_type in _GGML_QUANT_BLOCK:
|
||||
elems_per_block, bytes_per_block = _GGML_QUANT_BLOCK[ggml_type]
|
||||
return (n_elements // elems_per_block) * bytes_per_block
|
||||
raise ValueError(f"unsupported ggml type {ggml_type}")
|
||||
|
||||
|
||||
# ─── Sharded download (spec §4.1 / §4.2 / §4.7) ───────────────────────────
|
||||
#
|
||||
# The worker fetches only the byte ranges its stage actually needs
|
||||
# (§4.1), caches the partial GGUF idempotently (§4.2), and emits
|
||||
# `pp_download_progress` events with bounded latency while downloading
|
||||
# (§4.7). The cached file is a SPARSE file with the same apparent size
|
||||
# as the source — kept tensors live at their original byte offsets, so
|
||||
# `_load_sharded_transformer` opens it unchanged. Filesystem holes
|
||||
# absorb the non-kept regions so the actual disk usage is
|
||||
# O(per-stage shard size), not O(full model size).
|
||||
|
||||
_PP_DOWNLOAD_READ_CHUNK = 256 * 1024
|
||||
_PP_HEADER_INITIAL = 1024 * 1024
|
||||
_PP_HEADER_MAX = 64 * 1024 * 1024
|
||||
|
||||
|
||||
def _pp_round_up(n: int, align: int) -> int:
|
||||
if align <= 0:
|
||||
return n
|
||||
return ((n + align - 1) // align) * align
|
||||
|
||||
|
||||
class _PpHeaderTooShort(Exception):
|
||||
"""Raised mid-parse when the header buffer ran out — caller grows it."""
|
||||
|
||||
|
||||
def _pp_parse_gguf_header(buf: bytes) -> "tuple[list[tuple[str, tuple, int, int]], int, dict]":
|
||||
"""Parse a GGUF header from `buf`. Returns (t_infos, data_start, kv).
|
||||
|
||||
`t_infos` is a list of ``(name, dims, ggml_type, offset)`` tuples
|
||||
matching what ``_load_sharded_transformer``'s in-file parser
|
||||
produces. `data_start` is the absolute byte offset where tensor
|
||||
data begins. Raises `_PpHeaderTooShort` if the header is larger
|
||||
than `buf` — caller should re-fetch with a larger buffer.
|
||||
|
||||
Format reference: tinygrad 0.12.0's gguf reader. GGUF versions 2
|
||||
and 3 share the parse shape; the file's u32 version is checked.
|
||||
"""
|
||||
bio = io.BytesIO(buf)
|
||||
|
||||
def _need(n: int) -> bytes:
|
||||
start = bio.tell()
|
||||
out = bio.read(n)
|
||||
if len(out) != n:
|
||||
raise _PpHeaderTooShort(f"need {n} bytes at {start}, got {len(out)}")
|
||||
return out
|
||||
|
||||
def _unpack(fmt: str, nbytes: int):
|
||||
return struct.unpack(fmt, _need(nbytes))[0]
|
||||
|
||||
def _read_u32() -> int:
|
||||
return _unpack("<I", 4)
|
||||
|
||||
def _read_i32() -> int:
|
||||
return _unpack("<i", 4)
|
||||
|
||||
def _read_u64() -> int:
|
||||
return _unpack("<Q", 8)
|
||||
|
||||
def _read_str() -> str:
|
||||
length = _read_u64()
|
||||
return _need(length).decode("utf-8")
|
||||
|
||||
def _read_arr():
|
||||
elem_type = _read_i32()
|
||||
count = _read_u64()
|
||||
return [_readers[elem_type]() for _ in range(count)]
|
||||
|
||||
_readers = {
|
||||
0: lambda: _unpack("<b", 1),
|
||||
1: lambda: _unpack("<B", 1),
|
||||
2: lambda: _unpack("<h", 2),
|
||||
3: lambda: _unpack("<H", 2),
|
||||
4: _read_u32,
|
||||
5: _read_i32,
|
||||
6: lambda: _unpack("<f", 4),
|
||||
7: lambda: _unpack("<?", 1),
|
||||
8: _read_str,
|
||||
9: _read_arr,
|
||||
10: _read_u64,
|
||||
11: lambda: _unpack("<q", 8),
|
||||
12: lambda: _unpack("<d", 8),
|
||||
}
|
||||
|
||||
magic = _need(4)
|
||||
if magic != b"GGUF":
|
||||
raise ValueError(f"not a GGUF artifact (magic={magic!r})")
|
||||
version = _read_i32()
|
||||
if version not in (2, 3):
|
||||
raise ValueError(f"unsupported GGUF version {version}")
|
||||
n_tensors = _read_u64()
|
||||
n_kv = _read_u64()
|
||||
|
||||
kv: "dict[str, object]" = {}
|
||||
for _ in range(n_kv):
|
||||
key = _read_str()
|
||||
typ = _read_i32()
|
||||
kv[key] = _readers[typ]()
|
||||
|
||||
t_infos: "list[tuple[str, tuple, int, int]]" = []
|
||||
for _ in range(n_tensors):
|
||||
name = _read_str()
|
||||
n_dims = _read_u32()
|
||||
dims = tuple(_read_u64() for _ in range(n_dims))
|
||||
ggml_type = _read_i32()
|
||||
offset = _read_u64()
|
||||
t_infos.append((name, dims, ggml_type, offset))
|
||||
|
||||
alignment = int(kv.get("general.alignment", 32))
|
||||
data_start = _pp_round_up(bio.tell(), alignment)
|
||||
return t_infos, data_start, kv
|
||||
|
||||
|
||||
def _pp_kept_names(t_infos, stage: int, num_stages: int, kv: dict) -> "set[str]":
|
||||
"""Return the set of tensor names this stage requires. Matches the
|
||||
`_kept` predicate inside `_load_sharded_transformer` — both code
|
||||
paths must agree on the kept set or the loader would try to realize
|
||||
a tensor whose bytes were not fetched."""
|
||||
arch = str(kv["general.architecture"])
|
||||
total_blocks = int(kv[f"{arch}.block_count"])
|
||||
start, end = compute_layer_range(stage, num_stages, total_blocks)
|
||||
is_last = stage == num_stages - 1
|
||||
names = {info[0] for info in t_infos}
|
||||
tied_output = "output.weight" not in names
|
||||
kept: "set[str]" = set()
|
||||
for info in t_infos:
|
||||
name = info[0]
|
||||
keep = False
|
||||
for i in range(start, end):
|
||||
if name.startswith(f"blk.{i}."):
|
||||
keep = True
|
||||
break
|
||||
if not keep:
|
||||
if name == "token_embd.weight" and (stage == 0 or (is_last and tied_output)):
|
||||
keep = True
|
||||
elif name == "output_norm.weight" and is_last:
|
||||
keep = True
|
||||
elif name == "output.weight" and is_last and not tied_output:
|
||||
keep = True
|
||||
if keep:
|
||||
kept.add(name)
|
||||
return kept
|
||||
|
||||
|
||||
def _pp_kept_byte_ranges(t_infos, data_start: int, kept_names: "set[str]") -> "list[tuple[int, int]]":
|
||||
"""Return the ``[(absolute_offset, nbytes)]`` byte ranges this stage
|
||||
keeps, ordered by offset (so a streamed download writes ascending
|
||||
offsets and the filesystem allocates fewer fragmented holes)."""
|
||||
out: list[tuple[int, int]] = []
|
||||
for name, dims, ggml_type, offset in t_infos:
|
||||
if name not in kept_names:
|
||||
continue
|
||||
n_elements = 1
|
||||
for d in dims:
|
||||
n_elements *= int(d)
|
||||
nbytes = _ggml_tensor_nbytes(n_elements, ggml_type)
|
||||
out.append((data_start + int(offset), nbytes))
|
||||
out.sort()
|
||||
return out
|
||||
|
||||
|
||||
def _pp_cache_paths(url: str) -> "tuple[Path, Path, Path]":
|
||||
"""Return (cache_path, meta_path, partial_path) for ``url``.
|
||||
|
||||
Cache root defaults to ``$PP_MODEL_CACHE_DIR`` then ``~/.cache/pp-pipeline``.
|
||||
The filename is ``<short-url-hash>-<basename>`` so two URLs that share a
|
||||
basename cannot collide.
|
||||
"""
|
||||
raw = os.environ.get("PP_MODEL_CACHE_DIR", "").strip()
|
||||
cache_root = Path(raw).expanduser() if raw else Path.home() / ".cache" / "pp-pipeline"
|
||||
cache_root.mkdir(parents=True, exist_ok=True)
|
||||
url_hash = hashlib.sha256(url.encode("utf-8")).hexdigest()[:16]
|
||||
basename = os.path.basename(urllib.parse.urlparse(url).path) or "model.gguf"
|
||||
cache_path = cache_root / f"{url_hash}-{basename}"
|
||||
meta_path = cache_path.with_name(cache_path.name + ".pp_meta")
|
||||
partial_path = cache_path.with_name(cache_path.name + ".partial")
|
||||
return cache_path, meta_path, partial_path
|
||||
|
||||
|
||||
def _pp_head(url: str) -> "tuple[int, str]":
|
||||
"""HEAD request; return (Content-Length, Accept-Ranges header lowercased)."""
|
||||
req = urllib.request.Request(url, method="HEAD")
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
total = int(resp.headers.get("Content-Length", "0"))
|
||||
accept = (resp.headers.get("Accept-Ranges") or "").lower()
|
||||
return total, accept
|
||||
|
||||
|
||||
def _pp_range_get(url: str, start: int, end_inclusive: int) -> bytes:
|
||||
"""Issue a Range GET; return the body bytes."""
|
||||
req = urllib.request.Request(
|
||||
url, headers={"Range": f"bytes={start}-{end_inclusive}"}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=120) as resp:
|
||||
return resp.read()
|
||||
|
||||
|
||||
def _pp_emit_progress(stage: int, bytes_done: int, bytes_total: int, started_at_mono: float) -> None:
|
||||
"""Emit one `pp_download_progress` event with the spec's field set
|
||||
(§4.7). `started_at_mono` is `time.monotonic()` captured before the
|
||||
first event so `elapsed_ms` is monotonic across the fetch."""
|
||||
elapsed_ms = int((time.monotonic() - started_at_mono) * 1000)
|
||||
mbps = round(((bytes_done * 8) / 1_000_000) / max(elapsed_ms / 1000, 1e-3), 1)
|
||||
_emit_event(
|
||||
"pp_download_progress",
|
||||
stage_index=stage,
|
||||
bytes_done=bytes_done,
|
||||
bytes_total=bytes_total,
|
||||
elapsed_ms=elapsed_ms,
|
||||
mbps=mbps,
|
||||
)
|
||||
|
||||
|
||||
def _pp_fingerprint(f, offset: int, nbytes: int) -> str:
|
||||
"""Hash the first 4KB + last 4KB of a kept tensor (or the whole
|
||||
tensor if shorter). The §4.2 content-derived check: matches the
|
||||
fingerprint recorded in the sidecar at download time."""
|
||||
sample = 4096
|
||||
f.seek(offset)
|
||||
head = f.read(min(sample, nbytes))
|
||||
if nbytes > sample:
|
||||
f.seek(offset + nbytes - sample)
|
||||
tail = f.read(sample)
|
||||
else:
|
||||
tail = b""
|
||||
h = hashlib.sha256()
|
||||
h.update(head)
|
||||
h.update(tail)
|
||||
h.update(nbytes.to_bytes(8, "little"))
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def _pp_write_meta(
|
||||
meta_path: Path,
|
||||
cache_path: Path,
|
||||
stage: int,
|
||||
num_stages: int,
|
||||
url: str,
|
||||
total_size: int,
|
||||
kept_ranges: "list[tuple[int, int]]",
|
||||
) -> None:
|
||||
fingerprints = []
|
||||
with open(cache_path, "rb") as f:
|
||||
for offset, nbytes in kept_ranges:
|
||||
fingerprints.append({
|
||||
"offset": offset,
|
||||
"nbytes": nbytes,
|
||||
"fingerprint": _pp_fingerprint(f, offset, nbytes),
|
||||
})
|
||||
meta_path.write_text(json.dumps({
|
||||
"schema": 1,
|
||||
"url": url,
|
||||
"stage": stage,
|
||||
"num_stages": num_stages,
|
||||
"total_size": total_size,
|
||||
"kept": fingerprints,
|
||||
}))
|
||||
|
||||
|
||||
def _pp_verify_cache(
|
||||
cache_path: Path,
|
||||
meta_path: Path,
|
||||
stage: int,
|
||||
num_stages: int,
|
||||
) -> bool:
|
||||
"""Spec §4.2 integrity check: file present at the expected apparent
|
||||
size AND every recorded fingerprint re-matches the cached bytes.
|
||||
Returns False on any discrepancy (including missing files, missing
|
||||
sidecar, mismatched stage / num_stages, size mismatch, or any
|
||||
fingerprint mismatch). A passing cache is used as-is — no refetch."""
|
||||
if not cache_path.exists() or not meta_path.exists():
|
||||
return False
|
||||
try:
|
||||
meta = json.loads(meta_path.read_text())
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
if meta.get("schema") != 1:
|
||||
return False
|
||||
if meta.get("stage") != stage or meta.get("num_stages") != num_stages:
|
||||
return False
|
||||
if cache_path.stat().st_size != meta.get("total_size"):
|
||||
return False
|
||||
kept = meta.get("kept") or []
|
||||
if not kept:
|
||||
return False
|
||||
try:
|
||||
with open(cache_path, "rb") as f:
|
||||
for entry in kept:
|
||||
offset = int(entry["offset"])
|
||||
nbytes = int(entry["nbytes"])
|
||||
expected = entry["fingerprint"]
|
||||
if _pp_fingerprint(f, offset, nbytes) != expected:
|
||||
return False
|
||||
except (OSError, KeyError, ValueError):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _pp_download_sharded(url: str, stage: int, num_stages: int) -> str:
|
||||
"""Spec §4.1 + §4.2 + §4.7: fetch only this stage's tensor bytes,
|
||||
cache idempotently, emit progress events.
|
||||
|
||||
Returns the path to the on-disk file (sparse — apparent size matches
|
||||
the source; only the kept ranges occupy disk blocks). On cache hit,
|
||||
NO `pp_download_progress` events are emitted (spec §4.7).
|
||||
"""
|
||||
cache_path, meta_path, partial_path = _pp_cache_paths(url)
|
||||
|
||||
# §4.2: orphan-cleanup any leftover .partial from a previous killed
|
||||
# fetch BEFORE any new fetch is initiated. Emit an event so the
|
||||
# bundle reader can see that a stale temp was reaped.
|
||||
if partial_path.exists():
|
||||
try:
|
||||
partial_path.unlink()
|
||||
_emit_event(
|
||||
"pp_cache_orphan_cleaned",
|
||||
stage_index=stage,
|
||||
path=str(partial_path),
|
||||
)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# §4.2 cache hit — return the cached file as-is.
|
||||
if _pp_verify_cache(cache_path, meta_path, stage, num_stages):
|
||||
_emit_event(
|
||||
"pp_cache_hit",
|
||||
stage_index=stage,
|
||||
path=str(cache_path),
|
||||
)
|
||||
return str(cache_path)
|
||||
|
||||
# §4.1 fail-fast on no-range support.
|
||||
total_size, accept_ranges = _pp_head(url)
|
||||
if "bytes" not in accept_ranges:
|
||||
_emit_event(
|
||||
"pp_download_failed",
|
||||
stage_index=stage,
|
||||
reason="no_byte_range_support",
|
||||
url=url,
|
||||
accept_ranges=accept_ranges,
|
||||
)
|
||||
_die(f"source does not support byte-range requests: {url}")
|
||||
if total_size <= 0:
|
||||
_emit_event(
|
||||
"pp_download_failed",
|
||||
stage_index=stage,
|
||||
reason="no_content_length",
|
||||
url=url,
|
||||
)
|
||||
_die(f"source did not advertise Content-Length: {url}")
|
||||
|
||||
# Fetch the header in growing increments until we can parse it.
|
||||
header_size = min(_PP_HEADER_INITIAL, total_size)
|
||||
while True:
|
||||
try:
|
||||
header_bytes = _pp_range_get(url, 0, header_size - 1)
|
||||
t_infos, data_start, kv = _pp_parse_gguf_header(header_bytes)
|
||||
if data_start <= len(header_bytes):
|
||||
break
|
||||
# Parser succeeded structurally but data_start sits past our
|
||||
# buffer — re-fetch enough to include the tensor data start.
|
||||
header_size = min(data_start + 1024, total_size)
|
||||
except _PpHeaderTooShort:
|
||||
new_size = min(header_size * 2, total_size)
|
||||
if new_size == header_size or new_size > _PP_HEADER_MAX:
|
||||
_emit_event(
|
||||
"pp_download_failed",
|
||||
stage_index=stage,
|
||||
reason="header_too_large",
|
||||
header_size=header_size,
|
||||
)
|
||||
_die(f"GGUF header exceeded {_PP_HEADER_MAX} bytes")
|
||||
header_size = new_size
|
||||
|
||||
kept_names = _pp_kept_names(t_infos, stage, num_stages, kv)
|
||||
kept_ranges = _pp_kept_byte_ranges(t_infos, data_start, kept_names)
|
||||
|
||||
# bytes_total is the bytes this stage will pull from the network:
|
||||
# header + kept-tensor regions. Not the full file (spec §4.1 means
|
||||
# we never fetch the rest).
|
||||
header_keep_bytes = data_start
|
||||
kept_bytes_total = sum(nb for _, nb in kept_ranges)
|
||||
bytes_total = header_keep_bytes + kept_bytes_total
|
||||
|
||||
interval_s_raw = os.environ.get("PP_DOWNLOAD_PROGRESS_INTERVAL_SECS", "").strip()
|
||||
try:
|
||||
interval_s = float(interval_s_raw) if interval_s_raw else 10.0
|
||||
except ValueError:
|
||||
interval_s = 10.0
|
||||
if interval_s <= 0:
|
||||
interval_s = 10.0
|
||||
|
||||
# Write the sparse output to `.partial`; rename on success. Opening
|
||||
# with "wb" then truncate(total_size) creates a sparse file on
|
||||
# Linux: only blocks we actually `write()` allocate disk.
|
||||
started_at = time.monotonic()
|
||||
with open(partial_path, "wb") as f:
|
||||
f.truncate(total_size)
|
||||
f.seek(0)
|
||||
f.write(header_bytes[:data_start])
|
||||
bytes_done = data_start
|
||||
|
||||
# Spec §4.7: first event MUST be at start of fetch, AFTER we
|
||||
# know bytes_total. We have that now.
|
||||
_pp_emit_progress(stage, bytes_done, bytes_total, started_at)
|
||||
last_emit = time.monotonic()
|
||||
|
||||
for offset, nbytes in kept_ranges:
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={"Range": f"bytes={offset}-{offset + nbytes - 1}"},
|
||||
)
|
||||
try:
|
||||
resp = urllib.request.urlopen(req, timeout=300)
|
||||
except Exception as e:
|
||||
_emit_event(
|
||||
"pp_download_failed",
|
||||
stage_index=stage,
|
||||
reason="range_get_failed",
|
||||
offset=offset,
|
||||
nbytes=nbytes,
|
||||
error=str(e),
|
||||
)
|
||||
# Spec §4.7: final event MUST be emitted on fetch failure.
|
||||
_pp_emit_progress(stage, bytes_done, bytes_total, started_at)
|
||||
_die(f"range GET failed at offset {offset}: {e}")
|
||||
try:
|
||||
f.seek(offset)
|
||||
while True:
|
||||
chunk = resp.read(_PP_DOWNLOAD_READ_CHUNK)
|
||||
if not chunk:
|
||||
break
|
||||
f.write(chunk)
|
||||
bytes_done += len(chunk)
|
||||
now = time.monotonic()
|
||||
if now - last_emit >= interval_s:
|
||||
_pp_emit_progress(stage, bytes_done, bytes_total, started_at)
|
||||
last_emit = now
|
||||
finally:
|
||||
resp.close()
|
||||
|
||||
# Spec §4.7: final event at completion.
|
||||
_pp_emit_progress(stage, bytes_done, bytes_total, started_at)
|
||||
|
||||
# Sidecar before rename so a crash between rename + meta-write does
|
||||
# not leave a "valid file, no sidecar" → would fail _pp_verify and
|
||||
# refetch. Writing the sidecar first means a crash here leaves
|
||||
# cache_path absent and partial_path present (which orphan-cleanup
|
||||
# reaps on next boot).
|
||||
_pp_write_meta(
|
||||
meta_path, partial_path, stage, num_stages, url, total_size, kept_ranges
|
||||
)
|
||||
os.replace(partial_path, cache_path)
|
||||
return str(cache_path)
|
||||
|
||||
|
||||
def _load_sharded_transformer(gguf_path, stage: int, num_stages: int, max_context: int = 512):
|
||||
"""Load only this stage's slice of the model onto the compute device.
|
||||
|
||||
Stock ``Transformer.from_gguf`` copies the *entire* GGUF onto the compute
|
||||
device before any layer runs (it does ``gguf.to(None)``), so an 18 GB model
|
||||
OOMs a 12 GB GPU no matter how the layers are split. Instead we parse the
|
||||
GGUF header on the DISK device and copy only the tensors this stage needs —
|
||||
``blk[start:end]`` plus ``token_embd`` (stage 0) and ``output_norm`` /
|
||||
``output`` (last stage) — dequantizing each on the compute device. Returns
|
||||
``(model, kv, start, end)``. Vendored against tinygrad 0.12.0's gguf format.
|
||||
"""
|
||||
import io
|
||||
import struct
|
||||
import functools
|
||||
|
||||
from tinygrad import Tensor, Device, nn
|
||||
from tinygrad.helpers import prod, round_up, getenv
|
||||
from tinygrad.nn.state import TensorIO, ggml_data_to_tensor
|
||||
from tinygrad.apps.llm import Transformer
|
||||
|
||||
_t0 = time.monotonic()
|
||||
gguf = Tensor(gguf_path) # device is DISK:<path> — nothing is copied to the GPU yet
|
||||
|
||||
# --- parse the GGUF header (kv metadata + tensor directory) off disk ---
|
||||
reader = io.BufferedReader(TensorIO(gguf), 1_000_000)
|
||||
|
||||
def _unpack(fmt, nbytes):
|
||||
return struct.unpack(fmt, reader.read(nbytes))[0]
|
||||
|
||||
def _read_str():
|
||||
return str(reader.read(_read_u64()), "utf-8")
|
||||
|
||||
def _read_arr():
|
||||
elem_reader, count = _readers[_read_i32()], _read_u64()
|
||||
return [elem_reader() for _ in range(count)]
|
||||
|
||||
_readers = {8: _read_str, 9: _read_arr, **{t: functools.partial(_unpack, "<" + f, nb) for t, f, nb in
|
||||
[(0, "c", 1), (1, "b", 1), (2, "H", 2), (3, "h", 2), (4, "I", 4), (5, "i", 4),
|
||||
(6, "f", 4), (7, "?", 1), (10, "Q", 8), (11, "q", 8), (12, "d", 8)]}}
|
||||
_read_u32, _read_i32, _read_u64 = _readers[4], _readers[5], _readers[10]
|
||||
|
||||
magic, version = reader.read(4), _read_i32()
|
||||
n_tensors, n_kv = _read_u64(), _read_u64()
|
||||
if magic != b"GGUF" or version not in (2, 3):
|
||||
raise ValueError(f"invalid GGUF (magic={magic!r} version={version})")
|
||||
kv = {}
|
||||
for _ in range(n_kv):
|
||||
key, typ = _read_str(), _read_i32()
|
||||
kv[key] = _readers[typ]()
|
||||
t_infos = [(_read_str(), tuple(_read_u64() for _ in range(_read_u32())), _read_i32(), _read_u64())
|
||||
for _ in range(n_tensors)]
|
||||
data_start = round_up(reader.tell(), kv.get("general.alignment", 32))
|
||||
_t_header = time.monotonic()
|
||||
|
||||
arch = kv["general.architecture"]
|
||||
total_blocks = int(kv[f"{arch}.block_count"])
|
||||
start, end = compute_layer_range(stage, num_stages, total_blocks)
|
||||
is_last = stage == num_stages - 1
|
||||
names = {info[0] for info in t_infos}
|
||||
tied_output = "output.weight" not in names # small models tie output to token_embd
|
||||
|
||||
def _kept(name: str) -> bool:
|
||||
for i in range(start, end):
|
||||
if name.startswith(f"blk.{i}."):
|
||||
return True
|
||||
if name == "token_embd.weight" and (stage == 0 or (is_last and tied_output)):
|
||||
return True
|
||||
if name == "output_norm.weight" and is_last:
|
||||
return True
|
||||
if name == "output.weight" and is_last and not tied_output:
|
||||
return True
|
||||
return False
|
||||
|
||||
half, device = getenv("HALF", 1), Device.DEFAULT
|
||||
state_dict = {}
|
||||
bytes_copied = 0
|
||||
kept_count = 0
|
||||
for name, dims, ggml_type, offset in t_infos:
|
||||
n_elements = prod(dims)
|
||||
if _kept(name):
|
||||
nbytes = _ggml_tensor_nbytes(n_elements, ggml_type)
|
||||
bytes_copied += nbytes
|
||||
kept_count += 1
|
||||
raw = gguf[data_start + offset: data_start + offset + nbytes].to(device)
|
||||
tensor = ggml_data_to_tensor(raw, n_elements, ggml_type).reshape(*reversed(dims))
|
||||
if arch == "llama": # interleaved -> half-split RoPE layout (llama-style only)
|
||||
n_heads, n_kv_heads = kv[f"{arch}.attention.head_count"], kv[f"{arch}.attention.head_count_kv"]
|
||||
if "attn_q.weight" in name:
|
||||
tensor = tensor.rearrange("(n h two) d -> (n two h) d", n=n_heads, two=2)
|
||||
if "attn_k.weight" in name:
|
||||
tensor = tensor.rearrange("(n h two) d -> (n two h) d", n=n_kv_heads, two=2)
|
||||
state_dict[name] = tensor.cast("float16") if half else tensor
|
||||
else:
|
||||
# DISK-rooted lazy tensor: only its .shape is read (model construction); never realized.
|
||||
state_dict[name] = ggml_data_to_tensor(gguf[data_start + offset:], n_elements, ggml_type).reshape(*reversed(dims))
|
||||
if tied_output and is_last:
|
||||
state_dict["output.weight"] = state_dict["token_embd.weight"]
|
||||
_t_statedict = time.monotonic()
|
||||
|
||||
n_heads = kv[f"{arch}.attention.head_count"]
|
||||
model = Transformer(
|
||||
num_blocks=total_blocks, dim=kv[f"{arch}.embedding_length"],
|
||||
hidden_dim=kv.get(f"{arch}.expert_feed_forward_length", kv[f"{arch}.feed_forward_length"]),
|
||||
n_heads=n_heads, n_kv_heads=kv[f"{arch}.attention.head_count_kv"],
|
||||
norm_eps=kv[f"{arch}.attention.layer_norm_rms_epsilon"], vocab_size=len(kv["tokenizer.ggml.tokens"]),
|
||||
head_dim=kv.get(f"{arch}.attention.key_length", kv[f"{arch}.embedding_length"] // n_heads),
|
||||
rope_theta=kv[f"{arch}.rope.freq_base"], max_context=max_context,
|
||||
qk_norm=int(state_dict["blk.0.attn_q_norm.weight"].shape[0]) if "blk.0.attn_q_norm.weight" in state_dict else 0,
|
||||
num_experts=kv.get(f"{arch}.expert_count", 0), num_experts_per_tok=kv.get(f"{arch}.expert_used_count", 0))
|
||||
|
||||
_t_construct = time.monotonic()
|
||||
|
||||
# Provide only the kept weights; strict=False leaves the other blocks at their
|
||||
# (lazy, never-run) init so they never touch the compute device.
|
||||
kept = {name: tensor for name, tensor in state_dict.items()
|
||||
if _kept(name) or (tied_output and is_last and name == "output.weight")}
|
||||
# This is where the kept tensors are actually copied off disk and
|
||||
# dequantized on the compute device — the dominant load cost.
|
||||
nn.state.load_state_dict(model, kept, strict=False, verbose=False, consume=True, realize=True)
|
||||
_t_realize = time.monotonic()
|
||||
|
||||
realize_ms = (_t_realize - _t_construct) * 1000
|
||||
_emit_event(
|
||||
"model_load_breakdown",
|
||||
stage=stage,
|
||||
resident_blocks=end - start,
|
||||
total_blocks=total_blocks,
|
||||
kept_tensors=kept_count,
|
||||
bytes_copied=bytes_copied,
|
||||
mb_copied=round(bytes_copied / 1_000_000, 1),
|
||||
header_ms=round((_t_header - _t0) * 1000, 1),
|
||||
statedict_build_ms=round((_t_statedict - _t_header) * 1000, 1),
|
||||
construct_ms=round((_t_construct - _t_statedict) * 1000, 1),
|
||||
realize_ms=round(realize_ms, 1),
|
||||
realize_mb_per_s=round((bytes_copied / 1_000_000) / max(realize_ms / 1000, 1e-3), 1),
|
||||
rss_mb=_rss_mb(),
|
||||
)
|
||||
return model, kv, start, end
|
||||
|
||||
|
||||
def argmax_sample(logits: Sequence[float]) -> int:
|
||||
"""Return the index of the maximum element in ``logits``.
|
||||
|
||||
|
|
@ -229,6 +849,14 @@ def _uptime_ms() -> int:
|
|||
return int((time.monotonic() - _WORKER_START_MONOTONIC) * 1000)
|
||||
|
||||
|
||||
def _wall_ms() -> int:
|
||||
"""Epoch milliseconds. Lets the bundle align worker events across nodes
|
||||
and against the orchestrator's vast.ai create/lease timestamps — e.g.
|
||||
(worker `starting`.wall_ms − instance create_ms) is the image-pull +
|
||||
container-boot + worker-spawn cost the node can't see itself."""
|
||||
return int(time.time() * 1000)
|
||||
|
||||
|
||||
def _rss_mb() -> "int | None":
|
||||
"""Resident-set size in MB, read from /proc/self/status (Linux).
|
||||
Returns None on non-Linux or when the read fails — the field is
|
||||
|
|
@ -335,17 +963,27 @@ class _RealModelState:
|
|||
# the most likely crash site in real mode — emit lifecycle
|
||||
# events around the import so the bundle records exactly when
|
||||
# the worker started loading and how long it took.
|
||||
_emit_event("importing_tinygrad", stage=stage)
|
||||
_emit_event("importing_tinygrad", stage=stage, wall_ms=_wall_ms())
|
||||
_import_start = time.monotonic()
|
||||
import numpy as np
|
||||
from tinygrad import Tensor
|
||||
from tinygrad.helpers import fetch
|
||||
from tinygrad.apps.llm import Transformer, SimpleTokenizer, models
|
||||
from tinygrad import Tensor, Device
|
||||
from tinygrad.helpers import fetch, getenv
|
||||
from tinygrad.apps.llm import SimpleTokenizer, models
|
||||
|
||||
import_ms = int((time.monotonic() - _import_start) * 1000)
|
||||
_emit_event("tinygrad_imported", stage=stage, elapsed_ms=import_ms)
|
||||
# Device + precision context: which backend the shard lands on and the
|
||||
# toggles (HALF/JIT/BEAM) that dominate load + inference cost. Correlate
|
||||
# with model_load_breakdown / op events to attribute time to dequant vs
|
||||
# kernel compile vs steady-state matmul.
|
||||
_emit_event(
|
||||
"tinygrad_imported",
|
||||
"device",
|
||||
stage=stage,
|
||||
elapsed_ms=int((time.monotonic() - _import_start) * 1000),
|
||||
default_device=str(Device.DEFAULT),
|
||||
half=getenv("HALF", 1),
|
||||
jit=getenv("JIT", 1),
|
||||
beam=getenv("BEAM", 0),
|
||||
cuda_visible=os.environ.get("CUDA_VISIBLE_DEVICES"),
|
||||
)
|
||||
|
||||
if model_name not in models:
|
||||
|
|
@ -353,42 +991,90 @@ class _RealModelState:
|
|||
_die(f"unknown MODEL {model_name!r}; available: {available}")
|
||||
|
||||
url = models[model_name]
|
||||
_emit_event("fetching_model", stage=stage, model=model_name, url=url)
|
||||
_emit_event("fetching_model", stage=stage, model=model_name, url=url, wall_ms=_wall_ms())
|
||||
print(
|
||||
f"pp_tinygrad_worker: stage={stage}/{num_stages} fetching {model_name}",
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
_fetch_start = time.monotonic()
|
||||
gguf_path = fetch(url)
|
||||
# Spec §4.1: download only this stage's tensor byte ranges.
|
||||
# Spec §4.2: cache idempotently with an integrity check; a
|
||||
# complete-and-valid cache MUST NOT trigger a network fetch.
|
||||
# Spec §4.7: `pp_download_progress` events are emitted by
|
||||
# `_pp_download_sharded` while the fetch is in progress and
|
||||
# NEVER on a cache hit. The unused `fetch` import remains as
|
||||
# documentation of the prior code path; the sharded fetcher
|
||||
# replaces it.
|
||||
_ = fetch # silence the linter; kept for the diff reader
|
||||
gguf_path = _pp_download_sharded(url, stage, num_stages)
|
||||
fetch_ms = int((time.monotonic() - _fetch_start) * 1000)
|
||||
# `gguf_bytes` is the file's apparent size (matches the source's
|
||||
# total_size); actual on-disk usage is O(per-stage shard). Bundle
|
||||
# readers reading `model_fetched.gguf_bytes` see the same value
|
||||
# they did before the §4.1 change — the per-stage usage shows up
|
||||
# in `pp_download_progress.bytes_total` (header + kept ranges).
|
||||
try:
|
||||
gguf_bytes = os.path.getsize(gguf_path)
|
||||
except OSError:
|
||||
gguf_bytes = 0
|
||||
fetch_mb = gguf_bytes / 1_000_000
|
||||
# A near-instant return with a non-zero apparent size means the
|
||||
# sharded cache hit short-circuited the fetch. Distinguishable
|
||||
# in the bundle from a real download via the presence (or not)
|
||||
# of `pp_download_progress` events.
|
||||
cache_hit = gguf_bytes > 0 and fetch_ms < 2000
|
||||
download_mb_per_s = None if cache_hit else round(fetch_mb / max(fetch_ms / 1000, 1e-3), 1)
|
||||
_emit_event(
|
||||
"model_fetched",
|
||||
stage=stage,
|
||||
elapsed_ms=int((time.monotonic() - _fetch_start) * 1000),
|
||||
elapsed_ms=fetch_ms,
|
||||
gguf_path=str(gguf_path),
|
||||
gguf_bytes=gguf_bytes,
|
||||
gguf_mb=round(fetch_mb, 1),
|
||||
download_mb_per_s=download_mb_per_s,
|
||||
cache_hit=cache_hit,
|
||||
)
|
||||
print(
|
||||
f"pp_tinygrad_worker: stage={stage} loading model from {gguf_path}",
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
_emit_event("loading_model", stage=stage, model=model_name)
|
||||
_emit_event("loading_model", stage=stage, model=model_name, wall_ms=_wall_ms())
|
||||
_load_start = time.monotonic()
|
||||
model, kv = Transformer.from_gguf(Tensor(gguf_path), max_context=512)
|
||||
tokenizer = SimpleTokenizer.from_gguf_kv(kv)
|
||||
_emit_event(
|
||||
"model_loaded",
|
||||
stage=stage,
|
||||
elapsed_ms=int((time.monotonic() - _load_start) * 1000),
|
||||
rss_mb=_rss_mb(),
|
||||
# Shard at load time: only this stage's block range (+ embed/output on the
|
||||
# end stages) is copied to the compute device, so an 18 GB model fits on a
|
||||
# 12 GB GPU. See _load_sharded_transformer for why stock from_gguf can't.
|
||||
# The loader emits its own `model_load_breakdown` (header/dequant/realize).
|
||||
model, kv, start, end = _load_sharded_transformer(
|
||||
gguf_path, stage, num_stages, max_context=512
|
||||
)
|
||||
load_ms = int((time.monotonic() - _load_start) * 1000)
|
||||
tokenizer = SimpleTokenizer.from_gguf_kv(kv)
|
||||
|
||||
arch = kv["general.architecture"]
|
||||
hidden_dim = int(kv[f"{arch}.embedding_length"])
|
||||
total_blocks = int(kv[f"{arch}.block_count"])
|
||||
vocab_size = len(kv["tokenizer.ggml.tokens"])
|
||||
|
||||
start, end = compute_layer_range(stage, num_stages, total_blocks)
|
||||
_emit_event(
|
||||
"model_loaded",
|
||||
stage=stage,
|
||||
elapsed_ms=load_ms,
|
||||
rss_mb=_rss_mb(),
|
||||
blocks_resident=end - start,
|
||||
total_blocks=total_blocks,
|
||||
)
|
||||
# Stash the cold-start breakdown so main() can emit one `boot_profile`
|
||||
# summary once the worker is ready (import + fetch + load + total).
|
||||
self.timing = {
|
||||
"import_ms": import_ms,
|
||||
"fetch_ms": fetch_ms,
|
||||
"fetch_cache_hit": cache_hit,
|
||||
"gguf_bytes": gguf_bytes,
|
||||
"download_mb_per_s": download_mb_per_s,
|
||||
"load_ms": load_ms,
|
||||
}
|
||||
|
||||
# Pin EOS token ids (best-effort) for callers that want to detect
|
||||
# end-of-text from the sampled stream. We don't enforce stop here
|
||||
|
|
@ -414,6 +1100,9 @@ class _RealModelState:
|
|||
self.start = start
|
||||
self.end = end
|
||||
self.eos_ids = eos_ids
|
||||
# Per-op compute breakdown (deserialize / compute / host-copy ms), set by
|
||||
# the forward ops and folded into the serve loop's `op` timing event.
|
||||
self._last_compute: "dict | None" = None
|
||||
|
||||
print(
|
||||
f"pp_tinygrad_worker: stage={stage} ready "
|
||||
|
|
@ -427,16 +1116,29 @@ class _RealModelState:
|
|||
|
||||
def embed_and_forward(self, tokens: Sequence[int], position: int) -> tuple[bytes, int]:
|
||||
Tensor = self._Tensor
|
||||
t0 = time.monotonic()
|
||||
t = Tensor([list(tokens)], dtype="int32")
|
||||
x = self.model.token_embd(t)
|
||||
for block in self.model.blk[self.start : self.end]:
|
||||
x = block(x, position)
|
||||
# Cast to half (2 bytes/elem) for the wire format, matching the
|
||||
# stub. The model's weights are float16; the op output may have
|
||||
# promoted to float32, so an explicit cast normalises this.
|
||||
# promoted to float32, so an explicit cast normalises this. tinygrad is
|
||||
# lazy: embed + blocks + cast all *execute* at .realize() below (incl.
|
||||
# JIT kernel compile on the first call), so compute_ms captures them.
|
||||
t1 = time.monotonic()
|
||||
x = x.cast("half").realize()
|
||||
t2 = time.monotonic()
|
||||
arr = x.numpy() # shape (1, seq_len, hidden_dim), dtype float16
|
||||
return arr.tobytes(), int(arr.shape[1])
|
||||
out = arr.tobytes()
|
||||
t3 = time.monotonic()
|
||||
self._last_compute = {
|
||||
"build_ms": round((t1 - t0) * 1000, 2),
|
||||
"compute_ms": round((t2 - t1) * 1000, 2),
|
||||
"host_copy_ms": round((t3 - t2) * 1000, 2),
|
||||
"out_bytes": len(out),
|
||||
}
|
||||
return out, int(arr.shape[1])
|
||||
|
||||
def forward_range(
|
||||
self, hidden_bytes: bytes, position: int, seq_len: int
|
||||
|
|
@ -456,17 +1158,29 @@ class _RealModelState:
|
|||
f"seq_len*hidden_dim*2 ({seq_len}*{self.hidden_dim}*{BYTES_PER_ELEM} "
|
||||
f"= {expected})"
|
||||
)
|
||||
t0 = time.monotonic()
|
||||
arr = (
|
||||
np.frombuffer(hidden_bytes, dtype=np.float16)
|
||||
.reshape((1, seq_len, self.hidden_dim))
|
||||
.copy()
|
||||
)
|
||||
x = Tensor(arr)
|
||||
t1 = time.monotonic()
|
||||
for block in self.model.blk[self.start : self.end]:
|
||||
x = block(x, position)
|
||||
x = x.cast("half").realize()
|
||||
out = x.numpy()
|
||||
return out.tobytes(), int(out.shape[1])
|
||||
t2 = time.monotonic()
|
||||
out_arr = x.numpy()
|
||||
out = out_arr.tobytes()
|
||||
t3 = time.monotonic()
|
||||
self._last_compute = {
|
||||
"deserialize_ms": round((t1 - t0) * 1000, 2),
|
||||
"compute_ms": round((t2 - t1) * 1000, 2),
|
||||
"host_copy_ms": round((t3 - t2) * 1000, 2),
|
||||
"in_bytes": len(hidden_bytes),
|
||||
"out_bytes": len(out),
|
||||
}
|
||||
return out, int(out_arr.shape[1])
|
||||
|
||||
def forward_and_sample(
|
||||
self, hidden_bytes: bytes, position: int, seq_len: int
|
||||
|
|
@ -480,19 +1194,29 @@ class _RealModelState:
|
|||
f"seq_len*hidden_dim*2 ({seq_len}*{self.hidden_dim}*{BYTES_PER_ELEM} "
|
||||
f"= {expected})"
|
||||
)
|
||||
t0 = time.monotonic()
|
||||
arr = (
|
||||
np.frombuffer(hidden_bytes, dtype=np.float16)
|
||||
.reshape((1, seq_len, self.hidden_dim))
|
||||
.copy()
|
||||
)
|
||||
x = Tensor(arr)
|
||||
t1 = time.monotonic()
|
||||
for block in self.model.blk[self.start : self.end]:
|
||||
x = block(x, position)
|
||||
x = self.model.output_norm(x)
|
||||
logits = self.model.output(x)
|
||||
# Argmax on the last position's logits. Matches what
|
||||
# ``Transformer.forward`` does at llm.py:178.
|
||||
# ``Transformer.forward`` does at llm.py:178. The .item() forces the
|
||||
# blocks + output projection (over the full vocab) to execute here.
|
||||
token_id = int(logits[0, -1, :].argmax().item())
|
||||
t2 = time.monotonic()
|
||||
self._last_compute = {
|
||||
"deserialize_ms": round((t1 - t0) * 1000, 2),
|
||||
"compute_ms": round((t2 - t1) * 1000, 2),
|
||||
"in_bytes": len(hidden_bytes),
|
||||
"token_id": token_id,
|
||||
}
|
||||
return token_id
|
||||
|
||||
def generate_full(self, prompt: str, max_tokens: int) -> list[int]:
|
||||
|
|
@ -510,6 +1234,12 @@ class _RealModelState:
|
|||
as positions are revisited, so the same worker can serve multiple
|
||||
independent prompts.
|
||||
"""
|
||||
if self.start != 0 or self.end != self.total_blocks:
|
||||
raise RuntimeError(
|
||||
"generate_full needs the whole model resident, but this stage only "
|
||||
f"holds blk[{self.start}:{self.end}) of {self.total_blocks}. Use the "
|
||||
"per-stage ops (embed_and_forward / forward_range / forward_and_sample)."
|
||||
)
|
||||
Tensor = self._Tensor
|
||||
if max_tokens <= 0:
|
||||
return []
|
||||
|
|
@ -804,6 +1534,11 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|||
model=(args.model or os.environ.get("MODEL", "")).strip() or None,
|
||||
python_version=sys.version.split()[0],
|
||||
argv=list(sys.argv),
|
||||
# wall_ms anchors this worker's boot against the orchestrator's vast.ai
|
||||
# instance create/lease time — the only way to measure image-pull +
|
||||
# container-boot latency, which the worker can't observe directly.
|
||||
wall_ms=_wall_ms(),
|
||||
host=os.uname().nodename,
|
||||
)
|
||||
_start_heartbeat()
|
||||
|
||||
|
|
@ -845,19 +1580,35 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|||
ready["layer_range"] = [real_state.start, real_state.end]
|
||||
ready["eos_token_ids"] = real_state.eos_ids
|
||||
_write(ready)
|
||||
# Mirror ready as a structured event so the bundle records it under
|
||||
# the same `worker_*` kind family as the rest of the lifecycle. The
|
||||
# `status: "ready"` line above is kept for back-compat with the Rust
|
||||
# `parse_status_line` helper that drives the actor's ready signal.
|
||||
# Mirror ready as a structured event under the spec's event name
|
||||
# (§4.6 / §6.9: per-stage filter on `pp_worker_ready`). The
|
||||
# `pp_*` event kind is passed through unchanged by the Rust actor
|
||||
# so the bundle records `Custom("pp_worker_ready")`, matching the
|
||||
# name the orchestrator-side wired check filters on. The protocol
|
||||
# `status: "ready"` line above is unchanged for the actor's ready
|
||||
# signal.
|
||||
_emit_event(
|
||||
"ready",
|
||||
"pp_worker_ready",
|
||||
pid=os.getpid(),
|
||||
stage=stage,
|
||||
stage_index=stage,
|
||||
uptime_ms=_uptime_ms(),
|
||||
rss_mb=_rss_mb(),
|
||||
)
|
||||
# One-stop cold-start breakdown so a single event answers "where did
|
||||
# bring-up time go" per node: import + (fetch|cache) + load == time-to-ready.
|
||||
if real_state is not None:
|
||||
_emit_event(
|
||||
"boot_profile",
|
||||
stage=stage,
|
||||
total_to_ready_ms=_uptime_ms(),
|
||||
rss_mb=_rss_mb(),
|
||||
blocks_resident=real_state.end - real_state.start,
|
||||
**real_state.timing,
|
||||
)
|
||||
|
||||
global _REQUESTS_SERVED
|
||||
seen_ops: set = set()
|
||||
last_op_end = time.monotonic()
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
|
|
@ -870,6 +1621,14 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|||
if not isinstance(req, dict):
|
||||
_write({"error": f"request must be a JSON object, got {type(req).__name__}"})
|
||||
continue
|
||||
op = req.get("op")
|
||||
if real_state is not None:
|
||||
real_state._last_compute = None
|
||||
t_start = time.monotonic()
|
||||
# idle_ms_before is the pipeline bubble: how long this worker sat
|
||||
# blocked on its upstream stage between finishing the last op and
|
||||
# receiving this one. High idle => the bottleneck is elsewhere.
|
||||
idle_ms = round((t_start - last_op_end) * 1000, 2)
|
||||
try:
|
||||
reply = _handle_request(req, stage, num_stages, real_state=real_state)
|
||||
except Exception as e: # last-ditch safety net so the worker stays up
|
||||
|
|
@ -878,6 +1637,29 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|||
print(traceback.format_exc(), file=sys.stderr, flush=True)
|
||||
reply = {"request_id": req.get("request_id"), "error": f"internal: {e}"}
|
||||
_write(reply)
|
||||
t_end = time.monotonic()
|
||||
# Per-op trace — the granular signal for end-to-end latency. `first_call`
|
||||
# flags the JIT-compile-bearing first invocation of each op (kernels are
|
||||
# compiled once, then cached). `rid` (not `request_id`) keeps the Rust
|
||||
# actor from ever mistaking this event line for an op reply.
|
||||
is_first = op not in seen_ops
|
||||
seen_ops.add(op)
|
||||
_emit_event(
|
||||
"op",
|
||||
op=op,
|
||||
rid=req.get("request_id"),
|
||||
stage=stage,
|
||||
duration_ms=round((t_end - t_start) * 1000, 2),
|
||||
idle_ms_before=idle_ms,
|
||||
first_call=is_first,
|
||||
ok=isinstance(reply, dict) and "error" not in reply,
|
||||
in_tokens=len(req["tokens"]) if isinstance(req.get("tokens"), list) else None,
|
||||
in_seq_len=req.get("seq_len"),
|
||||
out_seq_len=reply.get("seq_len") if isinstance(reply, dict) else None,
|
||||
compute=(real_state._last_compute if real_state is not None else None),
|
||||
uptime_ms=_uptime_ms(),
|
||||
)
|
||||
last_op_end = t_end
|
||||
_REQUESTS_SERVED += 1
|
||||
|
||||
_emit_event(
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ use distribution::iroh_driver::{IrohDriver, IrohDriverConfig};
|
|||
use distribution::node::DistributedNodeConfig;
|
||||
use distribution::registry::RegistryConfig;
|
||||
use distribution::swim::probe::SwimConfig;
|
||||
use iroh::{PublicKey, RelayMode};
|
||||
use iroh::{PublicKey, RelayMode, SecretKey};
|
||||
|
||||
use swactor::actor::ActorAddress;
|
||||
use swactor::runtime::{Runtime, RuntimeConfig};
|
||||
|
|
@ -99,6 +99,36 @@ fn require_u32(name: &str) -> u32 {
|
|||
})
|
||||
}
|
||||
|
||||
/// A held cluster's stages must keep a STABLE node id across a redeploy
|
||||
/// bounce, or the pipeline name registry (pp-entry / pp-stage-N) keeps
|
||||
/// routing to the dead pre-bounce id and the response never returns.
|
||||
/// `PP_STAGE_SECRET` (64 hex = 32 bytes) pins this stage's keypair; it is
|
||||
/// injected at instance-create time, so it is re-read from PID 1's env on
|
||||
/// every restart and the stage id is unchanged. Unset → random identity
|
||||
/// (fine for a one-shot localhost `--seed` run).
|
||||
fn stage_secret_from_env() -> Option<SecretKey> {
|
||||
let hex = std::env::var("PP_STAGE_SECRET").ok()?;
|
||||
let hex = hex.trim();
|
||||
if hex.is_empty() {
|
||||
return None;
|
||||
}
|
||||
if hex.len() != 64 {
|
||||
eprintln!(
|
||||
"pp-gpu-node: PP_STAGE_SECRET must be 64 hex chars, got {}",
|
||||
hex.len()
|
||||
);
|
||||
std::process::exit(2);
|
||||
}
|
||||
let mut bytes = [0u8; 32];
|
||||
for (i, b) in bytes.iter_mut().enumerate() {
|
||||
*b = u8::from_str_radix(&hex[i * 2..i * 2 + 2], 16).unwrap_or_else(|_| {
|
||||
eprintln!("pp-gpu-node: PP_STAGE_SECRET is not valid hex");
|
||||
std::process::exit(2);
|
||||
});
|
||||
}
|
||||
Some(SecretKey::from_bytes(&bytes))
|
||||
}
|
||||
|
||||
fn node_config() -> DistributedNodeConfig {
|
||||
DistributedNodeConfig {
|
||||
swim: SwimConfig {
|
||||
|
|
@ -292,7 +322,133 @@ fn maybe_simulate_boot_delay(stage: u32) {
|
|||
}
|
||||
}
|
||||
|
||||
// ─── PROTOTYPE_BINARY_SWAP (spec §5.1) ────────────────────────────────
|
||||
//
|
||||
// Out-of-band binary swap scaffolding. Disabled by default — a freshly
|
||||
// pulled image with no `PP_BINARY_SWAP_*` env vars boots using the
|
||||
// binary it shipped with (spec §5.1: "The worker's normal boot path
|
||||
// MUST NOT consult this URL"). When both env vars are set, the worker
|
||||
// fetches the URL, verifies the SHA-256 digest, atomically renames the
|
||||
// new binary over the running binary, and exits so the container's
|
||||
// restart policy spawns the new binary.
|
||||
//
|
||||
// All wiring tagged `PROTOTYPE_BINARY_SWAP` for spec §5.3
|
||||
// grep-discoverability. Removal criterion: when image build/push is no
|
||||
// longer a binding constraint on iteration speed, delete:
|
||||
// - this function and its call site in `main()`
|
||||
// - the `sha2` dep added to Cargo.toml under the same comment
|
||||
// - the `PP_BINARY_SWAP_*` env vars from any deploy docs
|
||||
|
||||
fn prototype_binary_swap_maybe_apply() {
|
||||
let url = match std::env::var("PP_BINARY_SWAP_URL") {
|
||||
Ok(u) if !u.trim().is_empty() => u.trim().to_string(),
|
||||
_ => return, // PROTOTYPE_BINARY_SWAP: gate off — normal boot.
|
||||
};
|
||||
let expected_digest = match std::env::var("PP_BINARY_SWAP_SHA256") {
|
||||
Ok(d) if !d.trim().is_empty() => d.trim().to_lowercase(),
|
||||
_ => {
|
||||
eprintln!(
|
||||
"pp-gpu-node: PROTOTYPE_BINARY_SWAP — PP_BINARY_SWAP_URL is set \
|
||||
but PP_BINARY_SWAP_SHA256 is missing; refusing to swap (spec §5.1 \
|
||||
requires both URL and digest)"
|
||||
);
|
||||
std::process::exit(2);
|
||||
}
|
||||
};
|
||||
let our_path = std::env::current_exe()
|
||||
.expect("PROTOTYPE_BINARY_SWAP: current_exe failed");
|
||||
let mut new_os = our_path.clone().into_os_string();
|
||||
new_os.push(".new");
|
||||
let new_path = std::path::PathBuf::from(new_os);
|
||||
|
||||
eprintln!(
|
||||
"pp-gpu-node: PROTOTYPE_BINARY_SWAP fetching {url} → {}",
|
||||
new_path.display()
|
||||
);
|
||||
|
||||
let rt = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("PROTOTYPE_BINARY_SWAP: tokio runtime");
|
||||
let bytes = rt
|
||||
.block_on(async {
|
||||
let resp = reqwest::get(&url)
|
||||
.await
|
||||
.map_err(|e| format!("HTTP GET failed: {e}"))?;
|
||||
if !resp.status().is_success() {
|
||||
return Err(format!("HTTP {}", resp.status()));
|
||||
}
|
||||
resp.bytes().await.map_err(|e| format!("read body: {e}"))
|
||||
})
|
||||
.unwrap_or_else(|e| {
|
||||
eprintln!("pp-gpu-node: PROTOTYPE_BINARY_SWAP fetch failed: {e}");
|
||||
std::process::exit(1);
|
||||
});
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(&bytes);
|
||||
let actual_digest = format!("{:x}", hasher.finalize());
|
||||
if actual_digest != expected_digest {
|
||||
eprintln!(
|
||||
"pp-gpu-node: PROTOTYPE_BINARY_SWAP digest mismatch: expected {expected_digest}, \
|
||||
got {actual_digest} — refusing to install"
|
||||
);
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
if let Err(e) = std::fs::write(&new_path, &bytes) {
|
||||
eprintln!(
|
||||
"pp-gpu-node: PROTOTYPE_BINARY_SWAP write {} failed: {e}",
|
||||
new_path.display()
|
||||
);
|
||||
std::process::exit(1);
|
||||
}
|
||||
// Mark the staged binary executable (the URL host may serve it as
|
||||
// a plain file with no +x bit).
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
if let Err(e) = std::fs::set_permissions(
|
||||
&new_path,
|
||||
std::fs::Permissions::from_mode(0o755),
|
||||
) {
|
||||
eprintln!(
|
||||
"pp-gpu-node: PROTOTYPE_BINARY_SWAP chmod {} failed: {e}",
|
||||
new_path.display()
|
||||
);
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
// Spec §5.1: "Atomically rename the new binary over the running
|
||||
// binary." Linux rename() is atomic when both paths are on the
|
||||
// same filesystem; renaming over a memory-mapped ELF unlinks the
|
||||
// old inode but lets the running process continue (we exit
|
||||
// immediately below, so this is harmless).
|
||||
if let Err(e) = std::fs::rename(&new_path, &our_path) {
|
||||
eprintln!(
|
||||
"pp-gpu-node: PROTOTYPE_BINARY_SWAP rename {} → {} failed: {e}",
|
||||
new_path.display(),
|
||||
our_path.display()
|
||||
);
|
||||
std::process::exit(1);
|
||||
}
|
||||
eprintln!(
|
||||
"pp-gpu-node: PROTOTYPE_BINARY_SWAP installed {} bytes; exiting for container restart",
|
||||
bytes.len()
|
||||
);
|
||||
// Exit 0; the container's restart policy (typically `restart:
|
||||
// always`) re-execs the new binary.
|
||||
std::process::exit(0);
|
||||
}
|
||||
|
||||
fn main() {
|
||||
// PROTOTYPE_BINARY_SWAP (spec §5.1): inert when env vars unset.
|
||||
// Run before any other boot work so the spec's "normal boot path
|
||||
// MUST NOT consult this URL" holds — we either apply the swap and
|
||||
// exit, or return immediately and let normal boot proceed.
|
||||
prototype_binary_swap_maybe_apply();
|
||||
|
||||
let stage = require_u32("STAGE");
|
||||
let num_stages = require_u32("NUM_STAGES");
|
||||
if num_stages < 2 || stage >= num_stages {
|
||||
|
|
@ -328,7 +484,7 @@ fn main() {
|
|||
};
|
||||
|
||||
let mut driver = IrohDriver::new(IrohDriverConfig {
|
||||
secret_key: None,
|
||||
secret_key: stage_secret_from_env(),
|
||||
relay_mode,
|
||||
node: node_config(),
|
||||
peer_auth: None,
|
||||
|
|
@ -346,6 +502,26 @@ fn main() {
|
|||
.as_ref()
|
||||
.map(|d| d.subprocess_introspect().clone());
|
||||
|
||||
// Stamp the bundle the moment this process announces itself, so a
|
||||
// bundle reader can tell two pp-gpu-node incarnations of the same
|
||||
// stage apart: a --redeploy bounce pkills the old process and
|
||||
// setsid's a new one under the same PID-1 env, which means the same
|
||||
// run_id + node_id, but the pid differs. The event carries that
|
||||
// pid + wall clock as the slice point. No-op without diagnostics.
|
||||
driver.emit(distribution::diagnostics::event::Event::Custom {
|
||||
kind: "pp_stage_bounce".into(),
|
||||
fields: serde_json::json!({
|
||||
// Spec §4.8: stage worker events MUST carry stage_index
|
||||
// top-level. The legacy `stage` alias is kept for back-compat
|
||||
// with bundle consumers that filtered on it.
|
||||
"stage_index": stage,
|
||||
"stage": stage,
|
||||
"num_stages": num_stages,
|
||||
"pid": std::process::id(),
|
||||
"boot_wall_ms": distribution::diagnostics::wall_ms_now(),
|
||||
}),
|
||||
});
|
||||
|
||||
let my_id = driver.node_id();
|
||||
let my_hex: String = my_id.0.iter().map(|b| format!("{:02x}", b)).collect();
|
||||
let direct_addrs: Vec<String> = driver
|
||||
|
|
@ -446,8 +622,25 @@ fn main() {
|
|||
driver.node_mut().set_relay_url(Some(home.to_string()));
|
||||
}
|
||||
|
||||
if !wait_for_cluster(&mut driver, Duration::from_secs(120)) {
|
||||
eprintln!("pp-gpu-node: cluster did not converge in 120s");
|
||||
// SWIM convergence gate. The old hardcoded 120s was fatal on vast.ai:
|
||||
// every stage's only join target is the seed (the orchestrator), so a
|
||||
// stage can only converge once the orchestrator is ticking SWIM and acking
|
||||
// its pings. But the orchestrator is blocked in synchronous work for
|
||||
// minutes at a time — the vast.ai lease (HTTP polling in lease_chain) and
|
||||
// the redeploy scp/bounce loop — and never acks during those windows.
|
||||
// Stages that booted early would hit 120s with no alive peer and exit(1)
|
||||
// before the orchestrator ever became responsive (resolve loop), leaving
|
||||
// "running" containers with dead processes. The window a stage must outlast
|
||||
// is "however long the orchestrator stays busy", so the ceiling defaults
|
||||
// high and is tunable via PP_CONVERGE_TIMEOUT_SECS. Convergence is
|
||||
// near-instant once the orchestrator starts ticking, so a generous ceiling
|
||||
// only costs wall-clock in the genuine no-connectivity case.
|
||||
let converge_secs: u64 = std::env::var("PP_CONVERGE_TIMEOUT_SECS")
|
||||
.ok()
|
||||
.and_then(|s| s.trim().parse().ok())
|
||||
.unwrap_or(1200);
|
||||
if !wait_for_cluster(&mut driver, Duration::from_secs(converge_secs)) {
|
||||
eprintln!("pp-gpu-node: cluster did not converge in {converge_secs}s");
|
||||
std::process::exit(1);
|
||||
}
|
||||
eprintln!("pp-gpu-node: cluster converged");
|
||||
|
|
@ -587,6 +780,23 @@ fn run_stage(
|
|||
};
|
||||
let stage_actor_addr = rt.spawn(actor).unwrap();
|
||||
|
||||
// Effective worker-ready and neighbor-resolve timeouts. §4.3 couples
|
||||
// the neighbor resolve to worker-ready: a stage that has itself
|
||||
// become ready MUST be willing to wait for its downstream neighbor
|
||||
// for at least as long as it would wait for its own worker. With
|
||||
// the per-index registration deferred to post-worker-ready (below),
|
||||
// the neighbor's name is genuinely unavailable until the neighbor's
|
||||
// worker boots; the resolve must outlast that boot.
|
||||
let worker_ready_secs: u64 = std::env::var("PP_WORKER_READY_TIMEOUT_SECS")
|
||||
.ok()
|
||||
.and_then(|s| s.trim().parse().ok())
|
||||
.unwrap_or(1800);
|
||||
let neighbor_resolve_secs: u64 = std::env::var("PP_NEIGHBOR_RESOLVE_TIMEOUT_SECS")
|
||||
.ok()
|
||||
.and_then(|s| s.trim().parse().ok())
|
||||
.unwrap_or(worker_ready_secs);
|
||||
let neighbor_resolve_timeout = Duration::from_secs(neighbor_resolve_secs);
|
||||
|
||||
// Bridges are wired per role:
|
||||
// First: RequestBridge (entry) + NextTokenBridge (per-index).
|
||||
// Middle, Last: ActivationBridge (per-index).
|
||||
|
|
@ -622,20 +832,35 @@ fn run_stage(
|
|||
None
|
||||
};
|
||||
|
||||
// Register pp-stage-{stage} IMMEDIATELY so neighbouring stages can
|
||||
// resolve us. The per-index bridge is the role's *inbound-from-network*
|
||||
// adapter:
|
||||
// The per-index bridge is the role's *inbound-from-network* adapter:
|
||||
// First: receives NextTokens from Last → NextTokenBridge.
|
||||
// Middle, Last: receives StageActivations → ActivationBridge.
|
||||
//
|
||||
// Registration is deferred to AFTER the worker is ready (below) so
|
||||
// that the SWIM name `pp-stage-{stage}` is a real signal of stage
|
||||
// readiness — the orchestrator polls every per-index name to know
|
||||
// when to emit `pp_pipeline_wired` (spec §4.6). The per-index actor
|
||||
// address itself remains the bridge target.
|
||||
let per_index_bridge_addr = match role {
|
||||
StageRole::First => next_token_bridge_addr.unwrap(),
|
||||
StageRole::Middle | StageRole::Last => activation_bridge_addr.unwrap(),
|
||||
};
|
||||
register_name(&mut driver, &stage_name(stage), per_index_bridge_addr, stage);
|
||||
|
||||
// Worker boot can take time even in stub mode (Python startup +
|
||||
// tinygrad import on real mode). Generous timeout.
|
||||
if !wait_for_worker_ready(&rt, &mut driver, &status_inbox, Duration::from_secs(600)) {
|
||||
// tinygrad import on real mode). On a real run the worker also fetches
|
||||
// and realizes its model slice: an ~18 GB MoE GGUF (qwen3:30b-a3b) on a
|
||||
// cold node can spend many minutes downloading before it reports ready,
|
||||
// and the old hardcoded 600s wall would exit(1) a still-loading stage
|
||||
// before we ever learn whether the load succeeds. Default high and make
|
||||
// it tunable via PP_WORKER_READY_TIMEOUT_SECS; a generous ceiling only
|
||||
// costs wall-clock when a worker is genuinely wedged (which the
|
||||
// ProcessExited branch below already short-circuits).
|
||||
if !wait_for_worker_ready(
|
||||
&rt,
|
||||
&mut driver,
|
||||
&status_inbox,
|
||||
Duration::from_secs(worker_ready_secs),
|
||||
) {
|
||||
eprintln!("pp-gpu-node: stage-{stage} worker did not become ready");
|
||||
// The StageActor already emitted Custom("worker_exited") in
|
||||
// response to ProcessNotification::Exited. Give the HTTP-sink
|
||||
|
|
@ -648,15 +873,24 @@ fn run_stage(
|
|||
std::process::exit(1);
|
||||
}
|
||||
|
||||
// Now that this worker is ready, publish our per-index name. Spec
|
||||
// §4.6: the orchestrator uses each `pp-stage-{K}`'s availability as
|
||||
// a per-stage worker-ready signal when deciding to emit
|
||||
// `pp_pipeline_wired`. Spec §4.3: neighbour resolves wait on this
|
||||
// for as long as worker-ready takes.
|
||||
register_name(&mut driver, &stage_name(stage), per_index_bridge_addr, stage);
|
||||
|
||||
// Resolve neighbours and wire routes per role. Stage 3 keeps the
|
||||
// 2-stage resolution targets in place (Last looks up stage 0 as its
|
||||
// NextToken sink, which happens to be First in N=2).
|
||||
// NextToken sink, which happens to be First in N=2). Each neighbor
|
||||
// resolution uses the §4.3 coupled timeout so a slow-to-boot neighbor
|
||||
// cannot break a healthy chain.
|
||||
match role {
|
||||
StageRole::First => {
|
||||
let next_name = next_stage_name(stage, num_stages)
|
||||
.expect("first stage has a next neighbour for N>=2");
|
||||
let (next_addr, next_hex) =
|
||||
resolve_or_die(&mut driver, &next_name, Duration::from_secs(120));
|
||||
resolve_or_die(&mut driver, &next_name, neighbor_resolve_timeout);
|
||||
eprintln!(
|
||||
"pp-gpu-node: resolved {next_name} -> {next_addr:?} on {next_hex}"
|
||||
);
|
||||
|
|
@ -675,7 +909,7 @@ fn run_stage(
|
|||
let next_name = next_stage_name(stage, num_stages)
|
||||
.expect("middle stage has a next neighbour");
|
||||
let (next_addr, next_hex) =
|
||||
resolve_or_die(&mut driver, &next_name, Duration::from_secs(120));
|
||||
resolve_or_die(&mut driver, &next_name, neighbor_resolve_timeout);
|
||||
eprintln!(
|
||||
"pp-gpu-node: resolved {next_name} -> {next_addr:?} on {next_hex}"
|
||||
);
|
||||
|
|
@ -699,10 +933,10 @@ fn run_stage(
|
|||
let (feedback_addr, feedback_hex) = resolve_or_die(
|
||||
&mut driver,
|
||||
&feedback_name,
|
||||
Duration::from_secs(120),
|
||||
neighbor_resolve_timeout,
|
||||
);
|
||||
let (orch_addr, orch_hex) =
|
||||
resolve_or_die(&mut driver, ORCHESTRATOR_NAME, Duration::from_secs(120));
|
||||
resolve_or_die(&mut driver, ORCHESTRATOR_NAME, neighbor_resolve_timeout);
|
||||
eprintln!(
|
||||
"pp-gpu-node: resolved {feedback_name}={feedback_addr:?} on \
|
||||
{feedback_hex}, orch={orch_addr:?} on {orch_hex}"
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -105,13 +105,35 @@ impl DiagHandles {
|
|||
/// probe scheduler) run on the driver's tokio runtime and live until
|
||||
/// the process exits.
|
||||
pub fn install_from_env(driver: &mut IrohDriver, default_role: Role) -> Option<DiagHandles> {
|
||||
install_with_overrides(driver, default_role, None)
|
||||
}
|
||||
|
||||
/// Like [`install_from_env`] but lets the caller pin `run_id` explicitly
|
||||
/// rather than reading `SWACTOR_DIAG_RUN_ID` from the environment.
|
||||
///
|
||||
/// The orchestrator uses this to bind to the run_id persisted in the
|
||||
/// held-cluster handle — held stages baked their `SWACTOR_DIAG_RUN_ID`
|
||||
/// into PID-1's env at lease time and keep reusing it across bounces;
|
||||
/// the orchestrator (which runs in the operator's shell) cannot trust
|
||||
/// its own env to still match. Passing the handle's run_id here is the
|
||||
/// fix for that drift.
|
||||
///
|
||||
/// `run_id_override == None` falls back to `SWACTOR_DIAG_RUN_ID`.
|
||||
pub fn install_with_overrides(
|
||||
driver: &mut IrohDriver,
|
||||
default_role: Role,
|
||||
run_id_override: Option<&str>,
|
||||
) -> Option<DiagHandles> {
|
||||
let url = std::env::var(ENV_COLLECTOR_URL).ok()?;
|
||||
let url = url.trim().to_string();
|
||||
if url.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let run_id = env_string(ENV_RUN_ID).unwrap_or_else(|| DEFAULT_RUN_ID.to_string());
|
||||
let run_id = run_id_override
|
||||
.map(|s| s.to_string())
|
||||
.or_else(|| env_string(ENV_RUN_ID))
|
||||
.unwrap_or_else(|| DEFAULT_RUN_ID.to_string());
|
||||
let role = match env_string(ENV_NODE_ROLE).as_deref() {
|
||||
Some("orchestrator") => Role::orchestrator(),
|
||||
Some("stage") => Role::stage(),
|
||||
|
|
@ -318,8 +340,12 @@ pub fn emit_register_name(
|
|||
"our_node_id_hex": hex_of_bytes(&driver.node_id().0),
|
||||
"wall_ms": wall_ms_now(),
|
||||
});
|
||||
// Spec §4.8: events emitted by a stage worker process MUST carry
|
||||
// `stage_index` top-level once the index is known. The orchestrator's
|
||||
// own pp-orchestrator registration passes `None` here and so does not
|
||||
// carry the field — it is not a stage worker.
|
||||
if let Some(s) = stage {
|
||||
fields["stage"] = serde_json::json!(s);
|
||||
fields["stage_index"] = serde_json::json!(s);
|
||||
}
|
||||
driver.emit(Event::Custom {
|
||||
kind: "register_name".into(),
|
||||
|
|
|
|||
|
|
@ -9,6 +9,11 @@
|
|||
//! predecessor's announcement into the next child's environment.
|
||||
//! * [`await_convergence`] polls a closure that reports the current alive
|
||||
//! peer count and returns when the target is met (or times out).
|
||||
//! * [`resolve_roster`] polls SWIM until every `pp-stage-K` resolves, then
|
||||
//! returns the per-stage (node_id_hex, node_id_short) roster used by the
|
||||
//! `pp_stage_roster` diagnostic event (spec §4.5).
|
||||
//! * [`stage_roster_event_fields`] builds the JSON fields for a
|
||||
//! `pp_stage_roster` event from a resolved roster.
|
||||
//!
|
||||
//! Tests inject a fake command builder (e.g. `sh -c "echo PP_GPU_NODE_ADDR
|
||||
//! <hex> <direct>; sleep 60"`) so the chain can be exercised end-to-end
|
||||
|
|
@ -374,3 +379,132 @@ where
|
|||
std::thread::sleep(poll_interval);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Roster + pipeline-wired helpers (spec §4.5 / §4.6) ───────────────
|
||||
|
||||
/// One entry in the resolved stage roster: stage_index → node id.
|
||||
///
|
||||
/// Built by [`resolve_roster`] once every per-stage SWIM name resolves.
|
||||
/// The orchestrator emits these as the `stages` field of the
|
||||
/// `pp_stage_roster` event (spec §4.5).
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct StageRosterEntry {
|
||||
pub stage_index: u32,
|
||||
pub node_id_hex: String,
|
||||
pub node_id_short: String,
|
||||
}
|
||||
|
||||
/// Why [`resolve_roster`] gave up.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum RosterError {
|
||||
/// At least one `pp-stage-K` did not resolve within the timeout. The
|
||||
/// missing stage indices are reported in ascending order.
|
||||
Timeout {
|
||||
missing_stages: Vec<u32>,
|
||||
timeout: Duration,
|
||||
},
|
||||
}
|
||||
|
||||
impl std::fmt::Display for RosterError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
RosterError::Timeout {
|
||||
missing_stages,
|
||||
timeout,
|
||||
} => write!(
|
||||
f,
|
||||
"pipeline did not wire within {:.0}s: missing pp-stage-K for stages {:?}",
|
||||
timeout.as_secs_f32(),
|
||||
missing_stages,
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for RosterError {}
|
||||
|
||||
/// Poll until every `pp-stage-K` (K in `0..num_stages`) resolves, or
|
||||
/// `timeout` elapses. Returns the resolved roster ordered by
|
||||
/// `stage_index`. Each iteration calls `resolve_stage(K)` for any stage
|
||||
/// still missing — the callback returns `Some((actor_addr_unused,
|
||||
/// node_id_hex))` once SWIM has propagated the registration.
|
||||
///
|
||||
/// The callback's first tuple element is discarded by this helper; it
|
||||
/// exists because the orchestrator's per-name resolve returns
|
||||
/// `(ActorAddress, NodeId)` and most callers want the address too, so
|
||||
/// expressing the callback as "the resolve function" keeps adapter code
|
||||
/// short.
|
||||
pub fn resolve_roster<F>(
|
||||
num_stages: u32,
|
||||
timeout: Duration,
|
||||
poll_interval: Duration,
|
||||
mut resolve_stage: F,
|
||||
) -> Result<Vec<StageRosterEntry>, RosterError>
|
||||
where
|
||||
F: FnMut(u32) -> Option<String>,
|
||||
{
|
||||
let deadline = Instant::now() + timeout;
|
||||
let mut resolved: Vec<Option<String>> = vec![None; num_stages as usize];
|
||||
loop {
|
||||
for k in 0..num_stages {
|
||||
if resolved[k as usize].is_some() {
|
||||
continue;
|
||||
}
|
||||
if let Some(hex) = resolve_stage(k) {
|
||||
resolved[k as usize] = Some(hex);
|
||||
}
|
||||
}
|
||||
if resolved.iter().all(|o| o.is_some()) {
|
||||
let out: Vec<StageRosterEntry> = resolved
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(k, hex)| {
|
||||
let hex = hex.unwrap();
|
||||
let short = hex.chars().take(8).collect::<String>();
|
||||
StageRosterEntry {
|
||||
stage_index: k as u32,
|
||||
node_id_hex: hex,
|
||||
node_id_short: short,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
return Ok(out);
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
let missing: Vec<u32> = resolved
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(k, o)| o.is_none().then_some(k as u32))
|
||||
.collect();
|
||||
return Err(RosterError::Timeout {
|
||||
missing_stages: missing,
|
||||
timeout,
|
||||
});
|
||||
}
|
||||
std::thread::sleep(poll_interval);
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the `fields` JSON for a `pp_stage_roster` diagnostic event from
|
||||
/// a resolved roster, attaching the given `drive_seq`. Spec §4.5: the
|
||||
/// event MUST list every stage, ordered by `stage_index`, with each
|
||||
/// entry carrying `stage_index`, `node_id_hex`, and `node_id_short`.
|
||||
pub fn stage_roster_event_fields(
|
||||
drive_seq: u32,
|
||||
roster: &[StageRosterEntry],
|
||||
) -> serde_json::Value {
|
||||
let stages: Vec<serde_json::Value> = roster
|
||||
.iter()
|
||||
.map(|e| {
|
||||
serde_json::json!({
|
||||
"stage_index": e.stage_index,
|
||||
"node_id_hex": e.node_id_hex,
|
||||
"node_id_short": e.node_id_short,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
serde_json::json!({
|
||||
"drive_seq": drive_seq,
|
||||
"stages": stages,
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -515,8 +515,12 @@ impl StageActor {
|
|||
let Some(emitter) = &self.diagnostics else {
|
||||
return;
|
||||
};
|
||||
// Spec §4.8: every event emitted by a stage worker process MUST carry
|
||||
// `stage_index` as a top-level field whenever the worker is past the
|
||||
// point of knowing its index. The actor only ever has `stage_idx` set
|
||||
// post-construction, so when present, inject it under the spec's name.
|
||||
if let Some(stage) = self.stage_idx {
|
||||
fields["stage_idx"] = serde_json::json!(stage);
|
||||
fields["stage_index"] = serde_json::json!(stage);
|
||||
}
|
||||
fields["role"] = serde_json::json!(self.role_str());
|
||||
emitter.emit_event(Event::Custom {
|
||||
|
|
@ -645,17 +649,26 @@ impl StageActor {
|
|||
}
|
||||
|
||||
// Worker-side lifecycle event: any `{"event": "<kind>", ...}` line
|
||||
// is re-emitted as `Custom("worker_<kind>")` and otherwise ignored
|
||||
// (it carries no protocol payload). We stash the traceback off any
|
||||
// `uncaught_exception` so the eventual `worker_exited` event can
|
||||
// carry it even if the per-line event is truncated.
|
||||
// is re-emitted into the diagnostic stream. Spec-defined event
|
||||
// kinds — anything beginning with `pp_` — pass through verbatim
|
||||
// so the bundle reader sees the same kind the spec names (e.g.
|
||||
// `pp_download_progress`, spec §4.7). Generic worker events stay
|
||||
// under the `worker_*` namespace so they cannot collide with
|
||||
// orchestrator-emitted `pp_*` events. We stash the traceback off
|
||||
// any `uncaught_exception` so the eventual `worker_exited` event
|
||||
// can carry it even if the per-line event is truncated.
|
||||
if let Some(event_kind) = val.get("event").and_then(|v| v.as_str()) {
|
||||
if event_kind == "uncaught_exception" {
|
||||
if let Some(tb) = val.get("traceback").and_then(|v| v.as_str()) {
|
||||
self.last_python_traceback = Some(tb.to_string());
|
||||
}
|
||||
}
|
||||
self.emit_diag(&format!("worker_{event_kind}"), val.clone());
|
||||
let kind = if event_kind.starts_with("pp_") {
|
||||
event_kind.to_string()
|
||||
} else {
|
||||
format!("worker_{event_kind}")
|
||||
};
|
||||
self.emit_diag(&kind, val.clone());
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -72,6 +72,16 @@ impl DiagEnv {
|
|||
pub fn is_enabled(&self) -> bool {
|
||||
self.collector_url.is_some() || self.iroh_relay_url.is_some()
|
||||
}
|
||||
|
||||
/// Override the run_id that will be injected into every rented
|
||||
/// stage's `SWACTOR_DIAG_RUN_ID`. Used by the orchestrator's
|
||||
/// `--hold` path to pin the held cluster's run_id to whatever ends
|
||||
/// up in the on-disk cluster handle (rather than whatever happens
|
||||
/// to be in the operator's shell env at lease time).
|
||||
pub fn with_run_id(mut self, run_id: String) -> Self {
|
||||
self.run_id = Some(run_id);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// A vast.ai offer (GPU rental option) returned by [`find_offer`].
|
||||
|
|
@ -81,7 +91,23 @@ pub struct Offer {
|
|||
pub gpu_name: String,
|
||||
pub dph_total: f64,
|
||||
#[serde(default)]
|
||||
pub gpu_ram: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub geolocation: Option<String>,
|
||||
/// Inbound bandwidth price ($/TB). vast.ai bills *downloads to the
|
||||
/// instance* — i.e. every Docker image pull — at this rate, and it is
|
||||
/// excluded from `dph_total`, so a cheap-by-the-hour host can still
|
||||
/// double the bill on a fat image. Defaults to 0.0 if the offer omits it.
|
||||
#[serde(default, rename = "internet_down_cost_per_tb")]
|
||||
pub inet_down_cost_per_tb: f64,
|
||||
/// Outbound bandwidth price ($/TB). Surfaced alongside the download price
|
||||
/// so neither direction is hidden; uploads are usually negligible here.
|
||||
#[serde(default, rename = "internet_up_cost_per_tb")]
|
||||
pub inet_up_cost_per_tb: f64,
|
||||
/// Marketplace host that owns the machine. Used to blacklist providers
|
||||
/// that gouge on bandwidth, since bandwidth price is a per-host policy.
|
||||
#[serde(default)]
|
||||
pub host_id: Option<u64>,
|
||||
}
|
||||
|
||||
/// Connection details for a running instance.
|
||||
|
|
@ -136,8 +162,20 @@ pub async fn find_offer(
|
|||
// - cuda_max_good >= 12.6 matches our CUDA-12.6 base image. Cheaper
|
||||
// offers without a modern host CUDA stack were the source of the
|
||||
// `unresolvable CDI devices` failures we saw earlier.
|
||||
let query = serde_json::json!({
|
||||
"gpu_name": {"eq": gpu_name},
|
||||
// The cheap-card selector is normally `gpu_name == <model>` (e.g. "RTX
|
||||
// 3060"), which implicitly bounds cost because that model is cheap. When
|
||||
// `PP_GPU_MIN_RAM_MB` is set we instead select by VRAM so the pipeline can
|
||||
// span a *heterogeneous* set of cards: every PP stage is an independent
|
||||
// process exchanging fp16 hidden state over the wire, so stages need not
|
||||
// share a GPU model — only enough VRAM to hold their block slice. A VRAM
|
||||
// filter alone would pull in datacenter GPUs (4090/A100/H100…) and wreck
|
||||
// the median-cost pick, so `PP_GPU_MAX_DPH` caps $/hr to keep the pool in
|
||||
// the same cheap band the single-model filter gave us, and `num_gpus == 1`
|
||||
// keeps us from renting (and paying for) a multi-GPU rig per stage.
|
||||
// Everything else — reliability, CUDA floor, verified, ports, inet, the
|
||||
// non-CN geo filter, and the median-priced pick below — is identical to
|
||||
// the single-model path.
|
||||
let mut query = serde_json::json!({
|
||||
"rentable": {"eq": true},
|
||||
"rented": {"eq": false},
|
||||
"reliability2": {"gte": 0.995},
|
||||
|
|
@ -146,6 +184,18 @@ pub async fn find_offer(
|
|||
"direct_port_count": {"gte": 1},
|
||||
"inet_down": {"gte": 100.0},
|
||||
});
|
||||
match env_min_gpu_ram_mb() {
|
||||
Some(min_ram) => {
|
||||
query["gpu_ram"] = serde_json::json!({"gte": min_ram});
|
||||
query["num_gpus"] = serde_json::json!({"eq": 1});
|
||||
if let Some(max_dph) = env_max_dph() {
|
||||
query["dph_total"] = serde_json::json!({"lte": max_dph});
|
||||
}
|
||||
}
|
||||
None => {
|
||||
query["gpu_name"] = serde_json::json!({"eq": gpu_name});
|
||||
}
|
||||
}
|
||||
let url = format!(
|
||||
"{base_url}/api/v0/bundles/?q={}",
|
||||
urlencoding::encode(&query.to_string())
|
||||
|
|
@ -180,12 +230,16 @@ pub async fn find_offer(
|
|||
})
|
||||
.collect();
|
||||
|
||||
let blacklist = blacklisted_host_ids();
|
||||
let mut candidates: Vec<Offer> = filtered
|
||||
.into_iter()
|
||||
.filter(|o| !exclude_ids.contains(&o.id))
|
||||
.filter(|o| o.host_id.map_or(true, |h| !blacklist.contains(&h)))
|
||||
.collect();
|
||||
if candidates.is_empty() {
|
||||
return Err("no offers available (after geo/exclusion filter)".to_string());
|
||||
return Err(
|
||||
"no offers available (after geo/host-blacklist/exclusion filter)".to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
// Pick the median-priced offer rather than the cheapest. Cheap RTX 4090
|
||||
|
|
@ -196,9 +250,92 @@ pub async fn find_offer(
|
|||
// starts running). The median strikes a balance: it skips the bottom
|
||||
// tier of misconfigured hosts without paying for the most expensive
|
||||
// ones in the candidate set.
|
||||
candidates.sort_by(|a, b| a.dph_total.partial_cmp(&b.dph_total).unwrap());
|
||||
// Effective price = $/hr plus the amortized-as-one-time image-pull cost
|
||||
// (`image_GB * down_$/TB / 1000`) when PP_IMAGE_SIZE_GB is set. This makes
|
||||
// the median pick rank on true cost: a host that is cheap per-hour but
|
||||
// charges $40/TB sorts below a free-bandwidth host once a 20GB pull is
|
||||
// priced in. With the flag unset, `pull_cost` is 0 and this is the old
|
||||
// dph_total ordering.
|
||||
let image_gb = env_image_size_gb();
|
||||
let pull_cost = |o: &Offer| image_gb.map_or(0.0, |gb| gb * o.inet_down_cost_per_tb / 1000.0);
|
||||
let effective = |o: &Offer| o.dph_total + pull_cost(o);
|
||||
candidates.sort_by(|a, b| effective(a).partial_cmp(&effective(b)).unwrap());
|
||||
let median_idx = candidates.len() / 2;
|
||||
Ok(candidates.swap_remove(median_idx))
|
||||
let n_candidates = candidates.len();
|
||||
let picked = candidates.swap_remove(median_idx);
|
||||
// One line per stage (N small) so a heterogeneous lease is auditable: which
|
||||
// physical card each stage landed on and what it costs. Silent in the
|
||||
// single-model path too — handy when a lease picks an unexpected host.
|
||||
// When PP_IMAGE_SIZE_GB is set, append the priced-in one-time image pull so
|
||||
// the chosen $/hr and the cost it was actually ranked on are both visible.
|
||||
let pull_note = image_gb.map_or(String::new(), |gb| {
|
||||
format!(" +${:.2} pull ({:.0}GB)", pull_cost(&picked), gb)
|
||||
});
|
||||
eprintln!(
|
||||
"find_offer: selected offer {} — {} {} @ ${:.3}/hr [{}] \
|
||||
bw ${:.2}/TB down ${:.2}/TB up{} (median of {} candidates)",
|
||||
picked.id,
|
||||
picked.gpu_name,
|
||||
picked
|
||||
.gpu_ram
|
||||
.map(|r| format!("{:.0}MB", r))
|
||||
.unwrap_or_else(|| "?MB".into()),
|
||||
picked.dph_total,
|
||||
picked.geolocation.as_deref().unwrap_or("?"),
|
||||
picked.inet_down_cost_per_tb,
|
||||
picked.inet_up_cost_per_tb,
|
||||
pull_note,
|
||||
n_candidates,
|
||||
);
|
||||
Ok(picked)
|
||||
}
|
||||
|
||||
/// `PP_GPU_MIN_RAM_MB`: when set to a positive integer, the offer search
|
||||
/// selects cards by VRAM (`gpu_ram >= N` MB) instead of by exact GPU model,
|
||||
/// enabling a heterogeneous cluster. Unset / blank / zero → model-name mode
|
||||
/// (the historical default, unchanged).
|
||||
fn env_min_gpu_ram_mb() -> Option<u64> {
|
||||
std::env::var("PP_GPU_MIN_RAM_MB")
|
||||
.ok()
|
||||
.and_then(|s| s.trim().parse::<u64>().ok())
|
||||
.filter(|&n| n > 0)
|
||||
}
|
||||
|
||||
/// `PP_GPU_MAX_DPH`: optional $/hr cap, applied only in VRAM-filter mode, to
|
||||
/// keep the heterogeneous pool in the cheap band (otherwise datacenter GPUs
|
||||
/// dominate the median-cost pick). Unset → no cap.
|
||||
fn env_max_dph() -> Option<f64> {
|
||||
std::env::var("PP_GPU_MAX_DPH")
|
||||
.ok()
|
||||
.and_then(|s| s.trim().parse::<f64>().ok())
|
||||
.filter(|&v| v > 0.0)
|
||||
}
|
||||
|
||||
/// Hosts blacklisted regardless of env: providers caught gouging on bandwidth.
|
||||
/// host 59017 (machine 26024, Texas) lists $40/TB download — a ~20GB image pull
|
||||
/// costs ~$0.80/instance there, which doubled an earlier run's bill.
|
||||
const BLACKLISTED_HOST_IDS: &[u64] = &[59017];
|
||||
|
||||
/// `PP_BLACKLIST_HOSTS`: optional comma-separated host ids to exclude, merged
|
||||
/// with the always-on [`BLACKLISTED_HOST_IDS`]. Blank/garbage entries ignored.
|
||||
fn blacklisted_host_ids() -> std::collections::HashSet<u64> {
|
||||
let mut set: std::collections::HashSet<u64> = BLACKLISTED_HOST_IDS.iter().copied().collect();
|
||||
if let Ok(raw) = std::env::var("PP_BLACKLIST_HOSTS") {
|
||||
set.extend(raw.split(',').filter_map(|s| s.trim().parse::<u64>().ok()));
|
||||
}
|
||||
set
|
||||
}
|
||||
|
||||
/// `PP_IMAGE_SIZE_GB`: optional size of the deploy image, in GB. When set, the
|
||||
/// one-time cost of pulling the image (`image_GB * down_$/TB / 1000`) is folded
|
||||
/// into each offer's effective price so the median pick is ranked on true cost,
|
||||
/// not just $/hr — a host that is cheap-by-the-hour but gouges on bandwidth
|
||||
/// sorts down accordingly. Unset / non-positive → rank on `dph_total` alone.
|
||||
fn env_image_size_gb() -> Option<f64> {
|
||||
std::env::var("PP_IMAGE_SIZE_GB")
|
||||
.ok()
|
||||
.and_then(|s| s.trim().parse::<f64>().ok())
|
||||
.filter(|&v| v > 0.0)
|
||||
}
|
||||
|
||||
/// Poll vast.ai until the instance reaches `running`, then extract IP + port.
|
||||
|
|
@ -214,17 +351,41 @@ pub async fn wait_for_running(
|
|||
let url = format!("{base_url}/api/v0/instances/{contract_id}/");
|
||||
|
||||
for poll in 0..max_polls {
|
||||
let resp = client
|
||||
let resp = match client
|
||||
.get(&url)
|
||||
.header("Authorization", format!("Bearer {api_key}"))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("wait_for_running request failed: {e}"))?;
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
// A network blip is transient: re-poll rather than abort. The
|
||||
// caller treats a wait_for_running error as "this host is dead"
|
||||
// and tears the instance down, so failing over one dropped
|
||||
// request would needlessly kill a healthy, still-loading node.
|
||||
eprintln!(
|
||||
" contract {contract_id} poll {}/{max_polls}: request error: {e} (retrying)",
|
||||
poll + 1,
|
||||
);
|
||||
tokio::time::sleep(poll_interval).await;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
if !resp.status().is_success() {
|
||||
// 429 (rate-limit under the request burst of an N-node lease) and
|
||||
// 5xx are transient; re-poll instead of declaring the instance
|
||||
// dead. The loop is bounded by max_polls, so a persistently failing
|
||||
// endpoint still terminates with the poll-limit error below.
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
return Err(format!("wait_for_running HTTP {status}: {body}"));
|
||||
eprintln!(
|
||||
" contract {contract_id} poll {}/{max_polls}: HTTP {status} (retrying): {}",
|
||||
poll + 1,
|
||||
body.chars().take(80).collect::<String>(),
|
||||
);
|
||||
tokio::time::sleep(poll_interval).await;
|
||||
continue;
|
||||
}
|
||||
|
||||
let wrapper: InstanceResponse = resp
|
||||
|
|
@ -332,6 +493,7 @@ pub async fn create_instance(
|
|||
seed_relay: Option<&str>,
|
||||
image: &str,
|
||||
label: Option<&str>,
|
||||
stage_secret: Option<&str>,
|
||||
diag_env: Option<&DiagEnv>,
|
||||
) -> Result<InstanceInfo, String> {
|
||||
let url = format!("{base_url}/api/v0/asks/{offer_id}/");
|
||||
|
|
@ -343,6 +505,13 @@ pub async fn create_instance(
|
|||
if let Some(relay) = seed_relay {
|
||||
env["SEED_RELAY"] = serde_json::Value::String(relay.to_string());
|
||||
}
|
||||
// Pin this stage's iroh identity so it survives an in-place redeploy
|
||||
// bounce: re-read from PID 1's env on restart, the stage keeps the same
|
||||
// node id and the pipeline name registry stays valid. See
|
||||
// pp-gpu-node::stage_secret_from_env.
|
||||
if let Some(secret) = stage_secret {
|
||||
env["PP_STAGE_SECRET"] = serde_json::Value::String(secret.to_string());
|
||||
}
|
||||
// Pass through select orchestrator-side env to every rented stage so a
|
||||
// smoke run can flip e.g. stub mode or python interpreter without
|
||||
// rebuilding the docker image. Whitelist (not pass-everything) keeps
|
||||
|
|
@ -481,6 +650,7 @@ pub async fn create_pipeline_instances(
|
|||
seed_relay,
|
||||
image,
|
||||
None,
|
||||
None,
|
||||
diag_env,
|
||||
)
|
||||
.await
|
||||
|
|
@ -619,15 +789,227 @@ pub async fn find_offer_chain(
|
|||
Ok(chosen)
|
||||
}
|
||||
|
||||
/// Rent `num_stages` vast.ai instances and wait for each to reach
|
||||
/// `running`. Combines [`find_offer_chain`], [`create_pipeline_instances`]
|
||||
/// and [`wait_for_running`] into one all-or-nothing helper.
|
||||
/// Destroy one contract, retrying on transient failures (HTTP 429 rate-limit,
|
||||
/// 5xx, network blips). Rollback and stage-replacement paths use this so a
|
||||
/// throttled DELETE does not silently strand a billing instance — the bug that
|
||||
/// orphaned a Tesla T4 when a 12-node lease rolled back during a 429 storm.
|
||||
async fn destroy_instance_with_retry(
|
||||
client: &Client,
|
||||
base_url: &str,
|
||||
api_key: &str,
|
||||
contract_id: u64,
|
||||
) -> Result<(), String> {
|
||||
const ATTEMPTS: u32 = 5;
|
||||
let mut last = String::new();
|
||||
for attempt in 1..=ATTEMPTS {
|
||||
match destroy_instance(client, base_url, api_key, contract_id).await {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(e) => {
|
||||
last = e;
|
||||
// Back off proportionally; the endpoint threshold is ~4.5 req/s.
|
||||
tokio::time::sleep(Duration::from_millis(500 * attempt as u64)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(format!(
|
||||
"destroy {contract_id} failed after {ATTEMPTS} attempts: {last}"
|
||||
))
|
||||
}
|
||||
|
||||
/// Best-effort teardown of every contract created so far, used on the lease's
|
||||
/// failure paths. Each destroy is retried (see [`destroy_instance_with_retry`])
|
||||
/// so a 429 storm during rollback cannot leave a billing orphan. Logs but does
|
||||
/// not propagate errors — the caller is already returning the original failure.
|
||||
async fn rollback(client: &Client, base_url: &str, api_key: &str, created: &[InstanceInfo]) {
|
||||
for info in created {
|
||||
if let Err(e) =
|
||||
destroy_instance_with_retry(client, base_url, api_key, info.contract_id).await
|
||||
{
|
||||
eprintln!(
|
||||
"lease_chain: WARNING rollback could not destroy {}: {e}",
|
||||
info.contract_id
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Find an offer for `stage` (excluding everything in `tried_offer_ids`, which
|
||||
/// it appends to) and create one instance, retrying with the next-best offer
|
||||
/// when a create is throttled (429) or the offer was snatched between select
|
||||
/// and create. Returns the created [`InstanceInfo`], or an error after
|
||||
/// `MAX_CREATE_ATTEMPTS`. Pulls this stage's pinned identity from
|
||||
/// `stage_secrets[stage]` so a replacement keeps the same node id.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn provision_stage(
|
||||
client: &Client,
|
||||
base_url: &str,
|
||||
api_key: &str,
|
||||
gpu_name: &str,
|
||||
stage: u32,
|
||||
num_stages: u32,
|
||||
seed_addr: &str,
|
||||
seed_relay: Option<&str>,
|
||||
image: &str,
|
||||
label: Option<&str>,
|
||||
stage_secrets: Option<&[String]>,
|
||||
diag_env: Option<&DiagEnv>,
|
||||
tried_offer_ids: &mut Vec<u64>,
|
||||
// PROTOTYPE_PREFLIGHT_HF (spec §5.2): host ids already claimed by
|
||||
// earlier stages in this chain. Used (and updated) only when the
|
||||
// preflight gate is enabled.
|
||||
used_host_ids: &mut std::collections::HashSet<u64>,
|
||||
) -> Result<InstanceInfo, String> {
|
||||
const MAX_CREATE_ATTEMPTS: u32 = 5;
|
||||
let preflight = prototype_preflight_hf::enabled();
|
||||
let mut last_err: Option<String> = None;
|
||||
for attempt in 1..=MAX_CREATE_ATTEMPTS {
|
||||
let offer = match find_offer(client, base_url, api_key, gpu_name, tried_offer_ids).await {
|
||||
Ok(o) => o,
|
||||
Err(e) => {
|
||||
last_err = Some(format!("find_offer for stage {stage}: {e}"));
|
||||
break;
|
||||
}
|
||||
};
|
||||
tried_offer_ids.push(offer.id);
|
||||
|
||||
// PROTOTYPE_PREFLIGHT_HF (spec §5.2): "Candidate chains MUST
|
||||
// be filtered such that no two chain slots share the same
|
||||
// public network endpoint (e.g. the offer's public IP)." The
|
||||
// offer doesn't carry a resolved public IP yet, but `host_id`
|
||||
// (the physical machine) is the public-endpoint proxy: two
|
||||
// offers on the same host_id share the same NAT'd public IP.
|
||||
// Gate is off by default → behavior unchanged.
|
||||
if preflight {
|
||||
if let Some(h) = offer.host_id {
|
||||
if used_host_ids.contains(&h) {
|
||||
eprintln!(
|
||||
"PROTOTYPE_PREFLIGHT_HF: stage {stage} skipping offer {} \
|
||||
on host {h} (already used by an earlier chain slot)",
|
||||
offer.id,
|
||||
);
|
||||
last_err = Some(format!(
|
||||
"preflight rejected offer {} (host {h} already in chain)",
|
||||
offer.id,
|
||||
));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match create_instance(
|
||||
client,
|
||||
base_url,
|
||||
api_key,
|
||||
offer.id,
|
||||
stage,
|
||||
num_stages,
|
||||
seed_addr,
|
||||
seed_relay,
|
||||
image,
|
||||
label,
|
||||
stage_secrets
|
||||
.and_then(|ss| ss.get(stage as usize))
|
||||
.map(|s| s.as_str()),
|
||||
diag_env,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(info) => {
|
||||
if preflight {
|
||||
if let Some(h) = offer.host_id {
|
||||
used_host_ids.insert(h);
|
||||
}
|
||||
}
|
||||
return Ok(info);
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"lease_chain: stage {stage} create on offer {} failed (attempt {attempt}/{MAX_CREATE_ATTEMPTS}): {e}",
|
||||
offer.id,
|
||||
);
|
||||
last_err = Some(e);
|
||||
// Try the next-best offer; the loop excludes the already-tried
|
||||
// id via `tried_offer_ids`.
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(format!(
|
||||
"stage {stage} could not be created after {MAX_CREATE_ATTEMPTS} attempts: {}",
|
||||
last_err.unwrap_or_default(),
|
||||
))
|
||||
}
|
||||
|
||||
// ─── PROTOTYPE_PREFLIGHT_HF (spec §5.2) ──────────────────────────────
|
||||
//
|
||||
// Pre-deploy host-throughput probe scaffolding. Disabled by default —
|
||||
// when `PP_PREFLIGHT_HF` is unset or "0", host selection behaves as it
|
||||
// does today (spec §5.2 gate clause).
|
||||
//
|
||||
// When enabled:
|
||||
// - Candidate chains are filtered so that no two slots share the
|
||||
// same `host_id` (public-network-endpoint proxy, spec §5.2).
|
||||
// - The active per-host ranged-GET throughput probe is a MAY per
|
||||
// spec §5.2 and is currently a no-op stub: we expose the
|
||||
// threshold + sample-size knobs so future re-implementation has
|
||||
// a stable surface area, but the orchestrator does NOT issue
|
||||
// speculative leases just to probe. A future implementation
|
||||
// would brief-lease a candidate, ssh-curl the model URL with
|
||||
// `--range 0-PP_PREFLIGHT_HF_SAMPLE_MB`, and reject if measured
|
||||
// throughput < `PP_PREFLIGHT_HF_MIN_MBPS`.
|
||||
//
|
||||
// Removal criterion (spec §5.2): when a per-host quality data layer
|
||||
// exists outside this code path, delete this module, the
|
||||
// `used_host_ids` argument on `provision_stage`, the local set
|
||||
// threaded through `lease_chain`, and the `PP_PREFLIGHT_HF*` env
|
||||
// vars from any deploy docs.
|
||||
pub mod prototype_preflight_hf {
|
||||
/// Spec §5.2 gate. True when `PP_PREFLIGHT_HF` is set to anything
|
||||
/// other than empty, `0`, or `false` (case-insensitive).
|
||||
pub fn enabled() -> bool {
|
||||
match std::env::var("PP_PREFLIGHT_HF") {
|
||||
Ok(v) => {
|
||||
let v = v.trim();
|
||||
!v.is_empty() && v != "0" && !v.eq_ignore_ascii_case("false")
|
||||
}
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Minimum acceptable measured throughput in MB/s. Hosts whose
|
||||
/// measured throughput is below this MUST be rejected (spec §5.2).
|
||||
/// Default 50 MB/s.
|
||||
pub fn min_mbps() -> f64 {
|
||||
std::env::var("PP_PREFLIGHT_HF_MIN_MBPS")
|
||||
.ok()
|
||||
.and_then(|s| s.trim().parse::<f64>().ok())
|
||||
.filter(|v| *v > 0.0)
|
||||
.unwrap_or(50.0)
|
||||
}
|
||||
|
||||
/// Per-host probe sample size in MB. Spec §5.2 says "default
|
||||
/// sample size on the order of tens of MB". Default 20 MB.
|
||||
pub fn sample_mb() -> u64 {
|
||||
std::env::var("PP_PREFLIGHT_HF_SAMPLE_MB")
|
||||
.ok()
|
||||
.and_then(|s| s.trim().parse::<u64>().ok())
|
||||
.filter(|v| *v > 0)
|
||||
.unwrap_or(20)
|
||||
}
|
||||
}
|
||||
|
||||
/// Rent `num_stages` vast.ai instances and wait for each to reach `running`.
|
||||
/// Yields N running contracts or rolls everything back.
|
||||
///
|
||||
/// On any failure (no matching offers, partial creation, a contract that
|
||||
/// never reaches running), every instance that was created during this
|
||||
/// call is destroyed best-effort before the error returns. Destroy errors
|
||||
/// during rollback are swallowed — they would only mask the original
|
||||
/// failure the caller actually needs to see.
|
||||
/// An N-node heterogeneous lease routinely meets a flaky host, so the helper is
|
||||
/// resilient by construction:
|
||||
/// * each stage's create retries with the next-best offer on a 429 / snatched
|
||||
/// offer (see [`provision_stage`]);
|
||||
/// * a stage whose instance never reaches `running` (e.g. a host that loads the
|
||||
/// image, then stops) is destroyed and re-provisioned on a fresh offer — up
|
||||
/// to `MAX_REPLACE_ATTEMPTS` — rather than aborting the whole lease;
|
||||
/// * on give-up, every instance created during this call is rolled back with
|
||||
/// retried destroys (see [`rollback`]) so nothing is left billing.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn lease_chain(
|
||||
client: &Client,
|
||||
base_url: &str,
|
||||
|
|
@ -638,99 +1020,125 @@ pub async fn lease_chain(
|
|||
seed_relay: Option<&str>,
|
||||
image: &str,
|
||||
label: Option<&str>,
|
||||
stage_secrets: Option<&[String]>,
|
||||
poll_interval: Duration,
|
||||
max_polls: u32,
|
||||
diag_env: Option<&DiagEnv>,
|
||||
) -> Result<Vec<InstanceInfo>, String> {
|
||||
// Find + create per stage, retrying the find when an offer is snatched
|
||||
// between selection and creation. With tight reliability filters the
|
||||
// candidate pool is small enough that the race window matters at
|
||||
// N >= 3 — a single up-front `find_offer_chain` followed by a batch
|
||||
// `create_pipeline_instances` was losing the third offer to other
|
||||
// renters. Up to `max_create_attempts` per stage.
|
||||
const MAX_CREATE_ATTEMPTS: u32 = 5;
|
||||
let mut tried_offer_ids: Vec<u64> = Vec::new();
|
||||
let mut created: Vec<InstanceInfo> = Vec::with_capacity(num_stages as usize);
|
||||
// PROTOTYPE_PREFLIGHT_HF (spec §5.2): host ids in use by this
|
||||
// chain. Only consulted when the gate is on; the set is owned
|
||||
// here so it survives across both provisioning phases.
|
||||
let mut used_host_ids: std::collections::HashSet<u64> =
|
||||
std::collections::HashSet::new();
|
||||
|
||||
// Phase 1 — provision every stage (find + create, with per-stage retry).
|
||||
for stage in 0..num_stages {
|
||||
let mut last_err: Option<String> = None;
|
||||
let mut info_opt: Option<InstanceInfo> = None;
|
||||
for attempt in 1..=MAX_CREATE_ATTEMPTS {
|
||||
let offer = match find_offer(
|
||||
client,
|
||||
base_url,
|
||||
api_key,
|
||||
gpu_name,
|
||||
&tried_offer_ids,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(o) => o,
|
||||
Err(e) => {
|
||||
last_err = Some(format!("find_offer for stage {stage}: {e}"));
|
||||
break;
|
||||
}
|
||||
};
|
||||
tried_offer_ids.push(offer.id);
|
||||
match create_instance(
|
||||
client,
|
||||
base_url,
|
||||
api_key,
|
||||
offer.id,
|
||||
stage,
|
||||
num_stages,
|
||||
seed_addr,
|
||||
seed_relay,
|
||||
image,
|
||||
label,
|
||||
diag_env,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(info) => {
|
||||
info_opt = Some(info);
|
||||
break;
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"lease_chain: stage {stage} create on offer {} failed (attempt {attempt}/{MAX_CREATE_ATTEMPTS}): {e}",
|
||||
offer.id,
|
||||
);
|
||||
last_err = Some(e);
|
||||
// Try the next-best offer; the loop excludes the
|
||||
// already-tried id via `tried_offer_ids`.
|
||||
}
|
||||
}
|
||||
}
|
||||
match info_opt {
|
||||
Some(info) => created.push(info),
|
||||
None => {
|
||||
let ids: Vec<u64> = created.iter().map(|c| c.contract_id).collect();
|
||||
let _ = destroy_all_instances(client, base_url, api_key, &ids).await;
|
||||
return Err(format!(
|
||||
"lease_chain: stage {stage} could not be created after {MAX_CREATE_ATTEMPTS} attempts: {}",
|
||||
last_err.unwrap_or_default(),
|
||||
));
|
||||
match provision_stage(
|
||||
client,
|
||||
base_url,
|
||||
api_key,
|
||||
gpu_name,
|
||||
stage,
|
||||
num_stages,
|
||||
seed_addr,
|
||||
seed_relay,
|
||||
image,
|
||||
label,
|
||||
stage_secrets,
|
||||
diag_env,
|
||||
&mut tried_offer_ids,
|
||||
&mut used_host_ids,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(info) => created.push(info),
|
||||
Err(e) => {
|
||||
rollback(client, base_url, api_key, &created).await;
|
||||
return Err(format!("lease_chain: {e}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for info in &created {
|
||||
if let Err(e) = wait_for_running(
|
||||
client,
|
||||
base_url,
|
||||
api_key,
|
||||
info.contract_id,
|
||||
poll_interval,
|
||||
max_polls,
|
||||
)
|
||||
.await
|
||||
{
|
||||
let ids: Vec<u64> = created.iter().map(|c| c.contract_id).collect();
|
||||
let _ = destroy_all_instances(client, base_url, api_key, &ids).await;
|
||||
return Err(format!(
|
||||
"lease_chain: contract {} did not reach running: {e}",
|
||||
info.contract_id,
|
||||
));
|
||||
// Phase 2 — wait for each instance to reach `running`. A host that stops
|
||||
// after loading the image must not abort the lease: destroy it and
|
||||
// re-provision the SAME stage slot (same stage index + pinned identity)
|
||||
// on a fresh offer, up to MAX_REPLACE_ATTEMPTS, before giving up.
|
||||
const MAX_REPLACE_ATTEMPTS: u32 = 3;
|
||||
for stage in 0..num_stages {
|
||||
let idx = stage as usize;
|
||||
let mut replaced: u32 = 0;
|
||||
loop {
|
||||
let cid = created[idx].contract_id;
|
||||
match wait_for_running(client, base_url, api_key, cid, poll_interval, max_polls).await {
|
||||
Ok(_) => break,
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"lease_chain: stage {stage} contract {cid} did not reach running: {e}"
|
||||
);
|
||||
// Tear down the dead instance (retried, so a 429 can't orphan it).
|
||||
if let Err(de) =
|
||||
destroy_instance_with_retry(client, base_url, api_key, cid).await
|
||||
{
|
||||
eprintln!(
|
||||
"lease_chain: WARNING could not destroy dead contract {cid}: {de}"
|
||||
);
|
||||
}
|
||||
replaced += 1;
|
||||
if replaced > MAX_REPLACE_ATTEMPTS {
|
||||
// Give up on this stage; roll back the survivors (cid is
|
||||
// already destroyed, so exclude it).
|
||||
let survivors: Vec<InstanceInfo> = created
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(i, _)| *i != idx)
|
||||
.map(|(_, c)| c.clone())
|
||||
.collect();
|
||||
rollback(client, base_url, api_key, &survivors).await;
|
||||
return Err(format!(
|
||||
"lease_chain: stage {stage} never reached running after \
|
||||
{MAX_REPLACE_ATTEMPTS} replacements; last error: {e}"
|
||||
));
|
||||
}
|
||||
eprintln!(
|
||||
"lease_chain: replacing stage {stage} (replacement {replaced}/{MAX_REPLACE_ATTEMPTS})"
|
||||
);
|
||||
match provision_stage(
|
||||
client,
|
||||
base_url,
|
||||
api_key,
|
||||
gpu_name,
|
||||
stage,
|
||||
num_stages,
|
||||
seed_addr,
|
||||
seed_relay,
|
||||
image,
|
||||
label,
|
||||
stage_secrets,
|
||||
diag_env,
|
||||
&mut tried_offer_ids,
|
||||
&mut used_host_ids,
|
||||
)
|
||||
.await
|
||||
{
|
||||
// Loop re-waits on the replacement instance.
|
||||
Ok(info) => created[idx] = info,
|
||||
Err(pe) => {
|
||||
let survivors: Vec<InstanceInfo> = created
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(i, _)| *i != idx)
|
||||
.map(|(_, c)| c.clone())
|
||||
.collect();
|
||||
rollback(client, base_url, api_key, &survivors).await;
|
||||
return Err(format!(
|
||||
"lease_chain: stage {stage} replacement could not be provisioned: {pe}"
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
194
examples/pipeline-parallel-inference/tests/spec_probes.rs
Normal file
194
examples/pipeline-parallel-inference/tests/spec_probes.rs
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
//! Adversarial spec-anchored probes built per §4/§5 contracts of
|
||||
//! `PP_DEPLOY_FIX_SPEC.md`. Written to break the implementation, not echo it.
|
||||
//!
|
||||
//! Run with:
|
||||
//! \
|
||||
//! cargo test --test spec_probes -- --nocapture
|
||||
|
||||
use pipeline_parallel_inference::orchestrator::{
|
||||
spawn_chain, stage_roster_event_fields, resolve_roster, ChainGuard,
|
||||
StageRosterEntry, SpawnChainError,
|
||||
};
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
// ─── §4.5 stage roster event fields ─────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn s45_roster_lists_every_stage_in_index_order_with_required_fields() {
|
||||
let roster = vec![
|
||||
StageRosterEntry {
|
||||
stage_index: 0,
|
||||
node_id_hex: "00".repeat(32),
|
||||
node_id_short: "00".repeat(4),
|
||||
},
|
||||
StageRosterEntry {
|
||||
stage_index: 1,
|
||||
node_id_hex: "ff".repeat(32),
|
||||
node_id_short: "ff".repeat(4),
|
||||
},
|
||||
StageRosterEntry {
|
||||
stage_index: 2,
|
||||
node_id_hex: "aa".repeat(32),
|
||||
node_id_short: "aa".repeat(4),
|
||||
},
|
||||
];
|
||||
let fields = stage_roster_event_fields(7, &roster);
|
||||
assert_eq!(fields["drive_seq"], 7);
|
||||
let stages = fields["stages"].as_array().unwrap();
|
||||
assert_eq!(stages.len(), 3);
|
||||
for (k, s) in stages.iter().enumerate() {
|
||||
// spec §4.5 fields
|
||||
assert!(s["stage_index"].as_u64().is_some());
|
||||
assert!(s["node_id_hex"].as_str().is_some());
|
||||
assert!(s["node_id_short"].as_str().is_some());
|
||||
// ordered by stage_index ascending
|
||||
assert_eq!(s["stage_index"].as_u64().unwrap(), k as u64);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn s45_drive_seq_changes_per_drive() {
|
||||
// Just confirm: the helper takes drive_seq as a parameter, so the
|
||||
// orchestrator can vary it per drive (spec §4.5: "emitted on every
|
||||
// drive (including redeploys)").
|
||||
let roster = vec![StageRosterEntry {
|
||||
stage_index: 0,
|
||||
node_id_hex: "00".repeat(32),
|
||||
node_id_short: "00".repeat(4),
|
||||
}];
|
||||
assert_ne!(
|
||||
stage_roster_event_fields(1, &roster),
|
||||
stage_roster_event_fields(2, &roster),
|
||||
);
|
||||
}
|
||||
|
||||
// ─── §4.6 pipeline-wired ────────────────────────────────────────────
|
||||
//
|
||||
// Negative side: resolve_roster MUST NOT report all-resolved until every
|
||||
// pp-stage-K is up. Without all stages resolved, an emitter cannot emit
|
||||
// pp_pipeline_wired (spec §4.6 negative space).
|
||||
|
||||
#[test]
|
||||
fn s46_resolve_roster_times_out_when_a_stage_never_registers() {
|
||||
// Stage 1 (out of 3) is never resolvable.
|
||||
let started_at = std::time::Instant::now();
|
||||
let result = resolve_roster(
|
||||
3,
|
||||
Duration::from_millis(150),
|
||||
Duration::from_millis(10),
|
||||
|k| if k == 1 { None } else { Some("ab".repeat(32)) },
|
||||
);
|
||||
let elapsed = started_at.elapsed();
|
||||
match result {
|
||||
Err(e) => {
|
||||
let msg = format!("{e}");
|
||||
assert!(msg.contains("[1]"), "missing-stages list must name stage 1: {msg}");
|
||||
assert!(elapsed >= Duration::from_millis(100), "must wait full budget: {elapsed:?}");
|
||||
}
|
||||
Ok(r) => panic!("expected timeout, got Ok({:?})", r),
|
||||
}
|
||||
}
|
||||
|
||||
// ─── §4.9 redeploy reachable-set semantics ──────────────────────────
|
||||
//
|
||||
// The redeploy semantics live inline in pp_smoke_run::run_vastai (not
|
||||
// extractable from pipeline_parallel_inference::orchestrator), so the
|
||||
// contract is exercised here by a spawn_chain analogue: failures on one
|
||||
// stage MUST NOT prevent attempts on later stages, and the chain's
|
||||
// "did all succeed" predicate is what gates downstream drive.
|
||||
|
||||
#[test]
|
||||
fn s49_independent_per_host_attempt_is_visible_in_chain_guard() {
|
||||
// Spawn_chain treats each stage independently in the sense that
|
||||
// failure rolls back what was spawned. The negative case: if stage 1
|
||||
// is the one that fails, stage 0 (already spawned) is killed (no
|
||||
// orphaned subprocess) — proxy for §4.9's "per-host result" guarantee:
|
||||
// operation does not leak processes if any one host fails.
|
||||
let mut spawn_count = 0;
|
||||
let result = spawn_chain(3, Duration::from_millis(200), |_ctx| {
|
||||
spawn_count += 1;
|
||||
let mut cmd = std::process::Command::new("sh");
|
||||
if spawn_count == 2 {
|
||||
// Force stage 1 to never announce
|
||||
cmd.arg("-c").arg("sleep 10");
|
||||
} else {
|
||||
cmd.arg("-c").arg(
|
||||
"echo PP_GPU_NODE_ADDR aabbccddeeff0011223344556677889900aabbccddeeff00112233445566778899 127.0.0.1:9999; \
|
||||
sleep 10",
|
||||
);
|
||||
}
|
||||
cmd
|
||||
});
|
||||
// stage 1 timeout aborts
|
||||
match result {
|
||||
Err(SpawnChainError::AddressTimeout { stage, .. }) => {
|
||||
assert_eq!(stage, 1, "expected timeout on stage 1, got {stage}");
|
||||
}
|
||||
other => panic!("expected AddressTimeout(stage=1), got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
// ─── §4.5 negative space: redeployable roster ───────────────────────
|
||||
|
||||
#[test]
|
||||
fn s45_stages_list_carries_stage_index_node_id_hex_node_id_short() {
|
||||
let r = vec![StageRosterEntry {
|
||||
stage_index: 42,
|
||||
node_id_hex: "deadbeef".repeat(8),
|
||||
node_id_short: "dead".repeat(2),
|
||||
}];
|
||||
let v = stage_roster_event_fields(1, &r);
|
||||
let entry = &v["stages"][0];
|
||||
let keys: std::collections::HashSet<&str> = entry
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.keys()
|
||||
.map(|s| s.as_str())
|
||||
.collect();
|
||||
// Spec §4.5: at minimum these three fields
|
||||
for f in ["stage_index", "node_id_hex", "node_id_short"] {
|
||||
assert!(keys.contains(f), "missing field {f} in roster entry");
|
||||
}
|
||||
}
|
||||
|
||||
// ─── §5.1 binary swap default-off ───────────────────────────────────
|
||||
//
|
||||
// We can't exec pp-gpu-node from within a cargo test (and shouldn't —
|
||||
// it would attempt SWIM joins). But we can confirm the gate's surface:
|
||||
// `PP_BINARY_SWAP_URL` unset = inert. The function `prototype_binary_swap_maybe_apply`
|
||||
// is private to the binary; we observe its inertness indirectly by
|
||||
// confirming the worker binary boots its normal path when env unset.
|
||||
//
|
||||
// This is exercised in `s51_binary_swap_disabled_by_default` below by
|
||||
// invoking pp-gpu-node with no swap env and confirming it reaches the
|
||||
// normal STAGE-required check (exit 2), not a swap-related error.
|
||||
|
||||
// ─── §5.2 PP_PREFLIGHT_HF default-off ───────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn s52_preflight_hf_off_by_default() {
|
||||
use pipeline_parallel_inference::vastai::prototype_preflight_hf;
|
||||
// SAFETY: this test runs single-threaded under cargo test's default
|
||||
// (one #[test] at a time per process is not the default, but no
|
||||
// other test reads this var concurrently).
|
||||
unsafe {
|
||||
std::env::remove_var("PP_PREFLIGHT_HF");
|
||||
}
|
||||
assert!(!prototype_preflight_hf::enabled(), "default MUST be off");
|
||||
unsafe {
|
||||
std::env::set_var("PP_PREFLIGHT_HF", "0");
|
||||
}
|
||||
assert!(!prototype_preflight_hf::enabled(), "PP_PREFLIGHT_HF=0 MUST be off");
|
||||
unsafe {
|
||||
std::env::set_var("PP_PREFLIGHT_HF", "false");
|
||||
}
|
||||
assert!(!prototype_preflight_hf::enabled(), "PP_PREFLIGHT_HF=false MUST be off");
|
||||
unsafe {
|
||||
std::env::set_var("PP_PREFLIGHT_HF", "1");
|
||||
}
|
||||
assert!(prototype_preflight_hf::enabled(), "PP_PREFLIGHT_HF=1 should turn it on");
|
||||
unsafe {
|
||||
std::env::remove_var("PP_PREFLIGHT_HF");
|
||||
}
|
||||
}
|
||||
|
|
@ -364,6 +364,7 @@ async fn lease_chain_finds_n_distinct_offers() {
|
|||
None,
|
||||
IMAGE,
|
||||
None,
|
||||
None,
|
||||
Duration::from_millis(10),
|
||||
3,
|
||||
None,
|
||||
|
|
@ -371,14 +372,30 @@ async fn lease_chain_finds_n_distinct_offers() {
|
|||
.await
|
||||
.expect("lease_chain must succeed when N distinct offers exist");
|
||||
|
||||
// Contract: N pairwise-distinct instances, each drawn from the catalog.
|
||||
// Which offer a given stage lands on is an implementation detail
|
||||
// (find_offer takes the median-priced candidate, and a create that misses
|
||||
// an unmounted offer falls back to another), so we assert distinctness +
|
||||
// membership rather than a fixed id order.
|
||||
let ids: Vec<u64> = infos.iter().map(|i| i.contract_id).collect();
|
||||
assert_eq!(ids, vec![9000, 9001, 9002, 9003]);
|
||||
assert_eq!(ids.len(), 4, "must lease N=4 instances");
|
||||
let mut unique = ids.clone();
|
||||
unique.sort();
|
||||
unique.dedup();
|
||||
assert_eq!(unique.len(), 4, "the 4 leased contracts must be distinct");
|
||||
assert!(
|
||||
ids.iter().all(|c| (9000..=9003).contains(c)),
|
||||
"every leased contract must come from the mounted catalog, got {ids:?}",
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lease_chain_creates_n_instances_with_distinct_stage_env() {
|
||||
let server = MockServer::start().await;
|
||||
mount_offers(&server, 4).await;
|
||||
// Exactly N offers, all creatable, so the median-priced selection never
|
||||
// has to fall back to an unmounted offer — keeping the create count at one
|
||||
// PUT per stage regardless of which offer each stage picks.
|
||||
mount_offers(&server, 3).await;
|
||||
mount_creates_ok(&server, 3).await;
|
||||
mount_status_running(&server, &[9000, 9001, 9002]).await;
|
||||
|
||||
|
|
@ -393,6 +410,7 @@ async fn lease_chain_creates_n_instances_with_distinct_stage_env() {
|
|||
None,
|
||||
IMAGE,
|
||||
None,
|
||||
None,
|
||||
Duration::from_millis(10),
|
||||
3,
|
||||
None,
|
||||
|
|
@ -405,29 +423,32 @@ async fn lease_chain_creates_n_instances_with_distinct_stage_env() {
|
|||
.iter()
|
||||
.filter(|r| r.method.as_ref() == "PUT")
|
||||
.collect();
|
||||
assert_eq!(puts.len(), 3, "exactly one PUT per stage");
|
||||
assert_eq!(puts.len(), 3, "exactly one create (PUT) per stage");
|
||||
|
||||
// Pair each PUT body with its offer id so we can assert STAGE
|
||||
// independently of the on-wire ordering of the requests.
|
||||
let mut by_offer: std::collections::HashMap<String, serde_json::Value> =
|
||||
std::collections::HashMap::new();
|
||||
for r in puts {
|
||||
// Contract: the three creates collectively cover STAGE 0/1/2 exactly once
|
||||
// each, and every one carries NUM_STAGES=3. We assert STAGE as a set
|
||||
// rather than tying it to a specific offer id, since which offer hosts a
|
||||
// given stage is up to the (median-priced) selector.
|
||||
let mut stages: Vec<String> = Vec::new();
|
||||
for r in &puts {
|
||||
let body: serde_json::Value = serde_json::from_slice(&r.body).unwrap();
|
||||
by_offer.insert(r.url.path().to_string(), body);
|
||||
}
|
||||
for i in 0..3u32 {
|
||||
let path = format!("/api/v0/asks/{}/", 1000 + i);
|
||||
let body = by_offer
|
||||
.get(&path)
|
||||
.unwrap_or_else(|| panic!("no PUT to offer {}", 1000 + i));
|
||||
assert_eq!(
|
||||
body["env"]["STAGE"],
|
||||
i.to_string(),
|
||||
"offer {} must carry STAGE={i}",
|
||||
1000 + i,
|
||||
body["env"]["NUM_STAGES"], "3",
|
||||
"every create must carry NUM_STAGES=3",
|
||||
);
|
||||
stages.push(
|
||||
body["env"]["STAGE"]
|
||||
.as_str()
|
||||
.expect("STAGE env must be a string")
|
||||
.to_string(),
|
||||
);
|
||||
assert_eq!(body["env"]["NUM_STAGES"], "3");
|
||||
}
|
||||
stages.sort();
|
||||
assert_eq!(
|
||||
stages,
|
||||
vec!["0", "1", "2"],
|
||||
"the creates must cover STAGE 0,1,2 exactly once each",
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
|
@ -477,6 +498,7 @@ async fn lease_chain_rolls_back_on_partial_creation() {
|
|||
None,
|
||||
IMAGE,
|
||||
None,
|
||||
None,
|
||||
Duration::from_millis(10),
|
||||
3,
|
||||
None,
|
||||
|
|
@ -505,6 +527,7 @@ async fn lease_chain_waits_for_running_per_contract() {
|
|||
None,
|
||||
IMAGE,
|
||||
None,
|
||||
None,
|
||||
Duration::from_millis(10),
|
||||
3,
|
||||
None,
|
||||
|
|
@ -535,6 +558,86 @@ async fn lease_chain_waits_for_running_per_contract() {
|
|||
}
|
||||
}
|
||||
|
||||
/// A host that loads the image then stops (never reaching `running`) must not
|
||||
/// sink the whole lease: the stage it was filling is destroyed and
|
||||
/// re-provisioned on a fresh offer, and `lease_chain` still returns N distinct
|
||||
/// running contracts — none of them the dead one. (This is the real failure
|
||||
/// that aborted a 12-node lease: one instance reported "stopped: Successfully
|
||||
/// loaded <image>".)
|
||||
#[tokio::test]
|
||||
async fn lease_chain_replaces_a_stage_that_stops_before_running() {
|
||||
let server = MockServer::start().await;
|
||||
// Four offers (1000..1003) at ascending price; find_offer takes the
|
||||
// median, so stage 0 lands on offer 1002, stage 1 on 1001, and the
|
||||
// replacement for stage 0 on 1003.
|
||||
mount_offers(&server, 4).await;
|
||||
for (offer, contract) in [(1002u32, 9002u64), (1001, 9001), (1003, 9003)] {
|
||||
Mock::given(method("PUT"))
|
||||
.and(path_regex(format!("^/api/v0/asks/{offer}/$").as_str()))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.set_body_json(serde_json::json!({ "new_contract": contract })),
|
||||
)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
}
|
||||
// 9002 stops after loading the image (the failure we are guarding against).
|
||||
Mock::given(method("GET"))
|
||||
.and(path_regex("^/api/v0/instances/9002/$"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
|
||||
"instances": {
|
||||
"actual_status": "created",
|
||||
"intended_status": "stopped",
|
||||
"status_msg": "Successfully loaded zacheryasc/swactor-pp-gpu:latest",
|
||||
}
|
||||
})))
|
||||
.mount(&server)
|
||||
.await;
|
||||
// The replacement (9003) and the healthy stage 1 (9001) both come up.
|
||||
mount_status_running(&server, &[9003, 9001]).await;
|
||||
// The dead instance must be torn down so it stops billing.
|
||||
Mock::given(method("DELETE"))
|
||||
.and(path_regex("^/api/v0/instances/9002/$"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200).set_body_json(serde_json::json!({ "success": true })),
|
||||
)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let client = Client::new();
|
||||
let infos = vastai::lease_chain(
|
||||
&client,
|
||||
&server.uri(),
|
||||
API_KEY,
|
||||
"RTX 4090",
|
||||
2,
|
||||
SEED_ADDR,
|
||||
None,
|
||||
IMAGE,
|
||||
None,
|
||||
None,
|
||||
Duration::from_millis(10),
|
||||
3,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("lease_chain must recover by replacing the stopped stage");
|
||||
|
||||
let ids: Vec<u64> = infos.iter().map(|i| i.contract_id).collect();
|
||||
assert_eq!(ids.len(), 2, "lease must still yield N=2 running instances");
|
||||
assert!(
|
||||
!ids.contains(&9002),
|
||||
"the stopped instance must not appear in the lease, got {ids:?}",
|
||||
);
|
||||
let mut unique = ids.clone();
|
||||
unique.sort();
|
||||
unique.dedup();
|
||||
assert_eq!(unique.len(), 2, "leased contracts must be distinct");
|
||||
// The DELETE + create expectations are verified on server drop.
|
||||
}
|
||||
|
||||
// ─── §9.3 convergence wait ────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -557,8 +557,12 @@ async fn failure_to_create_kth_instance_triggers_destroy_of_prior_at_n_5() {
|
|||
#[tokio::test]
|
||||
async fn find_offer_excludes_all_prior_offer_ids() {
|
||||
let server = MockServer::start().await;
|
||||
// Catalog of 5 offers in ascending price; find_offer returns the
|
||||
// cheapest not in `exclude_ids`, so the chain picks 100, 101, 102, 103.
|
||||
// Catalog of 5 offers; each find_offer in the chain adds its pick to
|
||||
// `exclude_ids`, so the 4 chosen offers must be pairwise distinct and
|
||||
// all drawn from the catalog. Which of the eligible offers a single
|
||||
// find_offer returns is an implementation detail (it takes the
|
||||
// median-priced candidate, not the cheapest), so this asserts the
|
||||
// distinctness/membership contract rather than a fixed id order.
|
||||
Mock::given(method("GET"))
|
||||
.and(path_regex(r"^/api/v0/bundles/"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
|
||||
|
|
@ -578,7 +582,11 @@ async fn find_offer_excludes_all_prior_offer_ids() {
|
|||
.await
|
||||
.expect("4 distinct offers exist in the catalog");
|
||||
let ids: Vec<u64> = chosen.iter().map(|o| o.id).collect();
|
||||
assert_eq!(ids, vec![100, 101, 102, 103]);
|
||||
let catalog = [100u64, 101, 102, 103, 104];
|
||||
assert!(
|
||||
ids.iter().all(|id| catalog.contains(id)),
|
||||
"every chosen offer must come from the catalog, got {ids:?}"
|
||||
);
|
||||
let mut unique: Vec<u64> = ids.clone();
|
||||
unique.sort();
|
||||
unique.dedup();
|
||||
|
|
|
|||
Loading…
Reference in a new issue