2026-08-22 17:18:25 +00:00
|
|
|
use std::path::{Path, PathBuf};
|
|
|
|
|
use std::process::{Command, ExitCode, Stdio};
|
2026-08-17 22:51:08 +00:00
|
|
|
use std::time::Instant;
|
2026-02-15 17:03:31 +00:00
|
|
|
|
|
|
|
|
struct TestStep {
|
|
|
|
|
label: &'static str,
|
|
|
|
|
args: &'static [&'static str],
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-22 17:18:25 +00:00
|
|
|
const TEST_STEPS: &[TestStep] = &[
|
2026-06-25 12:30:18 +00:00
|
|
|
TestStep {
|
2026-08-22 17:18:25 +00:00
|
|
|
label: "strict workspace lint",
|
|
|
|
|
args: &["lint"],
|
2026-06-25 12:30:18 +00:00
|
|
|
},
|
|
|
|
|
TestStep {
|
2026-08-27 22:09:51 +00:00
|
|
|
label: "all Rust tests (180s per-test timeout)",
|
2026-08-22 17:18:25 +00:00
|
|
|
args: &["nextest", "run", "--workspace", "--all-features"],
|
2026-06-25 12:30:18 +00:00
|
|
|
},
|
|
|
|
|
TestStep {
|
2026-08-22 17:18:25 +00:00
|
|
|
label: "all Rust doctests",
|
|
|
|
|
args: &[
|
|
|
|
|
"test",
|
|
|
|
|
"--workspace",
|
|
|
|
|
"--all-features",
|
|
|
|
|
"--doc",
|
|
|
|
|
"--exclude",
|
|
|
|
|
"python",
|
|
|
|
|
"--exclude",
|
|
|
|
|
"wasm-runtime",
|
|
|
|
|
],
|
2026-06-25 12:30:18 +00:00
|
|
|
},
|
|
|
|
|
];
|
2026-02-24 09:12:28 +00:00
|
|
|
|
2026-06-25 12:30:18 +00:00
|
|
|
fn cargo_bin() -> String {
|
|
|
|
|
option_env!("CARGO")
|
|
|
|
|
.map(str::to_string)
|
|
|
|
|
.unwrap_or_else(|| "cargo".to_string())
|
2026-02-15 17:03:31 +00:00
|
|
|
}
|
|
|
|
|
|
feat(data-plane): add routed SPSC stream transport
Implement namespace-owned stream rendezvous with incarnation isolation, cancellation, recovery, bounded local byte rings, and host/child actor protocols.
Add a replaceable stream transport boundary plus an Iroh ALPN adapter with framed records, backpressure, reconnect handling, and terminal propagation.
Wire stream sources and sinks through Myelin job deployment and Python bindings, with guarantee coverage across blob, namespace, job, byte-ring, and transport paths.
Sanitize nested Cargo build context in xtask so lint, Nextest, doctest, and maturin invocations retain stable fingerprints.
2026-08-23 08:47:31 +00:00
|
|
|
fn remove_inherited_build_context(command: &mut Command) {
|
|
|
|
|
for (name, _) in std::env::vars_os() {
|
|
|
|
|
let name_text = name.to_string_lossy();
|
|
|
|
|
let package_scoped = name_text.starts_with("CARGO_PKG_")
|
|
|
|
|
|| name_text.starts_with("CARGO_FEATURE_")
|
|
|
|
|
|| name_text.starts_with("CARGO_CFG_")
|
|
|
|
|
|| name_text.starts_with("DEP_");
|
|
|
|
|
let build_scoped = matches!(
|
|
|
|
|
name_text.as_ref(),
|
|
|
|
|
"CARGO_BIN_NAME"
|
|
|
|
|
| "CARGO_CRATE_NAME"
|
|
|
|
|
| "CARGO_MANIFEST_DIR"
|
|
|
|
|
| "CARGO_MANIFEST_PATH"
|
|
|
|
|
| "CARGO_PRIMARY_PACKAGE"
|
|
|
|
|
| "DEBUG"
|
|
|
|
|
| "HOST"
|
|
|
|
|
| "NUM_JOBS"
|
|
|
|
|
| "OPT_LEVEL"
|
|
|
|
|
| "OUT_DIR"
|
|
|
|
|
| "PROFILE"
|
|
|
|
|
| "PYO3_ENVIRONMENT_SIGNATURE"
|
|
|
|
|
| "TARGET"
|
|
|
|
|
);
|
|
|
|
|
if package_scoped || build_scoped {
|
|
|
|
|
command.env_remove(name);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn cargo_command() -> Command {
|
|
|
|
|
let mut command = Command::new(cargo_bin());
|
|
|
|
|
remove_inherited_build_context(&mut command);
|
|
|
|
|
command
|
|
|
|
|
}
|
|
|
|
|
|
2026-02-15 17:03:31 +00:00
|
|
|
fn print_usage() {
|
|
|
|
|
println!(
|
|
|
|
|
"\
|
2026-07-05 09:59:51 +00:00
|
|
|
USAGE: cargo xtask <command>
|
2026-02-15 17:03:31 +00:00
|
|
|
|
2026-06-25 12:30:18 +00:00
|
|
|
COMMANDS:
|
demo: rename xtask demo command; dashboard-established data-plane edges
Rename `cargo xtask provisioning-reconciler-demo` to `cargo xtask demo`
(CLI dispatch, help, child re-exec argv, launch spec strings, module dir
xtask/src/provisioning_demo -> xtask/src/demo).
Add iteration-1 data-plane edges, established from Fleet Control:
- Fleet Control "edge" button -> POST /control/edge (new
ControlCommand::EstablishEdge) -> supervisor actor resolves the node's
advertised EndpointAddr (stashed in NodeRuntime by AnnounceActor) and
provisions a real outbound EdgeRuntime (arena ring lease, recorder
WorkerPort, EDGE_ALPN send pump) in a new edge pump thread.
- Node gains EDGE_ALPN, an actor bridge decoding EdgeProvision gossip,
and a NodeEdgeAgent that provisions its (single) inbound edge, polls
it, mirrors observations onto the node.edge telemetry channel
(render-only), and answers EdgeAck gossip which terminates the
supervisor's provision retries. Node teardown replaces its inbound on
re-provision; supervisor replaces sessions per node and tears them
down on node exit/replacement/shutdown.
- The edge pump runs on the engine's blocking pool with sole session
ownership (commands in, state mirror + feed lines out): the connect
handshake blocks its thread and must not run on a Tokio worker or
share a lock with the actor. Connects are bounded (10s) so a dead
node faults its session instead of wedging edge polling.
- iroh-driver: retain_telemetry_connections() opts an application out
of the driver-owned TELEMETRY_ALPN ingress so the node's pull server
can drain those connections itself (the actor-bridge pump would
otherwise claim them).
- Dashboard: edges array in the reconciler snapshot, per-node edge
badges and edge button in Fleet Control, node_edges render mirror.
2026-08-16 20:11:20 +00:00
|
|
|
demo [--port n] [--nodes n] [--docker]
|
2026-08-17 22:51:08 +00:00
|
|
|
Run the visual provisioning-reconciler demo.
|
|
|
|
|
check-telemetry-isolation
|
|
|
|
|
Verify no frame types appear in control-plane modules.
|
2026-08-27 22:09:51 +00:00
|
|
|
lint Run the Python timeout policy and strict workspace Clippy.
|
2026-08-22 17:18:25 +00:00
|
|
|
test Run strict lint plus every Rust and Python test."
|
2026-02-15 17:03:31 +00:00
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-22 17:18:25 +00:00
|
|
|
fn run_step(step: &TestStep, python: &Path) -> bool {
|
2026-06-25 12:30:18 +00:00
|
|
|
println!("\n=== {} ===", step.label);
|
|
|
|
|
println!(" cargo {}", step.args.join(" "));
|
|
|
|
|
println!();
|
2026-02-15 17:03:31 +00:00
|
|
|
|
2026-08-22 17:18:25 +00:00
|
|
|
match swactor_process::command_status(
|
feat(data-plane): add routed SPSC stream transport
Implement namespace-owned stream rendezvous with incarnation isolation, cancellation, recovery, bounded local byte rings, and host/child actor protocols.
Add a replaceable stream transport boundary plus an Iroh ALPN adapter with framed records, backpressure, reconnect handling, and terminal propagation.
Wire stream sources and sinks through Myelin job deployment and Python bindings, with guarantee coverage across blob, namespace, job, byte-ring, and transport paths.
Sanitize nested Cargo build context in xtask so lint, Nextest, doctest, and maturin invocations retain stable fingerprints.
2026-08-23 08:47:31 +00:00
|
|
|
cargo_command().args(step.args).env("PYO3_PYTHON", python),
|
2026-08-22 17:18:25 +00:00
|
|
|
) {
|
2026-06-25 12:30:18 +00:00
|
|
|
Ok(status) => status.success(),
|
|
|
|
|
Err(error) => {
|
|
|
|
|
eprintln!("Failed to execute cargo: {error}");
|
|
|
|
|
false
|
2026-02-15 17:03:31 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-25 12:30:18 +00:00
|
|
|
fn run_tests() -> ExitCode {
|
2026-02-15 17:03:31 +00:00
|
|
|
let start = Instant::now();
|
2026-08-22 17:18:25 +00:00
|
|
|
let Some(python) = PythonTestTools::discover() else {
|
|
|
|
|
return ExitCode::from(1);
|
|
|
|
|
};
|
|
|
|
|
if !nextest_available() {
|
|
|
|
|
return ExitCode::from(1);
|
|
|
|
|
}
|
|
|
|
|
if !check_telemetry_isolation() {
|
|
|
|
|
return ExitCode::from(1);
|
|
|
|
|
}
|
|
|
|
|
for (index, step) in TEST_STEPS.iter().enumerate() {
|
|
|
|
|
if !run_step(step, &python.python) {
|
2026-06-25 12:30:18 +00:00
|
|
|
eprintln!(
|
|
|
|
|
"\n--- FAILED after {:.1}s ({index} passed, 1 failed) ---",
|
|
|
|
|
start.elapsed().as_secs_f64()
|
|
|
|
|
);
|
|
|
|
|
return ExitCode::from(1);
|
2026-02-15 17:03:31 +00:00
|
|
|
}
|
|
|
|
|
}
|
2026-08-22 17:18:25 +00:00
|
|
|
if !python.run() {
|
|
|
|
|
eprintln!(
|
|
|
|
|
"\n--- FAILED after {:.1}s (Python tests failed) ---",
|
|
|
|
|
start.elapsed().as_secs_f64()
|
|
|
|
|
);
|
|
|
|
|
return ExitCode::from(1);
|
|
|
|
|
}
|
2026-02-15 17:03:31 +00:00
|
|
|
|
|
|
|
|
println!(
|
2026-06-25 12:30:18 +00:00
|
|
|
"\n--- All {} step(s) passed in {:.1}s ---",
|
2026-08-22 17:18:25 +00:00
|
|
|
TEST_STEPS.len() + 1,
|
2026-06-25 12:30:18 +00:00
|
|
|
start.elapsed().as_secs_f64()
|
2026-02-15 17:03:31 +00:00
|
|
|
);
|
2026-06-25 12:30:18 +00:00
|
|
|
ExitCode::SUCCESS
|
2026-02-15 17:03:31 +00:00
|
|
|
}
|
2026-02-16 15:39:27 +00:00
|
|
|
|
2026-08-27 22:09:51 +00:00
|
|
|
fn run_lint() -> ExitCode {
|
|
|
|
|
let policy = workspace_root().join("tools/timeout-policy.py");
|
|
|
|
|
let policy_status = swactor_process::command_status(Command::new("python3").arg(policy));
|
|
|
|
|
if !policy_status.is_ok_and(|status| status.success()) {
|
|
|
|
|
return ExitCode::from(1);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let mut clippy = cargo_command();
|
|
|
|
|
clippy.args(["clippy", "--workspace", "--all-targets", "--all-features"]);
|
|
|
|
|
// Pin pyo3 to the same interpreter the Python test flow uses: a newer
|
|
|
|
|
// system python exceeds the bundled pyo3's supported maximum and would
|
|
|
|
|
// fail the workspace build here even though tests build fine.
|
|
|
|
|
if let Some(tools) = PythonTestTools::discover() {
|
|
|
|
|
clippy.env("PYO3_PYTHON", tools.python);
|
|
|
|
|
}
|
|
|
|
|
match swactor_process::command_status(&mut clippy) {
|
|
|
|
|
Ok(status) if status.success() => ExitCode::SUCCESS,
|
|
|
|
|
Ok(_) => ExitCode::from(1),
|
|
|
|
|
Err(error) => {
|
|
|
|
|
eprintln!("Failed to execute cargo clippy: {error}");
|
|
|
|
|
ExitCode::from(1)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-22 17:18:25 +00:00
|
|
|
fn nextest_available() -> bool {
|
|
|
|
|
let available = swactor_process::command_status(
|
feat(data-plane): add routed SPSC stream transport
Implement namespace-owned stream rendezvous with incarnation isolation, cancellation, recovery, bounded local byte rings, and host/child actor protocols.
Add a replaceable stream transport boundary plus an Iroh ALPN adapter with framed records, backpressure, reconnect handling, and terminal propagation.
Wire stream sources and sinks through Myelin job deployment and Python bindings, with guarantee coverage across blob, namespace, job, byte-ring, and transport paths.
Sanitize nested Cargo build context in xtask so lint, Nextest, doctest, and maturin invocations retain stable fingerprints.
2026-08-23 08:47:31 +00:00
|
|
|
cargo_command()
|
2026-08-22 17:18:25 +00:00
|
|
|
.args(["nextest", "--version"])
|
|
|
|
|
.stdout(Stdio::null())
|
|
|
|
|
.stderr(Stdio::null()),
|
|
|
|
|
)
|
|
|
|
|
.is_ok_and(|status| status.success());
|
|
|
|
|
if !available {
|
|
|
|
|
eprintln!("cargo-nextest is required; install it from https://nexte.st/docs/installation/");
|
|
|
|
|
}
|
|
|
|
|
available
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
struct PythonTestTools {
|
|
|
|
|
directory: PathBuf,
|
|
|
|
|
maturin: PathBuf,
|
|
|
|
|
python: PathBuf,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
impl PythonTestTools {
|
|
|
|
|
fn discover() -> Option<Self> {
|
|
|
|
|
let directory = workspace_root().join("crates/bindings/python");
|
|
|
|
|
let tools = Self {
|
|
|
|
|
maturin: directory.join(".venv/bin/maturin"),
|
|
|
|
|
python: directory.join(".venv/bin/python"),
|
|
|
|
|
directory,
|
|
|
|
|
};
|
|
|
|
|
if tools.maturin.is_file() && tools.python.is_file() {
|
|
|
|
|
Some(tools)
|
|
|
|
|
} else {
|
|
|
|
|
eprintln!(
|
|
|
|
|
"Python test environment is missing; run `uv sync --project {}` first",
|
|
|
|
|
tools.directory.display()
|
|
|
|
|
);
|
|
|
|
|
None
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn run(&self) -> bool {
|
|
|
|
|
println!("\n=== all Python tests ===");
|
feat(data-plane): add routed SPSC stream transport
Implement namespace-owned stream rendezvous with incarnation isolation, cancellation, recovery, bounded local byte rings, and host/child actor protocols.
Add a replaceable stream transport boundary plus an Iroh ALPN adapter with framed records, backpressure, reconnect handling, and terminal propagation.
Wire stream sources and sinks through Myelin job deployment and Python bindings, with guarantee coverage across blob, namespace, job, byte-ring, and transport paths.
Sanitize nested Cargo build context in xtask so lint, Nextest, doctest, and maturin invocations retain stable fingerprints.
2026-08-23 08:47:31 +00:00
|
|
|
let mut build_command = Command::new(&self.maturin);
|
|
|
|
|
remove_inherited_build_context(&mut build_command);
|
2026-08-22 17:18:25 +00:00
|
|
|
let built = swactor_process::command_status(
|
feat(data-plane): add routed SPSC stream transport
Implement namespace-owned stream rendezvous with incarnation isolation, cancellation, recovery, bounded local byte rings, and host/child actor protocols.
Add a replaceable stream transport boundary plus an Iroh ALPN adapter with framed records, backpressure, reconnect handling, and terminal propagation.
Wire stream sources and sinks through Myelin job deployment and Python bindings, with guarantee coverage across blob, namespace, job, byte-ring, and transport paths.
Sanitize nested Cargo build context in xtask so lint, Nextest, doctest, and maturin invocations retain stable fingerprints.
2026-08-23 08:47:31 +00:00
|
|
|
build_command
|
2026-08-22 17:18:25 +00:00
|
|
|
.current_dir(&self.directory)
|
|
|
|
|
.arg("develop")
|
2026-08-27 22:09:51 +00:00
|
|
|
.arg("--features")
|
|
|
|
|
.arg("test-host"),
|
2026-08-22 17:18:25 +00:00
|
|
|
)
|
|
|
|
|
.is_ok_and(|status| status.success());
|
|
|
|
|
if !built {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
swactor_process::command_status(
|
|
|
|
|
Command::new(&self.python)
|
|
|
|
|
.current_dir(&self.directory)
|
|
|
|
|
.args([
|
|
|
|
|
"-m",
|
|
|
|
|
"pytest",
|
|
|
|
|
"-q",
|
|
|
|
|
"--timeout=60",
|
|
|
|
|
"tests/test_bootstrap.py",
|
|
|
|
|
"../../../tests/test_python.py",
|
|
|
|
|
]),
|
|
|
|
|
)
|
|
|
|
|
.is_ok_and(|status| status.success())
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn workspace_root() -> PathBuf {
|
|
|
|
|
Path::new(env!("CARGO_MANIFEST_DIR"))
|
|
|
|
|
.parent()
|
|
|
|
|
.expect("xtask must live directly under the workspace root")
|
|
|
|
|
.to_path_buf()
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-17 22:51:08 +00:00
|
|
|
/// Verify that control-plane modules never import telemetry frame/read-side
|
|
|
|
|
/// types. They may emit through the producer API only.
|
2026-08-22 17:18:25 +00:00
|
|
|
fn check_telemetry_isolation() -> bool {
|
2026-08-17 22:51:08 +00:00
|
|
|
const CONTROL_DIRS: &[&str] = &[
|
|
|
|
|
"apps/myelin/src/orchestration",
|
|
|
|
|
"crates/distribution/src",
|
|
|
|
|
"crates/data-plane/src",
|
|
|
|
|
"crates/provisioning/src",
|
|
|
|
|
];
|
|
|
|
|
const FORBIDDEN: &[&str] = &[
|
|
|
|
|
"telemetry::frame::",
|
|
|
|
|
"telemetry::store::",
|
|
|
|
|
"telemetry::ingest::",
|
|
|
|
|
"telemetry::views::",
|
|
|
|
|
"telemetry::transport::",
|
|
|
|
|
"CollectedTelemetryFrame",
|
|
|
|
|
];
|
2026-07-05 09:59:51 +00:00
|
|
|
|
2026-08-17 22:51:08 +00:00
|
|
|
let mut files = Vec::new();
|
|
|
|
|
for dir in CONTROL_DIRS {
|
|
|
|
|
collect_rs_files(dir, &mut files);
|
feat: mvp-chat benchmarking
Add end-to-end timing instrumentation and an xtask benchmark report for mvp-chat runs.
- benchmark_observability: add a shared stamping module — stamp(component) emitting schema/pid/monotonic+wall ms from a process-global start and sequence counter, plus unix_ms_now() — stamped onto every mvp-chat/orchestrator/worker-node event and frame-archive record
- mvp-chat: thread a run_id (new --run-id, defaults to 1) through config and the orchestrator CLI, add per-phase started/ready/failed emits for ensure_orch_binary/ensure_worker_binary/prepare_node_image, and a prompt_complete record carrying tokens_generated/elapsed_ms/final_text bytes
- orchestrator/worker-node: stamp bootstrap and prompt events, add arrival_unix_ms to archived frames, propagate MVP_RUN_ID/MVP_LOGICAL_NODE_ID/MVP_STAGE_INDEX into the tinygrad worker, default the device to CPU for the process provider, emit a prompt_rpc started span, and drop the MVP_TINYGRAD_TEST_MODE passthrough
- tinygrad_worker.py: stamp every control() event and tag it with run/node/stage env, add a CPU:X86 fallback when clang is absent, and remove the test_mode() short-circuits
- xtask: replace the flat dump-log fact assertions with a benchmark report builder (build_benchmark_report) that requires named spans (prepare_runtime, ensure_*_binary, weights_loaded, prompt_rpc) and emits per-prompt first-token/decode/tokens-per-second latency; wrap the cargo run in XtaskBenchmark synthetic frames and pass a unix-ms --run-id
Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-23 05:35:04 +00:00
|
|
|
}
|
|
|
|
|
|
2026-08-17 22:51:08 +00:00
|
|
|
let mut found = false;
|
|
|
|
|
for file in &files {
|
|
|
|
|
let Ok(src) = std::fs::read_to_string(file) else {
|
|
|
|
|
continue;
|
|
|
|
|
};
|
|
|
|
|
for (lineno, line) in src.lines().enumerate() {
|
|
|
|
|
for pattern in FORBIDDEN {
|
|
|
|
|
if line.contains(pattern) {
|
|
|
|
|
eprintln!(
|
|
|
|
|
"telemetry-isolation violation: {file}:{}: {}",
|
|
|
|
|
lineno + 1,
|
|
|
|
|
line.trim()
|
|
|
|
|
);
|
|
|
|
|
found = true;
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-07-05 09:59:51 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-17 22:51:08 +00:00
|
|
|
if found {
|
2026-07-24 19:05:11 +00:00
|
|
|
eprintln!(
|
2026-08-17 22:51:08 +00:00
|
|
|
"\ntelemetry-isolation: control-plane code must not import frame types \
|
|
|
|
|
or read-side modules. Use the telemetry producer API for emission."
|
2026-07-24 19:05:11 +00:00
|
|
|
);
|
2026-08-22 17:18:25 +00:00
|
|
|
false
|
feat: mvp-chat benchmarking
Add end-to-end timing instrumentation and an xtask benchmark report for mvp-chat runs.
- benchmark_observability: add a shared stamping module — stamp(component) emitting schema/pid/monotonic+wall ms from a process-global start and sequence counter, plus unix_ms_now() — stamped onto every mvp-chat/orchestrator/worker-node event and frame-archive record
- mvp-chat: thread a run_id (new --run-id, defaults to 1) through config and the orchestrator CLI, add per-phase started/ready/failed emits for ensure_orch_binary/ensure_worker_binary/prepare_node_image, and a prompt_complete record carrying tokens_generated/elapsed_ms/final_text bytes
- orchestrator/worker-node: stamp bootstrap and prompt events, add arrival_unix_ms to archived frames, propagate MVP_RUN_ID/MVP_LOGICAL_NODE_ID/MVP_STAGE_INDEX into the tinygrad worker, default the device to CPU for the process provider, emit a prompt_rpc started span, and drop the MVP_TINYGRAD_TEST_MODE passthrough
- tinygrad_worker.py: stamp every control() event and tag it with run/node/stage env, add a CPU:X86 fallback when clang is absent, and remove the test_mode() short-circuits
- xtask: replace the flat dump-log fact assertions with a benchmark report builder (build_benchmark_report) that requires named spans (prepare_runtime, ensure_*_binary, weights_loaded, prompt_rpc) and emits per-prompt first-token/decode/tokens-per-second latency; wrap the cargo run in XtaskBenchmark synthetic frames and pass a unix-ms --run-id
Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-23 05:35:04 +00:00
|
|
|
} else {
|
2026-08-17 22:51:08 +00:00
|
|
|
println!("telemetry-isolation: OK — no frame types in control-plane modules.");
|
2026-08-22 17:18:25 +00:00
|
|
|
true
|
2026-07-22 07:50:53 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-17 22:51:08 +00:00
|
|
|
fn collect_rs_files(dir: &str, out: &mut Vec<String>) {
|
|
|
|
|
let Ok(entries) = std::fs::read_dir(dir) else {
|
|
|
|
|
return;
|
feat: working 7B inferenced over 4 pipeline stages
Land end-to-end canonical benchmark observability and a synthetic datastream-connectivity preflight across the orchestrator, chat, worker-node, and Python tinygrad worker, plus an xtask validator, so a 4-stage 7B pipeline run is fully diagnosable.
- benchmark_observability: expand stamp() with canonical producer fields (producer_component/instance_id/process_id/sequence, wall_clock_unix_ms, monotonic_ms, clock_source) and schema_version so every event shares one envelope shape
- orchestrator_app + bin/{mvp_chat,worker_node}: stamp OrchBootstrap/OrchPromptEvent/ChatProgress/NodeEvent/SamplerHealth with the canonical fields plus span_id/parent_span_id, and add a 4-phase synthetic datastream preflight (ProducerConfigured/Connected/SyntheticEventSent/Observed) plus an endpoint_config_snapshot event on each process
- apps/mvp-node/tinygrad_worker: add apply_canonical_envelope()/datastream_endpoint_snapshot() and emit_python_datastream_preflight() mirroring the Rust preflight, and enrich benchmark_stamp() with the same producer fields
- bin/worker_node: pass MVP_DATASTREAM_ENDPOINT_ID/MVP_BENCHMARK_PRODUCER_INSTANCE/MVP_IROH_ENDPOINT_ADDR_MASK/MVP_IROH_RELAY_MODE env to the spawned tinygrad worker so its stamps identify the stage
- bin/mvp_chat: add --pipeline-parallel as an alias for --pipeline-stages (with a duplicate-guard) and bump recursion_limit
- xtask: add a benchmark-observability validator (ValidatorFinding/BenchmarkValidation, validate_benchmark_observability, canonical-stamp and stage/edge checks, evidence + gap-report builders) with tests for missing python datastream connectivity, wrong run_id, and missing span_id
Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-27 06:47:17 +00:00
|
|
|
};
|
2026-08-17 22:51:08 +00:00
|
|
|
for entry in entries.flatten() {
|
|
|
|
|
let path = entry.path();
|
|
|
|
|
if path.is_dir() {
|
|
|
|
|
if let Some(path) = path.to_str() {
|
|
|
|
|
collect_rs_files(path, out);
|
|
|
|
|
}
|
|
|
|
|
} else if path.extension().is_some_and(|extension| extension == "rs")
|
|
|
|
|
&& let Some(path) = path.to_str()
|
|
|
|
|
{
|
|
|
|
|
out.push(path.to_owned());
|
feat: successful 8 stage pipeline parallel run, more metrics
Complete an 8-stage pipeline-parallel run over VastAI by provisioning stages high-to-low, adding per-stage/per-step metrics, host anti-colocation, and provider state-timeout guardrails.
- orchestrator_app: select the next weight-load stage by max index (provision stages high-to-low for parallel spread), add a throttled "loaded N of M; waiting on stage X" stage_provision_wait headline, and surface min_compute_cap/state_timeout_secs in the config dump.
- orchestrator_app: enrich pipeline_token_in/out and tokenizer_decode events with token_count/token_ids/generated_index.
- worker_node: add timing metrics across the data path (helper_execute_ms, egress_ring_read_ms, send_ms, ingress_ring_write_ms, object_load_ms), refactor take_complete_ingress_record into IngressRecordBytes (object_id/sequence/extent/flags), and emit a new object_loaded event.
- vastai_provisioning: track leased host_ids and blacklist already-leased hosts in later ProvisionRequests so stages don't co-locate, and tag SSH-bootstrap retry logs with the attempt number.
- tools/vastai: add min_compute_cap (PP_MIN_COMPUTE_CAP) filter/search query and a LifecyclePolicy state_timeout (PP_STATE_TIMEOUT_SECS) that fails instances stuck in a non-running status instead of polling forever.
- xtask: raise the check timeout to 1800s/30s grace, drop --skip-rebuild for VastAI, aggregate per-stage StepExecuted metrics, add a vastai summary section, and write failure artifacts on abort.
Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-25 16:04:57 +00:00
|
|
|
}
|
2026-07-22 07:50:53 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-17 22:51:08 +00:00
|
|
|
mod demo;
|
2026-08-15 14:10:52 +00:00
|
|
|
|
2026-05-21 19:48:02 +00:00
|
|
|
fn main() -> ExitCode {
|
2026-06-25 12:30:18 +00:00
|
|
|
let mut args = std::env::args().skip(1);
|
2026-07-05 09:59:51 +00:00
|
|
|
match args.next().as_deref() {
|
2026-08-22 17:18:25 +00:00
|
|
|
Some("check-telemetry-isolation") => {
|
|
|
|
|
if check_telemetry_isolation() {
|
|
|
|
|
ExitCode::SUCCESS
|
|
|
|
|
} else {
|
|
|
|
|
ExitCode::from(1)
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-08-27 22:09:51 +00:00
|
|
|
Some("lint") if args.next().is_none() => run_lint(),
|
2026-07-05 09:59:51 +00:00
|
|
|
Some("test") if args.next().is_none() => run_tests(),
|
2026-08-17 22:51:08 +00:00
|
|
|
Some("demo") => demo::run(&args.collect::<Vec<_>>()),
|
2026-07-05 09:59:51 +00:00
|
|
|
Some("help" | "--help" | "-h") | None => {
|
2026-06-25 12:30:18 +00:00
|
|
|
print_usage();
|
2026-05-21 19:48:02 +00:00
|
|
|
ExitCode::SUCCESS
|
|
|
|
|
}
|
2026-06-25 12:30:18 +00:00
|
|
|
_ => {
|
2026-02-24 09:12:28 +00:00
|
|
|
print_usage();
|
2026-05-21 19:48:02 +00:00
|
|
|
ExitCode::from(1)
|
2026-02-16 15:39:27 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|