From 18f06e34f9c16ee60ea012a8f6aba4ca85977340 Mon Sep 17 00:00:00 2001 From: Zachery Aaron Shores-Chmielewski Date: Fri, 24 Jul 2026 11:04:40 +0400 Subject: [PATCH] feat: `--vastai` verification test target --- crates/mvp-system/src/bin/mvp_chat.rs | 10 +- crates/mvp-system/src/config.rs | 4 +- crates/mvp-system/src/orchestrator_app.rs | 9 +- xtask/src/main.rs | 149 ++++++++++++++++++++-- 4 files changed, 148 insertions(+), 24 deletions(-) diff --git a/crates/mvp-system/src/bin/mvp_chat.rs b/crates/mvp-system/src/bin/mvp_chat.rs index c619392..2117693 100644 --- a/crates/mvp-system/src/bin/mvp_chat.rs +++ b/crates/mvp-system/src/bin/mvp_chat.rs @@ -910,7 +910,7 @@ fn resolve_vastai_config( node_image: &str, ) -> Result { ResolvedVastAiConfig { - api_key: first_non_empty([env_optional("VASTAI_API_KEY")]).unwrap_or_default(), + api_key: first_non_empty([env_optional("VAST_API_KEY")]).unwrap_or_default(), relay_url: first_non_empty([file.relay_url.clone()]).unwrap_or_default(), image: node_image.to_owned(), bootstrap_command: first_non_empty([file.bootstrap_command.clone()]).unwrap_or_default(), @@ -2072,7 +2072,7 @@ mod tests { static NEXT_TEMP_ID: AtomicU64 = AtomicU64::new(1); const PROCESS_ENV_KEYS: &[&str] = &[ - "VASTAI_API_KEY", + "VAST_API_KEY", "MVP_PIPELINE_STAGES", "MVP_RUNTIME_CONFIG", "MVP_CHAT_GPU_RUN", @@ -2639,7 +2639,7 @@ bootstrap_command = "boot" "#, ); with_process_state( - &[("VASTAI_API_KEY", Some("secret"))], + &[("VAST_API_KEY", Some("secret"))], Some(missing_relay.path()), || { let config_arg = missing_relay_config.to_string_lossy().into_owned(); @@ -2664,7 +2664,7 @@ bootstrap_command = "boot" "#, ); with_process_state( - &[("VASTAI_API_KEY", Some("secret"))], + &[("VAST_API_KEY", Some("secret"))], Some(local_image.path()), || { let config_arg = local_image_config.to_string_lossy().into_owned(); @@ -2689,7 +2689,7 @@ bootstrap_command = "boot" "#, ); with_process_state( - &[("VASTAI_API_KEY", Some("secret"))], + &[("VAST_API_KEY", Some("secret"))], Some(valid.path()), || { let config_arg = valid_config.to_string_lossy().into_owned(); diff --git a/crates/mvp-system/src/config.rs b/crates/mvp-system/src/config.rs index 64b0cc9..aafd961 100644 --- a/crates/mvp-system/src/config.rs +++ b/crates/mvp-system/src/config.rs @@ -174,7 +174,7 @@ impl TomlConfigOverlay { impl ResolvedVastAiConfig { pub fn validate(self) -> Result { - require_non_empty("VASTAI_API_KEY", &self.api_key)?; + require_non_empty("VAST_API_KEY", &self.api_key)?; require_non_empty("relay.url", &self.relay_url)?; require_non_empty("vastai.image", &self.image)?; require_non_empty("vastai.bootstrap_command", &self.bootstrap_command)?; @@ -470,7 +470,7 @@ pipeline_stages = "many" fn resolved_vastai_config_validate_rejects_missing_required_fields() { let cases = [ ( - "VASTAI_API_KEY", + "VAST_API_KEY", ResolvedVastAiConfig { api_key: " ".to_owned(), ..valid_resolved_vastai_config() diff --git a/crates/mvp-system/src/orchestrator_app.rs b/crates/mvp-system/src/orchestrator_app.rs index 25accd1..770bf3f 100644 --- a/crates/mvp-system/src/orchestrator_app.rs +++ b/crates/mvp-system/src/orchestrator_app.rs @@ -1182,8 +1182,9 @@ impl ConfigBuilder { if let Some(mask) = env_optional(MVP_IROH_ENDPOINT_ADDR_MASK_ENV) { self.endpoint_addr_mask = Some(mask); } - if let Some(api_key) = - env_optional("MVP_VASTAI_API_KEY").or_else(|| env_optional("VASTAI_API_KEY")) + if let Some(api_key) = env_optional("VAST_API_KEY") + .or_else(|| env_optional("MVP_VASTAI_API_KEY")) + .or_else(|| env_optional("VASTAI_API_KEY")) { self.vastai_api_key = Some(api_key); } @@ -1663,7 +1664,7 @@ impl Config { .as_ref() .and_then(|vastai| vastai.api_key.as_deref()) .ok_or_else(|| { - "MVP_VASTAI_API_KEY or VASTAI_API_KEY is required when MVP_NODE_PROVIDER=vastai" + "VAST_API_KEY, MVP_VASTAI_API_KEY, or VASTAI_API_KEY is required when MVP_NODE_PROVIDER=vastai" .to_owned() })? .to_owned(); @@ -1726,7 +1727,7 @@ impl Config { ); } let api_key = vastai.api_key.clone().ok_or_else(|| { - "MVP_VASTAI_API_KEY or VASTAI_API_KEY is required when MVP_NODE_PROVIDER=vastai" + "VAST_API_KEY, MVP_VASTAI_API_KEY, or VASTAI_API_KEY is required when MVP_NODE_PROVIDER=vastai" .to_owned() })?; let ssh_identity = vastai diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 7d4014d..f1a563a 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -43,6 +43,7 @@ enum MvpChatCheckScenario { Gpu, Multinode, MultinodeDocker, + VastAi, } impl MvpChatCheckScenario { @@ -53,11 +54,12 @@ impl MvpChatCheckScenario { "--gpu" => Self::Gpu, "--multinode" => Self::Multinode, "--multinode-docker" => Self::MultinodeDocker, + "--vastai" => Self::VastAi, other => return Err(format!("unsupported mvp-chat-check argument {other:?}")), }; if scenario != Self::ProcessBaseline { return Err( - "mvp-chat-check accepts at most one scenario flag: --gpu, --multinode, or --multinode-docker" + "mvp-chat-check accepts at most one scenario flag: --gpu, --multinode, --multinode-docker, or --vastai" .to_owned(), ); } @@ -72,6 +74,7 @@ impl MvpChatCheckScenario { Self::Gpu => "gpu", Self::Multinode => "multinode", Self::MultinodeDocker => "multinode-docker", + Self::VastAi => "vastai", } } @@ -87,13 +90,23 @@ impl MvpChatCheckScenario { Self::MultinodeDocker => { args.push("--docker".to_owned()); } + Self::VastAi => { + args.push("--vastai".to_owned()); + } } if matches!(self, Self::Multinode | Self::MultinodeDocker) { args.extend(["--pipeline-stages".to_owned(), "2".to_owned()]); } - if !matches!(self, Self::Gpu) { + if !matches!(self, Self::Gpu | Self::VastAi) { args.push("--cached-model".to_owned()); } + if matches!(self, Self::VastAi) { + args.extend([ + "--yes".to_owned(), + "--endpoint-addr-mask".to_owned(), + "relay-only".to_owned(), + ]); + } args.extend([ "--run-id".to_owned(), run_id.to_string(), @@ -104,7 +117,11 @@ impl MvpChatCheckScenario { fn env_overrides(self) -> &'static [(&'static str, &'static str)] { match self { - Self::ProcessBaseline | Self::Gpu | Self::Multinode | Self::MultinodeDocker => &[], + Self::ProcessBaseline + | Self::Gpu + | Self::Multinode + | Self::MultinodeDocker + | Self::VastAi => &[], } } } @@ -165,7 +182,7 @@ USAGE: cargo xtask COMMANDS: mvp-chat [--gpu] [--process|--docker|--vastai] [--pipeline-stages n] [--cached-model] [-- args...] Run the human chat wrapper against the real orchestrator/worker bins. - mvp-chat-check [--gpu|--multinode|--multinode-docker] + mvp-chat-check [--gpu|--multinode|--multinode-docker|--vastai] Run real cargo mvp-chat acceptance check for one explicit scenario. test Run the basic non-binding test barrier: root crate plus each non-binding repository package with `cargo test -p`." @@ -1240,6 +1257,9 @@ fn assert_dump_log_facts( if scenario == MvpChatCheckScenario::MultinodeDocker { require_multinode_docker_network_facts(&facts)?; } + if scenario == MvpChatCheckScenario::VastAi { + require_vastai_network_facts(&facts)?; + } Ok(events) } @@ -1289,7 +1309,10 @@ fn build_benchmark_report( "ready", )?; } - if scenario != MvpChatCheckScenario::MultinodeDocker { + if !matches!( + scenario, + MvpChatCheckScenario::MultinodeDocker | MvpChatCheckScenario::VastAi + ) { facts.require_span( "mvp.chat.runtime", "ChatProgress", @@ -1591,8 +1614,11 @@ struct DumpLogFacts { gpu_pipeline_tokens_decoded: BTreeSet, gpu_pipeline_real_worker_step_seen: bool, docker_node_spec_worker_count: Option, - docker_worker_iroh_ready: BTreeSet, + worker_iroh_ready: BTreeSet, docker_worker_coordinator_join: BTreeSet, + vastai_node_spec_worker_count: Option, + vastai_provision_start_nodes: BTreeSet, + vastai_provider_start_nodes: BTreeSet, chat_config_ready: bool, prepare_runtime_ready: bool, prompt_rpc_ready: bool, @@ -1607,12 +1633,12 @@ struct DumpLogFacts { shutdown_requested: bool, orchestrator_stopped: bool, } - fn record_dump_log_event( channel: &str, event: &Value, facts: &mut DumpLogFacts, ) -> Result<(), String> { + record_vastai_provision_dump_log_event(channel, event, facts); record_gpu_dump_log_event(channel, event, facts); let event_type = event.get("type").and_then(Value::as_str); let phase = event.get("phase").and_then(Value::as_str); @@ -1642,12 +1668,18 @@ fn record_dump_log_event( (_, Some("NodeEvent"), Some("iroh_driver"), Some("ready")) => { facts.node_iroh_driver_ready = true; if let Some(node_id) = event_node_id(event) { - facts.docker_worker_iroh_ready.insert(node_id); + facts.worker_iroh_ready.insert(node_id); } } (_, Some("OrchBootstrap"), Some("node_spec"), Some("ready")) => { - if detail_str(event, "provider") == Some("docker") { - facts.docker_node_spec_worker_count = detail_u64(event, "worker_count"); + match detail_str(event, "provider") { + Some("docker") => { + facts.docker_node_spec_worker_count = detail_u64(event, "worker_count") + } + Some("vastai") => { + facts.vastai_node_spec_worker_count = detail_u64(event, "worker_count") + } + _ => {} } } (_, Some("NodeEvent"), Some("coordinator_join"), Some("started")) => { @@ -1657,6 +1689,13 @@ fn record_dump_log_event( facts.docker_worker_coordinator_join.insert(node_id); } } + (_, Some("OrchBootstrap"), Some("provider_start"), Some("started")) => { + if detail_str(event, "provider") == Some("vastai") + && let Some(node_id) = detail_u64(event, "node_id") + { + facts.vastai_provider_start_nodes.insert(node_id); + } + } (_, Some("NodeEvent"), Some("worker_initialize"), Some("ready")) => { facts.node_worker_initialize_ready = true; } @@ -1692,6 +1731,20 @@ fn record_dump_log_event( } Ok(()) } +fn record_vastai_provision_dump_log_event(channel: &str, event: &Value, facts: &mut DumpLogFacts) { + if channel != "mvp.provisioning.events" { + return; + } + let Some(provision) = event.get("event") else { + return; + }; + if provision.get("kind").and_then(Value::as_str) == Some("ProvisionStart") + && provision.get("provider").and_then(Value::as_str) == Some("vastai") + && let Some(node_id) = provision.get("node_id").and_then(Value::as_u64) + { + facts.vastai_provision_start_nodes.insert(node_id); + } +} fn record_gpu_dump_log_event(channel: &str, event: &Value, facts: &mut DumpLogFacts) { let event_type = event.get("type").and_then(Value::as_str); @@ -1873,7 +1926,7 @@ fn require_multinode_docker_network_facts(facts: &DumpLogFacts) -> Result<(), St "Docker node_spec with multiple workers", )?; require_dump_log_fact( - facts.docker_worker_iroh_ready.len() >= 2, + facts.worker_iroh_ready.len() >= 2, "Docker worker iroh_driver ready for multiple nodes", )?; require_dump_log_fact( @@ -1882,6 +1935,27 @@ fn require_multinode_docker_network_facts(facts: &DumpLogFacts) -> Result<(), St ) } +fn require_vastai_network_facts(facts: &DumpLogFacts) -> Result<(), String> { + require_dump_log_fact( + facts + .vastai_node_spec_worker_count + .is_some_and(|count| count >= 1), + "VastAI node_spec with workers", + )?; + require_dump_log_fact( + !facts.vastai_provision_start_nodes.is_empty(), + "VastAI ProvisionStart", + )?; + require_dump_log_fact( + !facts.vastai_provider_start_nodes.is_empty(), + "VastAI provider_start", + )?; + require_dump_log_fact( + !facts.worker_iroh_ready.is_empty(), + "VastAI worker iroh_driver ready", + ) +} + fn event_requested_device_is_cuda(event: &Value) -> bool { event .get("requested_device") @@ -2049,6 +2123,22 @@ mod tests { "--dump-logs=/tmp/mvp-chat-check.ndjson", ]) ); + + let vastai = + MvpChatCheckScenario::parse_args(strings(&["--vastai"])).expect("vastai parses"); + assert_eq!(vastai, MvpChatCheckScenario::VastAi); + assert_eq!( + vastai.mvp_chat_args(42, dump_log), + strings(&[ + "--vastai", + "--yes", + "--endpoint-addr-mask", + "relay-only", + "--run-id", + "42", + "--dump-logs=/tmp/mvp-chat-check.ndjson", + ]) + ); } #[test] @@ -2847,18 +2937,51 @@ mod tests { let _ = fs::remove_file(path); } + #[test] + fn benchmark_observability_vastai_dump_facts_require_remote_provider_events() { + let mut events = dump_log_fact_events(false, false); + events.extend([ + ( + "mvp.orch.bootstrap", + stamped( + json!({"type":"OrchBootstrap","phase":"node_spec","status":"ready","run_id":9,"node_id":1,"detail":{"endpoint_addr_mask":"relay-only","provider":"vastai","worker_count":1}}), + "mvp-orchestrator", + 1_071, + 71, + ), + ), + ( + "mvp.provisioning.events", + json!({"event":{"run_id":9,"node_id":3,"kind":"ProvisionStart","provider":"vastai","message":"starting vastai image registry.example/mvp-node:latest"}}), + ), + ( + "mvp.orch.bootstrap", + stamped( + json!({"type":"OrchBootstrap","phase":"provider_start","status":"started","run_id":9,"node_id":1,"detail":{"provider":"vastai","node_id":3,"stage_index":0}}), + "mvp-orchestrator", + 1_073, + 73, + ), + ), + ]); + let path = write_synthetic_event_dump("vastai-remote-provider", events); + assert_dump_log_facts(&path, MvpChatCheckScenario::VastAi) + .expect("VastAI remote provider facts pass"); + let _ = fs::remove_file(path); + } + #[test] fn benchmark_observability_multinode_docker_requires_direct_network_workers() { let mut valid = DumpLogFacts { docker_node_spec_worker_count: Some(2), ..DumpLogFacts::default() }; - valid.docker_worker_iroh_ready.extend([2, 3]); + valid.worker_iroh_ready.extend([2, 3]); valid.docker_worker_coordinator_join.extend([2, 3]); require_multinode_docker_network_facts(&valid).expect("direct-network Docker facts pass"); let mut missing_worker = valid; - missing_worker.docker_worker_iroh_ready.remove(&3); + missing_worker.worker_iroh_ready.remove(&3); let error = require_multinode_docker_network_facts(&missing_worker) .expect_err("single direct-network worker should fail"); assert!(