zacheryasc/swactor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: --vastai verification test target

Browse source
This commit is contained in:
Zachery Aaron Shores-Chmielewski 2026-07-24 11:04:40 +04:00
parent 6c11c4c2b4
commit 18f06e34f9
4 changed files with 148 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
crates/mvp-system/src/bin/mvp_chat.rs
View file

@ -910,7 +910,7 @@ fn resolve_vastai_config(
node_image: &str, node_image: &str,
) -> Result<ResolvedVastAiConfig, String> { ) -> Result<ResolvedVastAiConfig, String> {
ResolvedVastAiConfig { ResolvedVastAiConfig {
api_key: first_non_empty([env_optional("VASTAI_API_KEY")]).unwrap_or_default(), api_key: first_non_empty([env_optional("VAST_API_KEY")]).unwrap_or_default(),
relay_url: first_non_empty([file.relay_url.clone()]).unwrap_or_default(), relay_url: first_non_empty([file.relay_url.clone()]).unwrap_or_default(),
image: node_image.to_owned(), image: node_image.to_owned(),
bootstrap_command: first_non_empty([file.bootstrap_command.clone()]).unwrap_or_default(), bootstrap_command: first_non_empty([file.bootstrap_command.clone()]).unwrap_or_default(),
@ -2072,7 +2072,7 @@ mod tests {
static NEXT_TEMP_ID: AtomicU64 = AtomicU64::new(1); static NEXT_TEMP_ID: AtomicU64 = AtomicU64::new(1);
const PROCESS_ENV_KEYS: &[&str] = &[ const PROCESS_ENV_KEYS: &[&str] = &[
"VASTAI_API_KEY", "VAST_API_KEY",
"MVP_PIPELINE_STAGES", "MVP_PIPELINE_STAGES",
"MVP_RUNTIME_CONFIG", "MVP_RUNTIME_CONFIG",
"MVP_CHAT_GPU_RUN", "MVP_CHAT_GPU_RUN",
@ -2639,7 +2639,7 @@ bootstrap_command = "boot"
"#, "#,
); );
with_process_state( with_process_state(
&[("VASTAI_API_KEY", Some("secret"))], &[("VAST_API_KEY", Some("secret"))],
Some(missing_relay.path()), Some(missing_relay.path()),
|| { || {
let config_arg = missing_relay_config.to_string_lossy().into_owned(); let config_arg = missing_relay_config.to_string_lossy().into_owned();
@ -2664,7 +2664,7 @@ bootstrap_command = "boot"
"#, "#,
); );
with_process_state( with_process_state(
&[("VASTAI_API_KEY", Some("secret"))], &[("VAST_API_KEY", Some("secret"))],
Some(local_image.path()), Some(local_image.path()),
|| { || {
let config_arg = local_image_config.to_string_lossy().into_owned(); let config_arg = local_image_config.to_string_lossy().into_owned();
@ -2689,7 +2689,7 @@ bootstrap_command = "boot"
"#, "#,
); );
with_process_state( with_process_state(
&[("VASTAI_API_KEY", Some("secret"))], &[("VAST_API_KEY", Some("secret"))],
Some(valid.path()), Some(valid.path()),
|| { || {
let config_arg = valid_config.to_string_lossy().into_owned(); let config_arg = valid_config.to_string_lossy().into_owned();

4
crates/mvp-system/src/config.rs
View file

@ -174,7 +174,7 @@ impl TomlConfigOverlay {
impl ResolvedVastAiConfig { impl ResolvedVastAiConfig {
pub fn validate(self) -> Result<Self, String> { pub fn validate(self) -> Result<Self, String> {
require_non_empty("VASTAI_API_KEY", &self.api_key)?; require_non_empty("VAST_API_KEY", &self.api_key)?;
require_non_empty("relay.url", &self.relay_url)?; require_non_empty("relay.url", &self.relay_url)?;
require_non_empty("vastai.image", &self.image)?; require_non_empty("vastai.image", &self.image)?;
require_non_empty("vastai.bootstrap_command", &self.bootstrap_command)?; require_non_empty("vastai.bootstrap_command", &self.bootstrap_command)?;
@ -470,7 +470,7 @@ pipeline_stages = "many"
fn resolved_vastai_config_validate_rejects_missing_required_fields() { fn resolved_vastai_config_validate_rejects_missing_required_fields() {
let cases = [ let cases = [
( (
"VASTAI_API_KEY", "VAST_API_KEY",
ResolvedVastAiConfig { ResolvedVastAiConfig {
api_key: " ".to_owned(), api_key: " ".to_owned(),
..valid_resolved_vastai_config() ..valid_resolved_vastai_config()

9
crates/mvp-system/src/orchestrator_app.rs
View file

@ -1182,8 +1182,9 @@ impl ConfigBuilder {
if let Some(mask) = env_optional(MVP_IROH_ENDPOINT_ADDR_MASK_ENV) { if let Some(mask) = env_optional(MVP_IROH_ENDPOINT_ADDR_MASK_ENV) {
self.endpoint_addr_mask = Some(mask); self.endpoint_addr_mask = Some(mask);
} }
if let Some(api_key) = if let Some(api_key) = env_optional("VAST_API_KEY")
env_optional("MVP_VASTAI_API_KEY").or_else(|| env_optional("VASTAI_API_KEY")) .or_else(|| env_optional("MVP_VASTAI_API_KEY"))
.or_else(|| env_optional("VASTAI_API_KEY"))
{ {
self.vastai_api_key = Some(api_key); self.vastai_api_key = Some(api_key);
} }
@ -1663,7 +1664,7 @@ impl Config {
.as_ref() .as_ref()
.and_then(|vastai| vastai.api_key.as_deref()) .and_then(|vastai| vastai.api_key.as_deref())
.ok_or_else(|| { .ok_or_else(|| {
"MVP_VASTAI_API_KEY or VASTAI_API_KEY is required when MVP_NODE_PROVIDER=vastai" "VAST_API_KEY, MVP_VASTAI_API_KEY, or VASTAI_API_KEY is required when MVP_NODE_PROVIDER=vastai"
.to_owned() .to_owned()
})? })?
.to_owned(); .to_owned();
@ -1726,7 +1727,7 @@ impl Config {
); );
} }
let api_key = vastai.api_key.clone().ok_or_else(|| { let api_key = vastai.api_key.clone().ok_or_else(|| {
"MVP_VASTAI_API_KEY or VASTAI_API_KEY is required when MVP_NODE_PROVIDER=vastai" "VAST_API_KEY, MVP_VASTAI_API_KEY, or VASTAI_API_KEY is required when MVP_NODE_PROVIDER=vastai"
.to_owned() .to_owned()
})?; })?;
let ssh_identity = vastai let ssh_identity = vastai

149
xtask/src/main.rs
View file

@ -43,6 +43,7 @@ enum MvpChatCheckScenario {
Gpu, Gpu,
Multinode, Multinode,
MultinodeDocker, MultinodeDocker,
VastAi,
} }
impl MvpChatCheckScenario { impl MvpChatCheckScenario {
@ -53,11 +54,12 @@ impl MvpChatCheckScenario {
"--gpu" => Self::Gpu, "--gpu" => Self::Gpu,
"--multinode" => Self::Multinode, "--multinode" => Self::Multinode,
"--multinode-docker" => Self::MultinodeDocker, "--multinode-docker" => Self::MultinodeDocker,
"--vastai" => Self::VastAi,
other => return Err(format!("unsupported mvp-chat-check argument {other:?}")), other => return Err(format!("unsupported mvp-chat-check argument {other:?}")),
}; };
if scenario != Self::ProcessBaseline { if scenario != Self::ProcessBaseline {
return Err( return Err(
"mvp-chat-check accepts at most one scenario flag: --gpu, --multinode, or --multinode-docker" "mvp-chat-check accepts at most one scenario flag: --gpu, --multinode, --multinode-docker, or --vastai"
.to_owned(), .to_owned(),
); );
} }
@ -72,6 +74,7 @@ impl MvpChatCheckScenario {
Self::Gpu => "gpu", Self::Gpu => "gpu",
Self::Multinode => "multinode", Self::Multinode => "multinode",
Self::MultinodeDocker => "multinode-docker", Self::MultinodeDocker => "multinode-docker",
Self::VastAi => "vastai",
} }
} }
@ -87,13 +90,23 @@ impl MvpChatCheckScenario {
Self::MultinodeDocker => { Self::MultinodeDocker => {
args.push("--docker".to_owned()); args.push("--docker".to_owned());
} }
Self::VastAi => {
args.push("--vastai".to_owned());
}
} }
if matches!(self, Self::Multinode | Self::MultinodeDocker) { if matches!(self, Self::Multinode | Self::MultinodeDocker) {
args.extend(["--pipeline-stages".to_owned(), "2".to_owned()]); args.extend(["--pipeline-stages".to_owned(), "2".to_owned()]);
} }
if !matches!(self, Self::Gpu) { if !matches!(self, Self::Gpu | Self::VastAi) {
args.push("--cached-model".to_owned()); args.push("--cached-model".to_owned());
} }
if matches!(self, Self::VastAi) {
args.extend([
"--yes".to_owned(),
"--endpoint-addr-mask".to_owned(),
"relay-only".to_owned(),
]);
}
args.extend([ args.extend([
"--run-id".to_owned(), "--run-id".to_owned(),
run_id.to_string(), run_id.to_string(),
@ -104,7 +117,11 @@ impl MvpChatCheckScenario {
fn env_overrides(self) -> &'static [(&'static str, &'static str)] { fn env_overrides(self) -> &'static [(&'static str, &'static str)] {
match self { match self {
Self::ProcessBaseline | Self::Gpu | Self::Multinode | Self::MultinodeDocker => &[], Self::ProcessBaseline
| Self::Gpu
| Self::Multinode
| Self::MultinodeDocker
| Self::VastAi => &[],
} }
} }
} }
@ -165,7 +182,7 @@ USAGE: cargo xtask <command>
COMMANDS: COMMANDS:
mvp-chat [--gpu] [--process|--docker|--vastai] [--pipeline-stages n] [--cached-model] [-- args...] Run the human chat wrapper against the real orchestrator/worker bins. mvp-chat [--gpu] [--process|--docker|--vastai] [--pipeline-stages n] [--cached-model] [-- args...] Run the human chat wrapper against the real orchestrator/worker bins.
mvp-chat-check [--gpu|--multinode|--multinode-docker] mvp-chat-check [--gpu|--multinode|--multinode-docker|--vastai]
Run real cargo mvp-chat acceptance check for one explicit scenario. Run real cargo mvp-chat acceptance check for one explicit scenario.
test Run the basic non-binding test barrier: root crate plus each test Run the basic non-binding test barrier: root crate plus each
non-binding repository package with `cargo test -p`." non-binding repository package with `cargo test -p`."
@ -1240,6 +1257,9 @@ fn assert_dump_log_facts(
if scenario == MvpChatCheckScenario::MultinodeDocker { if scenario == MvpChatCheckScenario::MultinodeDocker {
require_multinode_docker_network_facts(&facts)?; require_multinode_docker_network_facts(&facts)?;
} }
if scenario == MvpChatCheckScenario::VastAi {
require_vastai_network_facts(&facts)?;
}
Ok(events) Ok(events)
} }
@ -1289,7 +1309,10 @@ fn build_benchmark_report(
"ready", "ready",
)?; )?;
} }
if scenario != MvpChatCheckScenario::MultinodeDocker { if !matches!(
scenario,
MvpChatCheckScenario::MultinodeDocker | MvpChatCheckScenario::VastAi
) {
facts.require_span( facts.require_span(
"mvp.chat.runtime", "mvp.chat.runtime",
"ChatProgress", "ChatProgress",
@ -1591,8 +1614,11 @@ struct DumpLogFacts {
gpu_pipeline_tokens_decoded: BTreeSet<u64>, gpu_pipeline_tokens_decoded: BTreeSet<u64>,
gpu_pipeline_real_worker_step_seen: bool, gpu_pipeline_real_worker_step_seen: bool,
docker_node_spec_worker_count: Option<u64>, docker_node_spec_worker_count: Option<u64>,
docker_worker_iroh_ready: BTreeSet<u64>, worker_iroh_ready: BTreeSet<u64>,
docker_worker_coordinator_join: BTreeSet<u64>, docker_worker_coordinator_join: BTreeSet<u64>,
vastai_node_spec_worker_count: Option<u64>,
vastai_provision_start_nodes: BTreeSet<u64>,
vastai_provider_start_nodes: BTreeSet<u64>,
chat_config_ready: bool, chat_config_ready: bool,
prepare_runtime_ready: bool, prepare_runtime_ready: bool,
prompt_rpc_ready: bool, prompt_rpc_ready: bool,
@ -1607,12 +1633,12 @@ struct DumpLogFacts {
shutdown_requested: bool, shutdown_requested: bool,
orchestrator_stopped: bool, orchestrator_stopped: bool,
} }
fn record_dump_log_event( fn record_dump_log_event(
channel: &str, channel: &str,
event: &Value, event: &Value,
facts: &mut DumpLogFacts, facts: &mut DumpLogFacts,
) -> Result<(), String> { ) -> Result<(), String> {
record_vastai_provision_dump_log_event(channel, event, facts);
record_gpu_dump_log_event(channel, event, facts); record_gpu_dump_log_event(channel, event, facts);
let event_type = event.get("type").and_then(Value::as_str); let event_type = event.get("type").and_then(Value::as_str);
let phase = event.get("phase").and_then(Value::as_str); let phase = event.get("phase").and_then(Value::as_str);
@ -1642,12 +1668,18 @@ fn record_dump_log_event(
(_, Some("NodeEvent"), Some("iroh_driver"), Some("ready")) => { (_, Some("NodeEvent"), Some("iroh_driver"), Some("ready")) => {
facts.node_iroh_driver_ready = true; facts.node_iroh_driver_ready = true;
if let Some(node_id) = event_node_id(event) { if let Some(node_id) = event_node_id(event) {
facts.docker_worker_iroh_ready.insert(node_id); facts.worker_iroh_ready.insert(node_id);
} }
} }
(_, Some("OrchBootstrap"), Some("node_spec"), Some("ready")) => { (_, Some("OrchBootstrap"), Some("node_spec"), Some("ready")) => {
if detail_str(event, "provider") == Some("docker") { match detail_str(event, "provider") {
facts.docker_node_spec_worker_count = detail_u64(event, "worker_count"); Some("docker") => {
facts.docker_node_spec_worker_count = detail_u64(event, "worker_count")
}
Some("vastai") => {
facts.vastai_node_spec_worker_count = detail_u64(event, "worker_count")
}
_ => {}
} }
} }
(_, Some("NodeEvent"), Some("coordinator_join"), Some("started")) => { (_, Some("NodeEvent"), Some("coordinator_join"), Some("started")) => {
@ -1657,6 +1689,13 @@ fn record_dump_log_event(
facts.docker_worker_coordinator_join.insert(node_id); facts.docker_worker_coordinator_join.insert(node_id);
} }
} }
(_, Some("OrchBootstrap"), Some("provider_start"), Some("started")) => {
if detail_str(event, "provider") == Some("vastai")
&& let Some(node_id) = detail_u64(event, "node_id")
{
facts.vastai_provider_start_nodes.insert(node_id);
}
}
(_, Some("NodeEvent"), Some("worker_initialize"), Some("ready")) => { (_, Some("NodeEvent"), Some("worker_initialize"), Some("ready")) => {
facts.node_worker_initialize_ready = true; facts.node_worker_initialize_ready = true;
} }
@ -1692,6 +1731,20 @@ fn record_dump_log_event(
} }
Ok(()) Ok(())
} }
fn record_vastai_provision_dump_log_event(channel: &str, event: &Value, facts: &mut DumpLogFacts) {
if channel != "mvp.provisioning.events" {
return;
}
let Some(provision) = event.get("event") else {
return;
};
if provision.get("kind").and_then(Value::as_str) == Some("ProvisionStart")
&& provision.get("provider").and_then(Value::as_str) == Some("vastai")
&& let Some(node_id) = provision.get("node_id").and_then(Value::as_u64)
{
facts.vastai_provision_start_nodes.insert(node_id);
}
}
fn record_gpu_dump_log_event(channel: &str, event: &Value, facts: &mut DumpLogFacts) { fn record_gpu_dump_log_event(channel: &str, event: &Value, facts: &mut DumpLogFacts) {
let event_type = event.get("type").and_then(Value::as_str); let event_type = event.get("type").and_then(Value::as_str);
@ -1873,7 +1926,7 @@ fn require_multinode_docker_network_facts(facts: &DumpLogFacts) -> Result<(), St
"Docker node_spec with multiple workers", "Docker node_spec with multiple workers",
)?; )?;
require_dump_log_fact( require_dump_log_fact(
facts.docker_worker_iroh_ready.len() >= 2, facts.worker_iroh_ready.len() >= 2,
"Docker worker iroh_driver ready for multiple nodes", "Docker worker iroh_driver ready for multiple nodes",
)?; )?;
require_dump_log_fact( require_dump_log_fact(
@ -1882,6 +1935,27 @@ fn require_multinode_docker_network_facts(facts: &DumpLogFacts) -> Result<(), St
) )
} }
fn require_vastai_network_facts(facts: &DumpLogFacts) -> Result<(), String> {
require_dump_log_fact(
facts
.vastai_node_spec_worker_count
.is_some_and(|count| count >= 1),
"VastAI node_spec with workers",
)?;
require_dump_log_fact(
!facts.vastai_provision_start_nodes.is_empty(),
"VastAI ProvisionStart",
)?;
require_dump_log_fact(
!facts.vastai_provider_start_nodes.is_empty(),
"VastAI provider_start",
)?;
require_dump_log_fact(
!facts.worker_iroh_ready.is_empty(),
"VastAI worker iroh_driver ready",
)
}
fn event_requested_device_is_cuda(event: &Value) -> bool { fn event_requested_device_is_cuda(event: &Value) -> bool {
event event
.get("requested_device") .get("requested_device")
@ -2049,6 +2123,22 @@ mod tests {
"--dump-logs=/tmp/mvp-chat-check.ndjson", "--dump-logs=/tmp/mvp-chat-check.ndjson",
]) ])
); );
let vastai =
MvpChatCheckScenario::parse_args(strings(&["--vastai"])).expect("vastai parses");
assert_eq!(vastai, MvpChatCheckScenario::VastAi);
assert_eq!(
vastai.mvp_chat_args(42, dump_log),
strings(&[
"--vastai",
"--yes",
"--endpoint-addr-mask",
"relay-only",
"--run-id",
"42",
"--dump-logs=/tmp/mvp-chat-check.ndjson",
])
);
} }
#[test] #[test]
@ -2847,18 +2937,51 @@ mod tests {
let _ = fs::remove_file(path); let _ = fs::remove_file(path);
} }
#[test]
fn benchmark_observability_vastai_dump_facts_require_remote_provider_events() {
let mut events = dump_log_fact_events(false, false);
events.extend([
(
"mvp.orch.bootstrap",
stamped(
json!({"type":"OrchBootstrap","phase":"node_spec","status":"ready","run_id":9,"node_id":1,"detail":{"endpoint_addr_mask":"relay-only","provider":"vastai","worker_count":1}}),
"mvp-orchestrator",
1_071,
71,
),
),
(
"mvp.provisioning.events",
json!({"event":{"run_id":9,"node_id":3,"kind":"ProvisionStart","provider":"vastai","message":"starting vastai image registry.example/mvp-node:latest"}}),
),
(
"mvp.orch.bootstrap",
stamped(
json!({"type":"OrchBootstrap","phase":"provider_start","status":"started","run_id":9,"node_id":1,"detail":{"provider":"vastai","node_id":3,"stage_index":0}}),
"mvp-orchestrator",
1_073,
73,
),
),
]);
let path = write_synthetic_event_dump("vastai-remote-provider", events);
assert_dump_log_facts(&path, MvpChatCheckScenario::VastAi)
.expect("VastAI remote provider facts pass");
let _ = fs::remove_file(path);
}
#[test] #[test]
fn benchmark_observability_multinode_docker_requires_direct_network_workers() { fn benchmark_observability_multinode_docker_requires_direct_network_workers() {
let mut valid = DumpLogFacts { let mut valid = DumpLogFacts {
docker_node_spec_worker_count: Some(2), docker_node_spec_worker_count: Some(2),
..DumpLogFacts::default() ..DumpLogFacts::default()
}; };
valid.docker_worker_iroh_ready.extend([2, 3]); valid.worker_iroh_ready.extend([2, 3]);
valid.docker_worker_coordinator_join.extend([2, 3]); valid.docker_worker_coordinator_join.extend([2, 3]);
require_multinode_docker_network_facts(&valid).expect("direct-network Docker facts pass"); require_multinode_docker_network_facts(&valid).expect("direct-network Docker facts pass");
let mut missing_worker = valid; let mut missing_worker = valid;
missing_worker.docker_worker_iroh_ready.remove(&3); missing_worker.worker_iroh_ready.remove(&3);
let error = require_multinode_docker_network_facts(&missing_worker) let error = require_multinode_docker_network_facts(&missing_worker)
.expect_err("single direct-network worker should fail"); .expect_err("single direct-network worker should fail");
assert!( assert!(