zacheryasc/swactor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: --vastai verification test target

Browse source 
Add a `--vastai` acceptance scenario to mvp-chat-check that provisions real VastAI nodes and verifies the remote provider data path, and standardize the VastAI API key on VAST_API_KEY.

- xtask: add the VastAi variant and `--vastai` flag to the mvp-chat-check scenario, passing `--vastai --yes --endpoint-addr-mask relay-only` (no --cached-model) and gating it on a new require_vastai_network_facts check (node_spec workers, ProvisionStart, provider_start, iroh_driver ready).
- xtask: track VastAI dump-log facts (vastai_node_spec_worker_count, vastai_provision_start_nodes, vastai_provider_start_nodes) via record_vastai_provision_dump_log_event and skip the local ChatProgress span assertion for the remote scenario.
- config: rename the VastAI key env var to VAST_API_KEY in ResolvedVastAiConfig validation while keeping MVP_VASTAI_API_KEY/VASTAI_API_KEY fallbacks.
- orchestrator_app: resolve the api key with VAST_API_KEY first, then MVP_VASTAI_API_KEY/VASTAI_API_KEY, and update the missing-key error messages.

Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
This commit is contained in:
Zachery Aaron Shores-Chmielewski 2026-07-24 11:04:40 +04:00
parent 3d18a4bd66
commit c526e929c3
4 changed files with 148 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
crates/mvp-system/src/bin/mvp_chat.rs
View file

@ -910,7 +910,7 @@ fn resolve_vastai_config(
node_image: &str,
) -> Result<ResolvedVastAiConfig, String> {
ResolvedVastAiConfig {
api_key: first_non_empty([env_optional("VASTAI_API_KEY")]).unwrap_or_default(),
api_key: first_non_empty([env_optional("VAST_API_KEY")]).unwrap_or_default(),
relay_url: first_non_empty([file.relay_url.clone()]).unwrap_or_default(),
image: node_image.to_owned(),
bootstrap_command: first_non_empty([file.bootstrap_command.clone()]).unwrap_or_default(),
@ -2072,7 +2072,7 @@ mod tests {
static NEXT_TEMP_ID: AtomicU64 = AtomicU64::new(1);
const PROCESS_ENV_KEYS: &[&str] = &[
"VASTAI_API_KEY",
"VAST_API_KEY",
"MVP_PIPELINE_STAGES",
"MVP_RUNTIME_CONFIG",
"MVP_CHAT_GPU_RUN",
@ -2639,7 +2639,7 @@ bootstrap_command = "boot"
"#,
);
with_process_state(
&[("VASTAI_API_KEY", Some("secret"))],
&[("VAST_API_KEY", Some("secret"))],
Some(missing_relay.path()),
|| {
let config_arg = missing_relay_config.to_string_lossy().into_owned();
@ -2664,7 +2664,7 @@ bootstrap_command = "boot"
"#,
);
with_process_state(
&[("VASTAI_API_KEY", Some("secret"))],
&[("VAST_API_KEY", Some("secret"))],
Some(local_image.path()),
|| {
let config_arg = local_image_config.to_string_lossy().into_owned();
@ -2689,7 +2689,7 @@ bootstrap_command = "boot"
"#,
);
with_process_state(
&[("VASTAI_API_KEY", Some("secret"))],
&[("VAST_API_KEY", Some("secret"))],
Some(valid.path()),
|| {
let config_arg = valid_config.to_string_lossy().into_owned();

4
crates/mvp-system/src/config.rs
View file

@ -174,7 +174,7 @@ impl TomlConfigOverlay {
impl ResolvedVastAiConfig {
pub fn validate(self) -> Result<Self, String> {
require_non_empty("VASTAI_API_KEY", &self.api_key)?;
require_non_empty("VAST_API_KEY", &self.api_key)?;
require_non_empty("relay.url", &self.relay_url)?;
require_non_empty("vastai.image", &self.image)?;
require_non_empty("vastai.bootstrap_command", &self.bootstrap_command)?;
@ -470,7 +470,7 @@ pipeline_stages = "many"
fn resolved_vastai_config_validate_rejects_missing_required_fields() {
let cases = [
(
"VASTAI_API_KEY",
"VAST_API_KEY",
ResolvedVastAiConfig {
api_key: " ".to_owned(),
..valid_resolved_vastai_config()

9
crates/mvp-system/src/orchestrator_app.rs
View file

@ -1182,8 +1182,9 @@ impl ConfigBuilder {
if let Some(mask) = env_optional(MVP_IROH_ENDPOINT_ADDR_MASK_ENV) {
self.endpoint_addr_mask = Some(mask);
}
if let Some(api_key) =
env_optional("MVP_VASTAI_API_KEY").or_else(|| env_optional("VASTAI_API_KEY"))
if let Some(api_key) = env_optional("VAST_API_KEY")
.or_else(|| env_optional("MVP_VASTAI_API_KEY"))
.or_else(|| env_optional("VASTAI_API_KEY"))
{
self.vastai_api_key = Some(api_key);
}
@ -1663,7 +1664,7 @@ impl Config {
.as_ref()
.and_then(|vastai| vastai.api_key.as_deref())
.ok_or_else(|| {
"MVP_VASTAI_API_KEY or VASTAI_API_KEY is required when MVP_NODE_PROVIDER=vastai"
"VAST_API_KEY, MVP_VASTAI_API_KEY, or VASTAI_API_KEY is required when MVP_NODE_PROVIDER=vastai"
.to_owned()
})?
.to_owned();
@ -1726,7 +1727,7 @@ impl Config {
);
}
let api_key = vastai.api_key.clone().ok_or_else(|| {
"MVP_VASTAI_API_KEY or VASTAI_API_KEY is required when MVP_NODE_PROVIDER=vastai"
"VAST_API_KEY, MVP_VASTAI_API_KEY, or VASTAI_API_KEY is required when MVP_NODE_PROVIDER=vastai"
.to_owned()
})?;
let ssh_identity = vastai

149
xtask/src/main.rs
View file

@ -43,6 +43,7 @@ enum MvpChatCheckScenario {
Gpu,
Multinode,
MultinodeDocker,
VastAi,
}
impl MvpChatCheckScenario {
@ -53,11 +54,12 @@ impl MvpChatCheckScenario {
"--gpu" => Self::Gpu,
"--multinode" => Self::Multinode,
"--multinode-docker" => Self::MultinodeDocker,
"--vastai" => Self::VastAi,
other => return Err(format!("unsupported mvp-chat-check argument {other:?}")),
};
if scenario != Self::ProcessBaseline {
return Err(
"mvp-chat-check accepts at most one scenario flag: --gpu, --multinode, or --multinode-docker"
"mvp-chat-check accepts at most one scenario flag: --gpu, --multinode, --multinode-docker, or --vastai"
.to_owned(),
);
}
@ -72,6 +74,7 @@ impl MvpChatCheckScenario {
Self::Gpu => "gpu",
Self::Multinode => "multinode",
Self::MultinodeDocker => "multinode-docker",
Self::VastAi => "vastai",
}
}
@ -87,13 +90,23 @@ impl MvpChatCheckScenario {
Self::MultinodeDocker => {
args.push("--docker".to_owned());
}
Self::VastAi => {
args.push("--vastai".to_owned());
}
}
if matches!(self, Self::Multinode | Self::MultinodeDocker) {
args.extend(["--pipeline-stages".to_owned(), "2".to_owned()]);
}
if !matches!(self, Self::Gpu) {
if !matches!(self, Self::Gpu | Self::VastAi) {
args.push("--cached-model".to_owned());
}
if matches!(self, Self::VastAi) {
args.extend([
"--yes".to_owned(),
"--endpoint-addr-mask".to_owned(),
"relay-only".to_owned(),
]);
}
args.extend([
"--run-id".to_owned(),
run_id.to_string(),
@ -104,7 +117,11 @@ impl MvpChatCheckScenario {
fn env_overrides(self) -> &'static [(&'static str, &'static str)] {
match self {
Self::ProcessBaseline | Self::Gpu | Self::Multinode | Self::MultinodeDocker => &[],
Self::ProcessBaseline
| Self::Gpu
| Self::Multinode
| Self::MultinodeDocker
| Self::VastAi => &[],
}
}
}
@ -165,7 +182,7 @@ USAGE: cargo xtask <command>
COMMANDS:
mvp-chat [--gpu] [--process|--docker|--vastai] [--pipeline-stages n] [--cached-model] [-- args...] Run the human chat wrapper against the real orchestrator/worker bins.
mvp-chat-check [--gpu|--multinode|--multinode-docker]
mvp-chat-check [--gpu|--multinode|--multinode-docker|--vastai]
Run real cargo mvp-chat acceptance check for one explicit scenario.
test Run the basic non-binding test barrier: root crate plus each
non-binding repository package with `cargo test -p`."
@ -1240,6 +1257,9 @@ fn assert_dump_log_facts(
if scenario == MvpChatCheckScenario::MultinodeDocker {
require_multinode_docker_network_facts(&facts)?;
}
if scenario == MvpChatCheckScenario::VastAi {
require_vastai_network_facts(&facts)?;
}
Ok(events)
}
@ -1289,7 +1309,10 @@ fn build_benchmark_report(
"ready",
)?;
}
if scenario != MvpChatCheckScenario::MultinodeDocker {
if !matches!(
scenario,
MvpChatCheckScenario::MultinodeDocker | MvpChatCheckScenario::VastAi
) {
facts.require_span(
"mvp.chat.runtime",
"ChatProgress",
@ -1591,8 +1614,11 @@ struct DumpLogFacts {
gpu_pipeline_tokens_decoded: BTreeSet<u64>,
gpu_pipeline_real_worker_step_seen: bool,
docker_node_spec_worker_count: Option<u64>,
docker_worker_iroh_ready: BTreeSet<u64>,
worker_iroh_ready: BTreeSet<u64>,
docker_worker_coordinator_join: BTreeSet<u64>,
vastai_node_spec_worker_count: Option<u64>,
vastai_provision_start_nodes: BTreeSet<u64>,
vastai_provider_start_nodes: BTreeSet<u64>,
chat_config_ready: bool,
prepare_runtime_ready: bool,
prompt_rpc_ready: bool,
@ -1607,12 +1633,12 @@ struct DumpLogFacts {
shutdown_requested: bool,
orchestrator_stopped: bool,
}
fn record_dump_log_event(
channel: &str,
event: &Value,
facts: &mut DumpLogFacts,
) -> Result<(), String> {
record_vastai_provision_dump_log_event(channel, event, facts);
record_gpu_dump_log_event(channel, event, facts);
let event_type = event.get("type").and_then(Value::as_str);
let phase = event.get("phase").and_then(Value::as_str);
@ -1642,12 +1668,18 @@ fn record_dump_log_event(
(_, Some("NodeEvent"), Some("iroh_driver"), Some("ready")) => {
facts.node_iroh_driver_ready = true;
if let Some(node_id) = event_node_id(event) {
facts.docker_worker_iroh_ready.insert(node_id);
facts.worker_iroh_ready.insert(node_id);
}
}
(_, Some("OrchBootstrap"), Some("node_spec"), Some("ready")) => {
if detail_str(event, "provider") == Some("docker") {
facts.docker_node_spec_worker_count = detail_u64(event, "worker_count");
match detail_str(event, "provider") {
Some("docker") => {
facts.docker_node_spec_worker_count = detail_u64(event, "worker_count")
}
Some("vastai") => {
facts.vastai_node_spec_worker_count = detail_u64(event, "worker_count")
}
_ => {}
}
}
(_, Some("NodeEvent"), Some("coordinator_join"), Some("started")) => {
@ -1657,6 +1689,13 @@ fn record_dump_log_event(
facts.docker_worker_coordinator_join.insert(node_id);
}
}
(_, Some("OrchBootstrap"), Some("provider_start"), Some("started")) => {
if detail_str(event, "provider") == Some("vastai")
&& let Some(node_id) = detail_u64(event, "node_id")
{
facts.vastai_provider_start_nodes.insert(node_id);
}
}
(_, Some("NodeEvent"), Some("worker_initialize"), Some("ready")) => {
facts.node_worker_initialize_ready = true;
}
@ -1692,6 +1731,20 @@ fn record_dump_log_event(
}
Ok(())
}
fn record_vastai_provision_dump_log_event(channel: &str, event: &Value, facts: &mut DumpLogFacts) {
if channel != "mvp.provisioning.events" {
return;
}
let Some(provision) = event.get("event") else {
return;
};
if provision.get("kind").and_then(Value::as_str) == Some("ProvisionStart")
&& provision.get("provider").and_then(Value::as_str) == Some("vastai")
&& let Some(node_id) = provision.get("node_id").and_then(Value::as_u64)
{
facts.vastai_provision_start_nodes.insert(node_id);
}
}
fn record_gpu_dump_log_event(channel: &str, event: &Value, facts: &mut DumpLogFacts) {
let event_type = event.get("type").and_then(Value::as_str);
@ -1873,7 +1926,7 @@ fn require_multinode_docker_network_facts(facts: &DumpLogFacts) -> Result<(), St
"Docker node_spec with multiple workers",
)?;
require_dump_log_fact(
facts.docker_worker_iroh_ready.len() >= 2,
facts.worker_iroh_ready.len() >= 2,
"Docker worker iroh_driver ready for multiple nodes",
)?;
require_dump_log_fact(
@ -1882,6 +1935,27 @@ fn require_multinode_docker_network_facts(facts: &DumpLogFacts) -> Result<(), St
)
}
fn require_vastai_network_facts(facts: &DumpLogFacts) -> Result<(), String> {
require_dump_log_fact(
facts
.vastai_node_spec_worker_count
.is_some_and(|count| count >= 1),
"VastAI node_spec with workers",
)?;
require_dump_log_fact(
!facts.vastai_provision_start_nodes.is_empty(),
"VastAI ProvisionStart",
)?;
require_dump_log_fact(
!facts.vastai_provider_start_nodes.is_empty(),
"VastAI provider_start",
)?;
require_dump_log_fact(
!facts.worker_iroh_ready.is_empty(),
"VastAI worker iroh_driver ready",
)
}
fn event_requested_device_is_cuda(event: &Value) -> bool {
event
.get("requested_device")
@ -2049,6 +2123,22 @@ mod tests {
"--dump-logs=/tmp/mvp-chat-check.ndjson",
])
);
let vastai =
MvpChatCheckScenario::parse_args(strings(&["--vastai"])).expect("vastai parses");
assert_eq!(vastai, MvpChatCheckScenario::VastAi);
assert_eq!(
vastai.mvp_chat_args(42, dump_log),
strings(&[
"--vastai",
"--yes",
"--endpoint-addr-mask",
"relay-only",
"--run-id",
"42",
"--dump-logs=/tmp/mvp-chat-check.ndjson",
])
);
}
#[test]
@ -2847,18 +2937,51 @@ mod tests {
let _ = fs::remove_file(path);
}
#[test]
fn benchmark_observability_vastai_dump_facts_require_remote_provider_events() {
let mut events = dump_log_fact_events(false, false);
events.extend([
(
"mvp.orch.bootstrap",
stamped(
json!({"type":"OrchBootstrap","phase":"node_spec","status":"ready","run_id":9,"node_id":1,"detail":{"endpoint_addr_mask":"relay-only","provider":"vastai","worker_count":1}}),
"mvp-orchestrator",
1_071,
71,
),
),
(
"mvp.provisioning.events",
json!({"event":{"run_id":9,"node_id":3,"kind":"ProvisionStart","provider":"vastai","message":"starting vastai image registry.example/mvp-node:latest"}}),
),
(
"mvp.orch.bootstrap",
stamped(
json!({"type":"OrchBootstrap","phase":"provider_start","status":"started","run_id":9,"node_id":1,"detail":{"provider":"vastai","node_id":3,"stage_index":0}}),
"mvp-orchestrator",
1_073,
73,
),
),
]);
let path = write_synthetic_event_dump("vastai-remote-provider", events);
assert_dump_log_facts(&path, MvpChatCheckScenario::VastAi)
.expect("VastAI remote provider facts pass");
let _ = fs::remove_file(path);
}
#[test]
fn benchmark_observability_multinode_docker_requires_direct_network_workers() {
let mut valid = DumpLogFacts {
docker_node_spec_worker_count: Some(2),
..DumpLogFacts::default()
};
valid.docker_worker_iroh_ready.extend([2, 3]);
valid.worker_iroh_ready.extend([2, 3]);
valid.docker_worker_coordinator_join.extend([2, 3]);
require_multinode_docker_network_facts(&valid).expect("direct-network Docker facts pass");
let mut missing_worker = valid;
missing_worker.docker_worker_iroh_ready.remove(&3);
missing_worker.worker_iroh_ready.remove(&3);
let error = require_multinode_docker_network_facts(&missing_worker)
.expect_err("single direct-network worker should fail");
assert!(