swactor/tools/vastai/src/teardown.rs

119 lines
3.5 KiB
Rust
Raw Normal View History

use std::time::Duration;
use crate::types::{InstanceInfo, InstanceListResponse, LabeledInstance};
/// Destroy one vast.ai instance by contract id.
pub async fn destroy_instance(
client: &reqwest::Client,
base_url: &str,
api_key: &str,
contract_id: u64,
) -> Result<(), String> {
feat: successful 8 stage pipeline parallel run, more metrics Complete an 8-stage pipeline-parallel run over VastAI by provisioning stages high-to-low, adding per-stage/per-step metrics, host anti-colocation, and provider state-timeout guardrails. - orchestrator_app: select the next weight-load stage by max index (provision stages high-to-low for parallel spread), add a throttled "loaded N of M; waiting on stage X" stage_provision_wait headline, and surface min_compute_cap/state_timeout_secs in the config dump. - orchestrator_app: enrich pipeline_token_in/out and tokenizer_decode events with token_count/token_ids/generated_index. - worker_node: add timing metrics across the data path (helper_execute_ms, egress_ring_read_ms, send_ms, ingress_ring_write_ms, object_load_ms), refactor take_complete_ingress_record into IngressRecordBytes (object_id/sequence/extent/flags), and emit a new object_loaded event. - vastai_provisioning: track leased host_ids and blacklist already-leased hosts in later ProvisionRequests so stages don't co-locate, and tag SSH-bootstrap retry logs with the attempt number. - tools/vastai: add min_compute_cap (PP_MIN_COMPUTE_CAP) filter/search query and a LifecyclePolicy state_timeout (PP_STATE_TIMEOUT_SECS) that fails instances stuck in a non-running status instead of polling forever. - xtask: raise the check timeout to 1800s/30s grace, drop --skip-rebuild for VastAI, aggregate per-stage StepExecuted metrics, add a vastai summary section, and write failure artifacts on abort. Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-25 16:04:57 +00:00
let url = format!(
"{base_url}/api/v0/instances/{contract_id}/?api_key={}",
urlencoding::encode(api_key)
);
let resp = client
.delete(&url)
.header("Authorization", format!("Bearer {api_key}"))
.send()
.await
.map_err(|e| format!("destroy_instance request failed: {e}"))?;
if !resp.status().is_success() {
let status = resp.status();
let body = resp.text().await.unwrap_or_default();
return Err(format!(
"destroy_instance {contract_id} HTTP {status}: {body}"
));
}
Ok(())
}
/// Destroy every contract and return per-id results in the same order.
pub async fn destroy_all_instances(
client: &reqwest::Client,
base_url: &str,
api_key: &str,
contract_ids: &[u64],
) -> Vec<Result<(), String>> {
let mut results = Vec::with_capacity(contract_ids.len());
for &id in contract_ids {
results.push(destroy_instance(client, base_url, api_key, id).await);
}
results
}
/// Destroy one contract, retrying transient failures so rollback does not strand billing instances.
pub async fn destroy_instance_with_retry(
client: &reqwest::Client,
base_url: &str,
api_key: &str,
contract_id: u64,
) -> Result<(), String> {
let mut attempt = 1_u64;
loop {
match destroy_instance(client, base_url, api_key, contract_id).await {
Ok(()) => return Ok(()),
Err(_) => {
let backoff = std::cmp::min(
Duration::from_millis(500_u64.saturating_mul(attempt)),
Duration::from_secs(30),
);
tokio::time::sleep(backoff).await;
attempt = attempt.saturating_add(1);
}
}
}
}
pub(crate) async fn rollback(
client: &reqwest::Client,
base_url: &str,
api_key: &str,
created: &[InstanceInfo],
) {
for info in created {
if let Err(e) =
destroy_instance_with_retry(client, base_url, api_key, info.contract_id).await
{
eprintln!(
"lease_chain: WARNING rollback could not destroy {}: {e}",
info.contract_id
);
}
}
}
/// List every instance on the account tagged with `label`, sorted by contract id.
pub async fn list_instances_by_label(
client: &reqwest::Client,
base_url: &str,
api_key: &str,
label: &str,
) -> Result<Vec<LabeledInstance>, String> {
let url = format!("{base_url}/api/v0/instances/");
let resp = client
.get(&url)
.header("Authorization", format!("Bearer {api_key}"))
.send()
.await
.map_err(|e| format!("list_instances request failed: {e}"))?;
if !resp.status().is_success() {
let status = resp.status();
let body = resp.text().await.unwrap_or_default();
return Err(format!("list_instances HTTP {status}: {body}"));
}
let body: InstanceListResponse = resp
.json()
.await
.map_err(|e| format!("list_instances parse failed: {e}"))?;
let mut out: Vec<LabeledInstance> = body
.instances
.into_iter()
.filter(|e| e.label.as_deref() == Some(label))
.map(Into::into)
.collect();
out.sort_by_key(|i| i.contract_id);
Ok(out)
}