2026-06-23 20:10:41 +00:00
|
|
|
use std::time::Duration;
|
|
|
|
|
|
|
|
|
|
use crate::types::{InstanceInfo, InstanceListResponse, LabeledInstance};
|
|
|
|
|
|
|
|
|
|
/// Destroy one vast.ai instance by contract id.
|
|
|
|
|
pub async fn destroy_instance(
|
|
|
|
|
client: &reqwest::Client,
|
|
|
|
|
base_url: &str,
|
|
|
|
|
api_key: &str,
|
|
|
|
|
contract_id: u64,
|
|
|
|
|
) -> Result<(), String> {
|
feat: successful 8 stage pipeline parallel run, more metrics
Complete an 8-stage pipeline-parallel run over VastAI by provisioning stages high-to-low, adding per-stage/per-step metrics, host anti-colocation, and provider state-timeout guardrails.
- orchestrator_app: select the next weight-load stage by max index (provision stages high-to-low for parallel spread), add a throttled "loaded N of M; waiting on stage X" stage_provision_wait headline, and surface min_compute_cap/state_timeout_secs in the config dump.
- orchestrator_app: enrich pipeline_token_in/out and tokenizer_decode events with token_count/token_ids/generated_index.
- worker_node: add timing metrics across the data path (helper_execute_ms, egress_ring_read_ms, send_ms, ingress_ring_write_ms, object_load_ms), refactor take_complete_ingress_record into IngressRecordBytes (object_id/sequence/extent/flags), and emit a new object_loaded event.
- vastai_provisioning: track leased host_ids and blacklist already-leased hosts in later ProvisionRequests so stages don't co-locate, and tag SSH-bootstrap retry logs with the attempt number.
- tools/vastai: add min_compute_cap (PP_MIN_COMPUTE_CAP) filter/search query and a LifecyclePolicy state_timeout (PP_STATE_TIMEOUT_SECS) that fails instances stuck in a non-running status instead of polling forever.
- xtask: raise the check timeout to 1800s/30s grace, drop --skip-rebuild for VastAI, aggregate per-stage StepExecuted metrics, add a vastai summary section, and write failure artifacts on abort.
Signed-off-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
2026-07-25 16:04:57 +00:00
|
|
|
let url = format!(
|
|
|
|
|
"{base_url}/api/v0/instances/{contract_id}/?api_key={}",
|
|
|
|
|
urlencoding::encode(api_key)
|
|
|
|
|
);
|
2026-06-23 20:10:41 +00:00
|
|
|
let resp = client
|
|
|
|
|
.delete(&url)
|
|
|
|
|
.header("Authorization", format!("Bearer {api_key}"))
|
|
|
|
|
.send()
|
|
|
|
|
.await
|
|
|
|
|
.map_err(|e| format!("destroy_instance request failed: {e}"))?;
|
|
|
|
|
if !resp.status().is_success() {
|
|
|
|
|
let status = resp.status();
|
|
|
|
|
let body = resp.text().await.unwrap_or_default();
|
|
|
|
|
return Err(format!(
|
|
|
|
|
"destroy_instance {contract_id} HTTP {status}: {body}"
|
|
|
|
|
));
|
|
|
|
|
}
|
|
|
|
|
Ok(())
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Destroy every contract and return per-id results in the same order.
|
|
|
|
|
pub async fn destroy_all_instances(
|
|
|
|
|
client: &reqwest::Client,
|
|
|
|
|
base_url: &str,
|
|
|
|
|
api_key: &str,
|
|
|
|
|
contract_ids: &[u64],
|
|
|
|
|
) -> Vec<Result<(), String>> {
|
|
|
|
|
let mut results = Vec::with_capacity(contract_ids.len());
|
|
|
|
|
for &id in contract_ids {
|
|
|
|
|
results.push(destroy_instance(client, base_url, api_key, id).await);
|
|
|
|
|
}
|
|
|
|
|
results
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Destroy one contract, retrying transient failures so rollback does not strand billing instances.
|
|
|
|
|
pub async fn destroy_instance_with_retry(
|
|
|
|
|
client: &reqwest::Client,
|
|
|
|
|
base_url: &str,
|
|
|
|
|
api_key: &str,
|
|
|
|
|
contract_id: u64,
|
|
|
|
|
) -> Result<(), String> {
|
2026-07-07 10:40:02 +00:00
|
|
|
let mut attempt = 1_u64;
|
|
|
|
|
loop {
|
2026-06-23 20:10:41 +00:00
|
|
|
match destroy_instance(client, base_url, api_key, contract_id).await {
|
|
|
|
|
Ok(()) => return Ok(()),
|
2026-07-07 10:40:02 +00:00
|
|
|
Err(_) => {
|
|
|
|
|
let backoff = std::cmp::min(
|
|
|
|
|
Duration::from_millis(500_u64.saturating_mul(attempt)),
|
|
|
|
|
Duration::from_secs(30),
|
|
|
|
|
);
|
|
|
|
|
tokio::time::sleep(backoff).await;
|
|
|
|
|
attempt = attempt.saturating_add(1);
|
2026-06-23 20:10:41 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub(crate) async fn rollback(
|
|
|
|
|
client: &reqwest::Client,
|
|
|
|
|
base_url: &str,
|
|
|
|
|
api_key: &str,
|
|
|
|
|
created: &[InstanceInfo],
|
|
|
|
|
) {
|
|
|
|
|
for info in created {
|
|
|
|
|
if let Err(e) =
|
|
|
|
|
destroy_instance_with_retry(client, base_url, api_key, info.contract_id).await
|
|
|
|
|
{
|
|
|
|
|
eprintln!(
|
|
|
|
|
"lease_chain: WARNING rollback could not destroy {}: {e}",
|
|
|
|
|
info.contract_id
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// List every instance on the account tagged with `label`, sorted by contract id.
|
|
|
|
|
pub async fn list_instances_by_label(
|
|
|
|
|
client: &reqwest::Client,
|
|
|
|
|
base_url: &str,
|
|
|
|
|
api_key: &str,
|
|
|
|
|
label: &str,
|
|
|
|
|
) -> Result<Vec<LabeledInstance>, String> {
|
|
|
|
|
let url = format!("{base_url}/api/v0/instances/");
|
|
|
|
|
let resp = client
|
|
|
|
|
.get(&url)
|
|
|
|
|
.header("Authorization", format!("Bearer {api_key}"))
|
|
|
|
|
.send()
|
|
|
|
|
.await
|
|
|
|
|
.map_err(|e| format!("list_instances request failed: {e}"))?;
|
|
|
|
|
if !resp.status().is_success() {
|
|
|
|
|
let status = resp.status();
|
|
|
|
|
let body = resp.text().await.unwrap_or_default();
|
|
|
|
|
return Err(format!("list_instances HTTP {status}: {body}"));
|
|
|
|
|
}
|
|
|
|
|
let body: InstanceListResponse = resp
|
|
|
|
|
.json()
|
|
|
|
|
.await
|
|
|
|
|
.map_err(|e| format!("list_instances parse failed: {e}"))?;
|
|
|
|
|
let mut out: Vec<LabeledInstance> = body
|
|
|
|
|
.instances
|
|
|
|
|
.into_iter()
|
|
|
|
|
.filter(|e| e.label.as_deref() == Some(label))
|
|
|
|
|
.map(Into::into)
|
|
|
|
|
.collect();
|
|
|
|
|
out.sort_by_key(|i| i.contract_id);
|
|
|
|
|
Ok(out)
|
|
|
|
|
}
|