120 lines
3.3 KiB
TOML
120 lines
3.3 KiB
TOML
|
[run]
|
||
|
|
name = "minimal_scaling_sweep_node0"
|
||
|
|
timeout_minutes = 1440
|
||
|
|
local_artifact_dir = "artifacts/remote_runs"
|
||
|
|
max_attempts = 1
|
||
|
|
artifact_sync_interval_seconds = 60
|
||
|
|
|
||
|
|
[provider]
|
||
|
|
kind = "vastai"
|
||
|
|
disk_gb = 192
|
||
|
|
max_price_per_hour = 0.80
|
||
|
|
image = "vastai/base:0.0.2"
|
||
|
|
|
||
|
|
[provider.gpu]
|
||
|
|
name = "RTX 4090"
|
||
|
|
count = 1
|
||
|
|
min_vram_gb = 20
|
||
|
|
|
||
|
|
[selection]
|
||
|
|
min_reliability = 0.95
|
||
|
|
min_down_mbps = 100
|
||
|
|
min_up_mbps = 25
|
||
|
|
require_verified = true
|
||
|
|
blocked_geos = ["CN"]
|
||
|
|
blacklist_hosts = [59017, 1647, 92578, 1276, 75481, 1256, 85323, 34031, 528237]
|
||
|
|
drop_cheap_frac = 0.30
|
||
|
|
image_size_gb = 5.0
|
||
|
|
base_url = "https://cloud.vast.ai"
|
||
|
|
|
||
|
|
[workspace]
|
||
|
|
workdir = "."
|
||
|
|
exclude = [
|
||
|
|
"/artifacts/remote_runs",
|
||
|
|
"/artifacts/runs",
|
||
|
|
"/artifacts/runs_minimal_scaling_20260728",
|
||
|
|
"/artifacts/data_cache",
|
||
|
|
"/artifacts/model_sanity",
|
||
|
|
"/artifacts/current_run",
|
||
|
|
"/artifacts/public_airfrans",
|
||
|
|
"/artifacts/notes",
|
||
|
|
"/artifacts/preflight_20260725",
|
||
|
|
"/artifacts/remote_state_20260725",
|
||
|
|
"/artifacts/sweep_20260725",
|
||
|
|
"/artifacts/model_sanity_decisions",
|
||
|
|
"/artifacts/model_sanity_verify",
|
||
|
|
"/artifacts/*.json",
|
||
|
|
"/artifacts/*.yaml",
|
||
|
|
"/artifacts/*.md",
|
||
|
|
"/data/raw",
|
||
|
|
"/data/processed",
|
||
|
|
"/.venv",
|
||
|
|
"/.git",
|
||
|
|
"/.airfrans_hf_resume",
|
||
|
|
"/sky_logs",
|
||
|
|
"/notebooks",
|
||
|
|
"__pycache__",
|
||
|
|
"*.pyc",
|
||
|
|
]
|
||
|
|
|
||
|
|
[bootstrap]
|
||
|
|
command = """
|
||
|
|
uv sync --no-dev
|
||
|
|
uv run --no-dev python -c "import torch; ok=torch.cuda.is_available(); print('torch_cuda_available=' + str(ok)); print(torch.cuda.get_device_name(0) if ok else 'no_cuda_device'); assert ok, 'torch CUDA unavailable'"
|
||
|
|
uv run --no-dev python -c "import huggingface_hub, wandb; print('hf_wandb_import_ok')"
|
||
|
|
"""
|
||
|
|
|
||
|
|
[data]
|
||
|
|
validation_command = """
|
||
|
|
uv run --no-dev python - <<'PY'
|
||
|
|
from airfrans_frontier.sweep import read_jobs
|
||
|
|
|
||
|
|
jobs = read_jobs('artifacts/minimal_scaling_sweep_20260728/jobs.jsonl')
|
||
|
|
pending = sum(1 for job in jobs if job['status'] == 'pending')
|
||
|
|
phases = [job.get('phase') for job in jobs]
|
||
|
|
transitions = []
|
||
|
|
for phase in phases:
|
||
|
|
if not transitions or transitions[-1] != phase:
|
||
|
|
transitions.append(phase)
|
||
|
|
assert len(jobs) == 23, len(jobs)
|
||
|
|
assert pending > 0
|
||
|
|
assert transitions == ['A_measurement_canary', 'B_family_viability', 'C_scaling_ladder'], transitions
|
||
|
|
print('minimal_scaling_pending_jobs=' + str(pending))
|
||
|
|
print('minimal_scaling_phase_order=' + ','.join(transitions))
|
||
|
|
PY
|
||
|
|
"""
|
||
|
|
|
||
|
|
[job]
|
||
|
|
command = """
|
||
|
|
uv run --no-dev python scripts/aggressive_oom_node_wrapper.py --jobs artifacts/minimal_scaling_sweep_20260728/jobs.jsonl --node node-0 --artifact-dir artifacts/current_run --utilization artifacts/minimal_scaling_sweep_20260728/utilization_node0.jsonl --sample-interval-seconds 30 --stale-after-seconds 21600 --max-attempts 2 --require-success
|
||
|
|
"""
|
||
|
|
artifact_dir = "artifacts/current_run"
|
||
|
|
heartbeat_file = "artifacts/current_run/heartbeat.json"
|
||
|
|
metrics_file = "artifacts/current_run/metrics.jsonl"
|
||
|
|
|
||
|
|
[artifacts]
|
||
|
|
mode = "object_store_upload"
|
||
|
|
required = [
|
||
|
|
"config.toml",
|
||
|
|
"job_manifest.json",
|
||
|
|
"metrics.jsonl",
|
||
|
|
"latest_metrics.json",
|
||
|
|
"heartbeat.json",
|
||
|
|
"checkpoint_latest.pt",
|
||
|
|
"checkpoint_best.pt",
|
||
|
|
"checkpoint_final.pt",
|
||
|
|
"final_metrics.json",
|
||
|
|
"run_manifest.json",
|
||
|
|
"environment_manifest.json",
|
||
|
|
"utilization.jsonl",
|
||
|
|
"node_summary.json",
|
||
|
|
"sweep_collect.json",
|
||
|
|
"sweep_jobs.jsonl",
|
||
|
|
"sweep_attempts.jsonl",
|
||
|
|
"sweep_utilization.jsonl",
|
||
|
|
]
|
||
|
|
|
||
|
|
[cleanup]
|
||
|
|
on_success = "sky_down"
|
||
|
|
on_failure = "collect_then_keep"
|