airfRANS-model-exploration/configs/remote_full_70gb.toml

[run]
name = "full_airfrans_incumbent_70gb_01"
timeout_minutes = 1440
local_artifact_dir = "artifacts/remote_runs"
max_attempts = 5
artifact_sync_interval_seconds = 60

[provider]
kind = "vastai"
disk_gb = 192
max_price_per_hour = 0.80
image = "vastai/base:0.0.2"

[provider.gpu]
name = "RTX 4090"
count = 1
min_vram_gb = 20

[selection]
min_reliability = 0.95
min_down_mbps = 100
min_up_mbps = 25
require_verified = true
blocked_geos = ["CN"]
blacklist_hosts = [59017, 1647, 92578, 1276, 75481, 1256, 85323, 34031]
drop_cheap_frac = 0.30
image_size_gb = 5.0
base_url = "https://cloud.vast.ai"

[workspace]
workdir = "."
exclude = [
  "/artifacts",
  "/data/raw",
  "/data/processed",
  "/.venv",
  "/notebooks",
  "__pycache__",
  "*.pyc",
]

[bootstrap]
command = """
uv sync --no-dev
uv run --no-dev python -c "import torch; ok=torch.cuda.is_available(); count=torch.cuda.device_count(); print('torch_cuda_available=' + str(ok)); print('torch_cuda_version=' + str(torch.version.cuda)); print('torch_device_count=' + str(count)); print('torch_device_name=' + (torch.cuda.get_device_name(0) if ok and count else 'none')); assert ok, 'torch CUDA unavailable'"
"""

[data]
validation_command = """
uv run --no-dev python -c "from airfrans_frontier.training.config import load_training_config; c=load_training_config('configs/full_airfrans_incumbent_70gb.toml'); assert c.data.source == 'huggingface'; assert c.data.hf_repo_id == 'zacheryasc/airfrans-processed'; assert c.data.hf_path_prefix == 'processed/full'; assert c.data.cache_dir is not None; assert c.data.train_cases == 900; assert c.data.val_cases == 50; assert c.data.test_cases == 50; assert c.data.batch_size == 4096; assert c.optim.steps == 5000; assert c.model.type == 'film_fourier_inr'; assert c.huggingface.enabled; print('data_source=' + c.data.source + ' hf_repo=' + str(c.data.hf_repo_id) + ' hf_path=' + c.data.hf_path_prefix + ' cache_dir=' + str(c.data.cache_dir))"
"""

[job]
command = """
uv run --no-dev remote-run smoke-train configs/full_airfrans_incumbent_70gb.toml --artifact-dir artifacts/current_run --run-id "$AIRFRANS_REMOTE_RUN_ID"
"""
artifact_dir = "artifacts/current_run"
heartbeat_file = "artifacts/current_run/heartbeat.json"
metrics_file = "artifacts/current_run/metrics.jsonl"

[artifacts]
mode = "object_store_upload"
required = [
  "config.toml",
  "metrics.jsonl",
  "latest_metrics.json",
  "heartbeat.json",
  "checkpoint_latest.pt",
  "checkpoint_best.pt",
  "checkpoint_final.pt",
  "final_metrics.json",
  "split_manifest.json",
  "data_manifest.json",
  "normalization.json",
  "environment_manifest.json",
  "calibration_manifest.json",
  "evaluation_protocol.json",
  "hf_upload_manifest.json",
  "run_manifest.json",
  "artifact_manifest.json",
  "checksums.txt",
  "verification_report.json",
]

[cleanup]
on_success = "sky_down"
on_failure = "sky_down"