7d11dfa688
Implementation of a mock scheduler for jobs/tasks that will evolve into a job manager for running training jobs over vast.ai instances.
222 lines
No EOL
16 KiB
HTML
222 lines
No EOL
16 KiB
HTML
<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<title>Training Run Report</title>
|
|
<style>
|
|
* { margin: 0; padding: 0; box-sizing: border-box; }
|
|
body {
|
|
background: #0f172a; color: #e2e8f0;
|
|
font-family: 'SF Mono', 'Cascadia Code', 'Consolas', 'Menlo', monospace;
|
|
padding: 32px; max-width: 960px; margin: 0 auto;
|
|
}
|
|
h1 { font-size: 20px; margin-bottom: 24px; }
|
|
.stats {
|
|
display: grid; grid-template-columns: repeat(auto-fit, minmax(140px, 1fr));
|
|
gap: 12px; margin-bottom: 32px;
|
|
}
|
|
.stat {
|
|
background: #1e293b; border-radius: 8px; padding: 16px;
|
|
border: 1px solid #334155;
|
|
}
|
|
.stat-value { font-size: 22px; font-weight: bold; }
|
|
.stat-label { font-size: 11px; color: #94a3b8; margin-top: 4px; }
|
|
.legend {
|
|
display: flex; gap: 20px; margin: 16px 0; flex-wrap: wrap;
|
|
}
|
|
.legend-item { display: flex; align-items: center; gap: 6px; font-size: 12px; }
|
|
.legend-dot { width: 12px; height: 12px; border-radius: 3px; flex-shrink: 0; }
|
|
.timeline { margin-top: 8px; }
|
|
.timeline svg { width: 100%; height: auto; }
|
|
.event-log {
|
|
margin-top: 32px; background: #1e293b; border-radius: 8px;
|
|
padding: 16px; border: 1px solid #334155;
|
|
max-height: 400px; overflow-y: auto;
|
|
}
|
|
.event-log pre {
|
|
font-size: 11px; line-height: 1.6; color: #94a3b8;
|
|
white-space: pre-wrap;
|
|
}
|
|
.event-log .err { color: #ef4444; }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<h1>Training Run Report</h1>
|
|
|
|
<div class="stats">
|
|
<div class="stat">
|
|
<div class="stat-value">3</div>
|
|
<div class="stat-label">Rounds</div>
|
|
</div>
|
|
<div class="stat">
|
|
<div class="stat-value">2</div>
|
|
<div class="stat-label">Workers</div>
|
|
</div>
|
|
<div class="stat">
|
|
<div class="stat-value">15.6s</div>
|
|
<div class="stat-label">Total Time</div>
|
|
</div>
|
|
<div class="stat">
|
|
<div class="stat-value">4.7MB</div>
|
|
<div class="stat-label">Data Transferred</div>
|
|
</div>
|
|
<div class="stat">
|
|
<div class="stat-value">2.3MB</div>
|
|
<div class="stat-label">Params Pushed</div>
|
|
</div>
|
|
<div class="stat">
|
|
<div class="stat-value">2.3MB</div>
|
|
<div class="stat-label">Weights Pulled</div>
|
|
</div>
|
|
<div class="stat">
|
|
<div class="stat-value">8.7</div>
|
|
<div class="stat-label">Final |W|</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="legend">
|
|
<div class="legend-item"><div class="legend-dot" style="background:#3b82f6"></div>Push params</div>
|
|
<div class="legend-item"><div class="legend-dot" style="background:#22c55e"></div>Train</div>
|
|
<div class="legend-item"><div class="legend-dot" style="background:#f97316"></div>Pull weights</div>
|
|
<div class="legend-item"><div class="legend-dot" style="background:#8b5cf6"></div>Aggregate</div>
|
|
<div class="legend-item"><div class="legend-dot" style="background:#ef4444"></div>Dead / Error</div>
|
|
</div>
|
|
|
|
<div class="timeline">
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 820 466" width="820" height="466">
|
|
<rect width="820" height="466" fill="#1e293b" rx="8"/>
|
|
<text x="12" y="26" font-size="13" font-family="monospace" fill="#e2e8f0" font-weight="bold">round 0 [2/2] 4.59s</text>
|
|
<line x1="100.0" y1="34" x2="100.0" y2="130" stroke="#334155" stroke-width="0.5" stroke-dasharray="4 4"/>
|
|
<text x="100.0" y="142" font-size="10" font-family="monospace" fill="#94a3b8" text-anchor="middle">0.0s</text>
|
|
<line x1="240.0" y1="34" x2="240.0" y2="130" stroke="#334155" stroke-width="0.5" stroke-dasharray="4 4"/>
|
|
<text x="240.0" y="142" font-size="10" font-family="monospace" fill="#94a3b8" text-anchor="middle">0.9s</text>
|
|
<line x1="380.0" y1="34" x2="380.0" y2="130" stroke="#334155" stroke-width="0.5" stroke-dasharray="4 4"/>
|
|
<text x="380.0" y="142" font-size="10" font-family="monospace" fill="#94a3b8" text-anchor="middle">1.8s</text>
|
|
<line x1="520.0" y1="34" x2="520.0" y2="130" stroke="#334155" stroke-width="0.5" stroke-dasharray="4 4"/>
|
|
<text x="520.0" y="142" font-size="10" font-family="monospace" fill="#94a3b8" text-anchor="middle">2.8s</text>
|
|
<line x1="660.0" y1="34" x2="660.0" y2="130" stroke="#334155" stroke-width="0.5" stroke-dasharray="4 4"/>
|
|
<text x="660.0" y="142" font-size="10" font-family="monospace" fill="#94a3b8" text-anchor="middle">3.7s</text>
|
|
<line x1="800.0" y1="34" x2="800.0" y2="130" stroke="#334155" stroke-width="0.5" stroke-dasharray="4 4"/>
|
|
<text x="800.0" y="142" font-size="10" font-family="monospace" fill="#94a3b8" text-anchor="middle">4.6s</text>
|
|
<text x="12" y="52" font-size="11" font-family="monospace" fill="#94a3b8">scheduler</text>
|
|
<rect x="100" y="38" width="700" height="20" fill="#0f172a" rx="3"/>
|
|
<rect x="100.00781517173503" y="40" width="3" height="16" fill="#3b82f6" rx="2" opacity="0.9"><title>push 399.7KB → node 0 (0.000s)</title></rect>
|
|
<rect x="100.0322995388102" y="40" width="3" height="16" fill="#3b82f6" rx="2" opacity="0.9"><title>push 399.7KB → node 1 (0.000s)</title></rect>
|
|
<rect x="799.338565795855" y="40" width="3" height="16" fill="#f97316" rx="2" opacity="0.9"><title>pull 399.7KB ← node 0 (0.000s)</title></rect>
|
|
<rect x="799.5730082941284" y="40" width="3" height="16" fill="#f97316" rx="2" opacity="0.9"><title>pull 399.7KB ← node 1 (0.000s)</title></rect>
|
|
<rect x="799.7174432972781" y="40" width="3" height="16" fill="#8b5cf6" rx="2" opacity="0.9"><title>aggregate: 2 workers, |W|=8.75, Δ=0.0891</title></rect>
|
|
<text x="12" y="84" font-size="11" font-family="monospace" fill="#94a3b8">worker 0</text>
|
|
<rect x="100" y="70" width="700" height="20" fill="#0f172a" rx="3"/>
|
|
<rect x="100.00781517173503" y="72" width="3" height="16" fill="#3b82f6" rx="2" opacity="0.9"><title>push 399.7KB → node 0 (0.000s)</title></rect>
|
|
<rect x="100.05669380279495" y="72" width="699.27049173338" height="16" fill="#22c55e" rx="2" opacity="0.9"><title>train node 0: 4.59s exit=0
|
|
cpu = _conversion_method_template(device=torch.device("cpu"))</title></rect>
|
|
<rect x="799.338565795855" y="72" width="3" height="16" fill="#f97316" rx="2" opacity="0.9"><title>pull 399.7KB ← node 0 (0.000s)</title></rect>
|
|
<text x="12" y="116" font-size="11" font-family="monospace" fill="#94a3b8">worker 1</text>
|
|
<rect x="100" y="102" width="700" height="20" fill="#0f172a" rx="3"/>
|
|
<rect x="100.0322995388102" y="104" width="3" height="16" fill="#3b82f6" rx="2" opacity="0.9"><title>push 399.7KB → node 1 (0.000s)</title></rect>
|
|
<rect x="100.25768249946535" y="104" width="657.6046307405937" height="16" fill="#22c55e" rx="2" opacity="0.9"><title>train node 1: 4.31s exit=0
|
|
cpu = _conversion_method_template(device=torch.device("cpu"))</title></rect>
|
|
<rect x="799.5730082941284" y="104" width="3" height="16" fill="#f97316" rx="2" opacity="0.9"><title>pull 399.7KB ← node 1 (0.000s)</title></rect>
|
|
<text x="12" y="168" font-size="13" font-family="monospace" fill="#e2e8f0" font-weight="bold">round 1 [2/2] 5.53s</text>
|
|
<line x1="100.0" y1="176" x2="100.0" y2="272" stroke="#334155" stroke-width="0.5" stroke-dasharray="4 4"/>
|
|
<text x="100.0" y="284" font-size="10" font-family="monospace" fill="#94a3b8" text-anchor="middle">0.0s</text>
|
|
<line x1="240.0" y1="176" x2="240.0" y2="272" stroke="#334155" stroke-width="0.5" stroke-dasharray="4 4"/>
|
|
<text x="240.0" y="284" font-size="10" font-family="monospace" fill="#94a3b8" text-anchor="middle">1.1s</text>
|
|
<line x1="380.0" y1="176" x2="380.0" y2="272" stroke="#334155" stroke-width="0.5" stroke-dasharray="4 4"/>
|
|
<text x="380.0" y="284" font-size="10" font-family="monospace" fill="#94a3b8" text-anchor="middle">2.2s</text>
|
|
<line x1="520.0" y1="176" x2="520.0" y2="272" stroke="#334155" stroke-width="0.5" stroke-dasharray="4 4"/>
|
|
<text x="520.0" y="284" font-size="10" font-family="monospace" fill="#94a3b8" text-anchor="middle">3.3s</text>
|
|
<line x1="660.0" y1="176" x2="660.0" y2="272" stroke="#334155" stroke-width="0.5" stroke-dasharray="4 4"/>
|
|
<text x="660.0" y="284" font-size="10" font-family="monospace" fill="#94a3b8" text-anchor="middle">4.4s</text>
|
|
<line x1="800.0" y1="176" x2="800.0" y2="272" stroke="#334155" stroke-width="0.5" stroke-dasharray="4 4"/>
|
|
<text x="800.0" y="284" font-size="10" font-family="monospace" fill="#94a3b8" text-anchor="middle">5.5s</text>
|
|
<text x="12" y="194" font-size="11" font-family="monospace" fill="#94a3b8">scheduler</text>
|
|
<rect x="100" y="180" width="700" height="20" fill="#0f172a" rx="3"/>
|
|
<rect x="100.00834405549375" y="182" width="3" height="16" fill="#3b82f6" rx="2" opacity="0.9"><title>push 399.6KB → node 0 (0.000s)</title></rect>
|
|
<rect x="100.03682653927218" y="182" width="3" height="16" fill="#3b82f6" rx="2" opacity="0.9"><title>push 399.6KB → node 1 (0.002s)</title></rect>
|
|
<rect x="799.4986844978123" y="182" width="3" height="16" fill="#f97316" rx="2" opacity="0.9"><title>pull 399.7KB ← node 0 (0.000s)</title></rect>
|
|
<rect x="799.6683723520547" y="182" width="3" height="16" fill="#f97316" rx="2" opacity="0.9"><title>pull 399.7KB ← node 1 (0.000s)</title></rect>
|
|
<rect x="799.8614614220614" y="182" width="3" height="16" fill="#8b5cf6" rx="2" opacity="0.9"><title>aggregate: 2 workers, |W|=8.74, Δ=0.0876</title></rect>
|
|
<text x="12" y="226" font-size="11" font-family="monospace" fill="#94a3b8">worker 0</text>
|
|
<rect x="100" y="212" width="700" height="20" fill="#0f172a" rx="3"/>
|
|
<rect x="100.00834405549375" y="214" width="3" height="16" fill="#3b82f6" rx="2" opacity="0.9"><title>push 399.6KB → node 0 (0.000s)</title></rect>
|
|
<rect x="100.30709872380446" y="214" width="676.9476932782056" height="16" fill="#22c55e" rx="2" opacity="0.9"><title>train node 0: 5.35s exit=0
|
|
cpu = _conversion_method_template(device=torch.device("cpu"))</title></rect>
|
|
<rect x="799.4986844978123" y="214" width="3" height="16" fill="#f97316" rx="2" opacity="0.9"><title>pull 399.7KB ← node 0 (0.000s)</title></rect>
|
|
<text x="12" y="258" font-size="11" font-family="monospace" fill="#94a3b8">worker 1</text>
|
|
<rect x="100" y="244" width="700" height="20" fill="#0f172a" rx="3"/>
|
|
<rect x="100.03682653927218" y="246" width="3" height="16" fill="#3b82f6" rx="2" opacity="0.9"><title>push 399.6KB → node 1 (0.002s)</title></rect>
|
|
<rect x="100.3581074116179" y="246" width="699.1320361372128" height="16" fill="#22c55e" rx="2" opacity="0.9"><title>train node 1: 5.53s exit=0
|
|
cpu = _conversion_method_template(device=torch.device("cpu"))</title></rect>
|
|
<rect x="799.6683723520547" y="246" width="3" height="16" fill="#f97316" rx="2" opacity="0.9"><title>pull 399.7KB ← node 1 (0.000s)</title></rect>
|
|
<text x="12" y="310" font-size="13" font-family="monospace" fill="#e2e8f0" font-weight="bold">round 2 [2/2] 5.50s</text>
|
|
<line x1="100.0" y1="318" x2="100.0" y2="414" stroke="#334155" stroke-width="0.5" stroke-dasharray="4 4"/>
|
|
<text x="100.0" y="426" font-size="10" font-family="monospace" fill="#94a3b8" text-anchor="middle">0.0s</text>
|
|
<line x1="240.0" y1="318" x2="240.0" y2="414" stroke="#334155" stroke-width="0.5" stroke-dasharray="4 4"/>
|
|
<text x="240.0" y="426" font-size="10" font-family="monospace" fill="#94a3b8" text-anchor="middle">1.1s</text>
|
|
<line x1="380.0" y1="318" x2="380.0" y2="414" stroke="#334155" stroke-width="0.5" stroke-dasharray="4 4"/>
|
|
<text x="380.0" y="426" font-size="10" font-family="monospace" fill="#94a3b8" text-anchor="middle">2.2s</text>
|
|
<line x1="520.0" y1="318" x2="520.0" y2="414" stroke="#334155" stroke-width="0.5" stroke-dasharray="4 4"/>
|
|
<text x="520.0" y="426" font-size="10" font-family="monospace" fill="#94a3b8" text-anchor="middle">3.3s</text>
|
|
<line x1="660.0" y1="318" x2="660.0" y2="414" stroke="#334155" stroke-width="0.5" stroke-dasharray="4 4"/>
|
|
<text x="660.0" y="426" font-size="10" font-family="monospace" fill="#94a3b8" text-anchor="middle">4.4s</text>
|
|
<line x1="800.0" y1="318" x2="800.0" y2="414" stroke="#334155" stroke-width="0.5" stroke-dasharray="4 4"/>
|
|
<text x="800.0" y="426" font-size="10" font-family="monospace" fill="#94a3b8" text-anchor="middle">5.5s</text>
|
|
<text x="12" y="336" font-size="11" font-family="monospace" fill="#94a3b8">scheduler</text>
|
|
<rect x="100" y="322" width="700" height="20" fill="#0f172a" rx="3"/>
|
|
<rect x="100.02026148389356" y="324" width="3" height="16" fill="#3b82f6" rx="2" opacity="0.9"><title>push 399.6KB → node 0 (0.000s)</title></rect>
|
|
<rect x="100.05808530320598" y="324" width="3" height="16" fill="#3b82f6" rx="2" opacity="0.9"><title>push 399.6KB → node 1 (0.000s)</title></rect>
|
|
<rect x="799.363116457372" y="324" width="3" height="16" fill="#f97316" rx="2" opacity="0.9"><title>pull 399.7KB ← node 0 (0.000s)</title></rect>
|
|
<rect x="799.5569707971623" y="324" width="3" height="16" fill="#f97316" rx="2" opacity="0.9"><title>pull 399.7KB ← node 1 (0.000s)</title></rect>
|
|
<rect x="799.7684761253557" y="324" width="3" height="16" fill="#8b5cf6" rx="2" opacity="0.9"><title>aggregate: 2 workers, |W|=8.74, Δ=0.0883</title></rect>
|
|
<text x="12" y="368" font-size="11" font-family="monospace" fill="#94a3b8">worker 0</text>
|
|
<rect x="100" y="354" width="700" height="20" fill="#0f172a" rx="3"/>
|
|
<rect x="100.02026148389356" y="356" width="3" height="16" fill="#3b82f6" rx="2" opacity="0.9"><title>push 399.6KB → node 0 (0.000s)</title></rect>
|
|
<rect x="100.09750932913751" y="356" width="699.2563629168552" height="16" fill="#22c55e" rx="2" opacity="0.9"><title>train node 0: 5.50s exit=0
|
|
cpu = _conversion_method_template(device=torch.device("cpu"))</title></rect>
|
|
<rect x="799.363116457372" y="356" width="3" height="16" fill="#f97316" rx="2" opacity="0.9"><title>pull 399.7KB ← node 0 (0.000s)</title></rect>
|
|
<text x="12" y="400" font-size="11" font-family="monospace" fill="#94a3b8">worker 1</text>
|
|
<rect x="100" y="386" width="700" height="20" fill="#0f172a" rx="3"/>
|
|
<rect x="100.05808530320598" y="388" width="3" height="16" fill="#3b82f6" rx="2" opacity="0.9"><title>push 399.6KB → node 1 (0.000s)</title></rect>
|
|
<rect x="100.15763886743558" y="388" width="684.0486400269059" height="16" fill="#22c55e" rx="2" opacity="0.9"><title>train node 1: 5.38s exit=0
|
|
cpu = _conversion_method_template(device=torch.device("cpu"))</title></rect>
|
|
<rect x="799.5569707971623" y="388" width="3" height="16" fill="#f97316" rx="2" opacity="0.9"><title>pull 399.7KB ← node 1 (0.000s)</title></rect>
|
|
</svg>
|
|
</div>
|
|
|
|
<div class="event-log">
|
|
<pre>[ 0.00s] push node=0 worker.py 1.8KB 0.000s
|
|
[ 0.00s] push node=0 job_module.py 1.4KB 0.000s
|
|
[ 0.00s] push node=1 worker.py 1.8KB 0.000s
|
|
[ 0.00s] push node=1 job_module.py 1.4KB 0.000s
|
|
[ 0.00s] ──── round 0 start (2 workers) ────────────────
|
|
[ 0.00s] push node=0 params.pt 399.7KB 0.000s
|
|
[ 0.00s] push node=1 params.pt 399.7KB 0.000s
|
|
[ 4.32s] exec node=1 exit=0 4.313s | cpu = _conversion_method_template(device=torch.device("cpu"))
|
|
[ 4.59s] exec node=0 exit=0 4.587s | cpu = _conversion_method_template(device=torch.device("cpu"))
|
|
[ 4.59s] pull node=0 weights.pt 399.7KB 0.000s
|
|
[ 4.59s] pull node=1 weights.pt 399.7KB 0.000s
|
|
[ 4.59s] agg 2 workers |W|=8.75 Δ=0.0891 0.001s
|
|
[ 4.59s] ──── round 0 end (2/2 survived, 4.59s) ────────
|
|
[ 4.59s] ──── round 1 start (2 workers) ────────────────
|
|
[ 4.59s] push node=0 params.pt 399.6KB 0.000s
|
|
[ 4.60s] push node=1 params.pt 399.6KB 0.002s
|
|
[ 9.95s] exec node=0 exit=0 5.350s | cpu = _conversion_method_template(device=torch.device("cpu"))
|
|
[ 10.12s] exec node=1 exit=0 5.525s | cpu = _conversion_method_template(device=torch.device("cpu"))
|
|
[ 10.12s] pull node=0 weights.pt 399.7KB 0.000s
|
|
[ 10.12s] pull node=1 weights.pt 399.7KB 0.000s
|
|
[ 10.13s] agg 2 workers |W|=8.74 Δ=0.0876 0.001s
|
|
[ 10.13s] ──── round 1 end (2/2 survived, 5.53s) ────────
|
|
[ 10.13s] ──── round 2 start (2 workers) ────────────────
|
|
[ 10.13s] push node=0 params.pt 399.6KB 0.000s
|
|
[ 10.13s] push node=1 params.pt 399.6KB 0.000s
|
|
[ 15.51s] exec node=1 exit=0 5.378s | cpu = _conversion_method_template(device=torch.device("cpu"))
|
|
[ 15.63s] exec node=0 exit=0 5.498s | cpu = _conversion_method_template(device=torch.device("cpu"))
|
|
[ 15.63s] pull node=0 weights.pt 399.7KB 0.000s
|
|
[ 15.63s] pull node=1 weights.pt 399.7KB 0.000s
|
|
[ 15.63s] agg 2 workers |W|=8.74 Δ=0.0883 0.001s
|
|
[ 15.63s] ──── round 2 end (2/2 survived, 5.50s) ────────</pre>
|
|
</div>
|
|
|
|
</body>
|
|
</html> |