Compare commits
2 commits
c42bf5e983
...
cd3c05c045
| Author | SHA1 | Date | |
|---|---|---|---|
| cd3c05c045 | |||
| f67dcbbec1 |
69 changed files with 6373 additions and 12899 deletions
|
|
@ -7,6 +7,13 @@ target/*
|
|||
!target/release/
|
||||
target/release/*
|
||||
!target/release/myelin-worker
|
||||
!target/x86_64-unknown-linux-musl/
|
||||
target/x86_64-unknown-linux-musl/*
|
||||
!target/x86_64-unknown-linux-musl/release/
|
||||
target/x86_64-unknown-linux-musl/release/*
|
||||
!target/x86_64-unknown-linux-musl/release/myelin-job-worker
|
||||
!.deploy-notes/
|
||||
!.deploy-notes/job_worker_entrypoint.sh
|
||||
!target/debug/
|
||||
target/debug/*
|
||||
!target/debug/myelin-worker
|
||||
|
|
|
|||
208
Cargo.lock
generated
208
Cargo.lock
generated
|
|
@ -953,6 +953,7 @@ name = "data-plane"
|
|||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"parking_lot",
|
||||
"serde",
|
||||
"swactor",
|
||||
"telemetry",
|
||||
|
|
@ -1217,15 +1218,6 @@ version = "0.6.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d"
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs"
|
||||
version = "0.8.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "enum-assoc"
|
||||
version = "1.3.0"
|
||||
|
|
@ -1271,6 +1263,16 @@ version = "0.3.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24"
|
||||
|
||||
[[package]]
|
||||
name = "filetime"
|
||||
version = "0.2.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
version = "0.1.9"
|
||||
|
|
@ -1299,21 +1301,6 @@ version = "0.2.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
|
||||
|
||||
[[package]]
|
||||
name = "foreign-types"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
|
||||
dependencies = [
|
||||
"foreign-types-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "foreign-types-shared"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
|
||||
|
||||
[[package]]
|
||||
name = "form_urlencoded"
|
||||
version = "1.2.2"
|
||||
|
|
@ -1782,22 +1769,6 @@ dependencies = [
|
|||
"webpki-roots 1.0.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-tls"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
"native-tls",
|
||||
"tokio",
|
||||
"tokio-native-tls",
|
||||
"tower-service",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-util"
|
||||
version = "0.1.20"
|
||||
|
|
@ -1816,11 +1787,9 @@ dependencies = [
|
|||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"socket2",
|
||||
"system-configuration",
|
||||
"tokio",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
"windows-registry",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2128,6 +2097,7 @@ name = "iroh-driver"
|
|||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"crossbeam-channel",
|
||||
"data-plane",
|
||||
"distribution",
|
||||
"iroh",
|
||||
"iroh-relay",
|
||||
|
|
@ -2342,9 +2312,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.102"
|
||||
version = "0.3.104"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31"
|
||||
checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"futures-util",
|
||||
|
|
@ -2522,10 +2492,12 @@ dependencies = [
|
|||
"signal-hook",
|
||||
"swactor",
|
||||
"swactor-engine",
|
||||
"swactor-job-runner",
|
||||
"swactor-process",
|
||||
"swactor-transport",
|
||||
"swactor-vastai",
|
||||
"telemetry",
|
||||
"tempfile",
|
||||
"tokio",
|
||||
"toml 0.8.23",
|
||||
"ureq",
|
||||
|
|
@ -2585,23 +2557,6 @@ dependencies = [
|
|||
"n0-future",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "native-tls"
|
||||
version = "0.2.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"log",
|
||||
"openssl",
|
||||
"openssl-probe",
|
||||
"openssl-sys",
|
||||
"schannel",
|
||||
"security-framework",
|
||||
"security-framework-sys",
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ndk-context"
|
||||
version = "0.1.1"
|
||||
|
|
@ -2972,49 +2927,12 @@ version = "0.3.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
|
||||
|
||||
[[package]]
|
||||
name = "openssl"
|
||||
version = "0.10.81"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cfg-if",
|
||||
"foreign-types",
|
||||
"libc",
|
||||
"openssl-macros",
|
||||
"openssl-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openssl-macros"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openssl-probe"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
|
||||
|
||||
[[package]]
|
||||
name = "openssl-sys"
|
||||
version = "0.9.117"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
"pkg-config",
|
||||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "papaya"
|
||||
version = "0.2.4"
|
||||
|
|
@ -3141,12 +3059,6 @@ dependencies = [
|
|||
"spki 0.8.0-rc.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pkg-config"
|
||||
version = "0.3.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
|
||||
|
||||
[[package]]
|
||||
name = "plist"
|
||||
version = "1.9.0"
|
||||
|
|
@ -3352,6 +3264,8 @@ dependencies = [
|
|||
"parking_lot",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"swactor",
|
||||
"swactor-engine",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3677,20 +3591,15 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
|
|||
dependencies = [
|
||||
"base64",
|
||||
"bytes",
|
||||
"encoding_rs",
|
||||
"futures-core",
|
||||
"h2",
|
||||
"http",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
"hyper-rustls",
|
||||
"hyper-tls",
|
||||
"hyper-util",
|
||||
"js-sys",
|
||||
"log",
|
||||
"mime",
|
||||
"native-tls",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"quinn",
|
||||
|
|
@ -3701,7 +3610,6 @@ dependencies = [
|
|||
"serde_urlencoded",
|
||||
"sync_wrapper",
|
||||
"tokio",
|
||||
"tokio-native-tls",
|
||||
"tokio-rustls",
|
||||
"tower",
|
||||
"tower-http",
|
||||
|
|
@ -4401,6 +4309,22 @@ dependencies = [
|
|||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "swactor-job-runner"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"parking_lot",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"swactor",
|
||||
"swactor-engine",
|
||||
"swactor-process",
|
||||
"swactor-transport",
|
||||
"tar",
|
||||
"tempfile",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "swactor-process"
|
||||
version = "0.1.0"
|
||||
|
|
@ -4495,6 +4419,17 @@ version = "0.2.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
|
||||
|
||||
[[package]]
|
||||
name = "tar"
|
||||
version = "0.4.46"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840"
|
||||
dependencies = [
|
||||
"filetime",
|
||||
"libc",
|
||||
"xattr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "target-lexicon"
|
||||
version = "0.12.16"
|
||||
|
|
@ -4671,16 +4606,6 @@ dependencies = [
|
|||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-native-tls"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
|
||||
dependencies = [
|
||||
"native-tls",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-rustls"
|
||||
version = "0.26.4"
|
||||
|
|
@ -5097,12 +5022,6 @@ version = "0.1.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
|
||||
|
||||
[[package]]
|
||||
name = "vcpkg"
|
||||
version = "0.2.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
|
||||
|
||||
[[package]]
|
||||
name = "vergen"
|
||||
version = "9.1.0"
|
||||
|
|
@ -5202,9 +5121,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.125"
|
||||
version = "0.2.127"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a"
|
||||
checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
|
|
@ -5215,9 +5134,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-futures"
|
||||
version = "0.4.75"
|
||||
version = "0.4.77"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280"
|
||||
checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
|
|
@ -5225,9 +5144,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.125"
|
||||
version = "0.2.127"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d"
|
||||
checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"wasm-bindgen-macro-support",
|
||||
|
|
@ -5235,9 +5154,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro-support"
|
||||
version = "0.2.125"
|
||||
version = "0.2.127"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd"
|
||||
checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"proc-macro2",
|
||||
|
|
@ -5248,9 +5167,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-shared"
|
||||
version = "0.2.125"
|
||||
version = "0.2.127"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f"
|
||||
checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
|
@ -5278,9 +5197,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "web-sys"
|
||||
version = "0.3.102"
|
||||
version = "0.3.104"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d"
|
||||
checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
|
|
@ -5818,6 +5737,16 @@ dependencies = [
|
|||
"time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "xattr"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"rustix",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "xml-rs"
|
||||
version = "0.8.28"
|
||||
|
|
@ -5837,8 +5766,8 @@ dependencies = [
|
|||
name = "xtask"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"blake3",
|
||||
"dashboard",
|
||||
"data-plane",
|
||||
"distribution",
|
||||
"iroh",
|
||||
"iroh-driver",
|
||||
|
|
@ -5852,7 +5781,6 @@ dependencies = [
|
|||
"swactor-process",
|
||||
"swactor-transport",
|
||||
"telemetry",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ members = [
|
|||
"crates/bindings/python",
|
||||
"crates/bindings/wasm-runtime",
|
||||
"crates/process",
|
||||
"crates/provisioning",
|
||||
"crates/job-runner",
|
||||
"crates/transport",
|
||||
"crates/distribution",
|
||||
"crates/iroh-driver",
|
||||
|
|
|
|||
|
|
@ -4,17 +4,18 @@ version = "0.1.0"
|
|||
edition = "2024"
|
||||
license = "AGPL-3.0-only"
|
||||
publish = false
|
||||
default-run = "myelin-orchestrator"
|
||||
autobins = false
|
||||
|
||||
[features]
|
||||
default = []
|
||||
default = ["dashboard"]
|
||||
dashboard = []
|
||||
|
||||
[dependencies]
|
||||
telemetry = { path = "../../crates/telemetry" }
|
||||
data-plane = { path = "../../crates/data-plane" }
|
||||
provisioning = { path = "../../crates/provisioning" }
|
||||
dashboard = { path = "../../crates/dashboard" }
|
||||
dashboard = { path = "../../crates/dashboard", features = ["demo-control"] }
|
||||
serde_json = "1"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
swactor = { path = "../..", features = ["serde", "transport"] }
|
||||
|
|
@ -22,6 +23,7 @@ swactor-engine = { path = "../../crates/engine" }
|
|||
swactor-transport = { path = "../../crates/transport" }
|
||||
swactor-process = { path = "../../crates/process" }
|
||||
distribution = { path = "../../crates/distribution" }
|
||||
swactor-job-runner = { path = "../../crates/job-runner" }
|
||||
iroh-driver = { path = "../../crates/iroh-driver" }
|
||||
iroh = "0.98"
|
||||
tokio.workspace = true
|
||||
|
|
@ -32,6 +34,7 @@ toml = "0.8"
|
|||
ureq = "2"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
wiremock = "0.6"
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
|
|
@ -47,6 +50,10 @@ name = "myelin-orchestrator"
|
|||
path = "src/bin/orchestrator.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "myelin-chat"
|
||||
path = "src/bin/chat.rs"
|
||||
name = "myelin-job-worker"
|
||||
path = "src/bin/job_worker.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "myelin-job"
|
||||
path = "src/bin/job.rs"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
# Myelin node CODE image — thin layer over the CUDA/tinygrad base.
|
||||
# Myelin node agent: a thin Rust control-plane layer over the CUDA base.
|
||||
# Workload frameworks are supplied by job payload images.
|
||||
# Build from workspace root after compiling the Rust binary:
|
||||
# cargo build --release -p myelin --bin myelin-worker
|
||||
# docker build -f apps/myelin/node-image/Dockerfile.base -t myelin-node-base:cuda12.6 .
|
||||
|
|
@ -10,6 +11,4 @@ FROM ${BASE_IMAGE}
|
|||
ARG MYELIN_NODE_BIN=target/release/myelin-worker
|
||||
|
||||
COPY ${MYELIN_NODE_BIN} /usr/local/bin/myelin-node
|
||||
COPY apps/myelin/node-image/tinygrad_worker.py /usr/local/share/myelin/tinygrad_worker.py
|
||||
|
||||
RUN chmod +x /usr/local/bin/myelin-node /usr/local/share/myelin/tinygrad_worker.py
|
||||
RUN chmod +x /usr/local/bin/myelin-node
|
||||
|
|
|
|||
|
|
@ -1,37 +1,12 @@
|
|||
# Myelin node BASE image — CUDA/Python/tinygrad/sshd/PID-1 foundation.
|
||||
# Myelin node base: CUDA runtime, SSH bootstrap, and the Myelin agent only.
|
||||
# Frameworks belong to job payload images, not the control-plane agent image.
|
||||
# Build from workspace root:
|
||||
# docker build -f apps/myelin/node-image/Dockerfile.base -t myelin-node-base:cuda12.6 .
|
||||
|
||||
FROM nvidia/cuda:12.6.3-runtime-ubuntu24.04 AS builder
|
||||
FROM nvidia/cuda:12.6.3-runtime-ubuntu24.04
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
python3 \
|
||||
python3-pip \
|
||||
ca-certificates \
|
||||
cuda-cudart-dev-12-6 && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN python3 -m pip install --no-cache-dir --break-system-packages \
|
||||
--target=/opt/myelin-pydeps \
|
||||
tinygrad==0.12.0 numpy && \
|
||||
find /opt/myelin-pydeps -depth -type d \
|
||||
\( -name '__pycache__' -o -name 'tests' -o -name 'test' \) \
|
||||
-exec rm -rf {} + && \
|
||||
find /opt/myelin-pydeps -name '*.pyc' -delete && \
|
||||
find /opt/myelin-pydeps -type d -name '*.dist-info' -exec rm -rf {} +
|
||||
|
||||
RUN mkdir -p /opt/myelin-nvrtc-include && \
|
||||
cp -rL /usr/local/cuda/include/. /opt/myelin-nvrtc-include/ && \
|
||||
test -f /opt/myelin-nvrtc-include/vector_types.h
|
||||
|
||||
FROM nvidia/cuda:12.6.3-base-ubuntu24.04 AS runtime
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
python3 \
|
||||
cuda-cudart-12-6 \
|
||||
cuda-nvrtc-12-6 \
|
||||
ca-certificates \
|
||||
procps \
|
||||
openssh-server && \
|
||||
|
|
@ -46,18 +21,7 @@ RUN apt-get update && \
|
|||
/usr/share/doc/* \
|
||||
/usr/share/man/* \
|
||||
/usr/share/info/* && \
|
||||
find /usr -depth -type d -name '__pycache__' -exec rm -rf {} + 2>/dev/null || true && \
|
||||
find /usr -type f -name '*.pyc' -delete 2>/dev/null || true && \
|
||||
mkdir -p /usr/local/share/myelin /var/cache/myelin-models /var/log
|
||||
|
||||
COPY --from=builder /opt/myelin-nvrtc-include/ /usr/local/cuda/include/
|
||||
COPY --from=builder /opt/myelin-pydeps /opt/myelin-pydeps
|
||||
ENV PYTHONPATH=/opt/myelin-pydeps
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
ENV CUDA=1
|
||||
ENV DEV=CUDA
|
||||
ENV MYELIN_MODEL_CACHE_DIR=/var/cache/myelin-models
|
||||
ENV MYELIN_TINYGRAD_WORKER=/usr/local/share/myelin/tinygrad_worker.py
|
||||
mkdir -p /usr/local/share/myelin /var/cache/myelin-jobs /var/log
|
||||
|
||||
COPY apps/myelin/node-image/myelin_entrypoint.sh /usr/local/bin/myelin_entrypoint.sh
|
||||
RUN chmod +x /usr/local/bin/myelin_entrypoint.sh
|
||||
|
|
|
|||
72
apps/myelin/specs/MYELIN_DAEMON.md
Normal file
72
apps/myelin/specs/MYELIN_DAEMON.md
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
# Myelin Fleet-Control Daemon
|
||||
|
||||
Myelin is a persistent, dashboard-first control plane for manually managed compute nodes. It boots an empty fleet, accepts explicit operator commands, records intent and observations, and never performs hidden replacement or teardown.
|
||||
|
||||
## Runtime shape
|
||||
|
||||
```text
|
||||
myelin-orchestrator
|
||||
-> load stable iroh identity and cluster snapshot
|
||||
-> start engine, iroh endpoint, telemetry collector, and dashboard
|
||||
-> adopt provider resources carrying this daemon's stable label
|
||||
-> idle event loop
|
||||
- pump telemetry and membership
|
||||
- dispatch add / kill / destroy commands
|
||||
- atomically persist every state transition
|
||||
-> on exit, detach provider handles without destroying resources
|
||||
```
|
||||
|
||||
There is no prompt RPC, chat loop, desired-shape reconciler, or automatic node replacement. GGUF pipeline code remains dormant for historical compatibility. Job submission and a node re-join handshake are deferred.
|
||||
|
||||
## Starting the daemon
|
||||
|
||||
From the workspace root:
|
||||
|
||||
```sh
|
||||
cargo run -p myelin
|
||||
```
|
||||
|
||||
The local default uses the process provider. Use Docker explicitly when required:
|
||||
|
||||
```sh
|
||||
cargo run -p myelin -- --provider docker
|
||||
```
|
||||
|
||||
`MYELIN_DASHBOARD_PORT` selects the dashboard port. The dashboard root is the fleet-control view. Its control routes dispatch the same provider-neutral commands intended for a future CLI:
|
||||
|
||||
- `Provision { command_id, count }`: add exactly `count` nodes, one transaction at a time.
|
||||
- `Kill { command_id, node }`: stop a node but retain its dead snapshot record.
|
||||
- `Remove { command_id, count }`: destroy the highest numbered managed nodes and remove their snapshot records.
|
||||
- `EstablishEdge { command_id, node }`: rejected; workload topology is not part of fleet control.
|
||||
|
||||
Every request carries a caller-generated command id. The daemon persists that id before any provider mutation; retries are ignored across restarts. This is deliberately at-most-once: a crash after acceptance may require a new operator command, but can never double-rent or double-destroy a resource. Node ids are monotonic and never reused.
|
||||
|
||||
## Durable state
|
||||
|
||||
The state directory contains:
|
||||
|
||||
- `identity.key`: 32-byte iroh secret key. Preserving it keeps the daemon endpoint stable across restarts.
|
||||
- `cluster.json`: schema-versioned snapshot containing the stable provider label, run id, next node id, accepted command ids, node specs, provider references, runtime facts, and observed status.
|
||||
|
||||
Writes use a temporary file plus rename. A corrupt identity or snapshot is a hard startup error. `--reset-state` explicitly clears both files; startup never treats corruption as an empty fleet.
|
||||
|
||||
Provider state is ground truth during adoption:
|
||||
|
||||
- snapshot + provider resource: adopt and observe it;
|
||||
- snapshot only: mark dead;
|
||||
- provider resource only: report as an orphan and take no action.
|
||||
|
||||
The current restart limitation is deliberate: an adopted node still has the prior orchestrator actor address in its environment. Provider monitoring and telemetry collection can resume, but actor-address re-join requires the deferred node handshake.
|
||||
|
||||
## Lifecycle policy
|
||||
|
||||
Graceful shutdown leaves Docker containers and Vast.ai leases running so a later daemon can adopt them. Local process children are different: they cannot be adopted, so Ctrl-C stops them and clears their snapshot records. They also exit when their daemon-owned stdin supervision pipe closes, preventing an abrupt daemon crash from leaving invisible local workers.
|
||||
|
||||
Destruction of durable provider resources occurs only through an explicit dashboard command or the development-only `--destroy-on-exit` flag. The default process path re-enters the running orchestrator executable in an internal worker mode, so `cargo run -p myelin` never depends on a separately built or stale `myelin-worker` binary.
|
||||
|
||||
## Node image contract
|
||||
|
||||
The standard image contains a uniform Myelin agent, SSH bootstrap, and the CUDA runtime. It does not contain tinygrad, NumPy, PyTorch, vLLM, or model weights. Frameworks and application dependencies belong to job payload images. Nodes launched by this daemon set `MYELIN_AGENT_ONLY=1`, so the agent joins membership, announces readiness, and exports telemetry without starting an inference helper.
|
||||
As part of runtime-ready bootstrap, the daemon dials the node's advertised iroh endpoint on `TELEMETRY_ALPN`, requests all telemetry channels, and retains that pull stream for the node's lifetime. The node serves the pull locally; it never needs to reverse-dial the dashboard. Pulled stream descriptors and frames feed both the Fleet view and the live telemetry explorer.
|
||||
|
||||
The retired chat/GGUF specification is archived at `archive/MYELIN_CHAT_SPEC.md`.
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
fn main() -> std::process::ExitCode {
|
||||
myelin::run_chat_from_args(std::env::args().skip(1))
|
||||
}
|
||||
5
apps/myelin/src/bin/job.rs
Normal file
5
apps/myelin/src/bin/job.rs
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
use std::process::ExitCode;
|
||||
|
||||
fn main() -> ExitCode {
|
||||
myelin::run_job_serve_from_args(std::env::args().skip(1))
|
||||
}
|
||||
5
apps/myelin/src/bin/job_worker.rs
Normal file
5
apps/myelin/src/bin/job_worker.rs
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
use std::process::ExitCode;
|
||||
|
||||
fn main() -> ExitCode {
|
||||
myelin::run_job_worker_from_args(std::env::args().skip(1))
|
||||
}
|
||||
|
|
@ -1,7 +1,12 @@
|
|||
use std::process::ExitCode;
|
||||
|
||||
fn main() -> ExitCode {
|
||||
match myelin::run_orchestrator_from_args(std::env::args().skip(1)) {
|
||||
let args = std::env::args().skip(1).collect::<Vec<_>>();
|
||||
if args.as_slice() == [myelin::ORCHESTRATOR_WORKER_MODE_ARG] {
|
||||
return myelin::run_worker_node_from_env();
|
||||
}
|
||||
|
||||
match myelin::run_orchestrator_from_args(args) {
|
||||
Ok(()) => ExitCode::SUCCESS,
|
||||
Err(error) => {
|
||||
eprintln!("myelin-orchestrator: {error}");
|
||||
|
|
|
|||
|
|
@ -1,4 +0,0 @@
|
|||
//! Myelin operator chat wrapper public surface.
|
||||
|
||||
mod node_image;
|
||||
pub(super) mod runtime;
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -2,12 +2,10 @@
|
|||
//!
|
||||
//! The runtime's [`CodecRegistry`](swactor_transport::CodecRegistry) needs an
|
||||
//! encoder/decoder entry for each inter-node message type. This is the single
|
||||
//! aggregator that wires up the node, orchestrator, prompt, and telemetry
|
||||
//! publisher codecs.
|
||||
//! aggregator that wires up the node, orchestrator, and prompt codecs.
|
||||
|
||||
pub(crate) fn register_myelin_actor_codecs(registry: &mut swactor_transport::CodecRegistry) {
|
||||
crate::node_actor::register_codecs(registry);
|
||||
crate::orchestration::actor::register_codecs(registry);
|
||||
telemetry::register_telemetry_publisher_codec(registry);
|
||||
crate::prompt::rpc::register_codecs(registry);
|
||||
swactor_job_runner::register_job_codecs(registry);
|
||||
}
|
||||
|
|
|
|||
512
apps/myelin/src/job_deploy.rs
Normal file
512
apps/myelin/src/job_deploy.rs
Normal file
|
|
@ -0,0 +1,512 @@
|
|||
//! Deployable job runner over real iroh. Two entrypoints share one module:
|
||||
//! `run_worker` (the GPU node) and `run_serve` (the operator). Each builds a
|
||||
//! swactor `Engine` + `IrohDriver` + distribution stack, registers its job actor
|
||||
//! in the directory, and exchanges its `EndpointAddr` + actor address
|
||||
//! out-of-band so each side can route to the other over the iroh actor plane.
|
||||
|
||||
use parking_lot::Mutex;
|
||||
use std::env;
|
||||
use std::io::{BufRead, BufReader};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use swactor::actor::ActorAddress;
|
||||
use swactor_engine::{Engine, EngineHandle, TokioBackend, TokioConfig};
|
||||
use swactor_job_runner::{
|
||||
JobDone, NodeJobActor, OUTPUTS_EDGE_ID, OrchestratorJobActor, OrchestratorJobMsg,
|
||||
WORKSPACE_EDGE_ID, register_job_codecs,
|
||||
};
|
||||
use swactor_transport::hex_encode;
|
||||
|
||||
use data_plane::edge_wire::WireEvent;
|
||||
use distribution::node::DistributedNodeConfig;
|
||||
use iroh::{EndpointAddr, RelayMode};
|
||||
use iroh_driver::{
|
||||
EDGE_ALPN, EndpointAddrMask, IrohDriver, IrohDriverConfig, MVP_IROH_ENDPOINT_ADDR_MASK_ENV,
|
||||
advertised_endpoint,
|
||||
};
|
||||
|
||||
use crate::orchestration::distribution_stack::DistributionRuntimeStack;
|
||||
|
||||
const POLL: Duration = Duration::from_millis(25);
|
||||
const CONVERGE_DEADLINE: Duration = Duration::from_secs(90);
|
||||
const JOB_DEADLINE: Duration = Duration::from_secs(60 * 45);
|
||||
const RELAY_WAIT_DEADLINE: Duration = Duration::from_secs(30);
|
||||
const MYELIN_IROH_RELAY_MODE_ENV: &str = "MYELIN_IROH_RELAY_MODE";
|
||||
const MYELIN_IROH_RELAY_URL_ENV: &str = "MYELIN_IROH_RELAY_URL";
|
||||
const SWACTOR_IROH_RELAY_URL_ENV: &str = "SWACTOR_IROH_RELAY_URL";
|
||||
|
||||
/// Out-of-band identity one side publishes so the other can route to it.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct NodeIdentity {
|
||||
pub endpoint: EndpointAddr,
|
||||
pub actor_hex: String,
|
||||
}
|
||||
|
||||
pub(crate) struct JobOrchestratorSession {
|
||||
_engine: Engine,
|
||||
driver: IrohDriver,
|
||||
stack: DistributionRuntimeStack,
|
||||
done: swactor::runtime::Inbox<JobDone>,
|
||||
orch: ActorAddress,
|
||||
identity: NodeIdentity,
|
||||
landing: PathBuf,
|
||||
}
|
||||
|
||||
type JobComposition = (Engine, IrohDriver, DistributionRuntimeStack);
|
||||
|
||||
pub(crate) fn build_composition() -> Result<JobComposition, String> {
|
||||
let (parts, runtime, codec, transport_router) =
|
||||
DistributionRuntimeStack::build_runtime(|c| register_job_codecs(c), None);
|
||||
let engine = Engine::new(
|
||||
parts,
|
||||
TokioBackend::new(TokioConfig::default()).expect("tokio backend"),
|
||||
)
|
||||
.expect("engine");
|
||||
let relay_mode = relay_mode_from_env()?;
|
||||
let mut driver = IrohDriver::with_engine(
|
||||
engine.handle(),
|
||||
IrohDriverConfig {
|
||||
secret_key: None,
|
||||
relay_mode,
|
||||
node: DistributedNodeConfig::default(),
|
||||
peer_auth: None,
|
||||
additional_alpns: vec![EDGE_ALPN.to_vec()],
|
||||
},
|
||||
)
|
||||
.map_err(|e| format!("iroh driver: {e}"))?;
|
||||
let stack = DistributionRuntimeStack::new_from_runtime(
|
||||
runtime.clone(),
|
||||
codec,
|
||||
transport_router,
|
||||
driver.node_id(),
|
||||
DistributedNodeConfig::default(),
|
||||
engine.handle(),
|
||||
);
|
||||
driver.enable_actor_bridge(
|
||||
stack.runtime.clone(),
|
||||
stack.codec.clone(),
|
||||
stack.actor_bridge_routes(),
|
||||
stack.actors.swim,
|
||||
stack.relay_mirror.clone(),
|
||||
stack.route_view.clone(),
|
||||
stack.outbox.clone(),
|
||||
);
|
||||
stack.spawn_protocol_ticker(POLL);
|
||||
driver.install_actor_bridge_pump(POLL);
|
||||
Ok((engine, driver, stack))
|
||||
}
|
||||
|
||||
fn identity_for(driver: &IrohDriver, actor: ActorAddress) -> Result<NodeIdentity, String> {
|
||||
let endpoint = advertised_endpoint_for(driver)?;
|
||||
Ok(NodeIdentity {
|
||||
endpoint,
|
||||
actor_hex: hex_encode(&actor.0),
|
||||
})
|
||||
}
|
||||
|
||||
fn advertised_endpoint_for(driver: &IrohDriver) -> Result<EndpointAddr, String> {
|
||||
let mask = endpoint_addr_mask_from_env()?;
|
||||
if !mask.requires_relay() {
|
||||
return advertised_endpoint(driver.endpoint_addr(), mask);
|
||||
}
|
||||
|
||||
let started = Instant::now();
|
||||
loop {
|
||||
let endpoint = driver.endpoint_addr();
|
||||
if endpoint.relay_urls().next().is_some() {
|
||||
return advertised_endpoint(endpoint, mask);
|
||||
}
|
||||
if started.elapsed() >= RELAY_WAIT_DEADLINE {
|
||||
return Err(format!(
|
||||
"relay-only endpoint address mask did not observe a relay URL within {RELAY_WAIT_DEADLINE:?}; last endpoint={endpoint:?}"
|
||||
));
|
||||
}
|
||||
std::thread::sleep(POLL);
|
||||
}
|
||||
}
|
||||
|
||||
fn endpoint_addr_mask_from_env() -> Result<EndpointAddrMask, String> {
|
||||
match env_optional(MVP_IROH_ENDPOINT_ADDR_MASK_ENV) {
|
||||
Some(mask) => EndpointAddrMask::parse(&mask),
|
||||
None => Ok(EndpointAddrMask::Full),
|
||||
}
|
||||
}
|
||||
|
||||
fn relay_mode_from_env() -> Result<RelayMode, String> {
|
||||
let mode = env_optional(MYELIN_IROH_RELAY_MODE_ENV).map(|value| value.to_ascii_lowercase());
|
||||
let relay_url = env_optional(MYELIN_IROH_RELAY_URL_ENV)
|
||||
.or_else(|| env_optional(SWACTOR_IROH_RELAY_URL_ENV));
|
||||
match mode.as_deref() {
|
||||
Some("disabled") => Ok(RelayMode::Disabled),
|
||||
None | Some("default") => match relay_url {
|
||||
Some(raw) => {
|
||||
let relay = raw
|
||||
.parse()
|
||||
.map_err(|e| format!("invalid relay URL {raw:?}: {e}"))?;
|
||||
Ok(RelayMode::custom(vec![relay]))
|
||||
}
|
||||
None => Ok(RelayMode::Default),
|
||||
},
|
||||
Some(other) => Err(format!(
|
||||
"unsupported {MYELIN_IROH_RELAY_MODE_ENV}={other:?}; use disabled or default"
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn env_optional(name: &str) -> Option<String> {
|
||||
env::var(name)
|
||||
.ok()
|
||||
.map(|value| value.trim().to_owned())
|
||||
.filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
pub(crate) fn parse_actor(hex: &str) -> Result<ActorAddress, String> {
|
||||
let bytes = swactor_transport::hex_decode(hex).ok_or_else(|| "bad actor hex".to_string())?;
|
||||
if bytes.len() != 32 {
|
||||
return Err(format!("actor hex must be 32 bytes, got {}", bytes.len()));
|
||||
}
|
||||
let mut arr = [0u8; 32];
|
||||
arr.copy_from_slice(&bytes);
|
||||
Ok(ActorAddress(arr))
|
||||
}
|
||||
|
||||
/// Worker: connect to the orchestrator, expose a `NodeJobActor`, run jobs it
|
||||
/// sends over iroh. Prints this node's identity as JSON on stdout, then runs
|
||||
/// until killed. Bulk bytes travel over EDGE_ALPN: the orchestrator pushes the
|
||||
/// workspace in (drained + extracted here), and this node ships outputs back.
|
||||
/// Small commands/lifecycle stay on the actor plane. The engine is held for the
|
||||
/// process lifetime.
|
||||
pub fn run_worker(orch_identity_json: String, workdir: PathBuf) -> Result<(), String> {
|
||||
let orch: NodeIdentity = serde_json::from_str(&orch_identity_json)
|
||||
.map_err(|e| format!("parse orch identity: {e}"))?;
|
||||
let orch_actor = parse_actor(&orch.actor_hex)?;
|
||||
let orch_endpoint = orch.endpoint.clone();
|
||||
let (_engine, driver, stack) = build_composition()?;
|
||||
let sender = stack.runtime.create_sender();
|
||||
|
||||
// Edge workspace: the orchestrator pushes the workspace tar over EDGE_ALPN
|
||||
// before submitting the job. A background thread drains those bytes,
|
||||
// extracts them into `workdir`, and signals readiness; the node actor waits
|
||||
// on that flag before announcing the workspace materialized.
|
||||
let workspace_ready = Arc::new(AtomicBool::new(false));
|
||||
let ws_events = driver.edge_events_handle();
|
||||
let ws_workdir = workdir.clone();
|
||||
let ws_ready = workspace_ready.clone();
|
||||
std::thread::Builder::new()
|
||||
.name("job-worker-ws-edge".to_owned())
|
||||
.spawn(move || {
|
||||
drain_workspace_edge(ws_events, ws_workdir, ws_ready);
|
||||
})
|
||||
.map_err(|e| format!("spawn workspace edge thread: {e}"))?;
|
||||
|
||||
// Edge outputs: a slot the main thread fills with an EDGE_ALPN sink to the
|
||||
// orchestrator once the iroh connection is up. The node actor ships
|
||||
// collected outputs through it.
|
||||
let output_sink_slot: Arc<Mutex<Option<Box<dyn swactor_job_runner::JobEdgeSink>>>> =
|
||||
Arc::new(Mutex::new(None));
|
||||
|
||||
let job_actor = stack
|
||||
.runtime
|
||||
.spawn(
|
||||
NodeJobActor::new(orch_actor, workdir, sender, 0)
|
||||
.with_workspace_ready(workspace_ready.clone())
|
||||
.with_output_sink_slot(output_sink_slot.clone()),
|
||||
)
|
||||
.map_err(|e| format!("spawn node job actor: {e}"))?;
|
||||
stack.register_local_actor(driver.register_actor(job_actor, 1));
|
||||
driver.join(std::slice::from_ref(&orch.endpoint));
|
||||
|
||||
let id = identity_for(&driver, job_actor)?;
|
||||
println!(
|
||||
"JOB_WORKER_IDENTITY {}",
|
||||
serde_json::to_string(&id).map_err(|e| e.to_string())?
|
||||
);
|
||||
let _ = std::io::Write::flush(&mut std::io::stdout());
|
||||
eprintln!(
|
||||
"job-worker: ready actor={} endpoint={:?}",
|
||||
id.actor_hex, id.endpoint
|
||||
);
|
||||
|
||||
// Wait for the iroh connection to the orchestrator, then arm the output
|
||||
// edge sink so it is ready before a CollectOutputs command can arrive.
|
||||
let orch_node = swactor_transport::NodeId(*orch_endpoint.id.as_bytes());
|
||||
let conn_started = Instant::now();
|
||||
while !driver.has_active_connection(&orch_node) {
|
||||
if conn_started.elapsed() >= CONVERGE_DEADLINE {
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(POLL);
|
||||
}
|
||||
match driver.spawn_edge_send_pump(orch_endpoint.clone(), OUTPUTS_EDGE_ID) {
|
||||
Ok(handle) => {
|
||||
*output_sink_slot.lock() = Some(Box::new(IrohEdgeSink(handle)));
|
||||
eprintln!("job-worker: output edge sink armed");
|
||||
}
|
||||
Err(e) => eprintln!("job-worker: failed to arm output edge sink: {e}"),
|
||||
}
|
||||
|
||||
loop {
|
||||
std::thread::sleep(Duration::from_secs(3600));
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts the operator-side job actor and publishes enough identity for a
|
||||
/// provisioned worker to join over iroh.
|
||||
pub(crate) fn start_orchestrator(landing: PathBuf) -> Result<JobOrchestratorSession, String> {
|
||||
let (engine, driver, stack) = build_composition()?;
|
||||
let done = stack
|
||||
.runtime
|
||||
.new_inbox::<JobDone>()
|
||||
.map_err(|e| format!("inbox: {e}"))?;
|
||||
let orch = stack
|
||||
.runtime
|
||||
.spawn(OrchestratorJobActor::new(*done.addr(), landing.clone()).with_edge_mode(true))
|
||||
.map_err(|e| format!("spawn orchestrator: {e}"))?;
|
||||
stack.register_local_actor(driver.register_actor(orch, 1));
|
||||
|
||||
let identity = identity_for(&driver, orch)?;
|
||||
Ok(JobOrchestratorSession {
|
||||
_engine: engine,
|
||||
driver,
|
||||
stack,
|
||||
done,
|
||||
orch,
|
||||
identity,
|
||||
landing,
|
||||
})
|
||||
}
|
||||
|
||||
impl JobOrchestratorSession {
|
||||
pub(crate) fn identity_json(&self) -> Result<String, String> {
|
||||
serde_json::to_string(&self.identity).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub(crate) fn runtime(&self) -> swactor::runtime::Runtime {
|
||||
self.stack.runtime.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn engine_handle(&self) -> EngineHandle {
|
||||
self.stack.engine.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn run_to_completion(
|
||||
&mut self,
|
||||
job: swactor_job_runner::Job,
|
||||
worker: NodeIdentity,
|
||||
) -> Result<JobDone, String> {
|
||||
let node_actor = parse_actor(&worker.actor_hex)?;
|
||||
self.driver.join(std::slice::from_ref(&worker.endpoint));
|
||||
|
||||
let started = Instant::now();
|
||||
while started.elapsed() < CONVERGE_DEADLINE {
|
||||
if self
|
||||
.stack
|
||||
.route_view
|
||||
.read()
|
||||
.map(|v| v.contains_key(&node_actor))
|
||||
.unwrap_or(false)
|
||||
{
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(POLL);
|
||||
}
|
||||
if !self
|
||||
.stack
|
||||
.route_view
|
||||
.read()
|
||||
.map(|v| v.contains_key(&node_actor))
|
||||
.unwrap_or(false)
|
||||
{
|
||||
return Err(
|
||||
"directory did not converge: orchestrator never learned the worker actor".into(),
|
||||
);
|
||||
}
|
||||
eprintln!("job-orch: directory converged; waiting for iroh connection to worker");
|
||||
let worker_node = swactor_transport::NodeId(*worker.endpoint.id.as_bytes());
|
||||
let conn_started = Instant::now();
|
||||
while !self.driver.has_active_connection(&worker_node) {
|
||||
if conn_started.elapsed() >= CONVERGE_DEADLINE {
|
||||
eprintln!(
|
||||
"job-orch: no iroh connection to worker after {CONVERGE_DEADLINE:?}; join_statuses={:?}; submitting best-effort",
|
||||
self.driver.join_statuses()
|
||||
);
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(POLL);
|
||||
}
|
||||
if self.driver.has_active_connection(&worker_node) {
|
||||
eprintln!(
|
||||
"job-orch: iroh connection to worker established after {:?}",
|
||||
conn_started.elapsed()
|
||||
);
|
||||
eprintln!("job-orch: submitting job");
|
||||
} else {
|
||||
eprintln!("job-orch: submitting job (no confirmed connection)");
|
||||
}
|
||||
|
||||
// EDGE: push the workspace tar over EDGE_ALPN before submitting. Small
|
||||
// commands/events still travel as actor messages; only bulk bytes move
|
||||
// onto the edge transport so they survive relay (NAT) traversal.
|
||||
let edge_events = self.driver.edge_events_handle();
|
||||
let workspace_bytes =
|
||||
swactor_job_runner::pack_workspace(&job).map_err(|e| format!("pack workspace: {e}"))?;
|
||||
if !workspace_bytes.is_empty() {
|
||||
let pump = self
|
||||
.driver
|
||||
.spawn_edge_send_pump(worker.endpoint.clone(), WORKSPACE_EDGE_ID)
|
||||
.map_err(|e| format!("workspace edge pump: {e}"))?;
|
||||
for record in workspace_bytes.chunks(swactor_job_runner::EDGE_RECORD_SIZE) {
|
||||
pump.send(record.to_vec())
|
||||
.map_err(|e| format!("workspace edge send: {e}"))?;
|
||||
}
|
||||
drop(pump); // finish the edge stream → receiver observes end-of-stream
|
||||
eprintln!("job-orch: workspace pushed over EDGE_ALPN");
|
||||
}
|
||||
|
||||
self.stack
|
||||
.runtime
|
||||
.send_to(self.orch, OrchestratorJobMsg::Submit { job, node_actor })
|
||||
.map_err(|e| format!("submit: {e}"))?;
|
||||
|
||||
// Drive lifecycle (actor messages) while draining the output edge stream.
|
||||
let started = Instant::now();
|
||||
let mut output_buf: Vec<u8> = Vec::new();
|
||||
let mut outputs_ended = false;
|
||||
// The orchestrator actor reports `JobDone` exactly once; hold it here
|
||||
// while we wait for the output edge stream to land so it is not lost.
|
||||
let mut pending_done: Option<JobDone> = None;
|
||||
loop {
|
||||
// Drain output edge bytes; extract the tar as soon as the stream ends
|
||||
// (release the edge-event lock before the potentially slow untar).
|
||||
let drained: Vec<WireEvent> = edge_events.lock().drain(..).collect();
|
||||
for ev in drained {
|
||||
match ev {
|
||||
WireEvent::BytesRead { edge_id, bytes, .. } if edge_id.0 == OUTPUTS_EDGE_ID => {
|
||||
output_buf.extend_from_slice(&bytes);
|
||||
}
|
||||
WireEvent::StreamEnded { edge_id, .. } if edge_id.0 == OUTPUTS_EDGE_ID => {
|
||||
if !output_buf.is_empty() {
|
||||
if let Err(e) =
|
||||
swactor_job_runner::extract_tar(&output_buf, &self.landing)
|
||||
{
|
||||
eprintln!("job-orch: untar edge outputs failed: {e}");
|
||||
}
|
||||
output_buf.clear();
|
||||
}
|
||||
outputs_ended = true;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
if pending_done.is_none() {
|
||||
pending_done = self.done.try_recv();
|
||||
}
|
||||
|
||||
// A job that ran (exit code observed) collected outputs over edge —
|
||||
// wait for that stream to land before returning so the landing dir is
|
||||
// populated. A pre-run fault (no exit code) ships no outputs.
|
||||
let ready = match &pending_done {
|
||||
Some(done) => {
|
||||
let need_outputs = done.exit_code.is_some() && !outputs_ended;
|
||||
!need_outputs || started.elapsed() >= JOB_DEADLINE
|
||||
}
|
||||
None => false,
|
||||
};
|
||||
if ready {
|
||||
let done = pending_done
|
||||
.take()
|
||||
.expect("pending_done observed Some in ready branch");
|
||||
if done.exit_code.is_some() && !outputs_ended {
|
||||
eprintln!("job-orch: output edge stream did not land before deadline");
|
||||
}
|
||||
return Ok(done);
|
||||
}
|
||||
|
||||
if started.elapsed() >= JOB_DEADLINE {
|
||||
return Err("job did not complete within deadline".into());
|
||||
}
|
||||
std::thread::sleep(POLL);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Orchestrator: expose an `OrchestratorJobActor`, print its identity, read the
|
||||
/// worker identity from stdin, drive the job to completion over iroh.
|
||||
pub fn run_serve(job: swactor_job_runner::Job, landing: PathBuf) -> Result<JobDone, String> {
|
||||
let mut session = start_orchestrator(landing)?;
|
||||
println!("JOB_ORCH_IDENTITY {}", session.identity_json()?);
|
||||
let _ = std::io::Write::flush(&mut std::io::stdout());
|
||||
eprintln!("job-orch: published identity; waiting for worker identity on stdin...");
|
||||
|
||||
let mut line = String::new();
|
||||
BufReader::new(std::io::stdin())
|
||||
.read_line(&mut line)
|
||||
.map_err(|e| format!("read worker identity: {e}"))?;
|
||||
let json = line
|
||||
.trim()
|
||||
.strip_prefix("JOB_WORKER_IDENTITY ")
|
||||
.or_else(|| line.trim().strip_prefix("JOB_ORCH_IDENTITY "))
|
||||
.unwrap_or(line.trim());
|
||||
let worker: NodeIdentity =
|
||||
serde_json::from_str(json).map_err(|e| format!("parse worker identity: {e}"))?;
|
||||
session.run_to_completion(job, worker)
|
||||
}
|
||||
|
||||
/// Bridges the job-runner substrate-agnostic [`JobEdgeSink`] to iroh-driver's
|
||||
/// EDGE_ALPN byte handle, so the node actor can ship outputs over the edge
|
||||
/// transport without the job-runner crate depending on iroh.
|
||||
struct IrohEdgeSink(iroh_driver::EdgeSendHandle);
|
||||
|
||||
impl swactor_job_runner::JobEdgeSink for IrohEdgeSink {
|
||||
fn send_bytes(&self, bytes: Vec<u8>) -> Result<(), String> {
|
||||
self.0.send(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
/// How long the worker waits for the orchestrator's workspace edge stream before
|
||||
/// giving up (the node actor faults on its own shorter timeout if this elapses
|
||||
/// without the readiness flag being set).
|
||||
const WORKSPACE_EDGE_WAIT: Duration = Duration::from_secs(60 * 30);
|
||||
|
||||
/// Drain EDGE_ALPN workspace bytes (edge id `WORKSPACE_EDGE_ID`) the orchestrator
|
||||
/// pushed, extract the tar into `workdir`, then signal readiness. Runs on a
|
||||
/// background worker thread; the driver auto-accepts EDGE_ALPN connections and
|
||||
/// pushes their bytes into the shared event queue drained here.
|
||||
fn drain_workspace_edge(
|
||||
events: Arc<Mutex<Vec<WireEvent>>>,
|
||||
workdir: PathBuf,
|
||||
ready: Arc<AtomicBool>,
|
||||
) {
|
||||
let mut buf = Vec::new();
|
||||
let started = Instant::now();
|
||||
loop {
|
||||
let drained: Vec<WireEvent> = events.lock().drain(..).collect();
|
||||
for ev in drained {
|
||||
match ev {
|
||||
WireEvent::BytesRead { edge_id, bytes, .. } if edge_id.0 == WORKSPACE_EDGE_ID => {
|
||||
buf.extend_from_slice(&bytes);
|
||||
}
|
||||
WireEvent::StreamEnded { edge_id, .. } if edge_id.0 == WORKSPACE_EDGE_ID => {
|
||||
if !buf.is_empty() {
|
||||
if let Err(e) = swactor_job_runner::extract_tar(&buf, &workdir) {
|
||||
eprintln!("job-worker: untar workspace failed: {e}");
|
||||
}
|
||||
}
|
||||
ready.store(true, Ordering::Release);
|
||||
return;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if started.elapsed() >= WORKSPACE_EDGE_WAIT {
|
||||
eprintln!("job-worker: workspace edge stream did not arrive");
|
||||
return;
|
||||
}
|
||||
std::thread::sleep(POLL);
|
||||
}
|
||||
}
|
||||
|
|
@ -12,15 +12,8 @@ extern crate self as myelin;
|
|||
const DEFAULT_PIPELINE_CACHED_MODEL_FILE: &str = "SmolLM2-135M-Instruct.Q4_0.gguf";
|
||||
const DEFAULT_PIPELINE_CACHED_MODEL_REPO: &str = "QuantFactory/SmolLM2-135M-Instruct-GGUF";
|
||||
const DEFAULT_PIPELINE_CACHED_MODEL_ID: &str = "smollm2-135m-instruct-q4";
|
||||
const DEFAULT_PIPELINE_CACHED_MODEL_MAX_CONTEXT: u32 = 256;
|
||||
|
||||
pub fn run_chat_from_args<I>(args: I) -> std::process::ExitCode
|
||||
where
|
||||
I: IntoIterator<Item = String>,
|
||||
{
|
||||
chat::runtime::run_from_args(args)
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
pub const ORCHESTRATOR_WORKER_MODE_ARG: &str = "--myelin-worker-node";
|
||||
pub fn run_orchestrator_from_args<I>(args: I) -> Result<(), String>
|
||||
where
|
||||
I: IntoIterator<Item = String>,
|
||||
|
|
@ -32,6 +25,372 @@ pub fn run_worker_node_from_env() -> std::process::ExitCode {
|
|||
node::worker_node_runtime::run_from_env()
|
||||
}
|
||||
|
||||
mod job_deploy;
|
||||
|
||||
/// `myelin-job-worker` — GPU-node side of the iroh job runner.
|
||||
pub fn run_job_worker_from_args<I>(args: I) -> std::process::ExitCode
|
||||
where
|
||||
I: IntoIterator<Item = String>,
|
||||
{
|
||||
let mut orch_identity = None;
|
||||
|
||||
let mut workdir = std::path::PathBuf::from("/root/workspace");
|
||||
let mut it = args.into_iter();
|
||||
while let Some(a) = it.next() {
|
||||
match a.as_str() {
|
||||
"--orch-identity" => orch_identity = it.next(),
|
||||
"--workdir" => workdir = std::path::PathBuf::from(it.next().unwrap_or_default()),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let identity = match orch_identity {
|
||||
Some(s) => s,
|
||||
None => {
|
||||
eprintln!("--orch-identity required");
|
||||
return std::process::ExitCode::from(2);
|
||||
}
|
||||
};
|
||||
match job_deploy::run_worker(identity, workdir) {
|
||||
Ok(()) => std::process::ExitCode::SUCCESS,
|
||||
Err(e) => {
|
||||
eprintln!("myelin-job-worker: {e}");
|
||||
std::process::ExitCode::from(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `myelin-job` — operator side: load a job, drive it over iroh to a worker.
|
||||
pub fn run_job_serve_from_args<I>(args: I) -> std::process::ExitCode
|
||||
where
|
||||
I: IntoIterator<Item = String>,
|
||||
{
|
||||
let mut job_path = None;
|
||||
|
||||
let mut landing = std::path::PathBuf::from("job-outputs");
|
||||
let mut provider = None;
|
||||
let mut vastai = match vastai_job_options_from_env() {
|
||||
Ok(options) => options,
|
||||
Err(error) => {
|
||||
eprintln!("myelin-job: {error}");
|
||||
return std::process::ExitCode::from(2);
|
||||
}
|
||||
};
|
||||
let mut it = args.into_iter();
|
||||
while let Some(a) = it.next() {
|
||||
match a.as_str() {
|
||||
"serve" => {}
|
||||
"reconcile-vastai" | "vastai" => provider = Some("vastai".to_owned()),
|
||||
"--job" => job_path = it.next(),
|
||||
"--landing" => landing = std::path::PathBuf::from(it.next().unwrap_or_default()),
|
||||
"--provider" => provider = it.next(),
|
||||
"--image" | "--node-image" => vastai.image = it.next(),
|
||||
"--vastai-api-key" => vastai.api_key = it.next(),
|
||||
"--vastai-ssh-identity" => match it.next() {
|
||||
Some(path) => match orchestration::app::expand_home_path(&path) {
|
||||
Ok(path) => vastai.ssh_identity = Some(path),
|
||||
Err(error) => {
|
||||
eprintln!("myelin-job: {error}");
|
||||
return std::process::ExitCode::from(2);
|
||||
}
|
||||
},
|
||||
None => {}
|
||||
},
|
||||
"--remote-worker-bin" => {
|
||||
vastai.remote_worker_bin = it.next().unwrap_or_default();
|
||||
}
|
||||
"--worker-workdir" => {
|
||||
vastai.worker_workdir = it.next().unwrap_or_default();
|
||||
}
|
||||
"--run-id" => match parse_next(&mut it, "--run-id") {
|
||||
Ok(value) => vastai.run_id = value,
|
||||
Err(error) => return job_arg_error(error),
|
||||
},
|
||||
"--node-id" => match parse_next(&mut it, "--node-id") {
|
||||
Ok(value) => vastai.node_id = value,
|
||||
Err(error) => return job_arg_error(error),
|
||||
},
|
||||
"--relay-mode" => vastai.relay_mode = it.next(),
|
||||
"--relay-url" => vastai.relay_url = it.next(),
|
||||
"--endpoint-addr-mask" => {
|
||||
vastai.endpoint_addr_mask = it.next().unwrap_or_default();
|
||||
}
|
||||
"--vastai-disk-gb" => match parse_next(&mut it, "--vastai-disk-gb") {
|
||||
Ok(value) => vastai.disk_gb = value,
|
||||
Err(error) => return job_arg_error(error),
|
||||
},
|
||||
"--vastai-ssh-user" => vastai.ssh_user = it.next().unwrap_or_default(),
|
||||
"--vastai-gpu-name" => vastai.gpu_name = it.next(),
|
||||
"--vastai-min-gpu-ram-mb" => match parse_next(&mut it, "--vastai-min-gpu-ram-mb") {
|
||||
Ok(value) => vastai.min_gpu_ram_mb = Some(value),
|
||||
Err(error) => return job_arg_error(error),
|
||||
},
|
||||
"--vastai-min-down-mbps" => match parse_next(&mut it, "--vastai-min-down-mbps") {
|
||||
Ok(value) => vastai.min_down_mbps = Some(value),
|
||||
Err(error) => return job_arg_error(error),
|
||||
},
|
||||
"--vastai-min-up-mbps" => match parse_next(&mut it, "--vastai-min-up-mbps") {
|
||||
Ok(value) => vastai.min_up_mbps = Some(value),
|
||||
Err(error) => return job_arg_error(error),
|
||||
},
|
||||
"--vastai-max-dph-total" => match parse_next(&mut it, "--vastai-max-dph-total") {
|
||||
Ok(value) => vastai.max_dph_total = Some(value),
|
||||
Err(error) => return job_arg_error(error),
|
||||
},
|
||||
"--vastai-min-reliability" => match parse_next(&mut it, "--vastai-min-reliability") {
|
||||
Ok(value) => vastai.min_reliability = Some(value),
|
||||
Err(error) => return job_arg_error(error),
|
||||
},
|
||||
"--vastai-require-verified" => {
|
||||
match parse_next_bool(&mut it, "--vastai-require-verified") {
|
||||
Ok(value) => vastai.require_verified = Some(value),
|
||||
Err(error) => return job_arg_error(error),
|
||||
}
|
||||
}
|
||||
"--vastai-blacklist-host" => match parse_next(&mut it, "--vastai-blacklist-host") {
|
||||
Ok(value) => vastai.blacklist_hosts.push(value),
|
||||
Err(error) => return job_arg_error(error),
|
||||
},
|
||||
"--vastai-blacklist-hosts" => match it.next() {
|
||||
Some(value) => match parse_csv::<u64>("MYELIN_VASTAI_BLACKLIST_HOSTS", &value) {
|
||||
Ok(values) => vastai.blacklist_hosts.extend(values),
|
||||
Err(error) => return job_arg_error(error),
|
||||
},
|
||||
None => {
|
||||
return job_arg_error("--vastai-blacklist-hosts requires a value".to_owned());
|
||||
}
|
||||
},
|
||||
"--vastai-poll-interval-secs" => {
|
||||
match parse_next::<u64>(&mut it, "--vastai-poll-interval-secs") {
|
||||
Ok(value) => vastai.poll_interval = Some(std::time::Duration::from_secs(value)),
|
||||
Err(error) => return job_arg_error(error),
|
||||
}
|
||||
}
|
||||
"--provision-timeout-secs" | "--wait-secs" => {
|
||||
match parse_next::<u64>(&mut it, "--provision-timeout-secs") {
|
||||
Ok(value) => vastai.provision_timeout = std::time::Duration::from_secs(value),
|
||||
Err(error) => return job_arg_error(error),
|
||||
}
|
||||
}
|
||||
"--vastai-onstart" => vastai.onstart = it.next(),
|
||||
"--vastai-confirm-lease" => vastai.confirm_lease = true,
|
||||
"--no-vastai-confirm-lease" => vastai.confirm_lease = false,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let job_path = match job_path {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
eprintln!("--job required");
|
||||
return std::process::ExitCode::from(2);
|
||||
}
|
||||
};
|
||||
let text = match std::fs::read_to_string(&job_path) {
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
eprintln!("read job {}: {e}", job_path);
|
||||
return std::process::ExitCode::from(2);
|
||||
}
|
||||
};
|
||||
#[derive(serde::Deserialize)]
|
||||
struct JobFile {
|
||||
job: swactor_job_runner::Job,
|
||||
}
|
||||
let file: JobFile = match toml::from_str(&text) {
|
||||
Ok(f) => f,
|
||||
Err(e) => {
|
||||
eprintln!("parse job toml: {e}");
|
||||
return std::process::ExitCode::from(2);
|
||||
}
|
||||
};
|
||||
let result = match provider.as_deref() {
|
||||
None | Some("manual") | Some("stdio") => job_deploy::run_serve(file.job, landing),
|
||||
Some("vastai") => orchestration::job_reconciler::run_vastai_job(file.job, landing, vastai),
|
||||
Some(other) => {
|
||||
eprintln!("unsupported myelin-job provider {other:?}; use vastai or omit --provider");
|
||||
return std::process::ExitCode::from(2);
|
||||
}
|
||||
};
|
||||
match result {
|
||||
Ok(done) => {
|
||||
eprintln!("job result: {:?} exit={:?}", done.state, done.exit_code);
|
||||
if done.state == swactor_job_runner::JobState::Completed {
|
||||
std::process::ExitCode::SUCCESS
|
||||
} else {
|
||||
std::process::ExitCode::from(1)
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("myelin-job: {e}");
|
||||
std::process::ExitCode::from(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn vastai_job_options_from_env() -> Result<orchestration::job_reconciler::VastAiJobOptions, String>
|
||||
{
|
||||
let mut options = orchestration::job_reconciler::VastAiJobOptions::default();
|
||||
options.api_key = first_env(["VAST_API_KEY", "MYELIN_VASTAI_API_KEY", "VASTAI_API_KEY"]);
|
||||
options.image = env_optional("MYELIN_NODE_IMAGE");
|
||||
options.ssh_identity = env_optional("MYELIN_VASTAI_SSH_IDENTITY")
|
||||
.map(|path| orchestration::app::expand_home_path(&path))
|
||||
.transpose()?;
|
||||
apply_env("MYELIN_JOB_REMOTE_WORKER_BIN", |value| {
|
||||
options.remote_worker_bin = value;
|
||||
Ok(())
|
||||
})?;
|
||||
apply_env("MYELIN_JOB_WORKER_WORKDIR", |value| {
|
||||
options.worker_workdir = value;
|
||||
Ok(())
|
||||
})?;
|
||||
apply_env_parse("MYELIN_RUN_ID", |value| options.run_id = value)?;
|
||||
apply_env_parse("MYELIN_LOGICAL_NODE_ID", |value| options.node_id = value)?;
|
||||
apply_env_parse("MYELIN_VASTAI_DISK_GB", |value| options.disk_gb = value)?;
|
||||
apply_env("MYELIN_VASTAI_SSH_USER", |value| {
|
||||
options.ssh_user = value;
|
||||
Ok(())
|
||||
})?;
|
||||
apply_env_parse_bool("MYELIN_VASTAI_CONFIRM_LEASE", |value| {
|
||||
options.confirm_lease = value;
|
||||
})?;
|
||||
options.onstart = env_optional("MYELIN_VASTAI_ONSTART");
|
||||
options.relay_mode = env_optional("MYELIN_IROH_RELAY_MODE").or(options.relay_mode);
|
||||
options.relay_url =
|
||||
first_env(["MYELIN_IROH_RELAY_URL", "SWACTOR_IROH_RELAY_URL"]).or(options.relay_url);
|
||||
if let Some(mask) = env_optional("MVP_IROH_ENDPOINT_ADDR_MASK") {
|
||||
options.endpoint_addr_mask = mask;
|
||||
}
|
||||
options.gpu_name = env_optional("MYELIN_VASTAI_GPU_NAME");
|
||||
apply_env_parse("MYELIN_VASTAI_MIN_GPU_RAM_MB", |value| {
|
||||
options.min_gpu_ram_mb = Some(value)
|
||||
})?;
|
||||
apply_env_parse("MYELIN_VASTAI_MIN_DOWN_MBPS", |value| {
|
||||
options.min_down_mbps = Some(value)
|
||||
})?;
|
||||
apply_env_parse("MYELIN_VASTAI_MIN_UP_MBPS", |value| {
|
||||
options.min_up_mbps = Some(value)
|
||||
})?;
|
||||
apply_env_parse("MYELIN_VASTAI_MAX_DPH_TOTAL", |value| {
|
||||
options.max_dph_total = Some(value)
|
||||
})?;
|
||||
apply_env_parse("MYELIN_VASTAI_MIN_RELIABILITY", |value| {
|
||||
options.min_reliability = Some(value)
|
||||
})?;
|
||||
apply_env_parse_bool("MYELIN_VASTAI_REQUIRE_VERIFIED", |value| {
|
||||
options.require_verified = Some(value)
|
||||
})?;
|
||||
if let Some(value) = env_optional("MYELIN_VASTAI_BLACKLIST_HOSTS") {
|
||||
options
|
||||
.blacklist_hosts
|
||||
.extend(parse_csv::<u64>("MYELIN_VASTAI_BLACKLIST_HOSTS", &value)?);
|
||||
}
|
||||
apply_env_parse("MYELIN_VASTAI_POLL_INTERVAL_SECS", |value: u64| {
|
||||
options.poll_interval = Some(std::time::Duration::from_secs(value))
|
||||
})?;
|
||||
apply_env_parse("MYELIN_JOB_PROVISION_TIMEOUT_SECS", |value: u64| {
|
||||
options.provision_timeout = std::time::Duration::from_secs(value)
|
||||
})?;
|
||||
Ok(options)
|
||||
}
|
||||
|
||||
fn job_arg_error(error: String) -> std::process::ExitCode {
|
||||
eprintln!("myelin-job: {error}");
|
||||
std::process::ExitCode::from(2)
|
||||
}
|
||||
|
||||
fn env_optional(name: &str) -> Option<String> {
|
||||
std::env::var(name)
|
||||
.ok()
|
||||
.map(|value| value.trim().to_owned())
|
||||
.filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
fn first_env<const N: usize>(names: [&str; N]) -> Option<String> {
|
||||
names.into_iter().find_map(env_optional)
|
||||
}
|
||||
|
||||
fn apply_env<F>(name: &str, mut apply: F) -> Result<(), String>
|
||||
where
|
||||
F: FnMut(String) -> Result<(), String>,
|
||||
{
|
||||
if let Some(value) = env_optional(name) {
|
||||
apply(value)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn apply_env_parse<T, F>(name: &str, mut apply: F) -> Result<(), String>
|
||||
where
|
||||
T: std::str::FromStr,
|
||||
T::Err: std::fmt::Display,
|
||||
F: FnMut(T),
|
||||
{
|
||||
if let Some(value) = env_optional(name) {
|
||||
apply(parse_value(name, &value)?);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn apply_env_parse_bool<F>(name: &str, mut apply: F) -> Result<(), String>
|
||||
where
|
||||
F: FnMut(bool),
|
||||
{
|
||||
if let Some(value) = env_optional(name) {
|
||||
apply(parse_bool(name, &value)?);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_next<T>(it: &mut impl Iterator<Item = String>, name: &str) -> Result<T, String>
|
||||
where
|
||||
T: std::str::FromStr,
|
||||
T::Err: std::fmt::Display,
|
||||
{
|
||||
let value = it
|
||||
.next()
|
||||
.ok_or_else(|| format!("{name} requires a value"))?;
|
||||
parse_value(name, &value)
|
||||
}
|
||||
|
||||
fn parse_next_bool(it: &mut impl Iterator<Item = String>, name: &str) -> Result<bool, String> {
|
||||
let value = it
|
||||
.next()
|
||||
.ok_or_else(|| format!("{name} requires a value"))?;
|
||||
parse_bool(name, &value)
|
||||
}
|
||||
|
||||
fn parse_value<T>(name: &str, value: &str) -> Result<T, String>
|
||||
where
|
||||
T: std::str::FromStr,
|
||||
T::Err: std::fmt::Display,
|
||||
{
|
||||
value
|
||||
.parse::<T>()
|
||||
.map_err(|e| format!("invalid {name}={value:?}: {e}"))
|
||||
}
|
||||
|
||||
fn parse_bool(name: &str, value: &str) -> Result<bool, String> {
|
||||
match value.to_ascii_lowercase().as_str() {
|
||||
"1" | "true" | "yes" | "on" => Ok(true),
|
||||
"0" | "false" | "no" | "off" => Ok(false),
|
||||
_ => Err(format!(
|
||||
"invalid {name}={value:?}; use 1/0, true/false, yes/no, or on/off"
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_csv<T>(name: &str, value: &str) -> Result<Vec<T>, String>
|
||||
where
|
||||
T: std::str::FromStr,
|
||||
T::Err: std::fmt::Display,
|
||||
{
|
||||
value
|
||||
.split(',')
|
||||
.map(str::trim)
|
||||
.filter(|part| !part.is_empty())
|
||||
.map(|part| parse_value(name, part))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[path = "staging/gguf_common.rs"]
|
||||
mod gguf_common;
|
||||
#[path = "staging/gguf_shard.rs"]
|
||||
|
|
@ -48,12 +407,10 @@ mod run_fsm;
|
|||
#[path = "orchestration/run_plan.rs"]
|
||||
mod run_plan;
|
||||
|
||||
mod chat;
|
||||
mod codecs;
|
||||
mod node;
|
||||
mod observability;
|
||||
mod orchestration;
|
||||
mod prompt;
|
||||
mod staging;
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
|
|||
|
|
@ -137,7 +137,6 @@ pub(crate) enum NodeAgentMsg {
|
|||
stage_index: u32,
|
||||
endpoint: EndpointAddr,
|
||||
node_actor: ActorAddress,
|
||||
telemetry_publisher: ActorAddress,
|
||||
readiness_id: u64,
|
||||
},
|
||||
RuntimeReadyAck {
|
||||
|
|
@ -450,7 +449,6 @@ impl NodeAgentActor {
|
|||
stage_index,
|
||||
endpoint,
|
||||
node_actor,
|
||||
telemetry_publisher,
|
||||
readiness_id,
|
||||
} => {
|
||||
self.core.observe(stage::StageEvent::WorkerReady);
|
||||
|
|
@ -462,7 +460,6 @@ impl NodeAgentActor {
|
|||
stage_index,
|
||||
endpoint,
|
||||
node_actor,
|
||||
telemetry_publisher,
|
||||
readiness_id,
|
||||
},
|
||||
);
|
||||
|
|
|
|||
|
|
@ -3,4 +3,5 @@
|
|||
//! Worker-node runtime behavior lives behind this module boundary; binaries
|
||||
//! only wire entrypoints into it.
|
||||
|
||||
pub(super) mod prompt_wire;
|
||||
pub(super) mod worker_node_runtime;
|
||||
|
|
|
|||
46
apps/myelin/src/node/prompt_wire.rs
Normal file
46
apps/myelin/src/node/prompt_wire.rs
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
//! Wire types for the dormant stage-pipeline prompt/tokenizer channels.
|
||||
//!
|
||||
//! The node runtime still hosts the pipeline stage channels (they are part of
|
||||
//! the node agent contract); the orchestrator no longer drives prompts, so
|
||||
//! only the wire structs live on.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use swactor_transport::NetworkMessage;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub(crate) enum PromptEvent {
|
||||
TextDelta {
|
||||
request_id: u64,
|
||||
text: String,
|
||||
},
|
||||
Done {
|
||||
request_id: u64,
|
||||
final_text: String,
|
||||
tokens_generated: u32,
|
||||
elapsed_ms: u64,
|
||||
},
|
||||
Fault {
|
||||
request_id: u64,
|
||||
error: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl NetworkMessage for PromptEvent {
|
||||
fn type_tag() -> &'static str {
|
||||
"myelin::PromptEvent"
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub(crate) enum TokenizerEvent {
|
||||
PromptEncoded { request_id: u64, tokens: Vec<u32> },
|
||||
TokensDecoded { request_id: u64, text: String },
|
||||
Fault { request_id: u64, error: String },
|
||||
}
|
||||
|
||||
impl NetworkMessage for TokenizerEvent {
|
||||
fn type_tag() -> &'static str {
|
||||
"myelin::TokenizerEvent"
|
||||
}
|
||||
}
|
||||
|
|
@ -18,13 +18,13 @@ use std::time::{Duration, Instant};
|
|||
|
||||
use telemetry::frame::TelemetryEvent;
|
||||
use telemetry::{
|
||||
ChannelContent, ChannelId, TELEMETRY_PUBLISHER_NAME, TelemetryEndpoint, TelemetryProducer,
|
||||
TelemetryPublisherActor, TelemetrySubscribe, TelemetrySubscription, Lifetime, NodeId,
|
||||
Record, StreamDescriptor, StreamId, StreamOrigin,
|
||||
ChannelContent, ChannelId, Lifetime, NodeId, Record, StreamDescriptor, StreamId, StreamOrigin,
|
||||
TelemetryEndpoint, TelemetryProducer, TelemetrySubscription,
|
||||
};
|
||||
|
||||
use crate::codecs::register_myelin_actor_codecs;
|
||||
use crate::gguf_shard::{StageShardPlan, materialize_stage_shard_http, validate_stage_shard_cache};
|
||||
use crate::node::prompt_wire::{PromptEvent, TokenizerEvent};
|
||||
use crate::node_actor::{
|
||||
NodeAgentActor, NodeAgentMsg, NodeAgentReport, StageCommandWire, StageInboundEdgeWire,
|
||||
StageObjectSpecWire, StageOutboundEdgeWire,
|
||||
|
|
@ -32,21 +32,17 @@ use crate::node_actor::{
|
|||
use crate::observability::benchmark;
|
||||
use crate::orchestration::distribution_stack::{DistributionRuntimeStack, duration_ms_u64};
|
||||
use crate::orchestration::provider_adapters::relay::relay_runtime_config_from_env;
|
||||
use crate::prompt::rpc::{PromptEvent, TokenizerEvent};
|
||||
use crate::run_plan::{GgufSource, TokenizerSource};
|
||||
use crate::staging::control as stage;
|
||||
use data_plane::arena;
|
||||
use data_plane::edge_lifecycle as edge;
|
||||
use data_plane::edge_runtime;
|
||||
use data_plane::object_record as ingress;
|
||||
use distribution::node::DistributedNodeConfig;
|
||||
use distribution::telemetry::{MembershipTransition, SwimProbeEvent};
|
||||
use distribution::types::{MemberState, NodeId as DistNodeId};
|
||||
use iroh::EndpointAddr;
|
||||
use data_plane::edge_runtime;
|
||||
use iroh_driver::{
|
||||
EDGE_ALPN, TELEMETRY_ALPN, TelemetryPublishHandle, TelemetryQuicHeader, IrohDriver,
|
||||
IrohDriverConfig,
|
||||
};
|
||||
use iroh_driver::{EDGE_ALPN, IrohDriver, IrohDriverConfig, TELEMETRY_ALPN, spawn_pull_server};
|
||||
use iroh_driver::{EndpointAddrMask, MVP_IROH_ENDPOINT_ADDR_MASK_ENV, advertised_endpoint};
|
||||
use parking_lot::Mutex;
|
||||
use serde_json::{Value, json};
|
||||
|
|
@ -863,9 +859,13 @@ impl edge_runtime::WorkerPort for TinygradRingPort<'_> {
|
|||
max_extent: spec.max_extent,
|
||||
alignment: spec.alignment.min(u64::from(u32::MAX)) as u32,
|
||||
};
|
||||
let loaded = self
|
||||
.worker
|
||||
.ring_readable(ring_id.0, edge_id.0, wire_spec, self.config, self.telemetry)?;
|
||||
let loaded = self.worker.ring_readable(
|
||||
ring_id.0,
|
||||
edge_id.0,
|
||||
wire_spec,
|
||||
self.config,
|
||||
self.telemetry,
|
||||
)?;
|
||||
Ok(edge_runtime::LoadedObject {
|
||||
object_id: loaded.object_id,
|
||||
sequence: loaded.sequence,
|
||||
|
|
@ -1087,7 +1087,15 @@ impl WorkerEdgeRuntime {
|
|||
config: &DeploymentConfig,
|
||||
telemetry: &mut NodeTelemetry,
|
||||
) -> Result<(), String> {
|
||||
self.poll_and_report(driver, stack, node_actor, worker, arena_manager, config, telemetry)
|
||||
self.poll_and_report(
|
||||
driver,
|
||||
stack,
|
||||
node_actor,
|
||||
worker,
|
||||
arena_manager,
|
||||
config,
|
||||
telemetry,
|
||||
)
|
||||
}
|
||||
|
||||
fn establish_inbound(
|
||||
|
|
@ -1125,7 +1133,15 @@ impl WorkerEdgeRuntime {
|
|||
parse_spec,
|
||||
);
|
||||
self.inbound_edge = Some(edge_wire);
|
||||
self.poll_and_report(driver, stack, node_actor, worker, arena_manager, config, telemetry)
|
||||
self.poll_and_report(
|
||||
driver,
|
||||
stack,
|
||||
node_actor,
|
||||
worker,
|
||||
arena_manager,
|
||||
config,
|
||||
telemetry,
|
||||
)
|
||||
}
|
||||
|
||||
fn establish_outbound(
|
||||
|
|
@ -1176,7 +1192,15 @@ impl WorkerEdgeRuntime {
|
|||
peer,
|
||||
);
|
||||
self.outbound_edge = Some(edge_wire);
|
||||
self.poll_and_report(driver, stack, node_actor, worker, arena_manager, config, telemetry)
|
||||
self.poll_and_report(
|
||||
driver,
|
||||
stack,
|
||||
node_actor,
|
||||
worker,
|
||||
arena_manager,
|
||||
config,
|
||||
telemetry,
|
||||
)
|
||||
}
|
||||
|
||||
fn execute_step(
|
||||
|
|
@ -1466,6 +1490,10 @@ fn run() -> Result<(), String> {
|
|||
return Err(format!("create iroh driver: {error}"));
|
||||
}
|
||||
};
|
||||
// Bootstrap supervisors pull telemetry from the node endpoint. Keep these
|
||||
// ALPN connections available to the node loop instead of the legacy push
|
||||
// reader path.
|
||||
driver.retain_telemetry_connections();
|
||||
let advertised_self_endpoint =
|
||||
advertised_endpoint(driver.endpoint_addr(), config.endpoint_addr_mask)?;
|
||||
boot(
|
||||
|
|
@ -1561,23 +1589,8 @@ fn run() -> Result<(), String> {
|
|||
let node_shutdown = |ds: &mut NodeTelemetry, phase: &str, status: &str, detail: Value| {
|
||||
emit_node_event(ds, &config, NODE_SHUTDOWN_CHANNEL, phase, status, detail)
|
||||
};
|
||||
let telemetry_transport = driver.telemetry_publish_handle();
|
||||
let telemetry_publisher = match stack
|
||||
.runtime
|
||||
.spawn(telemetry.publisher_actor(telemetry_transport))
|
||||
{
|
||||
Ok(actor) => actor,
|
||||
Err(error) => {
|
||||
node_boot(
|
||||
&mut telemetry,
|
||||
"telemetry_publisher",
|
||||
"failed",
|
||||
json!({"error":error.to_string()}),
|
||||
);
|
||||
return Err(format!("spawn telemetry publisher: {error}"));
|
||||
}
|
||||
};
|
||||
stack.register_local_actor(driver.register_actor(telemetry_publisher, 1));
|
||||
// Telemetry leaves this node exclusively through pull subscriptions served
|
||||
// by `serve_telemetry_pulls` on `TELEMETRY_ALPN`; no publisher actor.
|
||||
let sampler_health_channel = telemetry.channel_by_name(NODE_SAMPLER_CHANNEL);
|
||||
let sampler_health_context = SamplerHealthContext::from_config(&config);
|
||||
spawn_host_gpu_sampler(
|
||||
|
|
@ -1600,12 +1613,6 @@ fn run() -> Result<(), String> {
|
|||
telemetry.channels.arena,
|
||||
Arc::clone(&arena_manager),
|
||||
);
|
||||
node_boot(
|
||||
&mut telemetry,
|
||||
"telemetry_publisher",
|
||||
"ready",
|
||||
json!({"actor":telemetry_publisher,"name":TELEMETRY_PUBLISHER_NAME,"subscription_transport":"iroh"}),
|
||||
);
|
||||
let worker_synthetic_id = format!(
|
||||
"myelin-worker-{}-{}-telemetry-preflight",
|
||||
config.logical_node_id, config.stage_index
|
||||
|
|
@ -1625,7 +1632,7 @@ fn run() -> Result<(), String> {
|
|||
"producer_class":"rust-worker-node",
|
||||
"synthetic_id":worker_synthetic_id,
|
||||
"telemetry_endpoint":{
|
||||
"role":"worker-node-iroh-publisher",
|
||||
"role":"worker-node-iroh-pull-server",
|
||||
"transport":"iroh-telemetry",
|
||||
"endpoint_addr_mask":config.endpoint_addr_mask.as_str(),
|
||||
"relay_mode":format!("{:?}", config.relay_mode),
|
||||
|
|
@ -1718,6 +1725,114 @@ fn run() -> Result<(), String> {
|
|||
json!({"node_actor":node_actor,"network_reachable":true}),
|
||||
)?;
|
||||
|
||||
if config.agent_only {
|
||||
boot(
|
||||
"agent_mode",
|
||||
"ready",
|
||||
json!({"framework":"none","workloads":"external_jobs"}),
|
||||
)?;
|
||||
let mut pending_runtime_ready =
|
||||
PendingRuntimeReady::new(&config, advertised_self_endpoint.clone(), node_actor);
|
||||
let ready = json!({
|
||||
"type":"ready",
|
||||
"role":"node",
|
||||
"endpoint":advertised_self_endpoint.clone(),
|
||||
"node_actor":node_actor,
|
||||
"logical_node_id":config.logical_node_id,
|
||||
"stage_index":config.stage_index,
|
||||
});
|
||||
boot(
|
||||
"runtime_ready_local",
|
||||
"ready",
|
||||
json!({
|
||||
"endpoint":advertised_self_endpoint,
|
||||
"node_actor":node_actor,
|
||||
"logical_node_id":config.logical_node_id,
|
||||
"stage_index":config.stage_index,
|
||||
"readiness_id":pending_runtime_ready.readiness_id,
|
||||
}),
|
||||
)?;
|
||||
let shutdown_rx = spawn_stdin_shutdown_listener(config.exit_on_stdin_eof);
|
||||
node_runtime(
|
||||
&mut telemetry,
|
||||
"main_loop",
|
||||
"started",
|
||||
json!({
|
||||
"mode":"agent_only",
|
||||
"poll_interval_ms":PUMP_INTERVAL.as_millis(),
|
||||
"checks":["network","telemetry","node_reports","stdin_shutdown"],
|
||||
}),
|
||||
);
|
||||
loop {
|
||||
emit_swim_telemetry(&mut telemetry, &stack, "agent_loop");
|
||||
drain_debug_join_commands(&mut debug_join_rx, &mut driver, &config, &mut telemetry);
|
||||
telemetry.tick();
|
||||
serve_telemetry_pulls(&driver, &engine.handle(), &telemetry.endpoint);
|
||||
while let Some(report) = reports.try_recv() {
|
||||
if let NodeAgentReport::RuntimeReadyAck {
|
||||
run_id,
|
||||
node_id,
|
||||
stage_index,
|
||||
readiness_id,
|
||||
} = report
|
||||
&& pending_runtime_ready.observe_ack(run_id, node_id, stage_index, readiness_id)
|
||||
{
|
||||
node_boot(
|
||||
&mut telemetry,
|
||||
"runtime_ready_ack",
|
||||
"ready",
|
||||
json!({
|
||||
"readiness_id":readiness_id,
|
||||
"attempts":pending_runtime_ready.attempts,
|
||||
"endpoint":&pending_runtime_ready.endpoint,
|
||||
"node_actor":pending_runtime_ready.node_actor,
|
||||
}),
|
||||
);
|
||||
telemetry.submit_text(telemetry.channels.node_ready, ready.to_string());
|
||||
}
|
||||
}
|
||||
if !pending_runtime_ready.swim_logged && pending_runtime_ready.swim_ready(&stack) {
|
||||
node_runtime(
|
||||
&mut telemetry,
|
||||
"coordinator_swim",
|
||||
"ready",
|
||||
json!({
|
||||
"coordinator":pending_runtime_ready
|
||||
.coordinator
|
||||
.map(|node| format!("{node:?}"))
|
||||
.unwrap_or_else(|| "standalone".to_owned()),
|
||||
"readiness_id":pending_runtime_ready.readiness_id,
|
||||
}),
|
||||
);
|
||||
pending_runtime_ready.swim_logged = true;
|
||||
}
|
||||
if !pending_runtime_ready.acked
|
||||
&& pending_runtime_ready.maybe_send(&stack, node_actor)?
|
||||
{
|
||||
node_runtime(
|
||||
&mut telemetry,
|
||||
"runtime_ready_signal",
|
||||
"sent",
|
||||
json!({
|
||||
"readiness_id":pending_runtime_ready.readiness_id,
|
||||
"attempts":pending_runtime_ready.attempts,
|
||||
"next_backoff_ms":pending_runtime_ready.backoff.as_millis(),
|
||||
}),
|
||||
);
|
||||
}
|
||||
if shutdown_rx.try_recv().is_ok() {
|
||||
node_shutdown(
|
||||
&mut telemetry,
|
||||
"node_exit",
|
||||
"ready",
|
||||
json!({"result":"ok","mode":"agent_only"}),
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
thread::sleep(PUMP_INTERVAL);
|
||||
}
|
||||
}
|
||||
|
||||
worker_evt(
|
||||
"worker_process",
|
||||
"started",
|
||||
|
|
@ -1762,19 +1877,14 @@ fn run() -> Result<(), String> {
|
|||
}
|
||||
}
|
||||
let mut edge_runtime = WorkerEdgeRuntime::new(config.logical_node_id);
|
||||
let mut pending_runtime_ready = PendingRuntimeReady::new(
|
||||
&config,
|
||||
advertised_self_endpoint.clone(),
|
||||
node_actor,
|
||||
telemetry_publisher,
|
||||
);
|
||||
let mut pending_runtime_ready =
|
||||
PendingRuntimeReady::new(&config, advertised_self_endpoint.clone(), node_actor);
|
||||
|
||||
let ready = json!({
|
||||
"type":"ready",
|
||||
"role":"node",
|
||||
"endpoint": advertised_self_endpoint.clone(),
|
||||
"node_actor": node_actor,
|
||||
"telemetry_publisher": telemetry_publisher,
|
||||
"logical_node_id": config.logical_node_id,
|
||||
"stage_index": config.stage_index,
|
||||
});
|
||||
|
|
@ -1801,7 +1911,7 @@ fn run() -> Result<(), String> {
|
|||
)?;
|
||||
}
|
||||
|
||||
let shutdown_rx = spawn_stdin_shutdown_listener();
|
||||
let shutdown_rx = spawn_stdin_shutdown_listener(config.exit_on_stdin_eof);
|
||||
node_runtime(
|
||||
&mut telemetry,
|
||||
"stdin_shutdown_listener",
|
||||
|
|
@ -1821,6 +1931,7 @@ fn run() -> Result<(), String> {
|
|||
emit_swim_telemetry(&mut telemetry, &stack, "main_loop");
|
||||
drain_debug_join_commands(&mut debug_join_rx, &mut driver, &config, &mut telemetry);
|
||||
telemetry.tick();
|
||||
serve_telemetry_pulls(&driver, &engine.handle(), &telemetry.endpoint);
|
||||
drain_worker_stderr(&worker.stderr_rx, &config, &mut telemetry);
|
||||
edge_runtime.poll_iroh(
|
||||
&mut driver,
|
||||
|
|
@ -1916,12 +2027,7 @@ fn run() -> Result<(), String> {
|
|||
"ready",
|
||||
json!({"worker_event_type":"WorkerStopped"}),
|
||||
);
|
||||
node_shutdown(
|
||||
&mut telemetry,
|
||||
"node_exit",
|
||||
"ready",
|
||||
json!({"result":"ok"}),
|
||||
);
|
||||
node_shutdown(&mut telemetry, "node_exit", "ready", json!({"result":"ok"}));
|
||||
}
|
||||
Err(error) => node_shutdown(
|
||||
&mut telemetry,
|
||||
|
|
@ -2121,26 +2227,20 @@ impl NodeTelemetry {
|
|||
archive.drain(&self.by_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn publisher_actor(&self, transport: TelemetryPublishHandle) -> TelemetryPublisherActor {
|
||||
TelemetryPublisherActor::new(
|
||||
Arc::clone(&self.endpoint),
|
||||
move |subscribe: TelemetrySubscribe, subscription: TelemetrySubscription| {
|
||||
let Ok(header) = TelemetryQuicHeader::from_snapshot(
|
||||
subscribe.flow_id,
|
||||
subscribe.token,
|
||||
subscription.snapshot(),
|
||||
) else {
|
||||
return;
|
||||
};
|
||||
transport.publish_subscription(
|
||||
subscribe.collector,
|
||||
header,
|
||||
subscription,
|
||||
fn serve_telemetry_pulls(
|
||||
driver: &IrohDriver,
|
||||
engine: &EngineHandle,
|
||||
endpoint: &Arc<TelemetryEndpoint>,
|
||||
) {
|
||||
for (_node, connection) in driver.drain_accepted_for_alpn(TELEMETRY_ALPN) {
|
||||
spawn_pull_server(
|
||||
engine,
|
||||
connection,
|
||||
Arc::clone(endpoint),
|
||||
Duration::from_millis(10),
|
||||
);
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2223,7 +2323,6 @@ struct PendingRuntimeReady {
|
|||
stage_index: u32,
|
||||
endpoint: EndpointAddr,
|
||||
node_actor: ActorAddress,
|
||||
telemetry_publisher: ActorAddress,
|
||||
coordinator: Option<DistNodeId>,
|
||||
readiness_id: u64,
|
||||
attempts: u32,
|
||||
|
|
@ -2234,19 +2333,13 @@ struct PendingRuntimeReady {
|
|||
}
|
||||
|
||||
impl PendingRuntimeReady {
|
||||
fn new(
|
||||
config: &DeploymentConfig,
|
||||
endpoint: EndpointAddr,
|
||||
node_actor: ActorAddress,
|
||||
telemetry_publisher: ActorAddress,
|
||||
) -> Self {
|
||||
fn new(config: &DeploymentConfig, endpoint: EndpointAddr, node_actor: ActorAddress) -> Self {
|
||||
Self {
|
||||
run_id: config.run_id,
|
||||
node_id: config.logical_node_id,
|
||||
stage_index: config.stage_index,
|
||||
endpoint,
|
||||
node_actor,
|
||||
telemetry_publisher,
|
||||
coordinator: config
|
||||
.coordinator_endpoint
|
||||
.as_ref()
|
||||
|
|
@ -2308,7 +2401,6 @@ impl PendingRuntimeReady {
|
|||
stage_index: self.stage_index,
|
||||
endpoint: self.endpoint.clone(),
|
||||
node_actor: self.node_actor,
|
||||
telemetry_publisher: self.telemetry_publisher,
|
||||
readiness_id: self.readiness_id,
|
||||
},
|
||||
)
|
||||
|
|
@ -3098,8 +3190,7 @@ fn materialize_stage_shard_with_process(
|
|||
while let Some(report) = reports.try_recv() {
|
||||
match report {
|
||||
StageShardFetchReport::Progress(event) => {
|
||||
if let Err(error) = publish_stage_shard_fetch_event(telemetry, config, &event)
|
||||
{
|
||||
if let Err(error) = publish_stage_shard_fetch_event(telemetry, config, &event) {
|
||||
let _ = stack.runtime.stop_actor(actor);
|
||||
return Err(error);
|
||||
}
|
||||
|
|
@ -3457,6 +3548,8 @@ struct DeploymentConfig {
|
|||
debug_join_socket: Option<String>,
|
||||
relay_mode: iroh::RelayMode,
|
||||
endpoint_addr_mask: EndpointAddrMask,
|
||||
agent_only: bool,
|
||||
exit_on_stdin_eof: bool,
|
||||
worker_script: String,
|
||||
device: String,
|
||||
model_id: String,
|
||||
|
|
@ -3528,6 +3621,9 @@ impl DeploymentConfig {
|
|||
.map(EndpointAddrMask::parse)
|
||||
.transpose()?
|
||||
.unwrap_or_default(),
|
||||
agent_only: env_optional("MYELIN_AGENT_ONLY")
|
||||
.is_some_and(|value| matches!(value.as_str(), "1" | "true" | "yes" | "on")),
|
||||
exit_on_stdin_eof: provider == "process",
|
||||
worker_script: env_optional("MYELIN_TINYGRAD_WORKER")
|
||||
.unwrap_or_else(|| DEFAULT_WORKER_SCRIPT.to_owned()),
|
||||
device: env_optional("DEV").unwrap_or_else(|| default_device.to_owned()),
|
||||
|
|
@ -4252,16 +4348,19 @@ impl Drop for TinygradWorker {
|
|||
|
||||
// blocking user-stdin thread is process control, out of scope (ENGINE_SPEC.md §2)
|
||||
#[allow(clippy::disallowed_methods)]
|
||||
fn spawn_stdin_shutdown_listener() -> Receiver<()> {
|
||||
fn spawn_stdin_shutdown_listener(exit_on_eof: bool) -> Receiver<()> {
|
||||
let (tx, rx) = mpsc::channel();
|
||||
thread::spawn(move || {
|
||||
let stdin = std::io::stdin();
|
||||
for line in stdin.lock().lines().map_while(Result::ok) {
|
||||
if line.trim().eq_ignore_ascii_case("shutdown") {
|
||||
let _ = tx.send(());
|
||||
break;
|
||||
return;
|
||||
}
|
||||
}
|
||||
if exit_on_eof {
|
||||
let _ = tx.send(());
|
||||
}
|
||||
});
|
||||
rx
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,11 +4,11 @@ use std::collections::{BTreeMap, VecDeque};
|
|||
|
||||
use dashboard::FrameEvent;
|
||||
use dashboard::view::DashboardView;
|
||||
use telemetry::Record;
|
||||
use telemetry::frame::{Frame, StreamId};
|
||||
use serde::Serialize;
|
||||
use serde_json::{Value, json};
|
||||
use std::sync::RwLock;
|
||||
use telemetry::Record;
|
||||
use telemetry::frame::{Frame, StreamId};
|
||||
|
||||
use crate::observability::lifecycle as obs;
|
||||
use crate::observability::telemetry::{
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@ use std::fs::{self, File, OpenOptions};
|
|||
use std::io::{BufRead, BufReader, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use telemetry::frame::{Frame, StreamId};
|
||||
use serde_json::json;
|
||||
use telemetry::frame::{Frame, StreamId};
|
||||
|
||||
use crate::observability::benchmark;
|
||||
|
||||
|
|
|
|||
|
|
@ -8,12 +8,18 @@
|
|||
//! ([`StageLoadProgress`]) is co-located here because it is the one legitimate
|
||||
//! read of telemetry content for control decisions (weight-load liveness).
|
||||
|
||||
use telemetry::frame::{ChannelRef, TelemetryEvent, Frame, StreamId};
|
||||
use iroh_driver::IrohDriver;
|
||||
use iroh::EndpointAddr;
|
||||
use iroh_driver::{IrohDriver, TelemetryQuicHeader, spawn_pull_collector};
|
||||
use parking_lot::Mutex;
|
||||
use serde_json::Value;
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::mpsc;
|
||||
use std::sync::{Arc, mpsc};
|
||||
use std::time::Instant;
|
||||
use swactor_engine::EngineHandle;
|
||||
use telemetry::frame::{ChannelRef, Frame, StreamId, TelemetryEvent};
|
||||
use telemetry::{
|
||||
DeliveryFanout, StreamDescriptor, SubscriptionRequest, TelemetrySnapshot, TelemetrySubscription,
|
||||
};
|
||||
|
||||
use crate::observability::orch_telemetry::DashboardSupport;
|
||||
|
||||
|
|
@ -21,6 +27,7 @@ use crate::observability::orch_telemetry::DashboardSupport;
|
|||
#[derive(Clone, Debug)]
|
||||
struct CollectedTelemetryFrame {
|
||||
stream: StreamId,
|
||||
descriptor: Option<StreamDescriptor>,
|
||||
channel_name: String,
|
||||
frame: Frame,
|
||||
}
|
||||
|
|
@ -60,27 +67,143 @@ impl StageLoadProgress {
|
|||
pub(crate) struct FrameCollector {
|
||||
tx: mpsc::Sender<CollectedTelemetryFrame>,
|
||||
rx: mpsc::Receiver<CollectedTelemetryFrame>,
|
||||
pull_fanout: Arc<DeliveryFanout>,
|
||||
pull_subscription: TelemetrySubscription,
|
||||
pull_header_tx: mpsc::Sender<TelemetryQuicHeader>,
|
||||
pull_header_rx: mpsc::Receiver<TelemetryQuicHeader>,
|
||||
pull_channels: Mutex<BTreeMap<ChannelRef, String>>,
|
||||
pull_streams: Mutex<BTreeMap<StreamId, StreamDescriptor>>,
|
||||
}
|
||||
|
||||
impl FrameCollector {
|
||||
pub(crate) fn new() -> Self {
|
||||
let (tx, rx) = mpsc::channel();
|
||||
Self { tx, rx }
|
||||
let pull_fanout = Arc::new(DeliveryFanout::new(4096));
|
||||
let pull_subscription = pull_fanout.subscribe_all(
|
||||
"myelin-daemon",
|
||||
TelemetrySnapshot {
|
||||
streams: Vec::new(),
|
||||
channels: Vec::new(),
|
||||
},
|
||||
);
|
||||
let (pull_header_tx, pull_header_rx) = mpsc::channel();
|
||||
Self {
|
||||
tx,
|
||||
rx,
|
||||
pull_fanout,
|
||||
pull_subscription,
|
||||
pull_header_tx,
|
||||
pull_header_rx,
|
||||
pull_channels: Mutex::new(BTreeMap::new()),
|
||||
pull_streams: Mutex::new(BTreeMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Drain iroh telemetry connections into the internal queue.
|
||||
/// Dial a bootstrapped node and retain its live telemetry subscription.
|
||||
pub(crate) fn subscribe_node(
|
||||
&self,
|
||||
engine: &EngineHandle,
|
||||
endpoint: iroh::Endpoint,
|
||||
peer: EndpointAddr,
|
||||
run_id: u64,
|
||||
node_id: u64,
|
||||
) {
|
||||
let mut flow_id = [0_u8; 16];
|
||||
flow_id[..8].copy_from_slice(&run_id.to_le_bytes());
|
||||
flow_id[8..].copy_from_slice(&node_id.to_le_bytes());
|
||||
spawn_pull_collector(
|
||||
engine,
|
||||
endpoint,
|
||||
peer,
|
||||
flow_id,
|
||||
Vec::new(),
|
||||
SubscriptionRequest::all(),
|
||||
Arc::clone(&self.pull_fanout),
|
||||
self.pull_header_tx.clone(),
|
||||
);
|
||||
}
|
||||
|
||||
fn pump_pulls(&self) {
|
||||
while let Ok(header) = self.pull_header_rx.try_recv() {
|
||||
self.pull_streams
|
||||
.lock()
|
||||
.insert(header.stream.stream.clone(), header.stream.clone());
|
||||
let mut channels = self.pull_channels.lock();
|
||||
for descriptor in header.channels {
|
||||
channels.insert(
|
||||
ChannelRef {
|
||||
stream: descriptor.stream,
|
||||
channel: descriptor.id,
|
||||
},
|
||||
descriptor.name,
|
||||
);
|
||||
}
|
||||
}
|
||||
for event in self.pull_subscription.drain_available() {
|
||||
match event {
|
||||
TelemetryEvent::StreamDeclared(descriptor) => {
|
||||
self.pull_streams
|
||||
.lock()
|
||||
.insert(descriptor.stream.clone(), descriptor);
|
||||
}
|
||||
TelemetryEvent::ChannelDeclared(descriptor) => {
|
||||
self.pull_channels.lock().insert(
|
||||
ChannelRef {
|
||||
stream: descriptor.stream,
|
||||
channel: descriptor.id,
|
||||
},
|
||||
descriptor.name,
|
||||
);
|
||||
}
|
||||
TelemetryEvent::Frame(delivery) => {
|
||||
let stream = delivery.channel.stream;
|
||||
let channel_name = self
|
||||
.pull_channels
|
||||
.lock()
|
||||
.get(&ChannelRef {
|
||||
stream: stream.clone(),
|
||||
channel: delivery.channel.channel,
|
||||
})
|
||||
.cloned()
|
||||
.unwrap_or_else(|| format!("channel#{}", delivery.channel.channel.0));
|
||||
let descriptor = self.pull_streams.lock().get(&stream).cloned();
|
||||
let _ = self.tx.send(CollectedTelemetryFrame {
|
||||
stream,
|
||||
descriptor,
|
||||
channel_name,
|
||||
frame: Frame::new(
|
||||
delivery.channel.channel,
|
||||
delivery.position,
|
||||
delivery.payload,
|
||||
),
|
||||
});
|
||||
}
|
||||
TelemetryEvent::StreamEnded(stream) => {
|
||||
self.pull_streams.lock().remove(&stream);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Drain push-compatible and collector-initiated telemetry into the queue.
|
||||
pub(crate) fn pump(&self, driver: &IrohDriver) {
|
||||
drain_telemetry_connections(driver, &self.tx);
|
||||
self.pump_pulls();
|
||||
}
|
||||
|
||||
/// Drain queued frames, forwarding each via the closure. No progress extraction.
|
||||
pub(crate) fn drain<F>(&self, forward: F)
|
||||
where
|
||||
F: FnMut(&StreamId, &str, &Frame),
|
||||
F: FnMut(&StreamId, Option<&StreamDescriptor>, &str, &Frame),
|
||||
{
|
||||
let mut forward = forward;
|
||||
while let Ok(collected) = self.rx.try_recv() {
|
||||
forward(&collected.stream, &collected.channel_name, &collected.frame);
|
||||
forward(
|
||||
&collected.stream,
|
||||
collected.descriptor.as_ref(),
|
||||
&collected.channel_name,
|
||||
&collected.frame,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -90,13 +213,18 @@ impl FrameCollector {
|
|||
progress: &mut BTreeMap<u64, StageLoadProgress>,
|
||||
forward: F,
|
||||
) where
|
||||
F: FnMut(&StreamId, &str, &Frame),
|
||||
F: FnMut(&StreamId, Option<&StreamDescriptor>, &str, &Frame),
|
||||
{
|
||||
let mut forward = forward;
|
||||
let now = Instant::now();
|
||||
while let Ok(collected) = self.rx.try_recv() {
|
||||
update_load_progress_from_frame(progress, &collected, now);
|
||||
forward(&collected.stream, &collected.channel_name, &collected.frame);
|
||||
forward(
|
||||
&collected.stream,
|
||||
collected.descriptor.as_ref(),
|
||||
&collected.channel_name,
|
||||
&collected.frame,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -244,6 +372,10 @@ fn drain_telemetry_connections(
|
|||
frame_tx: &mpsc::Sender<CollectedTelemetryFrame>,
|
||||
) {
|
||||
for read in driver.drain_telemetry_reads() {
|
||||
let mut streams = BTreeMap::from([(
|
||||
read.header.stream.stream.clone(),
|
||||
read.header.stream.clone(),
|
||||
)]);
|
||||
let mut channels = read
|
||||
.header
|
||||
.channels
|
||||
|
|
@ -260,6 +392,9 @@ fn drain_telemetry_connections(
|
|||
.collect::<BTreeMap<_, _>>();
|
||||
for event in read.events {
|
||||
match event {
|
||||
TelemetryEvent::StreamDeclared(descriptor) => {
|
||||
streams.insert(descriptor.stream.clone(), descriptor);
|
||||
}
|
||||
TelemetryEvent::ChannelDeclared(descriptor) => {
|
||||
channels.insert(
|
||||
ChannelRef {
|
||||
|
|
@ -270,8 +405,12 @@ fn drain_telemetry_connections(
|
|||
);
|
||||
}
|
||||
TelemetryEvent::Frame(delivery) => {
|
||||
let stream = delivery.channel.stream;
|
||||
let channel_name = channels
|
||||
.get(&delivery.channel)
|
||||
.get(&ChannelRef {
|
||||
stream: stream.clone(),
|
||||
channel: delivery.channel.channel,
|
||||
})
|
||||
.cloned()
|
||||
.unwrap_or_else(|| format!("channel#{}", delivery.channel.channel.0));
|
||||
let frame = Frame::new(
|
||||
|
|
@ -281,7 +420,8 @@ fn drain_telemetry_connections(
|
|||
);
|
||||
if frame_tx
|
||||
.send(CollectedTelemetryFrame {
|
||||
stream: delivery.channel.stream,
|
||||
descriptor: streams.get(&stream).cloned(),
|
||||
stream,
|
||||
channel_name,
|
||||
frame,
|
||||
})
|
||||
|
|
@ -290,7 +430,9 @@ fn drain_telemetry_connections(
|
|||
return;
|
||||
}
|
||||
}
|
||||
TelemetryEvent::StreamDeclared(_) | TelemetryEvent::StreamEnded(_) => {}
|
||||
TelemetryEvent::StreamEnded(stream) => {
|
||||
streams.remove(&stream);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -308,3 +450,66 @@ pub(crate) fn ingest_dashboard_frame(
|
|||
dashboard.publish_frame(stream, channel, frame);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use telemetry::frame::{FrameDelivery, Lifetime, NodeId, Position, StreamOrigin};
|
||||
use telemetry::{ChannelContent, ChannelDescriptor, ChannelId};
|
||||
|
||||
#[test]
|
||||
fn pulled_frame_preserves_remote_stream_metadata() {
|
||||
let collector = FrameCollector::new();
|
||||
let stream = StreamId::new(NodeId::new("node-7"), Lifetime(3));
|
||||
let descriptor = StreamDescriptor {
|
||||
stream: stream.clone(),
|
||||
label: Some("worker seven".to_owned()),
|
||||
origin: StreamOrigin::RemoteNode,
|
||||
};
|
||||
let channel = ChannelDescriptor {
|
||||
stream: stream.clone(),
|
||||
id: ChannelId(9),
|
||||
name: "host.net".to_owned(),
|
||||
label: None,
|
||||
content: ChannelContent::JsonRecord { schema: None },
|
||||
};
|
||||
collector
|
||||
.pull_header_tx
|
||||
.send(TelemetryQuicHeader::new(
|
||||
[7; 16],
|
||||
Vec::new(),
|
||||
descriptor.clone(),
|
||||
vec![channel.clone()],
|
||||
))
|
||||
.unwrap();
|
||||
collector
|
||||
.pull_fanout
|
||||
.publish(TelemetryEvent::Frame(FrameDelivery {
|
||||
channel: ChannelRef {
|
||||
stream: stream.clone(),
|
||||
channel: channel.id,
|
||||
},
|
||||
position: Position(11),
|
||||
payload: br#"{"rx":1}"#.to_vec(),
|
||||
}));
|
||||
|
||||
collector.pump_pulls();
|
||||
let mut observed = None;
|
||||
collector.drain(|stream, descriptor, channel, frame| {
|
||||
observed = Some((
|
||||
stream.clone(),
|
||||
descriptor.cloned(),
|
||||
channel.to_owned(),
|
||||
frame.clone(),
|
||||
));
|
||||
});
|
||||
|
||||
let (observed_stream, observed_descriptor, observed_channel, observed_frame) =
|
||||
observed.expect("pulled frame");
|
||||
assert_eq!(observed_stream, stream);
|
||||
assert_eq!(observed_descriptor, Some(descriptor));
|
||||
assert_eq!(observed_channel, "host.net");
|
||||
assert_eq!(observed_frame.position, Position(11));
|
||||
assert_eq!(observed_frame.payload, br#"{"rx":1}"#);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,15 +5,15 @@
|
|||
//! optional live dashboard. Both are consumed by the control loop in
|
||||
//! `orchestration::app`; frame-bearing read paths live in `frame_collector`.
|
||||
|
||||
use telemetry::frame::{Frame, StreamId};
|
||||
use telemetry::{
|
||||
ChannelContent, ChannelId, TelemetryEndpoint, TelemetryProducer, Lifetime, NodeId, Record,
|
||||
StreamDescriptor, StreamOrigin,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use telemetry::frame::{Frame, StreamId};
|
||||
use telemetry::{
|
||||
ChannelContent, ChannelId, Lifetime, NodeId, Record, StreamDescriptor, StreamOrigin,
|
||||
TelemetryEndpoint, TelemetryProducer,
|
||||
};
|
||||
|
||||
use crate::observability::benchmark;
|
||||
#[cfg(feature = "dashboard")]
|
||||
|
|
@ -290,17 +290,27 @@ pub(crate) struct DashboardSupport {
|
|||
|
||||
#[cfg(feature = "dashboard")]
|
||||
impl DashboardSupport {
|
||||
pub(crate) fn start(enabled: bool, engine: &EngineHandle) -> Result<Option<Self>, String> {
|
||||
if !enabled {
|
||||
return Ok(None);
|
||||
}
|
||||
fn config() -> Result<dashboard::DashboardConfig, String> {
|
||||
let mut config = dashboard::DashboardConfig::default();
|
||||
if let Some(port) = env_optional("MYELIN_DASHBOARD_PORT") {
|
||||
config.port = port
|
||||
.parse::<u16>()
|
||||
.map_err(|e| format!("invalid MYELIN_DASHBOARD_PORT={port:?}: {e}"))?;
|
||||
}
|
||||
let handle = dashboard::DashboardHandle::new(config);
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
pub(crate) fn configured_url(enabled: bool) -> Result<Option<String>, String> {
|
||||
enabled
|
||||
.then(|| Self::config().map(|config| format!("http://127.0.0.1:{}/", config.port)))
|
||||
.transpose()
|
||||
}
|
||||
|
||||
pub(crate) fn start(enabled: bool, engine: &EngineHandle) -> Result<Option<Self>, String> {
|
||||
if !enabled {
|
||||
return Ok(None);
|
||||
}
|
||||
let handle = dashboard::DashboardHandle::new(Self::config()?);
|
||||
handle.register_view(Arc::new(MyelinClusterDashboardView::new()));
|
||||
engine.spawn(handle.http_server());
|
||||
Ok(Some(Self { handle }))
|
||||
|
|
@ -311,6 +321,35 @@ impl DashboardSupport {
|
|||
stream: dashboard::StreamEvent {
|
||||
node: stream.node.as_str().to_string(),
|
||||
life: stream.life.0,
|
||||
origin: None,
|
||||
label: None,
|
||||
},
|
||||
channel: channel.to_owned(),
|
||||
position: frame.position.0,
|
||||
payload: frame.payload.clone(),
|
||||
});
|
||||
}
|
||||
pub(crate) fn publish_collected_frame(
|
||||
&self,
|
||||
stream: &StreamId,
|
||||
descriptor: Option<&StreamDescriptor>,
|
||||
channel: &str,
|
||||
frame: &Frame,
|
||||
) {
|
||||
let origin = descriptor.map(|descriptor| {
|
||||
match descriptor.origin {
|
||||
StreamOrigin::Orchestrator => "orchestrator",
|
||||
StreamOrigin::Bootstrap => "bootstrap",
|
||||
StreamOrigin::RemoteNode => "remote-node",
|
||||
}
|
||||
.to_owned()
|
||||
});
|
||||
self.handle.publish(dashboard::FrameEvent {
|
||||
stream: dashboard::StreamEvent {
|
||||
node: stream.node.as_str().to_string(),
|
||||
life: stream.life.0,
|
||||
origin,
|
||||
label: descriptor.and_then(|descriptor| descriptor.label.clone()),
|
||||
},
|
||||
channel: channel.to_owned(),
|
||||
position: frame.position.0,
|
||||
|
|
@ -324,6 +363,16 @@ pub(crate) struct DashboardSupport;
|
|||
|
||||
#[cfg(not(feature = "dashboard"))]
|
||||
impl DashboardSupport {
|
||||
pub(crate) fn configured_url(enabled: bool) -> Result<Option<String>, String> {
|
||||
if enabled {
|
||||
return Err(
|
||||
"MYELIN_DASHBOARD requires building myelin-system with feature dashboard"
|
||||
.to_owned(),
|
||||
);
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
pub(crate) fn start(enabled: bool, _engine: &EngineHandle) -> Result<Option<Self>, String> {
|
||||
if enabled {
|
||||
return Err(
|
||||
|
|
@ -335,4 +384,12 @@ impl DashboardSupport {
|
|||
}
|
||||
|
||||
pub(crate) fn publish_frame(&self, _stream: &StreamId, _channel: &str, _frame: &Frame) {}
|
||||
pub(crate) fn publish_collected_frame(
|
||||
&self,
|
||||
_stream: &StreamId,
|
||||
_descriptor: Option<&StreamDescriptor>,
|
||||
_channel: &str,
|
||||
_frame: &Frame,
|
||||
) {
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
use std::io::{BufRead, BufReader, Read};
|
||||
use std::thread::{self, JoinHandle};
|
||||
|
||||
use telemetry::{ChannelContent, TelemetryProducer, Lifetime, NodeId, StreamId};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use telemetry::{ChannelContent, Lifetime, NodeId, StreamId, TelemetryProducer};
|
||||
|
||||
use crate::observability::telemetry::{
|
||||
MYELIN_PROVISIONING_LOGS, MyelinProvisionLogRecord, myelin_provision_log_channel,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
//! Myelin-system-owned telemetry channel records.
|
||||
|
||||
use telemetry::Record;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use telemetry::Record;
|
||||
|
||||
use crate::observability::lifecycle as obs;
|
||||
use crate::provisioning::{self, ProvisionLogStream};
|
||||
|
|
|
|||
|
|
@ -33,7 +33,6 @@ pub(crate) enum OrchestratorMsg {
|
|||
stage_index: u32,
|
||||
endpoint: EndpointAddr,
|
||||
node_actor: ActorAddress,
|
||||
telemetry_publisher: ActorAddress,
|
||||
readiness_id: u64,
|
||||
},
|
||||
ObserveNodeRuntimeReadyAck {
|
||||
|
|
@ -154,7 +153,6 @@ pub(crate) enum OrchestratorReport {
|
|||
stage_index: u32,
|
||||
endpoint: EndpointAddr,
|
||||
node_actor: ActorAddress,
|
||||
telemetry_publisher: ActorAddress,
|
||||
readiness_id: u64,
|
||||
},
|
||||
NodeRuntimeReadyAck {
|
||||
|
|
@ -326,7 +324,6 @@ impl ActorInterface for OrchestratorActor {
|
|||
stage_index,
|
||||
endpoint,
|
||||
node_actor,
|
||||
telemetry_publisher,
|
||||
readiness_id,
|
||||
} => {
|
||||
if let Some(report_to) = self.report_to {
|
||||
|
|
@ -338,7 +335,6 @@ impl ActorInterface for OrchestratorActor {
|
|||
stage_index,
|
||||
endpoint,
|
||||
node_actor,
|
||||
telemetry_publisher,
|
||||
readiness_id,
|
||||
},
|
||||
);
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -219,6 +219,23 @@ impl MyelinEffectBackend {
|
|||
(external_id, logical_id)
|
||||
}
|
||||
|
||||
fn detach_all(&self) {
|
||||
let nodes = lock_nodes_read(&self.nodes)
|
||||
.values()
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
for effects in nodes {
|
||||
let mut effects = lock_node(&effects);
|
||||
if let Some(live) = effects.live.take() {
|
||||
live.failure_sink.discard();
|
||||
}
|
||||
effects.plugin.detach_all();
|
||||
if let Some(staged) = effects.staged.as_mut() {
|
||||
staged.plugin.detach_all();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn stop_all(&self) -> Result<(), String> {
|
||||
let mut first_error = None;
|
||||
let nodes = lock_nodes_read(&self.nodes)
|
||||
|
|
@ -684,6 +701,14 @@ impl ProvisionedClusterGuard {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Releases the guard without touching any provider resource: no
|
||||
/// shutdown shape, no stop_all. The daemon exits; nodes keep running and
|
||||
/// are adopted by spec on the next boot.
|
||||
pub(crate) fn detach(&mut self) {
|
||||
self.executor.backend().detach_all();
|
||||
self.stopped = true;
|
||||
}
|
||||
|
||||
fn drain_wakes(&mut self, now: SystemTime) {
|
||||
loop {
|
||||
match self.wake_rx.try_recv() {
|
||||
|
|
@ -859,9 +884,9 @@ mod tests {
|
|||
|
||||
use crate::provisioning::ProviderMount;
|
||||
use ::provisioning::{
|
||||
BootSpec, BootstrapSessionSpec, ClusterShape, CreateLeaseRequest, TelemetryStreamId,
|
||||
DesiredNodeShape, LogicalNodeSpec, NodeGroupId, OperationId, ProviderKind, RetryPolicy,
|
||||
RoleId, RunId, RunNodeGroupSpec, SwarmJoinSpec, SwarmJoinTemplate,
|
||||
BootSpec, BootstrapSessionSpec, ClusterShape, CreateLeaseRequest, DesiredNodeShape,
|
||||
LogicalNodeSpec, NodeGroupId, OperationId, ProviderKind, RetryPolicy, RoleId, RunId,
|
||||
RunNodeGroupSpec, SwarmJoinSpec, SwarmJoinTemplate, TelemetryStreamId,
|
||||
};
|
||||
|
||||
use super::*;
|
||||
|
|
|
|||
|
|
@ -87,9 +87,8 @@ pub(crate) fn looks_remote_image(image: &str) -> bool {
|
|||
host == "localhost" || host.contains('.') || host.contains(':')
|
||||
}
|
||||
|
||||
/// Shared overlay for legacy and multi-binary configuration parsing. This accepts
|
||||
/// fields outside the fixed `myelin-chat` public config surface; `myelin-chat` uses a
|
||||
/// bin-local strict config loader instead.
|
||||
/// Shared overlay for daemon and worker configuration parsing. Unknown
|
||||
/// workload-specific fields remain available to dormant pipeline tooling.
|
||||
#[derive(Clone, Debug, Default, Deserialize, PartialEq)]
|
||||
#[serde(default)]
|
||||
pub(crate) struct TomlConfigOverlay {
|
||||
|
|
|
|||
653
apps/myelin/src/orchestration/daemon.rs
Normal file
653
apps/myelin/src/orchestration/daemon.rs
Normal file
|
|
@ -0,0 +1,653 @@
|
|||
//! Fleet-control daemon: persistent identity, cluster snapshot, and manual
|
||||
//! command dispatch.
|
||||
//!
|
||||
//! The daemon is deliberately manual: nothing is provisioned, replaced, or
|
||||
//! destroyed except in response to an operator command. Provider resources
|
||||
//! (docker containers, vastai leases) are ground truth; the snapshot records
|
||||
//! intent and facts so a restarted daemon can adopt what still exists and
|
||||
//! never silently re-provisions.
|
||||
|
||||
#[cfg(test)]
|
||||
use std::collections::BTreeMap;
|
||||
use std::collections::BTreeSet;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use crate::provisioning::NodeProvisionSpec;
|
||||
use distribution::types::NodeId as DistNodeId;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use swactor::actor::ActorAddress;
|
||||
|
||||
pub(crate) const SNAPSHOT_SCHEMA_VERSION: u32 = 1;
|
||||
pub(crate) const IDENTITY_FILE: &str = "identity.key";
|
||||
pub(crate) const SNAPSHOT_FILE: &str = "cluster.json";
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub(crate) enum NodeStatus {
|
||||
/// Provider resource exists and the runtime joined (or is expected to).
|
||||
Running,
|
||||
/// Tracked by the snapshot but gone from the provider.
|
||||
Dead,
|
||||
/// Exists at the provider under this daemon's label but was never added
|
||||
/// through this daemon's command surface.
|
||||
Orphan,
|
||||
}
|
||||
|
||||
/// Join/readiness facts captured when a node announced itself. Persisted so a
|
||||
/// restarted daemon can re-subscribe telemetry once routes recover.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub(crate) struct RuntimeFacts {
|
||||
pub endpoint: String,
|
||||
pub node_actor: ActorAddress,
|
||||
pub swim_node_id: DistNodeId,
|
||||
pub stage_index: u32,
|
||||
pub readiness_id: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub(crate) struct SnapshotNode {
|
||||
pub logical_node_id: u64,
|
||||
/// Provision intent for nodes added through this daemon. Absent for
|
||||
/// orphans (discovered, not managed).
|
||||
pub spec: Option<NodeProvisionSpec>,
|
||||
/// Provider-side address (e.g. docker container name) for records that
|
||||
/// exist without a full spec.
|
||||
pub provider_ref: Option<String>,
|
||||
pub status: NodeStatus,
|
||||
pub runtime: Option<RuntimeFacts>,
|
||||
pub last_seen_unix_ms: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub(crate) struct ClusterSnapshot {
|
||||
pub schema_version: u32,
|
||||
pub run_id: u64,
|
||||
pub label: String,
|
||||
pub next_node_id: u64,
|
||||
/// Durable at-most-once ledger for dashboard requests. A command id is
|
||||
/// recorded before provider mutation, so retrying after a timeout or crash
|
||||
/// cannot create or destroy a second resource.
|
||||
#[serde(default)]
|
||||
pub accepted_command_ids: BTreeSet<String>,
|
||||
pub nodes: Vec<SnapshotNode>,
|
||||
}
|
||||
|
||||
impl ClusterSnapshot {
|
||||
pub(crate) fn fresh(run_id: u64, label: impl Into<String>) -> Self {
|
||||
Self {
|
||||
schema_version: SNAPSHOT_SCHEMA_VERSION,
|
||||
run_id,
|
||||
label: label.into(),
|
||||
next_node_id: 1,
|
||||
accepted_command_ids: BTreeSet::new(),
|
||||
nodes: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn node(&self, logical_node_id: u64) -> Option<&SnapshotNode> {
|
||||
self.nodes
|
||||
.iter()
|
||||
.find(|node| node.logical_node_id == logical_node_id)
|
||||
}
|
||||
|
||||
pub(crate) fn node_mut(&mut self, logical_node_id: u64) -> Option<&mut SnapshotNode> {
|
||||
self.nodes
|
||||
.iter_mut()
|
||||
.find(|node| node.logical_node_id == logical_node_id)
|
||||
}
|
||||
|
||||
pub(crate) fn running_nodes(&self) -> impl Iterator<Item = &SnapshotNode> {
|
||||
self.nodes
|
||||
.iter()
|
||||
.filter(|node| node.status == NodeStatus::Running)
|
||||
}
|
||||
|
||||
/// Allocates the next logical node id. Ids are monotonic and never reused.
|
||||
pub(crate) fn allocate_node_id(&mut self) -> u64 {
|
||||
let id = self.next_node_id;
|
||||
self.next_node_id = self
|
||||
.next_node_id
|
||||
.checked_add(1)
|
||||
.expect("logical node id space exhausted");
|
||||
id
|
||||
}
|
||||
|
||||
pub(crate) fn upsert_node(&mut self, node: SnapshotNode) {
|
||||
if node.logical_node_id != 0 {
|
||||
self.next_node_id = self
|
||||
.next_node_id
|
||||
.max(node.logical_node_id.saturating_add(1));
|
||||
}
|
||||
match self
|
||||
.nodes
|
||||
.iter()
|
||||
.position(|existing| existing.logical_node_id == node.logical_node_id)
|
||||
{
|
||||
Some(index) => self.nodes[index] = node,
|
||||
None => self.nodes.push(node),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn remove_node(&mut self, logical_node_id: u64) -> Option<SnapshotNode> {
|
||||
let index = self
|
||||
.nodes
|
||||
.iter()
|
||||
.position(|node| node.logical_node_id == logical_node_id)?;
|
||||
Some(self.nodes.remove(index))
|
||||
}
|
||||
|
||||
/// Makes orphan records exactly match provider-only resources. Orphans are
|
||||
/// keyed by provider reference because they intentionally have no logical
|
||||
/// node id or provision spec.
|
||||
pub(crate) fn sync_orphans(
|
||||
&mut self,
|
||||
provider_refs: impl IntoIterator<Item = String>,
|
||||
) -> Vec<String> {
|
||||
let provider_refs = provider_refs
|
||||
.into_iter()
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
self.nodes.retain(|node| {
|
||||
node.status != NodeStatus::Orphan
|
||||
|| node
|
||||
.provider_ref
|
||||
.as_ref()
|
||||
.is_some_and(|provider_ref| provider_refs.contains(provider_ref))
|
||||
});
|
||||
let known = self
|
||||
.nodes
|
||||
.iter()
|
||||
.filter(|node| node.status == NodeStatus::Orphan)
|
||||
.filter_map(|node| node.provider_ref.clone())
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
let added = provider_refs
|
||||
.difference(&known)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
let now = unix_ms_now();
|
||||
for provider_ref in &added {
|
||||
self.nodes.push(SnapshotNode {
|
||||
logical_node_id: 0,
|
||||
spec: None,
|
||||
provider_ref: Some(provider_ref.clone()),
|
||||
status: NodeStatus::Orphan,
|
||||
runtime: None,
|
||||
last_seen_unix_ms: now,
|
||||
});
|
||||
}
|
||||
added
|
||||
}
|
||||
|
||||
pub(crate) fn accept_command(&mut self, command_id: &str) -> Result<bool, String> {
|
||||
let command_id = command_id.trim();
|
||||
if command_id.is_empty() {
|
||||
return Err("dashboard command_id must not be empty".to_owned());
|
||||
}
|
||||
Ok(self.accepted_command_ids.insert(command_id.to_owned()))
|
||||
}
|
||||
}
|
||||
|
||||
/// Operator commands accepted by the dispatcher. The dashboard (and later the
|
||||
/// CLI) is a transport into this surface.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub(crate) enum DaemonCommand {
|
||||
/// Provision exactly one node using the configured provider + selection
|
||||
/// policy. Fails (does not retry) if bring-up fails.
|
||||
AddNode,
|
||||
/// Terminate a node's provider resource; the record stays (status Dead).
|
||||
Kill { logical_node_id: u64 },
|
||||
/// Terminate and forget a node.
|
||||
Destroy { logical_node_id: u64 },
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub(crate) enum DaemonOutcome {
|
||||
Added { logical_node_id: u64 },
|
||||
Killed { logical_node_id: u64 },
|
||||
Destroyed { logical_node_id: u64 },
|
||||
NoSuchNode { logical_node_id: u64 },
|
||||
Failed { command: String, reason: String },
|
||||
}
|
||||
|
||||
/// Where the daemon keeps `identity.key` and `cluster.json`.
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct StateDir {
|
||||
root: PathBuf,
|
||||
}
|
||||
|
||||
impl StateDir {
|
||||
pub(crate) fn new(root: impl Into<PathBuf>) -> Self {
|
||||
Self { root: root.into() }
|
||||
}
|
||||
|
||||
pub(crate) fn root(&self) -> &Path {
|
||||
&self.root
|
||||
}
|
||||
|
||||
fn identity_path(&self) -> PathBuf {
|
||||
self.root.join(IDENTITY_FILE)
|
||||
}
|
||||
|
||||
fn snapshot_path(&self) -> PathBuf {
|
||||
self.root.join(SNAPSHOT_FILE)
|
||||
}
|
||||
|
||||
/// Loads the persisted iroh secret key, creating it on first boot. The
|
||||
/// endpoint address baked into every launched node's env stays valid
|
||||
/// across daemon restarts because of this.
|
||||
pub(crate) fn load_or_create_identity(&self) -> Result<iroh::SecretKey, String> {
|
||||
fs::create_dir_all(&self.root)
|
||||
.map_err(|error| format!("create state dir {}: {error}", self.root.display()))?;
|
||||
let path = self.identity_path();
|
||||
match fs::read(&path) {
|
||||
Ok(bytes) => {
|
||||
let array: [u8; 32] = bytes
|
||||
.try_into()
|
||||
.map_err(|_| format!("identity key {} is not 32 bytes", path.display()))?;
|
||||
Ok(iroh::SecretKey::from_bytes(&array))
|
||||
}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||
let key = iroh::SecretKey::generate();
|
||||
write_atomic(&path, &key.to_bytes())?;
|
||||
Ok(key)
|
||||
}
|
||||
Err(error) => Err(format!("read identity key {}: {error}", path.display())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Loads the cluster snapshot. Missing file is a fresh (empty) cluster; a
|
||||
/// corrupt file is a hard error so a stale state can never cause a silent
|
||||
/// re-provision.
|
||||
pub(crate) fn load_snapshot(&self) -> Result<ClusterSnapshot, String> {
|
||||
let path = self.snapshot_path();
|
||||
match fs::read_to_string(&path) {
|
||||
Ok(content) => {
|
||||
let snapshot: ClusterSnapshot = serde_json::from_str(&content).map_err(|e| {
|
||||
format!(
|
||||
"cluster snapshot {} is corrupt ({e}); inspect it or remove it with \
|
||||
--reset-state — refusing to silently re-provision",
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
if snapshot.schema_version != SNAPSHOT_SCHEMA_VERSION {
|
||||
return Err(format!(
|
||||
"cluster snapshot {} has unsupported schema_version {} (expected {}); \
|
||||
migrate or remove it with --reset-state",
|
||||
path.display(),
|
||||
snapshot.schema_version,
|
||||
SNAPSHOT_SCHEMA_VERSION
|
||||
));
|
||||
}
|
||||
Ok(snapshot)
|
||||
}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||
// Caller decides the run id/label for a fresh snapshot.
|
||||
Ok(ClusterSnapshot {
|
||||
schema_version: SNAPSHOT_SCHEMA_VERSION,
|
||||
run_id: 0,
|
||||
label: String::new(),
|
||||
next_node_id: 1,
|
||||
nodes: Vec::new(),
|
||||
accepted_command_ids: BTreeSet::new(),
|
||||
})
|
||||
}
|
||||
Err(error) => Err(format!("read cluster snapshot {}: {error}", path.display())),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn save_snapshot(&self, snapshot: &ClusterSnapshot) -> Result<(), String> {
|
||||
fs::create_dir_all(&self.root)
|
||||
.map_err(|error| format!("create state dir {}: {error}", self.root.display()))?;
|
||||
let bytes = serde_json::to_vec_pretty(snapshot)
|
||||
.map_err(|error| format!("serialize cluster snapshot: {error}"))?;
|
||||
write_atomic(&self.snapshot_path(), &bytes)
|
||||
}
|
||||
|
||||
/// Removes all state files. Explicit operator action only.
|
||||
pub(crate) fn reset(&self) -> Result<(), String> {
|
||||
for path in [self.identity_path(), self.snapshot_path()] {
|
||||
if let Err(error) = fs::remove_file(&path)
|
||||
&& error.kind() != std::io::ErrorKind::NotFound
|
||||
{
|
||||
return Err(format!("remove {}: {error}", path.display()));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn write_atomic(path: &Path, bytes: &[u8]) -> Result<(), String> {
|
||||
let tmp = path.with_extension("tmp");
|
||||
fs::write(&tmp, bytes).map_err(|error| format!("write {}: {error}", tmp.display()))?;
|
||||
fs::rename(&tmp, path).map_err(|error| {
|
||||
let _ = fs::remove_file(&tmp);
|
||||
format!("persist {}: {error}", path.display())
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn unix_ms_now() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|duration| duration.as_millis() as u64)
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Joins snapshot intent with provider ground truth at boot. Never takes a
|
||||
/// lifecycle action: live nodes are adopted for observation, missing ones are
|
||||
/// marked dead, provider-only resources are recorded as orphans.
|
||||
pub(crate) struct BootJoin {
|
||||
pub snapshot: ClusterSnapshot,
|
||||
}
|
||||
|
||||
pub(crate) struct JoinOutcome {
|
||||
pub adopted: Vec<u64>,
|
||||
pub dead: Vec<u64>,
|
||||
pub orphans: Vec<String>,
|
||||
}
|
||||
|
||||
impl BootJoin {
|
||||
/// `labeled` lists provider resources carrying this daemon's label that
|
||||
/// the snapshot does not account for.
|
||||
pub(crate) fn apply_provider_truthtable(
|
||||
&mut self,
|
||||
live_specs: &BTreeMap<u64, bool>,
|
||||
labeled_orphans: Vec<String>,
|
||||
) -> JoinOutcome {
|
||||
let mut adopted = Vec::new();
|
||||
let mut dead = Vec::new();
|
||||
let now = unix_ms_now();
|
||||
for node in &mut self.snapshot.nodes {
|
||||
if node.status == NodeStatus::Orphan {
|
||||
continue;
|
||||
}
|
||||
match live_specs.get(&node.logical_node_id) {
|
||||
Some(true) => {
|
||||
node.status = NodeStatus::Running;
|
||||
node.last_seen_unix_ms = now;
|
||||
adopted.push(node.logical_node_id);
|
||||
}
|
||||
_ => {
|
||||
node.status = NodeStatus::Dead;
|
||||
dead.push(node.logical_node_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
self.snapshot.sync_orphans(labeled_orphans.clone());
|
||||
JoinOutcome {
|
||||
adopted,
|
||||
dead,
|
||||
orphans: labeled_orphans,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
static NEXT_TEST_DIR: AtomicU64 = AtomicU64::new(1);
|
||||
|
||||
fn test_dir(name: &str) -> PathBuf {
|
||||
std::env::temp_dir().join(format!(
|
||||
"myelin-{name}-{}-{}",
|
||||
std::process::id(),
|
||||
NEXT_TEST_DIR.fetch_add(1, Ordering::Relaxed)
|
||||
))
|
||||
}
|
||||
|
||||
fn node(id: u64, status: NodeStatus) -> SnapshotNode {
|
||||
SnapshotNode {
|
||||
logical_node_id: id,
|
||||
spec: None,
|
||||
provider_ref: Some(format!("container-{id}")),
|
||||
status,
|
||||
runtime: None,
|
||||
last_seen_unix_ms: 0,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn node_ids_allocate_monotonically_and_never_reuse() {
|
||||
let mut snapshot = ClusterSnapshot::fresh(1, "test");
|
||||
assert_eq!(snapshot.allocate_node_id(), 1);
|
||||
assert_eq!(snapshot.allocate_node_id(), 2);
|
||||
snapshot.next_node_id = u64::MAX;
|
||||
assert!(
|
||||
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||
snapshot.allocate_node_id()
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remove_then_allocate_does_not_reuse_ids() {
|
||||
let mut snapshot = ClusterSnapshot::fresh(1, "test");
|
||||
snapshot.upsert_node(node(1, NodeStatus::Running));
|
||||
assert!(snapshot.remove_node(1).is_some());
|
||||
assert_eq!(snapshot.allocate_node_id(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn join_marks_live_missing_and_orphans_without_actions() {
|
||||
let mut join = BootJoin {
|
||||
snapshot: ClusterSnapshot::fresh(1, "test"),
|
||||
};
|
||||
join.snapshot.upsert_node(node(1, NodeStatus::Running));
|
||||
join.snapshot.upsert_node(node(2, NodeStatus::Running));
|
||||
let live = BTreeMap::from([(1_u64, true), (2_u64, false)]);
|
||||
let outcome = join.apply_provider_truthtable(&live, vec!["orphan-a".to_owned()]);
|
||||
assert_eq!(outcome.adopted, vec![1]);
|
||||
assert_eq!(outcome.dead, vec![2]);
|
||||
assert_eq!(outcome.orphans, vec!["orphan-a".to_owned()]);
|
||||
assert_eq!(join.snapshot.node(1).unwrap().status, NodeStatus::Running);
|
||||
assert_eq!(join.snapshot.node(2).unwrap().status, NodeStatus::Dead);
|
||||
assert_eq!(
|
||||
join.snapshot
|
||||
.nodes
|
||||
.iter()
|
||||
.find(|node| node.provider_ref.as_deref() == Some("orphan-a"))
|
||||
.unwrap()
|
||||
.status,
|
||||
NodeStatus::Orphan
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snapshot_round_trips_through_disk() {
|
||||
let dir = test_dir("snapshot");
|
||||
let state = StateDir::new(&dir);
|
||||
let mut snapshot = ClusterSnapshot::fresh(7, "label");
|
||||
snapshot.upsert_node(SnapshotNode {
|
||||
logical_node_id: 1,
|
||||
spec: Some(NodeProvisionSpec {
|
||||
run_id: 7,
|
||||
node_id: 1,
|
||||
attempt_id: 0,
|
||||
stage_index: Some(0),
|
||||
image: "myelin-node:latest".to_owned(),
|
||||
env: vec![("A".to_owned(), "B".to_owned())],
|
||||
args: vec![],
|
||||
mounts: vec![],
|
||||
}),
|
||||
provider_ref: Some("container-1".to_owned()),
|
||||
status: NodeStatus::Running,
|
||||
runtime: None,
|
||||
last_seen_unix_ms: 42,
|
||||
});
|
||||
state.save_snapshot(&snapshot).unwrap();
|
||||
let loaded = state.load_snapshot().unwrap();
|
||||
assert_eq!(loaded.run_id, 7);
|
||||
assert_eq!(loaded.nodes.len(), 1);
|
||||
assert_eq!(
|
||||
loaded.nodes[0].spec.as_ref().unwrap().image,
|
||||
"myelin-node:latest"
|
||||
);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corrupt_snapshot_is_a_hard_error() {
|
||||
let dir = test_dir("corrupt");
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
std::fs::write(dir.join(SNAPSHOT_FILE), b"{ not json").unwrap();
|
||||
let state = StateDir::new(&dir);
|
||||
let error = state.load_snapshot().unwrap_err();
|
||||
assert!(error.contains("corrupt"), "unexpected error: {error}");
|
||||
assert!(error.contains("refusing"), "unexpected error: {error}");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identity_key_is_stable_across_loads() {
|
||||
let dir = test_dir("identity");
|
||||
let state = StateDir::new(&dir);
|
||||
let first = state.load_or_create_identity().unwrap();
|
||||
let second = state.load_or_create_identity().unwrap();
|
||||
assert_eq!(first.to_bytes(), second.to_bytes());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn command_ids_are_durable_at_most_once_tokens() {
|
||||
let dir = test_dir("commands");
|
||||
let state = StateDir::new(&dir);
|
||||
let mut snapshot = ClusterSnapshot::fresh(1, "test");
|
||||
assert!(snapshot.accept_command("request-a").unwrap());
|
||||
assert!(!snapshot.accept_command("request-a").unwrap());
|
||||
assert!(snapshot.accept_command("request-b").unwrap());
|
||||
assert!(snapshot.accept_command(" ").is_err());
|
||||
state.save_snapshot(&snapshot).unwrap();
|
||||
let mut loaded = state.load_snapshot().unwrap();
|
||||
assert!(!loaded.accept_command("request-a").unwrap());
|
||||
assert!(!loaded.accept_command("request-b").unwrap());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn orphan_records_exactly_follow_provider_ground_truth() {
|
||||
let mut snapshot = ClusterSnapshot::fresh(1, "test");
|
||||
snapshot.sync_orphans(["b".to_owned(), "a".to_owned()]);
|
||||
snapshot.sync_orphans(["b".to_owned(), "c".to_owned()]);
|
||||
let refs = snapshot
|
||||
.nodes
|
||||
.iter()
|
||||
.filter(|node| node.status == NodeStatus::Orphan)
|
||||
.filter_map(|node| node.provider_ref.clone())
|
||||
.collect::<BTreeSet<_>>();
|
||||
assert_eq!(refs, BTreeSet::from(["b".to_owned(), "c".to_owned()]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_temp_write_never_replaces_last_snapshot() {
|
||||
let dir = test_dir("atomic");
|
||||
let state = StateDir::new(&dir);
|
||||
let mut snapshot = ClusterSnapshot::fresh(7, "stable");
|
||||
snapshot.allocate_node_id();
|
||||
state.save_snapshot(&snapshot).unwrap();
|
||||
std::fs::write(dir.join("cluster.tmp"), b"{partial").unwrap();
|
||||
let loaded = state.load_snapshot().unwrap();
|
||||
assert_eq!(loaded.run_id, 7);
|
||||
assert_eq!(loaded.next_node_id, 2);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identity_changes_only_after_explicit_reset() {
|
||||
let dir = test_dir("identity-reset");
|
||||
let state = StateDir::new(&dir);
|
||||
let first = state.load_or_create_identity().unwrap();
|
||||
assert_eq!(
|
||||
first.to_bytes(),
|
||||
state.load_or_create_identity().unwrap().to_bytes()
|
||||
);
|
||||
state.reset().unwrap();
|
||||
let replacement = state.load_or_create_identity().unwrap();
|
||||
assert_ne!(first.to_bytes(), replacement.to_bytes());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fuzzed_manual_interleavings_preserve_snapshot_invariants() {
|
||||
let dir = test_dir("state-fuzz");
|
||||
let state = StateDir::new(&dir);
|
||||
let mut snapshot = ClusterSnapshot::fresh(9, "fuzz");
|
||||
let mut model_nodes = BTreeMap::<u64, NodeStatus>::new();
|
||||
let mut provider_orphans = BTreeSet::<String>::new();
|
||||
let mut command_ids = BTreeSet::<String>::new();
|
||||
let mut rng = 0x6a09_e667_f3bc_c909_u64;
|
||||
|
||||
for step in 0..2_000_u64 {
|
||||
rng = rng
|
||||
.wrapping_mul(6_364_136_223_846_793_005)
|
||||
.wrapping_add(1_442_695_040_888_963_407);
|
||||
match rng % 7 {
|
||||
0 => {
|
||||
let id = snapshot.allocate_node_id();
|
||||
snapshot.upsert_node(node(id, NodeStatus::Running));
|
||||
model_nodes.insert(id, NodeStatus::Running);
|
||||
}
|
||||
1 => {
|
||||
let id = 1 + rng.rotate_left(17) % snapshot.next_node_id.max(2);
|
||||
if let Some(status) = model_nodes.get_mut(&id) {
|
||||
*status = NodeStatus::Dead;
|
||||
snapshot.node_mut(id).unwrap().status = NodeStatus::Dead;
|
||||
}
|
||||
}
|
||||
2 => {
|
||||
let id = 1 + rng.rotate_right(11) % snapshot.next_node_id.max(2);
|
||||
model_nodes.remove(&id);
|
||||
snapshot.remove_node(id);
|
||||
}
|
||||
3 => {
|
||||
provider_orphans.insert(format!("orphan-{}", rng % 19));
|
||||
}
|
||||
4 => {
|
||||
provider_orphans.remove(&format!("orphan-{}", rng % 19));
|
||||
}
|
||||
5 => {
|
||||
let command_id = format!("command-{}", rng % 31);
|
||||
let expected = command_ids.insert(command_id.clone());
|
||||
assert_eq!(snapshot.accept_command(&command_id).unwrap(), expected);
|
||||
}
|
||||
_ => {
|
||||
state.save_snapshot(&snapshot).unwrap();
|
||||
snapshot = state.load_snapshot().unwrap();
|
||||
}
|
||||
}
|
||||
snapshot.sync_orphans(provider_orphans.iter().cloned());
|
||||
|
||||
let managed_ids = snapshot
|
||||
.nodes
|
||||
.iter()
|
||||
.filter(|node| node.status != NodeStatus::Orphan)
|
||||
.map(|node| node.logical_node_id)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
managed_ids.iter().copied().collect::<BTreeSet<_>>().len(),
|
||||
managed_ids.len(),
|
||||
"duplicate managed id after step {step}"
|
||||
);
|
||||
for (id, expected) in &model_nodes {
|
||||
assert_eq!(
|
||||
snapshot.node(*id).map(|node| &node.status),
|
||||
Some(expected),
|
||||
"node model diverged after step {step}"
|
||||
);
|
||||
}
|
||||
assert!(managed_ids.iter().all(|id| *id < snapshot.next_node_id));
|
||||
let observed_orphans = snapshot
|
||||
.nodes
|
||||
.iter()
|
||||
.filter(|node| node.status == NodeStatus::Orphan)
|
||||
.filter_map(|node| node.provider_ref.clone())
|
||||
.collect::<BTreeSet<_>>();
|
||||
assert_eq!(observed_orphans, provider_orphans);
|
||||
assert_eq!(snapshot.accepted_command_ids, command_ids);
|
||||
}
|
||||
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
}
|
||||
548
apps/myelin/src/orchestration/job_reconciler.rs
Normal file
548
apps/myelin/src/orchestration/job_reconciler.rs
Normal file
|
|
@ -0,0 +1,548 @@
|
|||
//! Job-runner deployment through the existing Myelin cluster reconciler.
|
||||
//!
|
||||
//! SSH remains a provider bootstrap transport owned by `VastAiProvisioningPlugin`;
|
||||
//! the job workspace, commands, and outputs still travel through the job actors over
|
||||
//! the iroh actor plane.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex, mpsc};
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use iroh_driver::MVP_IROH_ENDPOINT_ADDR_MASK_ENV;
|
||||
use provisioning::{
|
||||
BootSpec, ClusterShape, DesiredNodeShape, LogicalNodeId, NodeAttemptId, NodeGroupId,
|
||||
ProviderKind, RetryPolicy, RoleId, RunId, RunNodeGroupSpec, SwactorId, SwarmJoinTemplate,
|
||||
};
|
||||
use swactor_job_runner::{Job, JobDone};
|
||||
use swactor_vastai::{LifecyclePolicy, SelectionPolicy};
|
||||
|
||||
use crate::job_deploy::{self, NodeIdentity};
|
||||
use crate::orchestration::app::{
|
||||
derive_ssh_public_key, ensure_vastai_account_ssh_key, resolve_vastai_ssh_identity,
|
||||
ssh_public_key_fingerprint,
|
||||
};
|
||||
use crate::orchestration::cluster_reconciler::{ProvisionedClusterGuard, ReconcilerNodeBinding};
|
||||
use crate::orchestration::provider_adapters::relay::{
|
||||
MYELIN_IROH_RELAY_MODE_ENV, MYELIN_IROH_RELAY_URL_ENV, SWACTOR_IROH_RELAY_URL_ENV,
|
||||
};
|
||||
use crate::orchestration::provider_adapters::vastai::{
|
||||
SshCommandBootstrapLauncher, ToolsVastAiLeaseClient, VastAiProvisioningConfig,
|
||||
VastAiProvisioningPlugin,
|
||||
};
|
||||
use crate::provisioning::{
|
||||
NodeProvisionSpec, PluginObservation, PluginObservationSink, PluginSink, ProvisionPlugin,
|
||||
};
|
||||
|
||||
const DEFAULT_NODE_ID: u64 = 2;
|
||||
const DEFAULT_STAGE_INDEX: u32 = 0;
|
||||
const DEFAULT_REMOTE_WORKER_BIN: &str = "/usr/local/bin/myelin-job-worker";
|
||||
const DEFAULT_WORKER_WORKDIR: &str = "/root/workspace";
|
||||
const DEFAULT_ENDPOINT_ADDR_MASK: &str = "relay-only";
|
||||
const DEFAULT_LABEL_PREFIX: &str = "myelin-job";
|
||||
const POLL: Duration = Duration::from_millis(100);
|
||||
const DEFAULT_PROVISION_TIMEOUT: Duration = Duration::from_secs(60 * 30);
|
||||
const RUNTIME_CONVERGENCE_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct VastAiJobOptions {
|
||||
pub api_key: Option<String>,
|
||||
pub image: Option<String>,
|
||||
pub ssh_identity: Option<PathBuf>,
|
||||
pub remote_worker_bin: String,
|
||||
pub worker_workdir: String,
|
||||
pub run_id: u64,
|
||||
pub node_id: u64,
|
||||
pub label_prefix: String,
|
||||
pub disk_gb: u32,
|
||||
pub ssh_user: String,
|
||||
pub confirm_lease: bool,
|
||||
pub onstart: Option<String>,
|
||||
pub relay_mode: Option<String>,
|
||||
pub relay_url: Option<String>,
|
||||
pub endpoint_addr_mask: String,
|
||||
pub gpu_name: Option<String>,
|
||||
pub min_gpu_ram_mb: Option<u64>,
|
||||
pub min_down_mbps: Option<f64>,
|
||||
pub min_up_mbps: Option<f64>,
|
||||
pub max_dph_total: Option<f64>,
|
||||
pub min_reliability: Option<f64>,
|
||||
pub require_verified: Option<bool>,
|
||||
pub blacklist_hosts: Vec<u64>,
|
||||
pub poll_interval: Option<Duration>,
|
||||
pub provision_timeout: Duration,
|
||||
}
|
||||
|
||||
impl Default for VastAiJobOptions {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
api_key: None,
|
||||
image: None,
|
||||
ssh_identity: None,
|
||||
remote_worker_bin: DEFAULT_REMOTE_WORKER_BIN.to_owned(),
|
||||
worker_workdir: DEFAULT_WORKER_WORKDIR.to_owned(),
|
||||
run_id: default_run_id(),
|
||||
node_id: DEFAULT_NODE_ID,
|
||||
label_prefix: DEFAULT_LABEL_PREFIX.to_owned(),
|
||||
disk_gb: 80,
|
||||
ssh_user: "root".to_owned(),
|
||||
confirm_lease: false,
|
||||
onstart: None,
|
||||
relay_mode: Some("default".to_owned()),
|
||||
relay_url: None,
|
||||
endpoint_addr_mask: DEFAULT_ENDPOINT_ADDR_MASK.to_owned(),
|
||||
gpu_name: None,
|
||||
min_gpu_ram_mb: None,
|
||||
min_down_mbps: None,
|
||||
min_up_mbps: None,
|
||||
max_dph_total: None,
|
||||
min_reliability: None,
|
||||
require_verified: None,
|
||||
blacklist_hosts: Vec::new(),
|
||||
poll_interval: None,
|
||||
provision_timeout: DEFAULT_PROVISION_TIMEOUT,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn run_vastai_job(
|
||||
job: Job,
|
||||
landing: PathBuf,
|
||||
options: VastAiJobOptions,
|
||||
) -> Result<JobDone, String> {
|
||||
let api_key = required_option(
|
||||
options.api_key.as_deref(),
|
||||
"VAST_API_KEY, MYELIN_VASTAI_API_KEY, VASTAI_API_KEY, or --vastai-api-key",
|
||||
)?
|
||||
.to_owned();
|
||||
let image = required_option(
|
||||
options.image.as_deref(),
|
||||
"MYELIN_NODE_IMAGE or --image for the job-worker image",
|
||||
)?
|
||||
.to_owned();
|
||||
validate_non_empty("remote worker binary", &options.remote_worker_bin)?;
|
||||
validate_non_empty("worker workdir", &options.worker_workdir)?;
|
||||
validate_non_empty("endpoint address mask", &options.endpoint_addr_mask)?;
|
||||
|
||||
let mut session = job_deploy::start_orchestrator(landing)?;
|
||||
let orch_json = session.identity_json()?;
|
||||
println!("JOB_ORCH_IDENTITY {orch_json}");
|
||||
let _ = std::io::Write::flush(&mut std::io::stdout());
|
||||
eprintln!(
|
||||
"job-reconcile: provisioning VastAI node through reconciler image={image} node_id={} run_id={}",
|
||||
options.node_id, options.run_id
|
||||
);
|
||||
|
||||
let (sink, observations) = observation_channel();
|
||||
let provisioner = build_vastai_provisioner(&api_key, &options, session.runtime())?;
|
||||
let spec = job_node_spec(&options, image, &orch_json)?;
|
||||
let mut cluster = build_cluster(&options, spec, provisioner, session.engine_handle(), sink)?;
|
||||
|
||||
let result = run_with_cluster(job, &mut session, &mut cluster, &observations, &options);
|
||||
let stop_result = cluster.stop();
|
||||
match (result, stop_result) {
|
||||
(Ok(done), Ok(())) => Ok(done),
|
||||
(Ok(_), Err(cleanup)) => Err(format!(
|
||||
"job completed but reconciler cleanup failed: {cleanup}"
|
||||
)),
|
||||
(Err(error), Ok(())) => Err(error),
|
||||
(Err(error), Err(cleanup)) => Err(format!("{error}; reconciler cleanup failed: {cleanup}")),
|
||||
}
|
||||
}
|
||||
|
||||
fn run_with_cluster(
|
||||
job: Job,
|
||||
session: &mut job_deploy::JobOrchestratorSession,
|
||||
cluster: &mut ProvisionedClusterGuard,
|
||||
observations: &mpsc::Receiver<PluginObservation>,
|
||||
options: &VastAiJobOptions,
|
||||
) -> Result<JobDone, String> {
|
||||
let worker = wait_for_worker_identity(cluster, observations, options)?;
|
||||
let actor = job_deploy::parse_actor(&worker.actor_hex)?;
|
||||
let attempt = cluster.current_attempt(options.node_id).ok_or_else(|| {
|
||||
format!(
|
||||
"reconciler has no active attempt for node {}",
|
||||
options.node_id
|
||||
)
|
||||
})?;
|
||||
if !cluster.observe_runtime_ready(
|
||||
options.node_id,
|
||||
NodeAttemptId(attempt.0),
|
||||
SwactorId(format!("{actor:?}")),
|
||||
SystemTime::now(),
|
||||
) {
|
||||
return Err(format!(
|
||||
"reconciler rejected runtime-ready observation for node {} attempt {}",
|
||||
options.node_id, attempt.0
|
||||
));
|
||||
}
|
||||
wait_for_cluster_convergence(cluster, observations)?;
|
||||
session.run_to_completion(job, worker)
|
||||
}
|
||||
|
||||
fn wait_for_worker_identity(
|
||||
cluster: &mut ProvisionedClusterGuard,
|
||||
observations: &mpsc::Receiver<PluginObservation>,
|
||||
options: &VastAiJobOptions,
|
||||
) -> Result<NodeIdentity, String> {
|
||||
let started = Instant::now();
|
||||
let mut worker = None;
|
||||
while started.elapsed() < options.provision_timeout {
|
||||
cluster
|
||||
.poll(SystemTime::now())
|
||||
.map_err(|error| format!("job reconciler poll: {error}"))?;
|
||||
drain_observations(observations, &mut worker)?;
|
||||
if let Some(worker) = worker.take() {
|
||||
eprintln!("job-reconcile: worker identity observed through reconciler stdout");
|
||||
return Ok(worker);
|
||||
}
|
||||
std::thread::sleep(POLL);
|
||||
}
|
||||
Err(format!(
|
||||
"timed out after {:?} waiting for reconciled job worker identity",
|
||||
options.provision_timeout
|
||||
))
|
||||
}
|
||||
|
||||
fn wait_for_cluster_convergence(
|
||||
cluster: &mut ProvisionedClusterGuard,
|
||||
observations: &mpsc::Receiver<PluginObservation>,
|
||||
) -> Result<(), String> {
|
||||
let started = Instant::now();
|
||||
let mut ignored = None;
|
||||
while started.elapsed() < RUNTIME_CONVERGENCE_TIMEOUT {
|
||||
cluster
|
||||
.poll(SystemTime::now())
|
||||
.map_err(|error| format!("job reconciler convergence poll: {error}"))?;
|
||||
drain_observations(observations, &mut ignored)?;
|
||||
if cluster.is_converged() {
|
||||
eprintln!("job-reconcile: reconciler accepted runtime-ready worker");
|
||||
return Ok(());
|
||||
}
|
||||
std::thread::sleep(POLL);
|
||||
}
|
||||
Err(format!(
|
||||
"timed out after {RUNTIME_CONVERGENCE_TIMEOUT:?} waiting for reconciler convergence"
|
||||
))
|
||||
}
|
||||
|
||||
fn build_vastai_provisioner(
|
||||
api_key: &str,
|
||||
options: &VastAiJobOptions,
|
||||
runtime: swactor::runtime::Runtime,
|
||||
) -> Result<Box<dyn ProvisionPlugin>, String> {
|
||||
let identity = resolve_vastai_ssh_identity(options.ssh_identity.clone())?;
|
||||
if !identity.is_file() {
|
||||
return Err(format!(
|
||||
"missing VastAI SSH identity {}; set --vastai-ssh-identity or MYELIN_VASTAI_SSH_IDENTITY",
|
||||
identity.display()
|
||||
));
|
||||
}
|
||||
let public_key = derive_ssh_public_key(&identity)?;
|
||||
ensure_vastai_account_ssh_key(api_key, &public_key)?;
|
||||
eprintln!(
|
||||
"job-reconcile: VastAI SSH identity {} fingerprint {} registered",
|
||||
identity.display(),
|
||||
ssh_public_key_fingerprint(&public_key)
|
||||
);
|
||||
|
||||
let mut config = VastAiProvisioningConfig::default();
|
||||
config.label_prefix = options.label_prefix.clone();
|
||||
config.disk_gb = options.disk_gb;
|
||||
config.ssh_user = options.ssh_user.clone();
|
||||
config.confirm_lease = options.confirm_lease;
|
||||
config.onstart = options.onstart.clone();
|
||||
config.ssh_public_key = Some(public_key);
|
||||
config.selection = selection_policy(options);
|
||||
if let Some(poll_interval) = options.poll_interval {
|
||||
config.lifecycle.poll_interval = poll_interval;
|
||||
}
|
||||
|
||||
Ok(Box::new(VastAiProvisioningPlugin::new(
|
||||
ToolsVastAiLeaseClient::from_api_key(api_key.to_owned())?,
|
||||
SshCommandBootstrapLauncher::new(Some(identity), runtime),
|
||||
config,
|
||||
)))
|
||||
}
|
||||
|
||||
fn job_node_spec(
|
||||
options: &VastAiJobOptions,
|
||||
image: String,
|
||||
orch_json: &str,
|
||||
) -> Result<NodeProvisionSpec, String> {
|
||||
let mut env = vec![
|
||||
("MYELIN_RUN_ID".to_owned(), options.run_id.to_string()),
|
||||
(
|
||||
"MYELIN_LOGICAL_NODE_ID".to_owned(),
|
||||
options.node_id.to_string(),
|
||||
),
|
||||
("MYELIN_NODE_PROVIDER".to_owned(), "vastai".to_owned()),
|
||||
(
|
||||
"MYELIN_STAGE_INDEX".to_owned(),
|
||||
DEFAULT_STAGE_INDEX.to_string(),
|
||||
),
|
||||
(
|
||||
MVP_IROH_ENDPOINT_ADDR_MASK_ENV.to_owned(),
|
||||
options.endpoint_addr_mask.clone(),
|
||||
),
|
||||
];
|
||||
if let Some(mode) = options
|
||||
.relay_mode
|
||||
.as_ref()
|
||||
.filter(|mode| !mode.trim().is_empty())
|
||||
{
|
||||
env.push((MYELIN_IROH_RELAY_MODE_ENV.to_owned(), mode.clone()));
|
||||
}
|
||||
if let Some(url) = options
|
||||
.relay_url
|
||||
.as_ref()
|
||||
.filter(|url| !url.trim().is_empty())
|
||||
{
|
||||
env.push((MYELIN_IROH_RELAY_URL_ENV.to_owned(), url.clone()));
|
||||
env.push((SWACTOR_IROH_RELAY_URL_ENV.to_owned(), url.clone()));
|
||||
}
|
||||
let command = worker_bootstrap_command(options, orch_json, &env);
|
||||
|
||||
Ok(NodeProvisionSpec {
|
||||
run_id: options.run_id,
|
||||
node_id: options.node_id,
|
||||
attempt_id: 0,
|
||||
stage_index: Some(DEFAULT_STAGE_INDEX),
|
||||
image,
|
||||
env,
|
||||
args: vec![command],
|
||||
mounts: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
fn build_cluster(
|
||||
options: &VastAiJobOptions,
|
||||
spec: NodeProvisionSpec,
|
||||
provisioner: Box<dyn ProvisionPlugin>,
|
||||
engine: swactor_engine::EngineHandle,
|
||||
sink: PluginSink,
|
||||
) -> Result<ProvisionedClusterGuard, String> {
|
||||
let group_id = NodeGroupId(format!("job-node-{}", spec.node_id));
|
||||
let logical_node_id = LogicalNodeId(format!("{}-0", group_id.0));
|
||||
let selection = selection_policy(options);
|
||||
let desired = ClusterShape {
|
||||
run_id: RunId(spec.run_id),
|
||||
generation: 1,
|
||||
groups: vec![RunNodeGroupSpec {
|
||||
run_id: RunId(spec.run_id),
|
||||
group_id,
|
||||
role: RoleId("job-worker".to_owned()),
|
||||
count: 1,
|
||||
provider: ProviderKind::new("vastai"),
|
||||
shape: DesiredNodeShape {
|
||||
image: spec.image.clone(),
|
||||
disk_gb: options.disk_gb,
|
||||
gpu_name: selection.gpu_name.clone(),
|
||||
min_gpu_ram_mb: selection.min_gpu_ram_mb,
|
||||
min_down_mbps: Some(selection.min_down_mbps),
|
||||
min_up_mbps: selection.min_up_mbps,
|
||||
min_reliability: Some(selection.min_reliability),
|
||||
require_verified: selection.require_verified,
|
||||
provider_labels: BTreeMap::from([(
|
||||
"myelin.job_runner".to_owned(),
|
||||
"reconciled-vastai".to_owned(),
|
||||
)]),
|
||||
},
|
||||
boot: BootSpec {
|
||||
ssh_user: options.ssh_user.clone(),
|
||||
verify_commands: Vec::new(),
|
||||
start_swactor_command: spec.args.join(" "),
|
||||
stdout_sources: Vec::new(),
|
||||
stderr_sources: Vec::new(),
|
||||
env: spec.env.clone(),
|
||||
args: spec.args.clone(),
|
||||
mounts: Vec::new(),
|
||||
},
|
||||
swarm_join: SwarmJoinTemplate {
|
||||
orch_swactor_addr: sessionless_orchestrator_ref(),
|
||||
join_token_ref: "myelin-job-worker-identity".to_owned(),
|
||||
},
|
||||
}],
|
||||
};
|
||||
let mut retry = RetryPolicy::default();
|
||||
retry.operation_timeout = options.provision_timeout;
|
||||
ProvisionedClusterGuard::new(
|
||||
desired,
|
||||
vec![ReconcilerNodeBinding {
|
||||
logical_node_id,
|
||||
provision: spec,
|
||||
plugin: provisioner,
|
||||
}],
|
||||
retry,
|
||||
engine,
|
||||
sink,
|
||||
)
|
||||
}
|
||||
|
||||
fn selection_policy(options: &VastAiJobOptions) -> SelectionPolicy {
|
||||
let mut selection = SelectionPolicy::default();
|
||||
if let Some(gpu_name) = &options.gpu_name {
|
||||
selection.gpu_name = Some(gpu_name.clone());
|
||||
}
|
||||
if let Some(min_gpu_ram_mb) = options.min_gpu_ram_mb {
|
||||
selection.min_gpu_ram_mb = Some(min_gpu_ram_mb);
|
||||
}
|
||||
if let Some(min_down_mbps) = options.min_down_mbps {
|
||||
selection.min_down_mbps = min_down_mbps;
|
||||
}
|
||||
if let Some(min_up_mbps) = options.min_up_mbps {
|
||||
selection.min_up_mbps = Some(min_up_mbps);
|
||||
}
|
||||
if let Some(max_dph_total) = options.max_dph_total {
|
||||
selection.max_dph_total = Some(max_dph_total);
|
||||
}
|
||||
if let Some(min_reliability) = options.min_reliability {
|
||||
selection.min_reliability = min_reliability;
|
||||
}
|
||||
if let Some(require_verified) = options.require_verified {
|
||||
selection.require_verified = require_verified;
|
||||
}
|
||||
for host_id in &options.blacklist_hosts {
|
||||
if !selection.blacklist_hosts.contains(host_id) {
|
||||
selection.blacklist_hosts.push(*host_id);
|
||||
}
|
||||
}
|
||||
selection
|
||||
}
|
||||
|
||||
fn worker_bootstrap_command(
|
||||
options: &VastAiJobOptions,
|
||||
orch_json: &str,
|
||||
env: &[(String, String)],
|
||||
) -> String {
|
||||
// The VastAI SSH bootstrap session does not inherit the container's
|
||||
// environment, so the relay + endpoint-address-mask env must be exported
|
||||
// inline. Without this the worker advertises container-local addresses
|
||||
// and is unreachable across the internet.
|
||||
let exports = env
|
||||
.iter()
|
||||
.map(|(key, value)| format!("export {}={};", shell_quote(key), shell_quote(value)))
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
format!(
|
||||
"{exports} exec {} --orch-identity {} --workdir {}",
|
||||
shell_quote(&options.remote_worker_bin),
|
||||
shell_quote(orch_json),
|
||||
shell_quote(&options.worker_workdir),
|
||||
)
|
||||
}
|
||||
|
||||
fn drain_observations(
|
||||
observations: &mpsc::Receiver<PluginObservation>,
|
||||
worker: &mut Option<NodeIdentity>,
|
||||
) -> Result<(), String> {
|
||||
while let Ok(observation) = observations.try_recv() {
|
||||
match observation {
|
||||
PluginObservation::StdoutLine { line, .. } => {
|
||||
eprintln!("job-reconcile worker stdout: {line}");
|
||||
if let Some(identity) = parse_worker_identity(&line)? {
|
||||
*worker = Some(identity);
|
||||
}
|
||||
}
|
||||
PluginObservation::StderrLine { line, .. } => {
|
||||
eprintln!("job-reconcile worker stderr: {line}");
|
||||
}
|
||||
PluginObservation::ProviderLine { line, .. } => {
|
||||
eprintln!("job-reconcile provider: {line}");
|
||||
}
|
||||
PluginObservation::TelemetryFrame { channel, .. } => {
|
||||
eprintln!("job-reconcile telemetry frame: {channel}");
|
||||
}
|
||||
PluginObservation::Exited {
|
||||
node_id, status, ..
|
||||
} => {
|
||||
return Err(format!(
|
||||
"reconciled job worker node {node_id} exited before job completion: {status:?}"
|
||||
));
|
||||
}
|
||||
PluginObservation::Failed {
|
||||
node_id, reason, ..
|
||||
} => {
|
||||
return Err(format!(
|
||||
"reconciled job worker node {node_id} failed before job completion: {reason}"
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_worker_identity(line: &str) -> Result<Option<NodeIdentity>, String> {
|
||||
let Some(json) = line.trim().strip_prefix("JOB_WORKER_IDENTITY ") else {
|
||||
return Ok(None);
|
||||
};
|
||||
serde_json::from_str(json)
|
||||
.map(Some)
|
||||
.map_err(|e| format!("parse JOB_WORKER_IDENTITY from reconciler stdout: {e}"))
|
||||
}
|
||||
|
||||
fn observation_channel() -> (PluginSink, mpsc::Receiver<PluginObservation>) {
|
||||
let (tx, rx) = mpsc::channel();
|
||||
(
|
||||
PluginSink::new(Arc::new(ChannelObservationSink { tx: Mutex::new(tx) })),
|
||||
rx,
|
||||
)
|
||||
}
|
||||
|
||||
struct ChannelObservationSink {
|
||||
tx: Mutex<mpsc::Sender<PluginObservation>>,
|
||||
}
|
||||
|
||||
impl PluginObservationSink for ChannelObservationSink {
|
||||
fn observe(&self, observation: PluginObservation) {
|
||||
if let Ok(tx) = self.tx.lock() {
|
||||
let _ = tx.send(observation);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn shell_quote(value: &str) -> String {
|
||||
if value.is_empty() {
|
||||
return "''".to_owned();
|
||||
}
|
||||
if value.bytes().all(|b| {
|
||||
b.is_ascii_alphanumeric()
|
||||
|| matches!(
|
||||
b,
|
||||
b'/' | b'.' | b'_' | b'-' | b':' | b'=' | b'@' | b'+' | b','
|
||||
)
|
||||
}) {
|
||||
return value.to_owned();
|
||||
}
|
||||
format!("'{}'", value.replace('\'', "'\"'\"'"))
|
||||
}
|
||||
|
||||
fn required_option<'a>(value: Option<&'a str>, label: &str) -> Result<&'a str, String> {
|
||||
value
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.ok_or_else(|| format!("missing required {label}"))
|
||||
}
|
||||
|
||||
fn validate_non_empty(label: &str, value: &str) -> Result<(), String> {
|
||||
if value.trim().is_empty() {
|
||||
Err(format!("missing required {label}"))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn default_run_id() -> u64 {
|
||||
let millis = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_millis();
|
||||
millis.try_into().unwrap_or(u64::MAX)
|
||||
}
|
||||
|
||||
fn sessionless_orchestrator_ref() -> String {
|
||||
"job-orchestrator-identity-exchanged-out-of-band".to_owned()
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn _assert_lifecycle_policy_is_linked(_: LifecyclePolicy) {}
|
||||
|
|
@ -9,10 +9,13 @@ pub(crate) mod actor;
|
|||
pub(crate) mod app;
|
||||
pub(crate) mod cluster_reconciler;
|
||||
pub(crate) mod config;
|
||||
pub(crate) mod daemon;
|
||||
pub(crate) mod distribution_stack;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod engine_builder;
|
||||
pub(crate) mod job_reconciler;
|
||||
|
||||
pub(crate) mod node_image;
|
||||
pub(crate) mod provider_adapters {
|
||||
pub(crate) mod relay;
|
||||
pub(super) mod vastai;
|
||||
|
|
|
|||
|
|
@ -7,10 +7,7 @@ use std::sync::mpsc;
|
|||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
const NODE_IMAGE_CONTENT_INPUTS: &[&str] = &[
|
||||
"apps/myelin/node-image/Dockerfile",
|
||||
"apps/myelin/node-image/tinygrad_worker.py",
|
||||
];
|
||||
const NODE_IMAGE_CONTENT_INPUTS: &[&str] = &["apps/myelin/node-image/Dockerfile"];
|
||||
|
||||
const BASE_IMAGE_SOURCE_INPUTS: &[&str] = &[
|
||||
"apps/myelin/node-image/Dockerfile.base",
|
||||
|
|
@ -18,7 +15,7 @@ const BASE_IMAGE_SOURCE_INPUTS: &[&str] = &[
|
|||
];
|
||||
const NODE_IMAGE_TAG_LABEL: &str = "org.swactor.myelin.node-image-tag";
|
||||
const NODE_IMAGE_SOURCE_HASH_LABEL: &str = "org.swactor.myelin.node.source-hash";
|
||||
const NODE_IMAGE_WORKER_HASH_LABEL: &str = "org.swactor.myelin.node.worker-hash";
|
||||
const NODE_IMAGE_AGENT_HASH_LABEL: &str = "org.swactor.myelin.node.agent-hash";
|
||||
const NODE_IMAGE_BASE_HASH_LABEL: &str = "org.swactor.myelin.node.base-hash";
|
||||
const BASE_IMAGE_SOURCE_HASH_LABEL: &str = "org.swactor.myelin.base.source-hash";
|
||||
|
||||
|
|
@ -117,18 +114,10 @@ fn prepare_node_image_inner(
|
|||
let tag = image_version_tag(&root, &image_content_hash)?;
|
||||
let image_ref = image.ref_for_tag(&tag);
|
||||
emit_image_reference(progress, "resolved", &image_ref);
|
||||
let worker_hash = hash_relative_files_with_salts(
|
||||
&root,
|
||||
vec![relative_path(
|
||||
&root,
|
||||
&root.join("apps/myelin/node-image/tinygrad_worker.py"),
|
||||
)?],
|
||||
&[],
|
||||
)?;
|
||||
let expected_node_labels = vec![
|
||||
(NODE_IMAGE_TAG_LABEL, tag.as_str()),
|
||||
(NODE_IMAGE_SOURCE_HASH_LABEL, image_content_hash.as_str()),
|
||||
(NODE_IMAGE_WORKER_HASH_LABEL, worker_hash.as_str()),
|
||||
(NODE_IMAGE_AGENT_HASH_LABEL, image_content_hash.as_str()),
|
||||
(NODE_IMAGE_BASE_HASH_LABEL, base_hash.as_str()),
|
||||
];
|
||||
let expected_base_labels = vec![(BASE_IMAGE_SOURCE_HASH_LABEL, base_hash.as_str())];
|
||||
|
|
@ -545,7 +534,7 @@ fn prune_old_dirty_images(root: &Path, image: &ImageName, keep_tag: &str) {
|
|||
};
|
||||
if labels.get(NODE_IMAGE_TAG_LABEL).map(String::as_str) != Some(tag.as_str())
|
||||
|| !labels.contains_key(NODE_IMAGE_SOURCE_HASH_LABEL)
|
||||
|| !labels.contains_key(NODE_IMAGE_WORKER_HASH_LABEL)
|
||||
|| !labels.contains_key(NODE_IMAGE_AGENT_HASH_LABEL)
|
||||
|| !labels.contains_key(NODE_IMAGE_BASE_HASH_LABEL)
|
||||
{
|
||||
continue;
|
||||
|
|
@ -12,7 +12,6 @@ use std::sync::atomic::{AtomicBool, Ordering};
|
|||
use std::thread::{self, JoinHandle};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use telemetry::TelemetryProducer;
|
||||
use swactor::actor::{ActorAddress, ActorInterface};
|
||||
use swactor::runtime::{
|
||||
Ctx, ExternalSender, Runtime, RuntimeConfig, RuntimeParts, SingleThreadRuntime,
|
||||
|
|
@ -21,10 +20,12 @@ use swactor_vastai::{
|
|||
CreateInstanceRequest, LifecyclePolicy, Offer, ProvisionRequest, ProvisionedInstance,
|
||||
SelectionPolicy, classify_vastai_error,
|
||||
};
|
||||
use telemetry::TelemetryProducer;
|
||||
|
||||
use crate::observability::provisioning_logs::{BootstrapTelemetryBridge, node_stream_id};
|
||||
use crate::provisioning::{
|
||||
NodeProvisionSpec, PluginNodeHandle, PluginObservation, PluginSink, ProvisionPlugin,
|
||||
AdoptedNode, NodeProvisionSpec, PluginNodeHandle, PluginObservation, PluginSink,
|
||||
ProvisionPlugin,
|
||||
};
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
|
|
@ -112,6 +113,9 @@ impl Drop for VastAiProviderMonitor {
|
|||
pub(crate) trait VastAiLeaseClient: Send {
|
||||
fn provision_one(&mut self, request: ProvisionRequest) -> Result<ProvisionedInstance, String>;
|
||||
|
||||
/// Resolves the live contract id carrying `label`, if any.
|
||||
fn contract_by_label(&mut self, label: &str) -> Result<Option<u64>, String>;
|
||||
|
||||
fn ssh_endpoint(
|
||||
&mut self,
|
||||
contract_id: u64,
|
||||
|
|
@ -197,22 +201,6 @@ impl ToolsVastAiLeaseClient {
|
|||
dph_total: offer.dph_total,
|
||||
})
|
||||
}
|
||||
|
||||
fn contract_by_label(&mut self, label: &str) -> Result<Option<u64>, String> {
|
||||
let instances = self.runtime.block_on(self.client.list_by_label(label))?;
|
||||
match instances.as_slice() {
|
||||
[] => Ok(None),
|
||||
[instance] => Ok(Some(instance.contract_id)),
|
||||
_ => Err(format!(
|
||||
"multiple VastAI contracts share stable label {label}: {}",
|
||||
instances
|
||||
.iter()
|
||||
.map(|instance| instance.contract_id.to_string())
|
||||
.collect::<Vec<_>>()
|
||||
.join(",")
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
|
|
@ -442,6 +430,22 @@ fn adopted_instance(contract_id: u64) -> ProvisionedInstance {
|
|||
}
|
||||
|
||||
impl VastAiLeaseClient for ToolsVastAiLeaseClient {
|
||||
fn contract_by_label(&mut self, label: &str) -> Result<Option<u64>, String> {
|
||||
let instances = self.runtime.block_on(self.client.list_by_label(label))?;
|
||||
match instances.as_slice() {
|
||||
[] => Ok(None),
|
||||
[instance] => Ok(Some(instance.contract_id)),
|
||||
_ => Err(format!(
|
||||
"multiple VastAI contracts share stable label {label}: {}",
|
||||
instances
|
||||
.iter()
|
||||
.map(|instance| instance.contract_id.to_string())
|
||||
.collect::<Vec<_>>()
|
||||
.join(",")
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn provision_one(&mut self, request: ProvisionRequest) -> Result<ProvisionedInstance, String> {
|
||||
if request.count != 1 {
|
||||
return Err(format!(
|
||||
|
|
@ -1465,6 +1469,63 @@ where
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn adopt_by_spec(
|
||||
&mut self,
|
||||
spec: &NodeProvisionSpec,
|
||||
sink: PluginSink,
|
||||
) -> Result<Option<AdoptedNode>, String> {
|
||||
let label = self.label_for(spec);
|
||||
if self.client.contract_by_label(&label)?.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
// create_node's provision path adopts an existing labeled contract;
|
||||
// the provider status monitor resumes and no agent restart occurs
|
||||
// (start_bootstrap is deliberately not called).
|
||||
let handle = self.create_node(spec.clone(), sink)?;
|
||||
Ok(Some(AdoptedNode {
|
||||
handle,
|
||||
provider_ref: label,
|
||||
}))
|
||||
}
|
||||
|
||||
fn provider_ref_for(&self, spec: &NodeProvisionSpec) -> String {
|
||||
self.label_for(spec)
|
||||
}
|
||||
|
||||
fn stop_by_spec(&mut self, spec: &NodeProvisionSpec, sink: PluginSink) -> Result<bool, String> {
|
||||
let label = self.label_for(spec);
|
||||
let Some(contract_id) = self.client.contract_by_label(&label)? else {
|
||||
return Ok(false);
|
||||
};
|
||||
let result = self.client.destroy_contract(contract_id);
|
||||
emit_node_line(
|
||||
&sink,
|
||||
spec.run_id,
|
||||
spec.node_id,
|
||||
serde_json::json!({
|
||||
"type": "VastAiContractCleanupBySpec",
|
||||
"run_id": spec.run_id,
|
||||
"node_id": spec.node_id,
|
||||
"label": &label,
|
||||
"contract_id": contract_id,
|
||||
"result": if result.is_ok() { "ok" } else { "failed" },
|
||||
"error": result.as_ref().err(),
|
||||
})
|
||||
.to_string(),
|
||||
);
|
||||
result.map(|_| true)
|
||||
}
|
||||
|
||||
fn detach_all(&mut self) {
|
||||
// Stop provider monitors (engine-hosted actors) without destroying
|
||||
// leases: daemon exit must leave instances running.
|
||||
for (_, mut node) in std::mem::take(&mut self.nodes) {
|
||||
if let Some(mut monitor) = node.provider_monitor.take() {
|
||||
monitor.stop();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -1540,6 +1601,10 @@ mod tests {
|
|||
}
|
||||
|
||||
impl VastAiLeaseClient for RetryDestroyClient {
|
||||
fn contract_by_label(&mut self, _label: &str) -> Result<Option<u64>, String> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn provision_one(
|
||||
&mut self,
|
||||
_request: ProvisionRequest,
|
||||
|
|
|
|||
|
|
@ -17,9 +17,9 @@ use std::time::{Duration, Instant, UNIX_EPOCH};
|
|||
use std::os::unix::process::CommandExt;
|
||||
|
||||
pub use ::provisioning::plugin::{
|
||||
NodeProvisionSpec, PluginNodeHandle, PluginObservation, PluginObservationSink, PluginSink,
|
||||
ProviderMount, ProvisionEvent, ProvisionEventKind, ProvisionLogLine, ProvisionLogStream,
|
||||
ProvisionPlugin,
|
||||
AdoptedNode, NodeProvisionSpec, PluginNodeHandle, PluginObservation, PluginObservationSink,
|
||||
PluginSink, ProviderMount, ProvisionEvent, ProvisionEventKind, ProvisionLogLine,
|
||||
ProvisionLogStream, ProvisionPlugin,
|
||||
};
|
||||
|
||||
use crate::observability::provisioning_logs::BootstrapTelemetryBridge;
|
||||
|
|
@ -81,6 +81,22 @@ fn docker_container_name(prefix: &str, spec: &NodeProvisionSpec) -> String {
|
|||
)
|
||||
}
|
||||
|
||||
/// Docker labels identifying every container this daemon owns. Orphan sweeps
|
||||
/// (`docker ps -a --filter label=myelin.daemon=<prefix>`) rely on these; the
|
||||
/// daemon label is the stable identity across restarts.
|
||||
fn docker_container_labels(prefix: &str, spec: &NodeProvisionSpec) -> Vec<String> {
|
||||
vec![
|
||||
format!("myelin.daemon={prefix}"),
|
||||
format!("myelin.run={}", spec.run_id),
|
||||
format!("myelin.node={}", spec.node_id),
|
||||
]
|
||||
}
|
||||
|
||||
fn docker_inspect_error_is_absent(stderr: &str) -> bool {
|
||||
let stderr = stderr.to_ascii_lowercase();
|
||||
stderr.contains("no such object") || stderr.contains("no such container")
|
||||
}
|
||||
|
||||
#[allow(clippy::disallowed_methods)]
|
||||
fn docker_container_is_absent(name: &str) -> Result<bool, String> {
|
||||
let output = Command::new("docker")
|
||||
|
|
@ -92,7 +108,7 @@ fn docker_container_is_absent(name: &str) -> Result<bool, String> {
|
|||
return Ok(false);
|
||||
}
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
if stderr.contains("No such object") || stderr.contains("No such container") {
|
||||
if docker_inspect_error_is_absent(&stderr) {
|
||||
Ok(true)
|
||||
} else {
|
||||
Err(format!(
|
||||
|
|
@ -103,6 +119,52 @@ fn docker_container_is_absent(name: &str) -> Result<bool, String> {
|
|||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::disallowed_methods)]
|
||||
fn docker_container_is_running(name: &str) -> Result<bool, String> {
|
||||
let output = Command::new("docker")
|
||||
.args(["inspect", "-f", "{{.State.Running}}"])
|
||||
.arg(name)
|
||||
.output()
|
||||
.map_err(|error| format!("inspect Docker container {name} state: {error}"))?;
|
||||
if !output.status.success() {
|
||||
return Err(format!(
|
||||
"inspect Docker container {name} state exited with {}: {}",
|
||||
output.status,
|
||||
String::from_utf8_lossy(&output.stderr).trim()
|
||||
));
|
||||
}
|
||||
Ok(String::from_utf8_lossy(&output.stdout).trim() == "true")
|
||||
}
|
||||
|
||||
/// Lists container names carrying this daemon's label, running or not.
|
||||
#[allow(clippy::disallowed_methods)]
|
||||
fn docker_labeled_containers(prefix: &str) -> Result<Vec<String>, String> {
|
||||
let output = Command::new("docker")
|
||||
.args([
|
||||
"ps",
|
||||
"-a",
|
||||
"--filter",
|
||||
&format!("label=myelin.daemon={prefix}"),
|
||||
"--format",
|
||||
"{{.Names}}",
|
||||
])
|
||||
.output()
|
||||
.map_err(|error| format!("list labeled Docker containers: {error}"))?;
|
||||
if !output.status.success() {
|
||||
return Err(format!(
|
||||
"list labeled Docker containers exited with {}: {}",
|
||||
output.status,
|
||||
String::from_utf8_lossy(&output.stderr).trim()
|
||||
));
|
||||
}
|
||||
Ok(String::from_utf8_lossy(&output.stdout)
|
||||
.lines()
|
||||
.map(str::trim)
|
||||
.filter(|line| !line.is_empty())
|
||||
.map(str::to_owned)
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn docker_mount_arg(mount: &ProviderMount) -> String {
|
||||
let mut arg = format!(
|
||||
"type=bind,src={},dst={}",
|
||||
|
|
@ -481,6 +543,14 @@ impl ProvisionPlugin for LocalProcessPlugin {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn detach_all(&mut self) {
|
||||
// Leak the children deliberately: daemon exit must not kill nodes.
|
||||
// Children become unmanageable (process provider has no cross-process
|
||||
// adoption surface); explicit destroy happened before this call or
|
||||
// not at all.
|
||||
self.nodes.clear();
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for LocalProcessPlugin {
|
||||
|
|
@ -543,8 +613,11 @@ impl ProvisionPlugin for LocalDockerPlugin {
|
|||
.arg("--add-host")
|
||||
.arg("host.docker.internal:host-gateway")
|
||||
.arg("--name")
|
||||
.arg(&container_name)
|
||||
.arg("-i");
|
||||
.arg(&container_name);
|
||||
for label in docker_container_labels(&self.container_name_prefix, &spec) {
|
||||
command.arg("--label").arg(label);
|
||||
}
|
||||
command.arg("-i");
|
||||
let docker_gpus = spec
|
||||
.env
|
||||
.iter()
|
||||
|
|
@ -669,6 +742,135 @@ impl ProvisionPlugin for LocalDockerPlugin {
|
|||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn adopt_by_spec(
|
||||
&mut self,
|
||||
spec: &NodeProvisionSpec,
|
||||
sink: PluginSink,
|
||||
) -> Result<Option<AdoptedNode>, String> {
|
||||
let container_name = docker_container_name(&self.container_name_prefix, spec);
|
||||
if docker_container_is_absent(&container_name)? {
|
||||
return Ok(None);
|
||||
}
|
||||
let running = docker_container_is_running(&container_name)?;
|
||||
let handle = PluginNodeHandle {
|
||||
id: self.next_handle_id,
|
||||
provider_process_id: None,
|
||||
};
|
||||
self.next_handle_id = self.next_handle_id.wrapping_add(1).max(1);
|
||||
if running {
|
||||
// Follow the adopted container's logs and wait for its exit; no
|
||||
// lifecycle action is taken — adoption is observation-only.
|
||||
for (_stream, stdout_flag) in [("stdout", true), ("stderr", false)] {
|
||||
let mut logs = Command::new("docker");
|
||||
logs.arg("logs")
|
||||
.arg("-f")
|
||||
.arg("--tail")
|
||||
.arg("0")
|
||||
.arg(if stdout_flag { "--stdout" } else { "--stderr" })
|
||||
.arg(&container_name);
|
||||
if let Ok(child) = logs.stdout(Stdio::piped()).stderr(Stdio::null()).spawn() {
|
||||
if let Some(out) = child.stdout {
|
||||
if stdout_flag {
|
||||
spawn_stdout_reader(spec.clone(), sink.clone(), out);
|
||||
} else {
|
||||
spawn_stderr_reader(spec.clone(), sink.clone(), out);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let wait_spec = spec.clone();
|
||||
let wait_sink = sink.clone();
|
||||
let wait_name = container_name.clone();
|
||||
thread::spawn(move || {
|
||||
let status = Command::new("docker").arg("wait").arg(&wait_name).output();
|
||||
let code = status.ok().and_then(|output| {
|
||||
String::from_utf8_lossy(&output.stdout)
|
||||
.trim()
|
||||
.parse::<i32>()
|
||||
.ok()
|
||||
});
|
||||
wait_sink.observe(PluginObservation::Exited {
|
||||
run_id: wait_spec.run_id,
|
||||
node_id: wait_spec.node_id,
|
||||
status: code,
|
||||
});
|
||||
});
|
||||
}
|
||||
let adopted_name = docker_container_name(&self.container_name_prefix, spec);
|
||||
self.nodes.insert(
|
||||
handle.id,
|
||||
LocalDockerNode {
|
||||
spec: spec.clone(),
|
||||
sink: sink.clone(),
|
||||
container_name,
|
||||
stdin: None,
|
||||
},
|
||||
);
|
||||
sink.observe(PluginObservation::TelemetryFrame {
|
||||
run_id: spec.run_id,
|
||||
node_id: spec.node_id,
|
||||
channel: "myelin.provisioning.events".to_owned(),
|
||||
payload: serde_json::json!({
|
||||
"type":"DockerContainerAdopted",
|
||||
"provider":"Docker",
|
||||
"container":adopted_name.clone(),
|
||||
"running":running,
|
||||
})
|
||||
.to_string(),
|
||||
});
|
||||
Ok(Some(AdoptedNode {
|
||||
handle,
|
||||
provider_ref: adopted_name,
|
||||
}))
|
||||
}
|
||||
|
||||
fn provider_ref_for(&self, spec: &NodeProvisionSpec) -> String {
|
||||
docker_container_name(&self.container_name_prefix, spec)
|
||||
}
|
||||
|
||||
fn list_managed_refs(&self) -> Result<Vec<String>, String> {
|
||||
docker_labeled_containers(&self.container_name_prefix)
|
||||
}
|
||||
|
||||
fn stop_by_spec(&mut self, spec: &NodeProvisionSpec, sink: PluginSink) -> Result<bool, String> {
|
||||
let container_name = docker_container_name(&self.container_name_prefix, spec);
|
||||
if docker_container_is_absent(&container_name)? {
|
||||
return Ok(false);
|
||||
}
|
||||
let status = Command::new("docker")
|
||||
.arg("rm")
|
||||
.arg("-f")
|
||||
.arg(&container_name)
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.status()
|
||||
.map_err(|error| format!("docker rm {container_name}: {error}"))?;
|
||||
let removed =
|
||||
status.success() || matches!(docker_container_is_absent(&container_name), Ok(true));
|
||||
sink.observe(PluginObservation::TelemetryFrame {
|
||||
run_id: spec.run_id,
|
||||
node_id: spec.node_id,
|
||||
channel: "myelin.provisioning.events".to_owned(),
|
||||
payload: serde_json::json!({
|
||||
"type":"DockerContainerRemoved",
|
||||
"provider":"Docker",
|
||||
"container":container_name,
|
||||
"removed":removed,
|
||||
"exit_ok":status.success(),
|
||||
})
|
||||
.to_string(),
|
||||
});
|
||||
if removed {
|
||||
Ok(true)
|
||||
} else {
|
||||
Err(format!("docker rm {container_name} exited with {status}"))
|
||||
}
|
||||
}
|
||||
|
||||
fn detach_all(&mut self) {
|
||||
self.nodes.clear();
|
||||
}
|
||||
}
|
||||
|
||||
fn spawn_stdout_reader(
|
||||
|
|
@ -692,6 +894,17 @@ mod tests {
|
|||
use super::*;
|
||||
use std::sync::mpsc;
|
||||
|
||||
#[test]
|
||||
fn docker_absence_detection_is_case_insensitive() {
|
||||
assert!(docker_inspect_error_is_absent(
|
||||
"Error: No such object: missing"
|
||||
));
|
||||
assert!(docker_inspect_error_is_absent(
|
||||
"error: no such container: missing"
|
||||
));
|
||||
assert!(!docker_inspect_error_is_absent("permission denied"));
|
||||
}
|
||||
|
||||
struct ChannelSink(mpsc::Sender<PluginObservation>);
|
||||
|
||||
impl PluginObservationSink for ChannelSink {
|
||||
|
|
|
|||
|
|
@ -1,3 +0,0 @@
|
|||
//! Myelin prompt protocol public surface.
|
||||
|
||||
pub(crate) mod rpc;
|
||||
|
|
@ -1,105 +0,0 @@
|
|||
use std::io::{BufRead, Write};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use swactor_transport::{CodecRegistry, NetworkMessage};
|
||||
|
||||
use swactor_transport::JsonCodec;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub(crate) struct SubmitPrompt {
|
||||
pub request_id: u64,
|
||||
pub prompt_text: String,
|
||||
pub max_tokens: u32,
|
||||
}
|
||||
|
||||
impl SubmitPrompt {
|
||||
pub(crate) fn with_defaults(mut self, max_tokens: u32) -> Self {
|
||||
if self.max_tokens == 0 {
|
||||
self.max_tokens = max_tokens;
|
||||
}
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub(crate) enum PromptEvent {
|
||||
TextDelta {
|
||||
request_id: u64,
|
||||
text: String,
|
||||
},
|
||||
Done {
|
||||
request_id: u64,
|
||||
final_text: String,
|
||||
tokens_generated: u32,
|
||||
elapsed_ms: u64,
|
||||
},
|
||||
Fault {
|
||||
request_id: u64,
|
||||
error: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl PromptEvent {
|
||||
pub(crate) fn request_id(&self) -> u64 {
|
||||
match self {
|
||||
Self::TextDelta { request_id, .. }
|
||||
| Self::Done { request_id, .. }
|
||||
| Self::Fault { request_id, .. } => *request_id,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_terminal(&self) -> bool {
|
||||
matches!(self, Self::Done { .. } | Self::Fault { .. })
|
||||
}
|
||||
}
|
||||
|
||||
impl NetworkMessage for PromptEvent {
|
||||
fn type_tag() -> &'static str {
|
||||
"myelin::PromptEvent"
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub(crate) enum TokenizerEvent {
|
||||
PromptEncoded { request_id: u64, tokens: Vec<u32> },
|
||||
TokensDecoded { request_id: u64, text: String },
|
||||
Fault { request_id: u64, error: String },
|
||||
}
|
||||
|
||||
impl NetworkMessage for TokenizerEvent {
|
||||
fn type_tag() -> &'static str {
|
||||
"myelin::TokenizerEvent"
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn register_codecs(registry: &mut CodecRegistry) {
|
||||
registry.register::<PromptEvent, _>(JsonCodec::<PromptEvent>::default());
|
||||
registry.register::<TokenizerEvent, _>(JsonCodec::<TokenizerEvent>::default());
|
||||
}
|
||||
|
||||
pub(crate) fn write_json_line<T: Serialize>(
|
||||
writer: &mut impl Write,
|
||||
value: &T,
|
||||
) -> Result<(), String> {
|
||||
serde_json::to_writer(&mut *writer, value).map_err(|e| format!("serialize JSON line: {e}"))?;
|
||||
writer
|
||||
.write_all(b"\n")
|
||||
.map_err(|e| format!("write JSON line: {e}"))?;
|
||||
writer.flush().map_err(|e| format!("flush JSON line: {e}"))
|
||||
}
|
||||
|
||||
pub(crate) fn read_submit_prompt(
|
||||
reader: &mut impl BufRead,
|
||||
) -> Result<Option<SubmitPrompt>, String> {
|
||||
let mut line = String::new();
|
||||
let n = reader
|
||||
.read_line(&mut line)
|
||||
.map_err(|e| format!("read prompt request: {e}"))?;
|
||||
if n == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
serde_json::from_str::<SubmitPrompt>(&line)
|
||||
.map(Some)
|
||||
.map_err(|e| format!("parse prompt request: {e}"))
|
||||
}
|
||||
127
apps/myelin/src/tests/job_runner_integration.rs
Normal file
127
apps/myelin/src/tests/job_runner_integration.rs
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
//! Proves the job runner runs through swactor **inside Myelin's real node
|
||||
//! composition**: a swactor `Engine` over Tokio owns the core runtime; a real
|
||||
//! `IrohDriver` is bound through the engine handle; the production distribution
|
||||
//! stack is built on the same runtime with the job wire codecs registered; an
|
||||
//! orchestrator FSM actor and a node job actor drive a real command via
|
||||
//! `swactor-process` over the actor plane, with workspace/output bytes traveling
|
||||
//! as chunked actor messages. No SSH for the job.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use swactor::actor::Message;
|
||||
use swactor::runtime::Inbox;
|
||||
use swactor_engine::{Engine, TokioBackend, TokioConfig};
|
||||
use swactor_job_runner::{
|
||||
register_job_codecs, Job, JobDone, JobState, NodeJobActor, OrchestratorJobActor,
|
||||
OrchestratorJobMsg, Workspace,
|
||||
};
|
||||
|
||||
use distribution::node::DistributedNodeConfig;
|
||||
use iroh::RelayMode;
|
||||
use iroh_driver::{IrohDriver, IrohDriverConfig};
|
||||
|
||||
use crate::orchestration::distribution_stack::DistributionRuntimeStack;
|
||||
|
||||
const POLL: Duration = Duration::from_millis(15);
|
||||
const DEADLINE: Duration = Duration::from_secs(20);
|
||||
|
||||
#[allow(clippy::disallowed_methods)]
|
||||
fn recv_within<T: Message>(inbox: &Inbox<T>, deadline: Duration) -> Option<T> {
|
||||
let started = Instant::now();
|
||||
loop {
|
||||
if let Some(v) = inbox.try_recv() {
|
||||
return Some(v);
|
||||
}
|
||||
if started.elapsed() >= deadline {
|
||||
return None;
|
||||
}
|
||||
std::thread::sleep(POLL);
|
||||
}
|
||||
}
|
||||
|
||||
fn build_composition() -> (Engine, IrohDriver, DistributionRuntimeStack) {
|
||||
let (parts, runtime, codec, transport_router) =
|
||||
DistributionRuntimeStack::build_runtime(|c| register_job_codecs(c), None);
|
||||
let engine = Engine::new(parts, TokioBackend::new(TokioConfig::default()).expect("tokio backend"))
|
||||
.expect("engine");
|
||||
let mut driver = IrohDriver::with_engine(
|
||||
engine.handle(),
|
||||
IrohDriverConfig {
|
||||
secret_key: None,
|
||||
relay_mode: RelayMode::Disabled,
|
||||
node: DistributedNodeConfig::default(),
|
||||
peer_auth: None,
|
||||
additional_alpns: vec![],
|
||||
},
|
||||
)
|
||||
.expect("iroh driver");
|
||||
let stack = DistributionRuntimeStack::new_from_runtime(
|
||||
runtime.clone(),
|
||||
codec,
|
||||
transport_router,
|
||||
driver.node_id(),
|
||||
DistributedNodeConfig::default(),
|
||||
engine.handle(),
|
||||
);
|
||||
driver.enable_actor_bridge(
|
||||
stack.runtime.clone(),
|
||||
stack.codec.clone(),
|
||||
stack.actor_bridge_routes(),
|
||||
stack.actors.swim,
|
||||
stack.relay_mirror.clone(),
|
||||
stack.route_view.clone(),
|
||||
stack.outbox.clone(),
|
||||
);
|
||||
stack.spawn_protocol_ticker(POLL);
|
||||
driver.install_actor_bridge_pump(POLL);
|
||||
(engine, driver, stack)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn job_runs_through_swactor_inside_myelin_composition() {
|
||||
let (engine, _driver, stack) = build_composition();
|
||||
let runtime = stack.runtime.clone();
|
||||
let sender = runtime.create_sender();
|
||||
|
||||
let ws = tempfile::tempdir().expect("ws");
|
||||
std::fs::write(ws.path().join("seed.txt"), "seed-value").expect("seed");
|
||||
let node_workdir = tempfile::tempdir().expect("node workdir");
|
||||
let landing = tempfile::tempdir().expect("landing");
|
||||
|
||||
let done = runtime.new_inbox::<JobDone>().expect("done inbox");
|
||||
let orch = runtime
|
||||
.spawn(OrchestratorJobActor::new(*done.addr(), landing.path().to_path_buf()))
|
||||
.expect("spawn orchestrator");
|
||||
let node = runtime
|
||||
.spawn(NodeJobActor::new(orch, node_workdir.path().to_path_buf(), sender, 0))
|
||||
.expect("spawn node");
|
||||
|
||||
let job = Job {
|
||||
name: "myelin-probe".to_owned(),
|
||||
setup: Some("echo setup-ok > setup_done.txt".to_owned()),
|
||||
run: "echo hello-from-myelin-swactor > greeting.txt".to_owned(),
|
||||
workspace: Some(Workspace { workdir: ws.path().to_path_buf(), exclude: vec![] }),
|
||||
outputs: vec![
|
||||
"greeting.txt".to_owned(),
|
||||
"setup_done.txt".to_owned(),
|
||||
"seed.txt".to_owned(),
|
||||
],
|
||||
env: BTreeMap::new(),
|
||||
};
|
||||
runtime
|
||||
.send_to(orch, OrchestratorJobMsg::Submit { job, node_actor: node })
|
||||
.expect("submit");
|
||||
|
||||
let result = recv_within(&done, DEADLINE);
|
||||
drop(engine);
|
||||
let done = result.expect("job did not reach a terminal state");
|
||||
assert_eq!(done.state, JobState::Completed, "expected COMPLETED, got {:?}", done);
|
||||
assert_eq!(done.exit_code, Some(0));
|
||||
|
||||
let greeting = std::fs::read_to_string(landing.path().join("greeting.txt"))
|
||||
.expect("collected greeting.txt");
|
||||
assert!(greeting.contains("hello-from-myelin-swactor"), "greeting: {greeting}");
|
||||
let seed = std::fs::read_to_string(landing.path().join("seed.txt")).expect("collected seed.txt");
|
||||
assert_eq!(seed, "seed-value", "workspace materialized + collected through swactor");
|
||||
}
|
||||
194
apps/myelin/src/tests/job_runner_iroh.rs
Normal file
194
apps/myelin/src/tests/job_runner_iroh.rs
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
//! Real-iroh two-node proof: an orchestrator composition and a worker
|
||||
//! composition, each with its own `Engine` + `IrohDriver` + distribution stack,
|
||||
//! connected over **actual iroh**. The worker registers its `NodeJobActor` in the
|
||||
//! directory (the production `driver.register_actor` + `register_local_actor`
|
||||
//! path); the orchestrator joins, the directory converges, and the orchestrator
|
||||
//! drives the job across iroh — control, workspace bytes, output bytes, and the
|
||||
//! supervised-process exit all cross the iroh actor plane.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use swactor_engine::{Engine, TokioBackend, TokioConfig};
|
||||
use swactor_job_runner::{
|
||||
Job, JobDone, JobState, NodeJobActor, OrchestratorJobActor, OrchestratorJobMsg, Workspace,
|
||||
register_job_codecs,
|
||||
};
|
||||
|
||||
use distribution::node::DistributedNodeConfig;
|
||||
use iroh::RelayMode;
|
||||
use iroh_driver::{IrohDriver, IrohDriverConfig};
|
||||
|
||||
use crate::orchestration::distribution_stack::DistributionRuntimeStack;
|
||||
|
||||
const POLL: Duration = Duration::from_millis(25);
|
||||
const CONVERGE_DEADLINE: Duration = Duration::from_secs(30);
|
||||
const JOB_DEADLINE: Duration = Duration::from_secs(30);
|
||||
|
||||
fn build_composition() -> (Engine, IrohDriver, DistributionRuntimeStack) {
|
||||
let (parts, runtime, codec, transport_router) =
|
||||
DistributionRuntimeStack::build_runtime(|c| register_job_codecs(c), None);
|
||||
let engine = Engine::new(
|
||||
parts,
|
||||
TokioBackend::new(TokioConfig::default()).expect("tokio backend"),
|
||||
)
|
||||
.expect("engine");
|
||||
let mut driver = IrohDriver::with_engine(
|
||||
engine.handle(),
|
||||
IrohDriverConfig {
|
||||
secret_key: None,
|
||||
relay_mode: RelayMode::Disabled,
|
||||
node: DistributedNodeConfig::default(),
|
||||
peer_auth: None,
|
||||
additional_alpns: vec![],
|
||||
},
|
||||
)
|
||||
.expect("iroh driver");
|
||||
let stack = DistributionRuntimeStack::new_from_runtime(
|
||||
runtime.clone(),
|
||||
codec,
|
||||
transport_router,
|
||||
driver.node_id(),
|
||||
DistributedNodeConfig::default(),
|
||||
engine.handle(),
|
||||
);
|
||||
driver.enable_actor_bridge(
|
||||
stack.runtime.clone(),
|
||||
stack.codec.clone(),
|
||||
stack.actor_bridge_routes(),
|
||||
stack.actors.swim,
|
||||
stack.relay_mirror.clone(),
|
||||
stack.route_view.clone(),
|
||||
stack.outbox.clone(),
|
||||
);
|
||||
stack.spawn_protocol_ticker(POLL);
|
||||
driver.install_actor_bridge_pump(POLL);
|
||||
(engine, driver, stack)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn job_runs_across_two_nodes_over_real_iroh() {
|
||||
let node_workdir = tempfile::tempdir().expect("node workdir");
|
||||
let landing = tempfile::tempdir().expect("landing");
|
||||
let ws = tempfile::tempdir().expect("ws");
|
||||
std::fs::write(ws.path().join("seed.txt"), "seed-value").expect("seed");
|
||||
|
||||
// Worker composition (B): spawn + register the NodeJobActor so peers can
|
||||
// route to it through the converged directory.
|
||||
let (engine_b, driver_b, stack_b) = build_composition();
|
||||
let sender_b = stack_b.runtime.create_sender();
|
||||
// Placeholder orchestrator address: the real one is on A; the node only
|
||||
// needs it once the orchestrator submits. We point the node at A's
|
||||
// orchestrator after it exists (address is fixed below), but the node actor
|
||||
// captures the address at construction — so spawn it after A's orchestrator.
|
||||
let (_engine_a, _driver_a, stack_a) = build_composition();
|
||||
|
||||
let done = stack_a
|
||||
.runtime
|
||||
.new_inbox::<JobDone>()
|
||||
.expect("orchestrator done inbox");
|
||||
let orch = stack_a
|
||||
.runtime
|
||||
.spawn(OrchestratorJobActor::new(
|
||||
*done.addr(),
|
||||
landing.path().to_path_buf(),
|
||||
))
|
||||
.expect("spawn orchestrator on A");
|
||||
|
||||
let job_actor = stack_b
|
||||
.runtime
|
||||
.spawn(NodeJobActor::new(
|
||||
orch,
|
||||
node_workdir.path().to_path_buf(),
|
||||
sender_b,
|
||||
0,
|
||||
))
|
||||
.expect("spawn node job actor on B");
|
||||
stack_b.register_local_actor(driver_b.register_actor(job_actor, 1));
|
||||
|
||||
// Both control directions must be routable before Submit. A learning the
|
||||
// worker route does not imply B has already learned the orchestrator route;
|
||||
// submitting at that one-way boundary loses the first node event.
|
||||
driver_a_join(&stack_a, &_driver_a, &driver_b);
|
||||
stack_a.register_local_actor(_driver_a.register_actor(orch, 1));
|
||||
|
||||
let converged = wait_until(CONVERGE_DEADLINE, || {
|
||||
stack_a.route_owner(job_actor).is_some() && stack_b.route_owner(orch).is_some()
|
||||
});
|
||||
assert!(
|
||||
converged,
|
||||
"directory did not converge bidirectionally for orchestrator and worker actors"
|
||||
);
|
||||
|
||||
let job = Job {
|
||||
name: "iroh-probe".to_owned(),
|
||||
setup: Some("echo setup-ok > setup_done.txt".to_owned()),
|
||||
run: "echo hello-over-iroh > greeting.txt".to_owned(),
|
||||
workspace: Some(Workspace {
|
||||
workdir: ws.path().to_path_buf(),
|
||||
exclude: vec![],
|
||||
}),
|
||||
outputs: vec![
|
||||
"greeting.txt".to_owned(),
|
||||
"setup_done.txt".to_owned(),
|
||||
"seed.txt".to_owned(),
|
||||
],
|
||||
env: BTreeMap::new(),
|
||||
};
|
||||
stack_a
|
||||
.runtime
|
||||
.send_to(
|
||||
orch,
|
||||
OrchestratorJobMsg::Submit {
|
||||
job,
|
||||
node_actor: job_actor,
|
||||
},
|
||||
)
|
||||
.expect("submit");
|
||||
let mut outcome = None;
|
||||
let started = Instant::now();
|
||||
while started.elapsed() < JOB_DEADLINE {
|
||||
if let Some(d) = done.try_recv() {
|
||||
outcome = Some(d);
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(POLL);
|
||||
}
|
||||
drop(_engine_a);
|
||||
drop(engine_b);
|
||||
let done = outcome.expect("job did not complete across iroh within deadline");
|
||||
assert_eq!(
|
||||
done.state,
|
||||
JobState::Completed,
|
||||
"expected COMPLETED over iroh, got {:?}",
|
||||
done
|
||||
);
|
||||
assert_eq!(done.exit_code, Some(0));
|
||||
let greeting =
|
||||
std::fs::read_to_string(landing.path().join("greeting.txt")).expect("collected greeting");
|
||||
assert!(greeting.contains("hello-over-iroh"), "greeting: {greeting}");
|
||||
let seed = std::fs::read_to_string(landing.path().join("seed.txt")).expect("collected seed");
|
||||
assert_eq!(seed, "seed-value", "workspace crossed iroh through swactor");
|
||||
}
|
||||
|
||||
fn driver_a_join(
|
||||
_stack_a: &DistributionRuntimeStack,
|
||||
driver_a: &IrohDriver,
|
||||
driver_b: &IrohDriver,
|
||||
) {
|
||||
driver_a.join(std::slice::from_ref(&driver_b.endpoint_addr()));
|
||||
}
|
||||
|
||||
#[allow(clippy::disallowed_methods)]
|
||||
fn wait_until(deadline: Duration, mut check: impl FnMut() -> bool) -> bool {
|
||||
let started = Instant::now();
|
||||
loop {
|
||||
if check() {
|
||||
return true;
|
||||
}
|
||||
if started.elapsed() >= deadline {
|
||||
return false;
|
||||
}
|
||||
std::thread::sleep(POLL);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
mod engine_composition;
|
||||
mod harness;
|
||||
mod job_runner_iroh;
|
||||
mod local_e2e_guarantees;
|
||||
mod local_mock;
|
||||
mod node_guarantees;
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ fn node_agent_runtime_loaded_reports_orchestrator() {
|
|||
.expect("orchestrator inbox");
|
||||
let orchestrator = *orchestrator_inbox.addr();
|
||||
let node_actor = ActorAddress::new_random();
|
||||
let telemetry_publisher = ActorAddress::new_random();
|
||||
let endpoint = EndpointAddr::new(SecretKey::from_bytes(&[9; 32]).public());
|
||||
let actor = runtime
|
||||
.spawn(NodeAgentActor::new(stage::NodeId(11), orchestrator, None))
|
||||
|
|
@ -36,7 +35,6 @@ fn node_agent_runtime_loaded_reports_orchestrator() {
|
|||
stage_index: 3,
|
||||
endpoint: endpoint.clone(),
|
||||
node_actor,
|
||||
telemetry_publisher,
|
||||
readiness_id: 99,
|
||||
},
|
||||
)
|
||||
|
|
@ -51,7 +49,6 @@ fn node_agent_runtime_loaded_reports_orchestrator() {
|
|||
stage_index: 3,
|
||||
endpoint,
|
||||
node_actor,
|
||||
telemetry_publisher,
|
||||
readiness_id: 99,
|
||||
})
|
||||
);
|
||||
|
|
|
|||
|
|
@ -1,42 +1,25 @@
|
|||
//! Behavior guarantees for the `prompt` module.
|
||||
//! Behavior guarantees for the prompt wire types.
|
||||
|
||||
use myelin::prompt::rpc::{PromptEvent, SubmitPrompt};
|
||||
use myelin::node::prompt_wire::PromptEvent;
|
||||
|
||||
#[test]
|
||||
fn zero_request_limits_take_loop_defaults() {
|
||||
let request = SubmitPrompt {
|
||||
request_id: 7,
|
||||
prompt_text: "hello".to_owned(),
|
||||
max_tokens: 0,
|
||||
}
|
||||
.with_defaults(32);
|
||||
|
||||
assert_eq!(request.max_tokens, 32);
|
||||
fn is_terminal(event: &PromptEvent) -> bool {
|
||||
matches!(event, PromptEvent::Done { .. } | PromptEvent::Fault { .. })
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn event_terminal_state_is_explicit() {
|
||||
assert!(
|
||||
!PromptEvent::TextDelta {
|
||||
assert!(!is_terminal(&PromptEvent::TextDelta {
|
||||
request_id: 1,
|
||||
text: "a".to_owned(),
|
||||
}
|
||||
.is_terminal()
|
||||
);
|
||||
assert!(
|
||||
PromptEvent::Done {
|
||||
}));
|
||||
assert!(is_terminal(&PromptEvent::Done {
|
||||
request_id: 1,
|
||||
final_text: "a".to_owned(),
|
||||
tokens_generated: 1,
|
||||
elapsed_ms: 2,
|
||||
}
|
||||
.is_terminal()
|
||||
);
|
||||
assert!(
|
||||
PromptEvent::Fault {
|
||||
}));
|
||||
assert!(is_terminal(&PromptEvent::Fault {
|
||||
request_id: 1,
|
||||
error: "boom".to_owned(),
|
||||
}
|
||||
.is_terminal()
|
||||
);
|
||||
}));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,8 +7,8 @@
|
|||
//!
|
||||
//! Routes (only present when the feature is enabled and a control sink is
|
||||
//! installed):
|
||||
//! - `POST /control/kill` body `{"node": "<stream node id>"}`
|
||||
//! - `POST /control/provision` body `{"count": 1}`
|
||||
//! - `POST /control/kill` body `{"Kill":{"command_id":"...","node":"..."}}`
|
||||
//! - `POST /control/provision` body `{"Provision":{"command_id":"...","count":1}}`
|
||||
|
||||
use std::sync::mpsc::Sender;
|
||||
use std::sync::OnceLock;
|
||||
|
|
@ -19,16 +19,25 @@ use serde::Deserialize;
|
|||
#[derive(Clone, Debug, Deserialize)]
|
||||
pub enum ControlCommand {
|
||||
/// Kill the process backing the fleet card identified by its stream node.
|
||||
Kill { node: String },
|
||||
Kill { command_id: String, node: String },
|
||||
/// Ask the reconciler to provision `count` additional nodes.
|
||||
Provision { count: u32 },
|
||||
Provision { command_id: String, count: u32 },
|
||||
/// Lower the desired cluster size by `count` nodes (graceful scale
|
||||
/// down: teardown through the reconciler, not a kill).
|
||||
Remove { count: u32 },
|
||||
/// Establish (or replace) the data-plane edge toward one node. The
|
||||
/// supervisor provisions the node's inbound edge over the control
|
||||
/// plane and dials it over EDGE_ALPN.
|
||||
EstablishEdge { node: String },
|
||||
Remove { command_id: String, count: u32 },
|
||||
/// Establish (or replace) the data-plane edge toward one node.
|
||||
EstablishEdge { command_id: String, node: String },
|
||||
}
|
||||
|
||||
impl ControlCommand {
|
||||
pub fn command_id(&self) -> &str {
|
||||
match self {
|
||||
Self::Kill { command_id, .. }
|
||||
| Self::Provision { command_id, .. }
|
||||
| Self::Remove { command_id, .. }
|
||||
| Self::EstablishEdge { command_id, .. } => command_id,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static CONTROL_SENDER: OnceLock<Sender<ControlCommand>> = OnceLock::new();
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@
|
|||
#status { font-family: var(--mono); font-size: 13px; color: var(--amber); }
|
||||
a.node-card { color: inherit; text-decoration: none; display: block; }
|
||||
.node-card.orchestrator { grid-column: 1 / -1; border-color: var(--amber); background: var(--card-hover); }
|
||||
.node-card .role { display: inline-flex; margin-left: 8px; padding: 2px 6px; border: 1px solid var(--amber); border-radius: var(--r); color: var(--amber); font: 600 10px/1.4 var(--mono); text-transform: uppercase; letter-spacing: .07em; vertical-align: 2px; }
|
||||
.node-card.orchestrator .role { margin-bottom: 6px; color: var(--muted); font: 600 9px/1.2 var(--mono); text-transform: uppercase; letter-spacing: .12em; }
|
||||
:is(a, button, input, summary):focus-visible, tr[data-addr]:focus-visible { outline: 2px solid var(--cyan); outline-offset: 2px; }
|
||||
.grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(300px, 1fr)); gap: 12px; }
|
||||
.node-card { background: var(--panel); border: 1px solid var(--border); border-radius: var(--r); padding: 14px; cursor: pointer; transition: border-color var(--t), background var(--t); }
|
||||
|
|
@ -136,6 +136,25 @@ let selectedActor = params.get('actor') || null;
|
|||
let rosterFilter = '';
|
||||
let lastSnapshot = null;
|
||||
let detailTimer = null;
|
||||
// A1: last-rendered HTML per region. A poll that yields identical markup
|
||||
// must not swap innerHTML — the swap destroyed hover/selection/presses and
|
||||
// restarted animations every second even in a fully converged steady state.
|
||||
let lastPageHtml = null;
|
||||
let lastRosterHtml = null;
|
||||
let lastDossierHtml = null;
|
||||
|
||||
// Receipt ages (dossier) render as empty spans carrying an absolute epoch —
|
||||
// stable across polls — and a 1 Hz pass rewrites their textContent. All other
|
||||
// ticking text (card seen/stats) is written per-poll by updateCardTexts.
|
||||
function agoSpan(epochMs) {
|
||||
return `<span data-ago-at="${Math.round(epochMs)}"></span>`;
|
||||
}
|
||||
function updateAgo() {
|
||||
const now = Date.now();
|
||||
for (const span of document.querySelectorAll('[data-ago-at]')) {
|
||||
span.textContent = ago(now - Number(span.dataset.agoAt));
|
||||
}
|
||||
}
|
||||
|
||||
function esc(value) {
|
||||
return String(value ?? '').replace(/[&<>"']/g, c => ({'&':'&','<':'<','>':'>','"':'"','\'':'''}[c]));
|
||||
|
|
@ -205,6 +224,7 @@ function render() {
|
|||
} else {
|
||||
renderFleet(page, live, stale, totals);
|
||||
}
|
||||
updateAgo();
|
||||
}
|
||||
|
||||
function renderFleet(page, live, stale, totals) {
|
||||
|
|
@ -223,8 +243,12 @@ function renderFleet(page, live, stale, totals) {
|
|||
</details>`);
|
||||
}
|
||||
}
|
||||
page.innerHTML = parts.join('');
|
||||
bindCards();
|
||||
const html = parts.join('');
|
||||
if (html !== lastPageHtml) {
|
||||
page.innerHTML = html;
|
||||
lastPageHtml = html;
|
||||
}
|
||||
updateCardTexts(orchestrators.concat(workers, stale));
|
||||
}
|
||||
|
||||
function nodeCard(node) {
|
||||
|
|
@ -233,29 +257,54 @@ function nodeCard(node) {
|
|||
const origin = node.stream.origin || '';
|
||||
const isOrchestrator = origin === 'orchestrator';
|
||||
const role = isOrchestrator ? '<span class="role">orchestrator</span>' : '';
|
||||
const stats = [
|
||||
const label = node.stream.label ? ` · ${esc(node.stream.label)}` : '';
|
||||
// The stats line (actors · msg/s · uptime) jitters every frame on busy
|
||||
// supervisors; it renders as a pass-filled span so the card markup — and
|
||||
// with it the whole grid — stays byte-identical between polls.
|
||||
return `<a class="node-card${isOrchestrator ? ' orchestrator' : ''}" data-stream="${esc(node.stream.key)}" data-live="${node.live ? 'true' : 'false'}" href="?stream=${encodeURIComponent(node.stream.key)}">
|
||||
${role}
|
||||
<h3>${esc(node.stream.node)} <span class="muted" style="font-size:12px">life ${fmt(node.stream.life)}</span></h3>
|
||||
<div class="meta">${proc}seen <span data-seen></span> ago · <span data-stats></span>${label}</div>
|
||||
${summary.poisoned ? `<div class="notice err">${fmt(summary.poisoned)} poisoned actor(s)</div>` : ''}
|
||||
</a>`;
|
||||
}
|
||||
|
||||
function cardStats(node) {
|
||||
const summary = node.actor_summary || {};
|
||||
return [
|
||||
summary.actors ? fmt(summary.actors) + ' actors' : null,
|
||||
summary.msg_per_sec ? fmtRate(summary.msg_per_sec) + ' msg/s' : null,
|
||||
summary.num_workers != null ? fmt(summary.num_workers) + ' workers' : null,
|
||||
summary.uptime_ms != null ? 'up ' + ago(summary.uptime_ms) : null,
|
||||
].filter(Boolean).join(' · ');
|
||||
const label = node.stream.label ? ` · ${esc(node.stream.label)}` : '';
|
||||
return `<a class="node-card${isOrchestrator ? ' orchestrator' : ''}" data-stream="${esc(node.stream.key)}" data-live="${node.live ? 'true' : 'false'}" href="?stream=${encodeURIComponent(node.stream.key)}">
|
||||
<h3>${esc(node.stream.node)}${role} <span class="muted" style="font-size:12px">life ${fmt(node.stream.life)}</span></h3>
|
||||
<div class="meta">${proc}seen ${ago(node.last_seen_ms_ago)} ago · ${stats || 'no runtime stats'}${label}</div>
|
||||
${summary.poisoned ? `<div class="notice err">${fmt(summary.poisoned)} poisoned actor(s)</div>` : ''}
|
||||
</a>`;
|
||||
].filter(Boolean).join(' · ') || 'no runtime stats';
|
||||
}
|
||||
|
||||
function updateCardTexts(nodes) {
|
||||
const byStream = new Map(nodes.map(node => [node.stream.key, node]));
|
||||
for (const card of document.querySelectorAll('a.node-card')) {
|
||||
const node = byStream.get(card.getAttribute('data-stream'));
|
||||
const seen = card.querySelector('[data-seen]');
|
||||
if (seen) seen.textContent = node ? ago(node.last_seen_ms_ago) : '—';
|
||||
const stats = card.querySelector('[data-stats]');
|
||||
if (stats) stats.textContent = node ? cardStats(node) : 'no runtime stats';
|
||||
}
|
||||
}
|
||||
|
||||
function renderNode(page, node, live, stale) {
|
||||
const summary = node.actor_summary || {};
|
||||
const machine = machineDetail(node);
|
||||
page.innerHTML = `
|
||||
// The machine panel churns with live metrics, so swaps here are real; keep
|
||||
// the roster filter usable by restoring focus and caret across them.
|
||||
const active = document.activeElement;
|
||||
const refocus = active && active.id === 'roster-filter' && page.contains(active)
|
||||
? { start: active.selectionStart, end: active.selectionEnd, dir: active.selectionDirection }
|
||||
: null;
|
||||
const html = `
|
||||
<a class="focus-back" href="/">← Fleet (${fmt(live.length)} live, ${fmt(stale.length)} stale)</a>
|
||||
<div class="layout ${selectedActor ? 'with-dossier' : ''}">
|
||||
<div class="node-view">
|
||||
<section class="panel">
|
||||
<h2 style="margin:0 0 8px">${esc(node.stream.node)}${node.stream.label ? ` <span class="muted" style="font-size:13px">${esc(node.stream.label)}</span>` : ''} <span class="muted" style="font-size:13px">life ${fmt(node.stream.life)} · ${node.live ? 'live' : 'stale ' + ago(node.last_seen_ms_ago) + ' ago'}</span></h2>
|
||||
<h2 style="margin:0 0 8px">${esc(node.stream.node)}${node.stream.label ? ` <span class="muted" style="font-size:13px">${esc(node.stream.label)}</span>` : ''} <span class="muted" style="font-size:13px">life ${fmt(node.stream.life)} · <span data-seen></span></span></h2>
|
||||
${machine}
|
||||
</section>
|
||||
<section class="panel">
|
||||
|
|
@ -266,9 +315,19 @@ function renderNode(page, node, live, stale) {
|
|||
</div>
|
||||
<div id="dossier-slot"></div>
|
||||
</div>`;
|
||||
renderRoster(node);
|
||||
if (html !== lastPageHtml) {
|
||||
page.innerHTML = html;
|
||||
lastPageHtml = html;
|
||||
const filter = document.getElementById('roster-filter');
|
||||
filter.addEventListener('input', e => { rosterFilter = e.target.value; renderRoster(node); });
|
||||
if (refocus) {
|
||||
filter.focus();
|
||||
try { filter.setSelectionRange(refocus.start, refocus.end, refocus.dir); } catch { /* caret shape changed */ }
|
||||
}
|
||||
}
|
||||
const seen = document.querySelector('#page h2 [data-seen]');
|
||||
if (seen) seen.textContent = node.live ? 'live' : 'stale ' + ago(node.last_seen_ms_ago) + ' ago';
|
||||
renderRoster(node);
|
||||
if (selectedActor) startDetailPolling();
|
||||
}
|
||||
|
||||
|
|
@ -337,11 +396,11 @@ function renderRoster(node) {
|
|||
const notice = rows.length > capped.length
|
||||
? `<div class="notice">showing ${fmt(capped.length)} of ${fmt(rows.length)} — refine the filter to see more</div>`
|
||||
: '';
|
||||
let html;
|
||||
if (!rows.length) {
|
||||
wrap.innerHTML = '<div class="empty">No actors on this stream (or none match the filter).</div>';
|
||||
return;
|
||||
}
|
||||
wrap.innerHTML = notice + `<table>
|
||||
html = '<div class="empty">No actors on this stream (or none match the filter).</div>';
|
||||
} else {
|
||||
html = notice + `<table>
|
||||
<thead><tr><th>Actor</th><th>Type</th><th>State</th><th>Mailbox</th><th>Msg/s</th><th>Processed</th><th>Worker</th><th>Last message</th></tr></thead>
|
||||
<tbody>${capped.map(a => `<tr data-addr="${esc(a.address)}" tabindex="0" ${a.address === selectedActor ? 'data-selected="true"' : ''}>
|
||||
<td class="mono">${esc(shortAddr(a.address))}${a.name ? `<br><span class="muted">${esc(a.name)}</span>` : ''}</td>
|
||||
|
|
@ -354,15 +413,10 @@ function renderRoster(node) {
|
|||
<td><span class="muted" title="${esc(a.last_msg_type || '')}">${typeShort(a.last_msg_type)}</span></td>
|
||||
</tr>`).join('')}</tbody>
|
||||
</table>`;
|
||||
wrap.querySelectorAll('tr[data-addr]').forEach(tr => {
|
||||
const select = () => {
|
||||
selectedActor = tr.getAttribute('data-addr');
|
||||
pushUrl();
|
||||
render();
|
||||
};
|
||||
tr.addEventListener('click', select);
|
||||
tr.addEventListener('keydown', e => { if (e.key === 'Enter') select(); });
|
||||
});
|
||||
}
|
||||
if (html === lastRosterHtml) return;
|
||||
wrap.innerHTML = html;
|
||||
lastRosterHtml = html;
|
||||
}
|
||||
|
||||
function shortAddr(address) {
|
||||
|
|
@ -393,6 +447,7 @@ function startDetailPolling() {
|
|||
function renderDossier(d) {
|
||||
const slot = document.getElementById('dossier-slot');
|
||||
if (!slot) return;
|
||||
|
||||
const diet = (d.message_type_counts || []).slice().sort((a, b) => b.count - a.count);
|
||||
const dietMax = Math.max(1, ...diet.map(x => x.count));
|
||||
const lastName = d.last_msg_type;
|
||||
|
|
@ -405,14 +460,14 @@ function renderDossier(d) {
|
|||
const receiptsHtml = receipts.length ? `<table class="receipts">
|
||||
<thead><tr><th>When</th><th>Message type</th><th>Folded</th></tr></thead>
|
||||
<tbody>${receipts.map(r => `<tr>
|
||||
<td class="muted">${ago(r.ms_ago)} ago</td>
|
||||
<td class="muted">${agoSpan(Date.now() - r.ms_ago)} ago</td>
|
||||
<td><span class="mono" title="${esc(r.ty)}">${typeShort(r.ty)}</span></td>
|
||||
<td class="muted">${r.folded ? '+' + fmt(r.folded) + ' sampled' : '—'}</td>
|
||||
</tr>`).join('')}</tbody>
|
||||
</table>
|
||||
<div class="notice">showing last ${fmt(receipts.length)} receipts · ${fmt(d.sampled_out)} messages sampled out · ${fmt(d.messages_processed)} processed total</div>`
|
||||
: '<div class="empty">no message receipts folded yet</div>';
|
||||
slot.innerHTML = `<section class="panel dossier">
|
||||
const html = `<section class="panel dossier">
|
||||
<div style="display:flex;justify-content:space-between;align-items:baseline;gap:8px">
|
||||
<h2 style="margin:0">${typeShort(d.actor_type)} <span class="muted mono" style="font-size:12px">${esc(shortAddr(d.address))}</span></h2>
|
||||
<button id="dossier-close" aria-label="Close dossier" style="background:none;border:none;color:var(--muted);cursor:pointer;font-size:16px">×</button>
|
||||
|
|
@ -437,31 +492,56 @@ function renderDossier(d) {
|
|||
<h2>Message history</h2>
|
||||
${receiptsHtml}
|
||||
</section>`;
|
||||
document.getElementById('dossier-close').addEventListener('click', () => {
|
||||
selectedActor = null;
|
||||
pushUrl();
|
||||
render();
|
||||
});
|
||||
if (html !== lastDossierHtml) {
|
||||
slot.innerHTML = html;
|
||||
lastDossierHtml = html;
|
||||
}
|
||||
const history = d.history || [];
|
||||
const canvas = document.getElementById('dossier-spark');
|
||||
if (canvas && history.length > 1) {
|
||||
const ctx = canvas.getContext('2d');
|
||||
drawLine(ctx, history.map(h => h.msg_per_sec), '#60a5fa');
|
||||
}
|
||||
updateAgo();
|
||||
}
|
||||
|
||||
function bindCards() {
|
||||
document.querySelectorAll('.node-card').forEach(card => {
|
||||
card.addEventListener('click', e => {
|
||||
|
||||
// A2: delegation on the persistent #page container. Cards, roster rows and
|
||||
// the dossier close button are rebuilt by render passes; listeners bound at
|
||||
// startup mean a press straddling a rebuild still lands on the right target.
|
||||
const pageEl = document.getElementById('page');
|
||||
pageEl.addEventListener('click', e => {
|
||||
if (e.target.closest('#dossier-close')) {
|
||||
selectedActor = null;
|
||||
pushUrl();
|
||||
render();
|
||||
return;
|
||||
}
|
||||
const rosterRow = e.target.closest('tr[data-addr]');
|
||||
if (rosterRow) {
|
||||
selectedActor = rosterRow.getAttribute('data-addr');
|
||||
pushUrl();
|
||||
render();
|
||||
return;
|
||||
}
|
||||
const card = e.target.closest('a.node-card');
|
||||
if (card) {
|
||||
e.preventDefault();
|
||||
selectedStream = card.getAttribute('data-stream');
|
||||
selectedActor = null;
|
||||
pushUrl();
|
||||
render();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
});
|
||||
pageEl.addEventListener('keydown', e => {
|
||||
if (e.key !== 'Enter') return;
|
||||
const rosterRow = e.target.closest('tr[data-addr]');
|
||||
if (rosterRow) {
|
||||
selectedActor = rosterRow.getAttribute('data-addr');
|
||||
pushUrl();
|
||||
render();
|
||||
}
|
||||
});
|
||||
refresh();
|
||||
setInterval(refresh, POLL_MS);
|
||||
</script>
|
||||
|
|
|
|||
|
|
@ -65,12 +65,38 @@ impl DashboardView for DemoControlView {
|
|||
}
|
||||
|
||||
fn ingest(&self, _stream: &StreamId, _frame: &Frame, event: &FrameEvent) {
|
||||
if !is_lifecycle_channel(&event.channel) && event.channel != "node.status" {
|
||||
return;
|
||||
}
|
||||
let Ok(payload) = serde_json::from_slice::<Value>(&event.payload) else {
|
||||
return;
|
||||
};
|
||||
if event.channel == "myelin.provisioning.events" {
|
||||
let provision = payload.get("event").unwrap_or(&payload);
|
||||
let Some(run_id) = provision.get("run_id").and_then(Value::as_u64) else {
|
||||
return;
|
||||
};
|
||||
let Some(node_id) = provision.get("node_id").and_then(Value::as_u64) else {
|
||||
return;
|
||||
};
|
||||
if node_id == 0 {
|
||||
return;
|
||||
}
|
||||
let node = format!("myelin-node-{run_id}-{node_id}");
|
||||
let mut processes = self.processes.lock();
|
||||
let entry = processes.entry(node).or_default();
|
||||
entry.seen = Some(Instant::now());
|
||||
entry.state = match provision.get("kind").and_then(Value::as_str) {
|
||||
Some("ProvisionStart") => "provisioning",
|
||||
Some("NodeLive") => "running",
|
||||
Some("ProvisionFailed") => "failed",
|
||||
Some("NodeStopped") => "exited",
|
||||
Some(other) => other,
|
||||
None => return,
|
||||
}
|
||||
.to_owned();
|
||||
return;
|
||||
}
|
||||
if !is_lifecycle_channel(&event.channel) && event.channel != "node.status" {
|
||||
return;
|
||||
}
|
||||
let node = event.stream.node.clone();
|
||||
let mut processes = self.processes.lock();
|
||||
let entry = processes.entry(node).or_default();
|
||||
|
|
|
|||
|
|
@ -62,6 +62,7 @@
|
|||
.seg .seg-v { position: relative; }
|
||||
.seg-sm { font-size: 15px; }
|
||||
.blink { animation: seg-blink 1.1s steps(2, start) infinite; }
|
||||
.counter[hidden] { display: none; }
|
||||
@keyframes seg-blink { to { visibility: hidden; } }
|
||||
.controls { display: flex; align-items: center; flex-wrap: wrap; gap: 8px; }
|
||||
.controls input {
|
||||
|
|
@ -356,6 +357,11 @@ RISK: terminal cosplay if decoration creeps past data; held by the palette law.
|
|||
return Math.max(1, Math.min(8, Number(document.getElementById('provision-count').value) || 1));
|
||||
}
|
||||
|
||||
function commandId() {
|
||||
return globalThis.crypto?.randomUUID?.()
|
||||
|| `${Date.now()}-${Math.random().toString(16).slice(2)}`;
|
||||
}
|
||||
|
||||
function post(path, body, action) {
|
||||
fetch(path, {
|
||||
method: 'POST',
|
||||
|
|
@ -367,11 +373,11 @@ RISK: terminal cosplay if decoration creeps past data; held by the palette law.
|
|||
}
|
||||
|
||||
document.getElementById('provision').addEventListener('click', () => {
|
||||
post('/control/provision', { Provision: { count: nodeCount() } }, 'provision');
|
||||
post('/control/provision', { Provision: { command_id: commandId(), count: nodeCount() } }, 'provision');
|
||||
});
|
||||
|
||||
document.getElementById('remove').addEventListener('click', () => {
|
||||
post('/control/remove', { Remove: { count: nodeCount() } }, 'remove');
|
||||
post('/control/remove', { Remove: { command_id: commandId(), count: nodeCount() } }, 'remove');
|
||||
});
|
||||
|
||||
// A2: one delegated listener on the persistent tbody. Per-render binding
|
||||
|
|
@ -381,9 +387,10 @@ RISK: terminal cosplay if decoration creeps past data; held by the palette law.
|
|||
if (!button || !rows.contains(button)) return;
|
||||
const action = button.dataset.action || 'kill';
|
||||
const route = action === 'edge' ? '/control/edge' : '/control/kill';
|
||||
const command_id = commandId();
|
||||
const body = action === 'edge'
|
||||
? { EstablishEdge: { node: button.dataset.node } }
|
||||
: { Kill: { node: button.dataset.node } };
|
||||
? { EstablishEdge: { command_id, node: button.dataset.node } }
|
||||
: { Kill: { command_id, node: button.dataset.node } };
|
||||
const label = action === 'edge' ? 'edge' : 'kill';
|
||||
if (action === 'edge') button.disabled = false; // edges can be re-issued
|
||||
else button.disabled = true;
|
||||
|
|
|
|||
|
|
@ -104,11 +104,11 @@ pub fn spawn_pull_server(
|
|||
});
|
||||
}
|
||||
|
||||
/// Supervisor side: dial a node on `TELEMETRY_ALPN`, send the pull request,
|
||||
/// and stream answering events into `fanout` as they arrive (incrementally,
|
||||
/// not buffered until stream end). The header is reported through
|
||||
/// `on_header` first so the caller can register stream/channel metadata
|
||||
/// before any frame lands.
|
||||
/// Supervisor side: retain a pull subscription to a node on `TELEMETRY_ALPN`.
|
||||
///
|
||||
/// A transport interruption reconnects with bounded backoff. Returning after
|
||||
/// the first EOF leaves a healthy node permanently stale, which is especially
|
||||
/// easy to trigger while several freshly-bootstrapped nodes answer at once.
|
||||
pub fn spawn_pull_collector(
|
||||
engine: &EngineHandle,
|
||||
endpoint: Endpoint,
|
||||
|
|
@ -119,35 +119,74 @@ pub fn spawn_pull_collector(
|
|||
fanout: std::sync::Arc<telemetry::DeliveryFanout>,
|
||||
on_header: std::sync::mpsc::Sender<TelemetryQuicHeader>,
|
||||
) {
|
||||
let engine_handle = engine.clone();
|
||||
engine.spawn(async move {
|
||||
let peer_id = peer.id.to_string();
|
||||
let Ok(conn) = endpoint.connect(peer, TELEMETRY_ALPN).await else {
|
||||
eprintln!("telemetry-pull: connect to {peer_id} failed");
|
||||
return;
|
||||
};
|
||||
let Ok(mut req) = conn.open_uni().await else {
|
||||
eprintln!("telemetry-pull: open request stream to {peer_id} failed");
|
||||
return;
|
||||
};
|
||||
if let Err(error) = write_pull_request(&mut req, flow_id, &token, &request).await {
|
||||
eprintln!("telemetry-pull: write request to {peer_id} failed: {error}");
|
||||
return;
|
||||
let mut retry_delay = Duration::from_millis(250);
|
||||
loop {
|
||||
match collect_pull_once(
|
||||
&endpoint, &peer, flow_id, &token, &request, &fanout, &on_header,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => return,
|
||||
Err(error) => {
|
||||
eprintln!(
|
||||
"telemetry-pull: {peer_id}: {error}; retrying in {} ms",
|
||||
retry_delay.as_millis()
|
||||
);
|
||||
}
|
||||
let Ok(mut recv) = conn.accept_uni().await else {
|
||||
eprintln!("telemetry-pull: no answer stream from {peer_id}");
|
||||
return;
|
||||
};
|
||||
let Ok(header) = read_header(&mut recv).await else {
|
||||
eprintln!("telemetry-pull: answer header from {peer_id} unreadable");
|
||||
return;
|
||||
};
|
||||
let _ = on_header.send(header.clone());
|
||||
let stream = header.stream.clone();
|
||||
while let Ok(Some(event)) = read_next_event(&mut recv, &stream).await {
|
||||
fanout.publish(event);
|
||||
}
|
||||
engine_handle.timer(retry_delay).await;
|
||||
retry_delay = retry_delay
|
||||
.checked_mul(2)
|
||||
.unwrap_or(Duration::from_secs(5))
|
||||
.min(Duration::from_secs(5));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async fn collect_pull_once(
|
||||
endpoint: &Endpoint,
|
||||
peer: &EndpointAddr,
|
||||
flow_id: [u8; 16],
|
||||
token: &[u8],
|
||||
request: &telemetry::SubscriptionRequest,
|
||||
fanout: &telemetry::DeliveryFanout,
|
||||
on_header: &std::sync::mpsc::Sender<TelemetryQuicHeader>,
|
||||
) -> Result<(), String> {
|
||||
let conn = endpoint
|
||||
.connect(peer.clone(), TELEMETRY_ALPN)
|
||||
.await
|
||||
.map_err(|error| format!("connect failed: {error}"))?;
|
||||
let mut req = conn
|
||||
.open_uni()
|
||||
.await
|
||||
.map_err(|error| format!("open request stream failed: {error}"))?;
|
||||
write_pull_request(&mut req, flow_id, token, request)
|
||||
.await
|
||||
.map_err(|error| format!("write request failed: {error}"))?;
|
||||
let mut recv = conn
|
||||
.accept_uni()
|
||||
.await
|
||||
.map_err(|error| format!("no answer stream: {error}"))?;
|
||||
let header = read_header(&mut recv)
|
||||
.await
|
||||
.map_err(|error| format!("answer header unreadable: {error}"))?;
|
||||
if on_header.send(header.clone()).is_err() {
|
||||
return Ok(());
|
||||
}
|
||||
let stream = header.stream;
|
||||
loop {
|
||||
match read_next_event(&mut recv, &stream).await {
|
||||
Ok(Some(event)) => {
|
||||
fanout.publish(event);
|
||||
}
|
||||
Ok(None) => return Err("answer stream closed".to_owned()),
|
||||
Err(error) => return Err(format!("read answer stream failed: {error}")),
|
||||
}
|
||||
}
|
||||
}
|
||||
const MAX_RECORD_BYTES: usize = 16 * 1024 * 1024;
|
||||
|
||||
type BoxError = Box<dyn Error + Send + Sync + 'static>;
|
||||
|
|
|
|||
|
|
@ -49,7 +49,6 @@ use distribution::types::{MemberState, NodeId};
|
|||
|
||||
use super::test_config;
|
||||
|
||||
|
||||
// ── Membership fanout (copied verbatim from main.rs) ────────────────────────
|
||||
// Adapts the SwimActor's `MembershipChanged` stream (its sole observable) into
|
||||
// the registry/metadata/directory actors' `Membership` control messages, and
|
||||
|
|
@ -123,8 +122,8 @@ impl IrohNode {
|
|||
)
|
||||
.expect("build test engine");
|
||||
|
||||
let mut driver = IrohDriver::with_engine(engine.handle(), config)
|
||||
.expect("failed to create iroh driver");
|
||||
let mut driver =
|
||||
IrohDriver::with_engine(engine.handle(), config).expect("failed to create iroh driver");
|
||||
let node_id = driver.node_id();
|
||||
|
||||
// The node's distribution config (SWIM/registry/metadata params).
|
||||
|
|
@ -262,7 +261,6 @@ impl IrohNode {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
// ── Passthroughs to the driver (keep consumer churn small) ──────────────
|
||||
|
||||
pub fn join(&mut self, seeds: &[EndpointAddr]) {
|
||||
|
|
@ -338,7 +336,6 @@ pub fn make_driver_with_relay(relay_url: iroh::RelayUrl) -> IrohNode {
|
|||
})
|
||||
}
|
||||
|
||||
|
||||
/// Poll until `check_fn` holds over `a` and `b` or `timeout` elapses, sleeping
|
||||
/// ~10ms between checks. Progression is engine-hosted; this only waits for
|
||||
/// wall-clock SWIM convergence.
|
||||
|
|
|
|||
|
|
@ -1,9 +1,5 @@
|
|||
use std::net::{IpAddr, SocketAddr};
|
||||
|
||||
use telemetry::frame::TelemetryEvent;
|
||||
use telemetry::{
|
||||
ChannelContent, TelemetryEndpoint, Lifetime, NodeId, Position, StreamId,
|
||||
};
|
||||
use iroh::{Endpoint, EndpointAddr, RelayMode};
|
||||
use iroh_driver::{
|
||||
TELEMETRY_ALPN, TelemetryQuicHeader, read_next_uni_from_connection,
|
||||
|
|
@ -12,6 +8,8 @@ use iroh_driver::{
|
|||
use swactor::config::RuntimeConfig;
|
||||
use swactor::runtime::RuntimeParts;
|
||||
use swactor_engine::{Engine, TokioBackend, TokioConfig};
|
||||
use telemetry::frame::TelemetryEvent;
|
||||
use telemetry::{ChannelContent, Lifetime, NodeId, Position, StreamId, TelemetryEndpoint};
|
||||
|
||||
/// Telemetry transport test scheduled through `EngineHandle`, not an ambient
|
||||
/// `#[tokio::test]` runtime (ENGINE_SPEC.md).
|
||||
|
|
|
|||
24
crates/job-runner/Cargo.toml
Normal file
24
crates/job-runner/Cargo.toml
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
[package]
|
||||
name = "swactor-job-runner"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
license = "AGPL-3.0-only"
|
||||
publish = false
|
||||
|
||||
[lib]
|
||||
name = "swactor_job_runner"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[dependencies]
|
||||
swactor = { path = "../..", features = ["serde"] }
|
||||
swactor-transport = { path = "../transport" }
|
||||
swactor-process = { path = "../process" }
|
||||
swactor-engine = { path = "../engine" }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
tar = "0.4"
|
||||
parking_lot = "0.12"
|
||||
|
||||
[dev-dependencies]
|
||||
tokio.workspace = true
|
||||
tempfile = "3"
|
||||
215
crates/job-runner/src/fsm.rs
Normal file
215
crates/job-runner/src/fsm.rs
Normal file
|
|
@ -0,0 +1,215 @@
|
|||
//! Job lifecycle FSM — spec §6.
|
||||
//!
|
||||
//! States: `PENDING → RUNNING → COMPLETED | FAILED`.
|
||||
//! The transition function is pure: given the current state, an observed event,
|
||||
//! the job shape (workspace/setup presence), and the last observed exit code,
|
||||
//! it returns the next state and the command (if any) the orchestrator emits.
|
||||
//! This keeps the lifecycle logic fully testable and independent of transport.
|
||||
|
||||
/// Lifecycle states. Spec §6.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum JobState {
|
||||
Pending,
|
||||
Running,
|
||||
Completed,
|
||||
Failed,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for JobState {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
JobState::Pending => f.write_str("PENDING"),
|
||||
JobState::Running => f.write_str("RUNNING"),
|
||||
JobState::Completed => f.write_str("COMPLETED"),
|
||||
JobState::Failed => f.write_str("FAILED"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Commands the orchestrator emits, routed to the node. Spec §6 command table.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum JobCommand {
|
||||
MaterializeWorkspace,
|
||||
RunSetup,
|
||||
RunJob,
|
||||
CollectOutputs,
|
||||
}
|
||||
|
||||
/// Events the orchestrator observes. Spec §6 event table.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum JobEvent {
|
||||
JobSubmitted,
|
||||
/// Reconciler reports the bootstrapped node reached ready.
|
||||
NodeReady,
|
||||
WorkspaceMaterialized,
|
||||
SetupCompleted,
|
||||
/// Supervised run process exited with `code`.
|
||||
JobExited(i32),
|
||||
OutputsCollected,
|
||||
NodeFault(String),
|
||||
NodeLost,
|
||||
OperatorStop,
|
||||
}
|
||||
|
||||
/// Context the transition needs that is not in the event itself: whether the
|
||||
/// job declares a workspace / setup, and the last observed run exit code
|
||||
/// (decides COMPLETED vs FAILED after output collection).
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct TransitionCtx {
|
||||
pub has_workspace: bool,
|
||||
pub has_setup: bool,
|
||||
pub prior_exit: Option<i32>,
|
||||
}
|
||||
|
||||
impl TransitionCtx {
|
||||
pub fn new(job_has_workspace: bool, job_has_setup: bool) -> Self {
|
||||
Self { has_workspace: job_has_workspace, has_setup: job_has_setup, prior_exit: None }
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply one event. Returns `(new_state, optional_command_to_emit)`.
|
||||
///
|
||||
/// Faithful to the spec §6 transition table:
|
||||
/// - `JobSubmitted` → `PENDING`
|
||||
/// - `NodeReady` → emit `MaterializeWorkspace` (or `RunSetup`/`RunJob` when
|
||||
/// there is no workspace) → `RUNNING`
|
||||
/// - `WorkspaceMaterialized` → emit `RunSetup`, or `RunJob` when no setup
|
||||
/// - `SetupCompleted` → emit `RunJob`
|
||||
/// - `JobExited{0}` → emit `CollectOutputs`
|
||||
/// - `JobExited{non-zero}` → emit `CollectOutputs` (best-effort)
|
||||
/// - `OutputsCollected` → `COMPLETED` if the run exited 0 (or never ran),
|
||||
/// else `FAILED`
|
||||
/// - `NodeFault` / `NodeLost` / `OperatorStop` → `FAILED`
|
||||
pub fn transition(
|
||||
state: JobState,
|
||||
event: &JobEvent,
|
||||
ctx: TransitionCtx,
|
||||
) -> (JobState, Option<JobCommand>) {
|
||||
use JobCommand::*;
|
||||
use JobEvent::*;
|
||||
use JobState::*;
|
||||
|
||||
let failing = matches!(ctx.prior_exit, Some(c) if c != 0);
|
||||
|
||||
match (state, event) {
|
||||
// Submission.
|
||||
(_, JobSubmitted) => (Pending, None),
|
||||
|
||||
// Node reached ready: begin work, skipping stages the job does not need.
|
||||
(Pending, NodeReady) => {
|
||||
let cmd = if ctx.has_workspace {
|
||||
MaterializeWorkspace
|
||||
} else if ctx.has_setup {
|
||||
RunSetup
|
||||
} else {
|
||||
RunJob
|
||||
};
|
||||
(Running, Some(cmd))
|
||||
}
|
||||
|
||||
// Workspace materialized: run setup, or jump straight to the job.
|
||||
(Running, WorkspaceMaterialized) => {
|
||||
let cmd = if ctx.has_setup { RunSetup } else { RunJob };
|
||||
(Running, Some(cmd))
|
||||
}
|
||||
|
||||
// Setup done: run the job.
|
||||
(Running, SetupCompleted) => (Running, Some(RunJob)),
|
||||
|
||||
// Job exited zero: collect outputs, then complete.
|
||||
(Running, JobExited(0)) => (Running, Some(CollectOutputs)),
|
||||
// Job exited non-zero: best-effort collection, then fail.
|
||||
(Running, JobExited(_)) => (Running, Some(CollectOutputs)),
|
||||
|
||||
// Outputs collected: terminal state decided by the run's exit code.
|
||||
(Running, OutputsCollected) => {
|
||||
if failing {
|
||||
(Failed, None)
|
||||
} else {
|
||||
(Completed, None)
|
||||
}
|
||||
}
|
||||
|
||||
// Faults abort immediately.
|
||||
(_, NodeFault(_) | NodeLost | OperatorStop) => (Failed, None),
|
||||
|
||||
// Any other (state, event) pairing is not reachable in the v1 flow.
|
||||
(s, e) => (s, event_noop(e)),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn event_noop(_: &JobEvent) -> Option<JobCommand> {
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn ctx(ws: bool, setup: bool, exit: Option<i32>) -> TransitionCtx {
|
||||
TransitionCtx { has_workspace: ws, has_setup: setup, prior_exit: exit }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn happy_path_with_workspace_and_setup() {
|
||||
let c = ctx(true, true, None);
|
||||
let (s, cmd) = transition(JobState::Pending, &JobEvent::JobSubmitted, c);
|
||||
assert_eq!(s, JobState::Pending);
|
||||
assert_eq!(cmd, None);
|
||||
|
||||
let (s, cmd) = transition(s, &JobEvent::NodeReady, c);
|
||||
assert_eq!(s, JobState::Running);
|
||||
assert_eq!(cmd, Some(JobCommand::MaterializeWorkspace));
|
||||
|
||||
let (s, cmd) = transition(s, &JobEvent::WorkspaceMaterialized, c);
|
||||
assert_eq!(s, JobState::Running);
|
||||
assert_eq!(cmd, Some(JobCommand::RunSetup));
|
||||
|
||||
let (s, cmd) = transition(s, &JobEvent::SetupCompleted, c);
|
||||
assert_eq!(s, JobState::Running);
|
||||
assert_eq!(cmd, Some(JobCommand::RunJob));
|
||||
|
||||
let c2 = ctx(true, true, Some(0));
|
||||
let (s, cmd) = transition(s, &JobEvent::JobExited(0), c2);
|
||||
assert_eq!(s, JobState::Running);
|
||||
assert_eq!(cmd, Some(JobCommand::CollectOutputs));
|
||||
|
||||
let (s, cmd) = transition(s, &JobEvent::OutputsCollected, c2);
|
||||
assert_eq!(s, JobState::Completed);
|
||||
assert_eq!(cmd, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_workspace_jumps_setup_or_run() {
|
||||
let c = ctx(false, true, None);
|
||||
let (s, cmd) = transition(JobState::Pending, &JobEvent::NodeReady, c);
|
||||
assert_eq!(cmd, Some(JobCommand::RunSetup));
|
||||
|
||||
let c = ctx(false, false, None);
|
||||
let (s, cmd) = transition(JobState::Pending, &JobEvent::NodeReady, c);
|
||||
assert_eq!(cmd, Some(JobCommand::RunJob));
|
||||
assert_eq!(s, JobState::Running);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nonzero_exit_collects_then_fails() {
|
||||
let c = ctx(true, false, None);
|
||||
let (s, _) = transition(JobState::Pending, &JobEvent::NodeReady, c);
|
||||
let (s, _) = transition(s, &JobEvent::WorkspaceMaterialized, c);
|
||||
let c2 = ctx(true, false, Some(3));
|
||||
let (s, cmd) = transition(s, &JobEvent::JobExited(3), c2);
|
||||
assert_eq!(cmd, Some(JobCommand::CollectOutputs));
|
||||
let (s, cmd) = transition(s, &JobEvent::OutputsCollected, c2);
|
||||
assert_eq!(s, JobState::Failed);
|
||||
assert_eq!(cmd, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn node_fault_aborts() {
|
||||
let c = ctx(true, true, None);
|
||||
let (s, cmd) = transition(JobState::Running, &JobEvent::NodeFault("oom".into()), c);
|
||||
assert_eq!(s, JobState::Failed);
|
||||
assert_eq!(cmd, None);
|
||||
}
|
||||
}
|
||||
22
crates/job-runner/src/lib.rs
Normal file
22
crates/job-runner/src/lib.rs
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
//! swactor job runner — drives one command job end to end **through swactor**.
|
||||
//!
|
||||
//! Implements the Synaptic Job Runner Specification (spec id 4) as a plugin on
|
||||
//! swactor's substrate: orchestrator↔node control and bulk transfer (workspace
|
||||
//! push / output pull) travel as actor messages over the runtime/iroh plane; the
|
||||
//! node runs `setup`/`run` as supervised processes via `swactor-process`; the
|
||||
//! exit code is authoritative.
|
||||
|
||||
pub mod fsm;
|
||||
pub mod model;
|
||||
pub mod node;
|
||||
pub mod orchestrator;
|
||||
pub mod wire;
|
||||
|
||||
pub use fsm::{transition, JobCommand, JobEvent, JobState, TransitionCtx};
|
||||
pub use model::{ClusterConfig, Job, ProviderConfig, Workspace};
|
||||
pub use node::{extract_tar, JobPhase, NodeJobActor, ProcessExitRelay};
|
||||
pub use orchestrator::{pack_workspace, JobDone, OrchestratorJobActor, OrchestratorJobMsg};
|
||||
pub use wire::{
|
||||
register_job_codecs, EDGE_RECORD_SIZE, JobEdgeSink, NodeJobCommand, NodeJobEvent, OutputChunk,
|
||||
CHUNK_SIZE, OUTPUTS_EDGE_ID, WORKSPACE_EDGE_ID,
|
||||
};
|
||||
139
crates/job-runner/src/model.rs
Normal file
139
crates/job-runner/src/model.rs
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
//! Job and cluster-config model — spec §3 (gpu-agnostic job) and §4 (fields).
|
||||
//!
|
||||
//! A [`Job`] describes *what to do* and never what hardware. Hardware lives in
|
||||
//! a separate [`ClusterConfig`] bootstrapped before the job is submitted.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// A single-node command job. Spec §4 fields, nothing else.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Job {
|
||||
/// Job identity.
|
||||
pub name: String,
|
||||
/// The command that does the work.
|
||||
pub run: String,
|
||||
/// One-time command run before `run` (e.g. env install).
|
||||
#[serde(default)]
|
||||
pub setup: Option<String>,
|
||||
/// `{ workdir, exclude }` — pushed to the node first.
|
||||
#[serde(default)]
|
||||
pub workspace: Option<Workspace>,
|
||||
/// Paths to collect back after run, relative to the outputs root
|
||||
/// (the job's workdir on the node). Spec §4 outputs-root convention.
|
||||
#[serde(default)]
|
||||
pub outputs: Vec<String>,
|
||||
/// Environment variables injected into setup and run.
|
||||
#[serde(default)]
|
||||
pub env: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
impl Job {
|
||||
pub fn has_workspace(&self) -> bool {
|
||||
self.workspace.is_some()
|
||||
}
|
||||
pub fn has_setup(&self) -> bool {
|
||||
self.setup.as_deref().map(|s| !s.trim().is_empty()).unwrap_or(false)
|
||||
}
|
||||
}
|
||||
|
||||
/// Workspace push target: the operator's workdir tree, minus `exclude` globs.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Workspace {
|
||||
pub workdir: PathBuf,
|
||||
#[serde(default)]
|
||||
pub exclude: Vec<String>,
|
||||
}
|
||||
|
||||
/// Hardware (provider, GPU, image, disk, selection). Spec §3: lives in a
|
||||
/// separate cluster config bootstrapped before any job is submitted.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ClusterConfig {
|
||||
pub provider: ProviderConfig,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ProviderConfig {
|
||||
/// Provider kind. v1 realizes `"vastai"`.
|
||||
pub kind: String,
|
||||
/// Docker image launched on the node.
|
||||
pub image: String,
|
||||
#[serde(default = "default_disk_gb")]
|
||||
pub disk_gb: u32,
|
||||
#[serde(default)]
|
||||
pub gpu: GpuSpec,
|
||||
#[serde(default)]
|
||||
pub selection: SelectionSpec,
|
||||
/// Vast.ai API base URL.
|
||||
#[serde(default = "default_base_url")]
|
||||
pub base_url: String,
|
||||
}
|
||||
|
||||
fn default_disk_gb() -> u32 {
|
||||
64
|
||||
}
|
||||
fn default_base_url() -> String {
|
||||
"https://cloud.vast.ai".to_string()
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct GpuSpec {
|
||||
#[serde(default = "default_gpu_name")]
|
||||
pub name: String,
|
||||
#[serde(default = "default_gpu_count")]
|
||||
pub count: u32,
|
||||
#[serde(default)]
|
||||
pub min_vram_gb: Option<u32>,
|
||||
}
|
||||
|
||||
fn default_gpu_name() -> String {
|
||||
"RTX 3090".to_string()
|
||||
}
|
||||
fn default_gpu_count() -> u32 {
|
||||
1
|
||||
}
|
||||
|
||||
impl Default for GpuSpec {
|
||||
fn default() -> Self {
|
||||
Self { name: default_gpu_name(), count: default_gpu_count(), min_vram_gb: None }
|
||||
}
|
||||
}
|
||||
|
||||
/// Offer selection knobs, mirrored from the swactor-vastai `SelectionPolicy`.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SelectionSpec {
|
||||
#[serde(default = "default_min_reliability")]
|
||||
pub min_reliability: f64,
|
||||
#[serde(default = "default_min_down_mbps")]
|
||||
pub min_down_mbps: f64,
|
||||
#[serde(default)]
|
||||
pub min_up_mbps: Option<f64>,
|
||||
#[serde(default = "default_require_verified")]
|
||||
pub require_verified: bool,
|
||||
#[serde(default)]
|
||||
pub max_price_per_hour: Option<f64>,
|
||||
}
|
||||
|
||||
fn default_min_reliability() -> f64 {
|
||||
0.97
|
||||
}
|
||||
fn default_min_down_mbps() -> f64 {
|
||||
100.0
|
||||
}
|
||||
fn default_require_verified() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
impl Default for SelectionSpec {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
min_reliability: default_min_reliability(),
|
||||
min_down_mbps: default_min_down_mbps(),
|
||||
min_up_mbps: None,
|
||||
require_verified: default_require_verified(),
|
||||
max_price_per_hour: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
386
crates/job-runner/src/node.rs
Normal file
386
crates/job-runner/src/node.rs
Normal file
|
|
@ -0,0 +1,386 @@
|
|||
//! Node-side job actor — the spec §5/§7 executor. It receives orchestrator
|
||||
//! commands over the actor plane and runs `setup`/`run` as supervised processes
|
||||
//! via `swactor-process`. Bulk bytes (workspace push / output pull) travel either
|
||||
//! as chunked actor messages (the in-process test path) or over the EDGE_ALPN
|
||||
//! byte transport driven by the integration layer (the real iroh path); which
|
||||
//! path is used is decided by which optional edge capabilities the constructor is
|
||||
//! given. Lifecycle stays on the actor plane either way.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::io::Cursor;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use parking_lot::Mutex;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use swactor::actor::{ActorAddress, ActorInterface, Ctx};
|
||||
use swactor::runtime::ExternalSender;
|
||||
use swactor_process::{
|
||||
ExitStatus, ProcessOutput, ProcessOutputConfig, ProcessSpec, spawn_local_process,
|
||||
};
|
||||
use crate::orchestrator::OrchestratorJobMsg;
|
||||
use crate::wire::{EDGE_RECORD_SIZE, JobEdgeSink, NodeJobCommand, NodeJobEvent, OutputChunk, CHUNK_SIZE};
|
||||
|
||||
/// Which supervised phase a process exit belongs to.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum JobPhase {
|
||||
Setup,
|
||||
Run,
|
||||
}
|
||||
|
||||
/// The node-side executor. `Incoming` is the orchestrator↔node wire command.
|
||||
pub struct NodeJobActor {
|
||||
orchestrator: ActorAddress,
|
||||
workdir: PathBuf,
|
||||
sender: ExternalSender,
|
||||
job_id: u64,
|
||||
workspace_buf: Vec<u8>,
|
||||
/// Edge-mode workspace-ready flag. When set, the orchestrator pushed the
|
||||
/// workspace tar over EDGE_ALPN (drained + extracted by the integration
|
||||
/// layer); `MaterializeWorkspace` waits for it before emitting
|
||||
/// `WorkspaceMaterialized`. `None` ⇒ in-process chunk path.
|
||||
workspace_ready: Option<Arc<AtomicBool>>,
|
||||
/// Edge-mode output sink slot, filled by the integration layer once the iroh
|
||||
/// connection to the orchestrator is up. `None` ⇒ chunk path (outputs stream
|
||||
/// back as `OutputChunk` actor messages).
|
||||
output_sink: Option<Arc<Mutex<Option<Box<dyn JobEdgeSink>>>>>,
|
||||
}
|
||||
|
||||
/// How long the node waits for an edge-mode workspace transfer to land before
|
||||
/// faulting. Generous because the workspace is pushed over a relay.
|
||||
const WORKSPACE_EDGE_WAIT: Duration = Duration::from_secs(60 * 10);
|
||||
/// How long the node waits for the integration layer to arm the output edge sink.
|
||||
const OUTPUT_EDGE_ARM_WAIT: Duration = Duration::from_secs(60 * 5);
|
||||
/// Poll interval for edge-mode spin-waits inside the actor.
|
||||
const EDGE_SPIN: Duration = Duration::from_millis(25);
|
||||
|
||||
impl NodeJobActor {
|
||||
pub fn new(
|
||||
orchestrator: ActorAddress,
|
||||
workdir: PathBuf,
|
||||
sender: ExternalSender,
|
||||
job_id: u64,
|
||||
) -> Self {
|
||||
Self {
|
||||
orchestrator,
|
||||
workdir,
|
||||
sender,
|
||||
job_id,
|
||||
workspace_buf: Vec::new(),
|
||||
workspace_ready: None,
|
||||
output_sink: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Edge mode: workspace bytes arrive over EDGE_ALPN and are extracted by the
|
||||
/// integration layer, which sets `flag` once the workspace is on disk.
|
||||
pub fn with_workspace_ready(mut self, flag: Arc<AtomicBool>) -> Self {
|
||||
self.workspace_ready = Some(flag);
|
||||
self
|
||||
}
|
||||
|
||||
/// Edge mode: ship collected outputs over EDGE_ALPN through `slot`, which the
|
||||
/// integration layer fills with a byte sink once the connection is up.
|
||||
pub fn with_output_sink_slot(
|
||||
mut self,
|
||||
slot: Arc<Mutex<Option<Box<dyn JobEdgeSink>>>>,
|
||||
) -> Self {
|
||||
self.output_sink = Some(slot);
|
||||
self
|
||||
}
|
||||
|
||||
fn emit(&self, ctx: &Ctx, event: NodeJobEvent) {
|
||||
let _ = ctx.send(self.orchestrator, OrchestratorJobMsg::NodeEvent(event));
|
||||
}
|
||||
|
||||
fn spawn_supervised(&self, ctx: &Ctx, phase: JobPhase, command: String, env: &BTreeMap<String, String>) {
|
||||
let relay = match ctx.spawn(ProcessExitRelay {
|
||||
orchestrator: self.orchestrator,
|
||||
phase,
|
||||
job_id: self.job_id,
|
||||
}) {
|
||||
Ok(addr) => addr,
|
||||
Err(e) => {
|
||||
self.emit(ctx, NodeJobEvent::NodeFault {
|
||||
job_id: self.job_id,
|
||||
reason: format!("spawn relay: {e}"),
|
||||
});
|
||||
return;
|
||||
}
|
||||
};
|
||||
let spec = ProcessSpec {
|
||||
command: "bash".to_owned(),
|
||||
args: vec!["-c".to_owned(), command],
|
||||
env: env.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
|
||||
working_dir: Some(self.workdir.clone()),
|
||||
label: Some(format!("job-runner-{:?}", phase).to_lowercase()),
|
||||
};
|
||||
if let Err(e) = spawn_local_process(ctx, &self.sender, spec, ProcessOutputConfig::disabled(relay)) {
|
||||
self.emit(ctx, NodeJobEvent::NodeFault {
|
||||
job_id: self.job_id,
|
||||
reason: format!("spawn process: {e}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Edge-mode output collection: pack every declared output into a single tar
|
||||
/// and ship it over EDGE_ALPN, then announce collection. Dropping the sink
|
||||
/// finishes the edge stream so the orchestrator observes end-of-stream.
|
||||
fn collect_outputs_edge(
|
||||
&self,
|
||||
ctx: &Ctx,
|
||||
job_id: u64,
|
||||
outputs: &[String],
|
||||
slot: Arc<Mutex<Option<Box<dyn JobEdgeSink>>>>,
|
||||
) {
|
||||
let sink = match self.take_output_sink(ctx, job_id, &slot) {
|
||||
Some(sink) => sink,
|
||||
None => return, // already faulted while waiting for the sink
|
||||
};
|
||||
let bytes = match pack_outputs_tar(&self.workdir, outputs) {
|
||||
Ok(b) => b,
|
||||
Err(e) => {
|
||||
self.emit(
|
||||
ctx,
|
||||
NodeJobEvent::NodeFault {
|
||||
job_id,
|
||||
reason: format!("pack outputs: {e}"),
|
||||
},
|
||||
);
|
||||
// Finish the stream + announce so the lifecycle FSM can terminate.
|
||||
let _ = sink.send_bytes(Vec::new());
|
||||
drop(sink);
|
||||
self.emit(ctx, NodeJobEvent::OutputsCollected { job_id });
|
||||
return;
|
||||
}
|
||||
};
|
||||
let mut send_err: Option<String> = None;
|
||||
for record in bytes.chunks(EDGE_RECORD_SIZE) {
|
||||
if let Err(e) = sink.send_bytes(record.to_vec()) {
|
||||
send_err = Some(e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if let Some(e) = send_err {
|
||||
self.emit(
|
||||
ctx,
|
||||
NodeJobEvent::NodeFault {
|
||||
job_id,
|
||||
reason: format!("edge output send: {e}"),
|
||||
},
|
||||
);
|
||||
}
|
||||
drop(sink);
|
||||
self.emit(ctx, NodeJobEvent::OutputsCollected { job_id });
|
||||
}
|
||||
|
||||
/// Take the edge output sink from the shared slot, waiting briefly for the
|
||||
/// integration layer to arm it. Emits a fault and returns `None` on timeout.
|
||||
fn take_output_sink(
|
||||
&self,
|
||||
ctx: &Ctx,
|
||||
job_id: u64,
|
||||
slot: &Arc<Mutex<Option<Box<dyn JobEdgeSink>>>>,
|
||||
) -> Option<Box<dyn JobEdgeSink>> {
|
||||
let deadline = Instant::now() + OUTPUT_EDGE_ARM_WAIT;
|
||||
loop {
|
||||
{
|
||||
let mut guard = slot.lock();
|
||||
if guard.is_some() {
|
||||
return guard.take();
|
||||
}
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
self.emit(
|
||||
ctx,
|
||||
NodeJobEvent::NodeFault {
|
||||
job_id,
|
||||
reason: "output edge sink was never armed".to_owned(),
|
||||
},
|
||||
);
|
||||
return None;
|
||||
}
|
||||
std::thread::sleep(EDGE_SPIN);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
impl ActorInterface for NodeJobActor {
|
||||
type Incoming = NodeJobCommand;
|
||||
type Response = ();
|
||||
|
||||
fn handle(&mut self, ctx: &Ctx, cmd: NodeJobCommand) {
|
||||
match cmd {
|
||||
NodeJobCommand::MaterializeWorkspace { job_id } => {
|
||||
if let Some(flag) = self.workspace_ready.as_ref() {
|
||||
// Edge mode: the workspace tar traveled over EDGE_ALPN and
|
||||
// was extracted into `workdir` by the integration layer.
|
||||
// Wait for its readiness signal before announcing ready.
|
||||
let deadline = Instant::now() + WORKSPACE_EDGE_WAIT;
|
||||
while !flag.load(Ordering::Acquire) {
|
||||
if Instant::now() >= deadline {
|
||||
self.emit(
|
||||
ctx,
|
||||
NodeJobEvent::NodeFault {
|
||||
job_id,
|
||||
reason: "workspace edge transfer did not land".to_owned(),
|
||||
},
|
||||
);
|
||||
return;
|
||||
}
|
||||
std::thread::sleep(EDGE_SPIN);
|
||||
}
|
||||
self.emit(ctx, NodeJobEvent::WorkspaceMaterialized { job_id });
|
||||
} else {
|
||||
// Chunk mode: clear the buffer; bytes arrive as WorkspaceChunk.
|
||||
self.workspace_buf.clear();
|
||||
}
|
||||
}
|
||||
NodeJobCommand::WorkspaceChunk { job_id, data, eof, .. } => {
|
||||
self.workspace_buf.extend_from_slice(&data);
|
||||
if eof {
|
||||
let buf = std::mem::take(&mut self.workspace_buf);
|
||||
match extract_tar(&buf, &self.workdir) {
|
||||
Ok(()) => self.emit(ctx, NodeJobEvent::WorkspaceMaterialized { job_id }),
|
||||
Err(e) => self.emit(ctx, NodeJobEvent::NodeFault { job_id, reason: format!("untar workspace: {e}") }),
|
||||
}
|
||||
}
|
||||
}
|
||||
NodeJobCommand::RunSetup { command, env, .. } => {
|
||||
self.spawn_supervised(ctx, JobPhase::Setup, command, &env);
|
||||
}
|
||||
NodeJobCommand::RunJob { command, env, .. } => {
|
||||
self.spawn_supervised(ctx, JobPhase::Run, command, &env);
|
||||
}
|
||||
NodeJobCommand::CollectOutputs { job_id, outputs } => {
|
||||
if let Some(slot) = self.output_sink.clone() {
|
||||
// Edge mode: pack all outputs into one tar and ship over
|
||||
// EDGE_ALPN, then announce collection. Dropping the sink
|
||||
// finishes the stream so the orchestrator sees end-of-stream.
|
||||
self.collect_outputs_edge(ctx, job_id, &outputs, slot);
|
||||
} else {
|
||||
for name in &outputs {
|
||||
let path = self.workdir.join(name);
|
||||
let bytes = match pack_path(&path) {
|
||||
Ok(b) => b,
|
||||
Err(e) => {
|
||||
self.emit(ctx, NodeJobEvent::NodeFault { job_id, reason: format!("pack output {name}: {e}") });
|
||||
continue;
|
||||
}
|
||||
};
|
||||
stream_chunks(ctx, self.orchestrator, self.job_id, name, &bytes);
|
||||
}
|
||||
self.emit(ctx, NodeJobEvent::OutputsCollected { job_id });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Relays `swactor-process` exits to the orchestrator as lifecycle events.
|
||||
pub struct ProcessExitRelay {
|
||||
pub orchestrator: ActorAddress,
|
||||
pub phase: JobPhase,
|
||||
pub job_id: u64,
|
||||
}
|
||||
|
||||
impl ActorInterface for ProcessExitRelay {
|
||||
type Incoming = ProcessOutput;
|
||||
type Response = ();
|
||||
|
||||
fn handle(&mut self, ctx: &Ctx, output: ProcessOutput) {
|
||||
if let ProcessOutput::Exited { status } = output {
|
||||
let event = match (self.phase, status) {
|
||||
(JobPhase::Setup, ExitStatus::Code(0)) => NodeJobEvent::SetupCompleted { job_id: self.job_id },
|
||||
(JobPhase::Setup, ExitStatus::Code(c)) => NodeJobEvent::NodeFault {
|
||||
job_id: self.job_id,
|
||||
reason: format!("setup exited {c}"),
|
||||
},
|
||||
(JobPhase::Setup, _) => NodeJobEvent::NodeFault {
|
||||
job_id: self.job_id,
|
||||
reason: "setup exited without a code".to_owned(),
|
||||
},
|
||||
(JobPhase::Run, ExitStatus::Code(c)) => NodeJobEvent::JobExited { job_id: self.job_id, code: c },
|
||||
(JobPhase::Run, _) => NodeJobEvent::JobExited { job_id: self.job_id, code: 1 },
|
||||
};
|
||||
let _ = ctx.send(self.orchestrator, OrchestratorJobMsg::NodeEvent(event));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn extract_tar(bytes: &[u8], dst: &std::path::Path) -> Result<(), String> {
|
||||
std::fs::create_dir_all(dst).map_err(|e| e.to_string())?;
|
||||
let mut archive = tar::Archive::new(Cursor::new(bytes));
|
||||
archive.unpack(dst).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
fn pack_path(path: &std::path::Path) -> Result<Vec<u8>, String> {
|
||||
let mut buf = Vec::new();
|
||||
let mut builder = tar::Builder::new(&mut buf);
|
||||
let name = path.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| "output".into());
|
||||
if path.is_dir() {
|
||||
builder.append_dir_all(&name, path).map_err(|e| e.to_string())?;
|
||||
} else if path.is_file() {
|
||||
let mut f = std::fs::File::open(path).map_err(|e| e.to_string())?;
|
||||
builder.append_file(&name, &mut f).map_err(|e| e.to_string())?;
|
||||
} else {
|
||||
// Missing outputs are skipped (best-effort): emit an empty eof chunk.
|
||||
}
|
||||
builder.finish().map_err(|e| e.to_string())?;
|
||||
drop(builder);
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
/// Pack every declared output (relative to `workdir`) into a single tar, each
|
||||
/// entry named by its declared output path. Missing outputs are skipped
|
||||
/// (best-effort). Used by the edge-mode output path to ship all outputs in one
|
||||
/// EDGE_ALPN stream.
|
||||
fn pack_outputs_tar(workdir: &Path, outputs: &[String]) -> Result<Vec<u8>, String> {
|
||||
let mut buf = Vec::new();
|
||||
let mut builder = tar::Builder::new(&mut buf);
|
||||
for name in outputs {
|
||||
let path = workdir.join(name);
|
||||
if path.is_dir() {
|
||||
builder.append_dir_all(name, &path).map_err(|e| e.to_string())?;
|
||||
} else if path.is_file() {
|
||||
let mut f = std::fs::File::open(&path).map_err(|e| e.to_string())?;
|
||||
builder.append_file(name, &mut f).map_err(|e| e.to_string())?;
|
||||
}
|
||||
// Missing outputs are skipped (best-effort).
|
||||
}
|
||||
builder.finish().map_err(|e| e.to_string())?;
|
||||
drop(builder);
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
fn stream_chunks(ctx: &Ctx, orchestrator: ActorAddress, job_id: u64, name: &str, bytes: &[u8]) {
|
||||
if bytes.is_empty() {
|
||||
let _ = ctx.send(
|
||||
orchestrator,
|
||||
OrchestratorJobMsg::OutputChunk(OutputChunk {
|
||||
job_id,
|
||||
name: name.to_owned(),
|
||||
seq: 0,
|
||||
data: Vec::new(),
|
||||
eof: true,
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
let chunks: Vec<&[u8]> = bytes.chunks(CHUNK_SIZE).collect();
|
||||
let total = chunks.len() as u64;
|
||||
for (i, chunk) in chunks.iter().enumerate() {
|
||||
let _ = ctx.send(
|
||||
orchestrator,
|
||||
OrchestratorJobMsg::OutputChunk(OutputChunk {
|
||||
job_id,
|
||||
name: name.to_owned(),
|
||||
seq: i as u64,
|
||||
data: chunk.to_vec(),
|
||||
eof: i as u64 + 1 == total,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
361
crates/job-runner/src/orchestrator.rs
Normal file
361
crates/job-runner/src/orchestrator.rs
Normal file
|
|
@ -0,0 +1,361 @@
|
|||
//! Orchestrator-side job FSM actor — spec §6. It owns the job's lifecycle,
|
||||
//! claims a ready node, drives the transition table, routes commands to the node
|
||||
//! job actor over the actor plane, streams the workspace to the node, and
|
||||
//! reassembles output chunks the node streams back.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::io::Cursor;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use swactor::actor::{ActorAddress, ActorInterface, Ctx};
|
||||
use swactor_transport::NetworkMessage;
|
||||
|
||||
use crate::fsm::{transition, JobCommand, JobEvent, JobState, TransitionCtx};
|
||||
use crate::model::Job;
|
||||
use crate::wire::{NodeJobCommand, NodeJobEvent, OutputChunk, CHUNK_SIZE};
|
||||
|
||||
/// Reports a terminal job state to an observer address.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct JobDone {
|
||||
pub state: JobState,
|
||||
pub exit_code: Option<i32>,
|
||||
}
|
||||
|
||||
/// Messages the orchestrator job actor accepts. A `NetworkMessage` so it can
|
||||
/// cross runtime boundaries (the node reports events/chunks back over iroh).
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub enum OrchestratorJobMsg {
|
||||
Submit { job: Job, node_actor: ActorAddress },
|
||||
NodeEvent(NodeJobEvent),
|
||||
OutputChunk(OutputChunk),
|
||||
}
|
||||
|
||||
impl NetworkMessage for OrchestratorJobMsg {
|
||||
fn type_tag() -> &'static str {
|
||||
"job_runner::OrchestratorJobMsg"
|
||||
}
|
||||
}
|
||||
|
||||
pub struct OrchestratorJobActor {
|
||||
report_to: ActorAddress,
|
||||
node: Option<ActorAddress>,
|
||||
state: JobState,
|
||||
ctx_fsm: TransitionCtx,
|
||||
job: Option<Job>,
|
||||
exit_code: Option<i32>,
|
||||
landing: PathBuf,
|
||||
output_bufs: HashMap<String, Vec<u8>>,
|
||||
pending_outputs: HashSet<String>,
|
||||
outputs_collected: bool,
|
||||
/// Edge mode: bulk bytes (workspace + outputs) travel over EDGE_ALPN,
|
||||
/// driven by the integration layer, not as actor messages. `false` keeps the
|
||||
/// in-process chunk path (what the tests exercise).
|
||||
edge_mode: bool,
|
||||
}
|
||||
|
||||
impl OrchestratorJobActor {
|
||||
const JOB_ID: u64 = 0;
|
||||
|
||||
pub fn new(report_to: ActorAddress, landing: PathBuf) -> Self {
|
||||
Self {
|
||||
report_to,
|
||||
node: None,
|
||||
state: JobState::Pending,
|
||||
ctx_fsm: TransitionCtx::new(false, false),
|
||||
job: None,
|
||||
exit_code: None,
|
||||
landing,
|
||||
output_bufs: HashMap::new(),
|
||||
pending_outputs: HashSet::new(),
|
||||
outputs_collected: false,
|
||||
edge_mode: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Edge mode: bulk bytes (workspace + outputs) travel over the EDGE_ALPN
|
||||
/// transport driven by the integration layer, not as actor messages. The
|
||||
/// orchestrator still emits the small `MaterializeWorkspace`/`CollectOutputs`
|
||||
/// commands and drives the lifecycle FSM. Defaults to `false` (chunk path),
|
||||
/// which is what the in-process tests exercise.
|
||||
pub fn with_edge_mode(mut self, edge: bool) -> Self {
|
||||
self.edge_mode = edge;
|
||||
self
|
||||
}
|
||||
|
||||
fn emit_command(&mut self, ctx: &Ctx, cmd: JobCommand) {
|
||||
let Some(node) = self.node else { return };
|
||||
let job_id = Self::JOB_ID;
|
||||
match cmd {
|
||||
JobCommand::MaterializeWorkspace => {
|
||||
let _ = ctx.send(node, NodeJobCommand::MaterializeWorkspace { job_id });
|
||||
if self.edge_mode {
|
||||
// Workspace bytes travel over EDGE_ALPN (the integration layer
|
||||
// pushed them before the job was submitted).
|
||||
return;
|
||||
}
|
||||
if let Some(job) = self.job.as_ref() {
|
||||
match pack_workspace(job) {
|
||||
Ok(bytes) => stream_workspace(ctx, node, job_id, &bytes),
|
||||
Err(e) => {
|
||||
eprintln!("job-runner: pack workspace failed: {e}");
|
||||
self.state = JobState::Failed;
|
||||
self.finish(ctx);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
JobCommand::RunSetup => {
|
||||
let _ = ctx.send(node, NodeJobCommand::RunSetup {
|
||||
job_id,
|
||||
command: self.job.as_ref().and_then(|j| j.setup.clone()).unwrap_or_default(),
|
||||
env: self.job.as_ref().map(|j| j.env.clone()).unwrap_or_default(),
|
||||
});
|
||||
}
|
||||
JobCommand::RunJob => {
|
||||
let _ = ctx.send(node, NodeJobCommand::RunJob {
|
||||
job_id,
|
||||
command: self.job.as_ref().map(|j| j.run.clone()).unwrap_or_default(),
|
||||
env: self.job.as_ref().map(|j| j.env.clone()).unwrap_or_default(),
|
||||
});
|
||||
}
|
||||
JobCommand::CollectOutputs => {
|
||||
let outputs = self.job.as_ref().map(|j| j.outputs.clone()).unwrap_or_default();
|
||||
self.outputs_collected = false;
|
||||
if !self.edge_mode {
|
||||
// Chunk mode: track each output until its eof chunk lands.
|
||||
// Edge mode ships outputs over EDGE_ALPN; the integration
|
||||
// layer drains them, so there are no chunks to wait for.
|
||||
self.pending_outputs = outputs.iter().cloned().collect();
|
||||
}
|
||||
let _ = ctx.send(node, NodeJobCommand::CollectOutputs { job_id, outputs });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn finish(&mut self, ctx: &Ctx) {
|
||||
let _ = ctx.send(
|
||||
self.report_to,
|
||||
JobDone {
|
||||
state: self.state,
|
||||
exit_code: self.exit_code,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
fn observe_outputs_collected(&mut self, ctx: &Ctx) {
|
||||
self.outputs_collected = true;
|
||||
self.finish_if_output_collection_complete(ctx);
|
||||
}
|
||||
|
||||
fn finish_if_output_collection_complete(&mut self, ctx: &Ctx) {
|
||||
if !self.outputs_collected || !self.pending_outputs.is_empty() {
|
||||
return;
|
||||
}
|
||||
self.outputs_collected = false;
|
||||
let (new_state, cmd) = transition(self.state, &JobEvent::OutputsCollected, self.ctx_fsm);
|
||||
self.state = new_state;
|
||||
if matches!(self.state, JobState::Completed | JobState::Failed) {
|
||||
self.finish(ctx);
|
||||
return;
|
||||
}
|
||||
if let Some(cmd) = cmd {
|
||||
self.emit_command(ctx, cmd);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ActorInterface for OrchestratorJobActor {
|
||||
type Incoming = OrchestratorJobMsg;
|
||||
type Response = ();
|
||||
|
||||
fn handle(&mut self, ctx: &Ctx, msg: OrchestratorJobMsg) {
|
||||
match msg {
|
||||
OrchestratorJobMsg::Submit { job, node_actor } => {
|
||||
self.node = Some(node_actor);
|
||||
self.ctx_fsm = TransitionCtx::new(job.has_workspace(), job.has_setup());
|
||||
self.job = Some(job);
|
||||
let (s, _) = transition(self.state, &JobEvent::JobSubmitted, self.ctx_fsm);
|
||||
self.state = s;
|
||||
let (s, cmd) = transition(self.state, &JobEvent::NodeReady, self.ctx_fsm);
|
||||
self.state = s;
|
||||
if let Some(cmd) = cmd {
|
||||
self.emit_command(ctx, cmd);
|
||||
}
|
||||
}
|
||||
OrchestratorJobMsg::NodeEvent(ev) => {
|
||||
if let NodeJobEvent::JobExited { code, .. } = ev {
|
||||
self.exit_code = Some(code);
|
||||
self.ctx_fsm.prior_exit = Some(code);
|
||||
}
|
||||
let Some(fsm_ev) = (match ev {
|
||||
NodeJobEvent::WorkspaceMaterialized { .. } => {
|
||||
Some(JobEvent::WorkspaceMaterialized)
|
||||
}
|
||||
NodeJobEvent::SetupCompleted { .. } => Some(JobEvent::SetupCompleted),
|
||||
NodeJobEvent::JobExited { code, .. } => Some(JobEvent::JobExited(code)),
|
||||
NodeJobEvent::OutputsCollected { .. } => {
|
||||
self.observe_outputs_collected(ctx);
|
||||
None
|
||||
}
|
||||
NodeJobEvent::NodeFault { reason, .. } => Some(JobEvent::NodeFault(reason)),
|
||||
}) else {
|
||||
return;
|
||||
};
|
||||
let (new_state, cmd) = transition(self.state, &fsm_ev, self.ctx_fsm);
|
||||
self.state = new_state;
|
||||
if matches!(self.state, JobState::Completed | JobState::Failed) {
|
||||
self.finish(ctx);
|
||||
return;
|
||||
}
|
||||
if let Some(cmd) = cmd {
|
||||
self.emit_command(ctx, cmd);
|
||||
}
|
||||
}
|
||||
OrchestratorJobMsg::OutputChunk(chunk) => {
|
||||
let entry = self.output_bufs.entry(chunk.name.clone()).or_default();
|
||||
entry.extend_from_slice(&chunk.data);
|
||||
if chunk.eof {
|
||||
let bytes = self.output_bufs.remove(&chunk.name).unwrap_or_default();
|
||||
if !bytes.is_empty() {
|
||||
if let Err(e) = extract_tar(&bytes, &self.landing) {
|
||||
eprintln!("job-runner: untar output `{}` failed: {e}", chunk.name);
|
||||
}
|
||||
}
|
||||
self.pending_outputs.remove(&chunk.name);
|
||||
self.finish_if_output_collection_complete(ctx);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Pack the job's workspace tree (minus `exclude` globs) into a tar. Returns an
|
||||
/// empty buffer when the job declares no workspace. Exposed so the integration
|
||||
/// layer can push the workspace over EDGE_ALPN without touching the actor plane.
|
||||
pub fn pack_workspace(job: &Job) -> Result<Vec<u8>, String> {
|
||||
match job.workspace.as_ref() {
|
||||
Some(ws) => pack_dir(&ws.workdir, &ws.exclude),
|
||||
None => Ok(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
fn pack_dir(src: &Path, exclude: &[String]) -> Result<Vec<u8>, String> {
|
||||
let mut buf = Vec::new();
|
||||
let mut builder = tar::Builder::new(&mut buf);
|
||||
append_dir_filtered(&mut builder, src, src, exclude)?;
|
||||
builder.finish().map_err(|e| e.to_string())?;
|
||||
drop(builder);
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
fn append_dir_filtered(
|
||||
builder: &mut tar::Builder<&mut Vec<u8>>,
|
||||
root: &Path,
|
||||
dir: &Path,
|
||||
exclude: &[String],
|
||||
) -> Result<(), String> {
|
||||
let mut entries = std::fs::read_dir(dir)
|
||||
.map_err(|e| format!("read dir {}: {e}", dir.display()))?
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(|e| e.to_string())?;
|
||||
entries.sort_by_key(|entry| entry.path());
|
||||
for entry in entries {
|
||||
let path = entry.path();
|
||||
let rel = path.strip_prefix(root).map_err(|e| e.to_string())?;
|
||||
if is_excluded(rel, exclude) {
|
||||
continue;
|
||||
}
|
||||
let file_type = entry.file_type().map_err(|e| e.to_string())?;
|
||||
if file_type.is_dir() {
|
||||
append_dir_filtered(builder, root, &path, exclude)?;
|
||||
} else if file_type.is_file() {
|
||||
builder.append_path_with_name(&path, rel).map_err(|e| e.to_string())?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn is_excluded(rel: &Path, exclude: &[String]) -> bool {
|
||||
let rel_text = path_slash(rel);
|
||||
let name = rel.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
|
||||
let components = rel.iter().map(|c| c.to_string_lossy().to_string()).collect::<Vec<_>>();
|
||||
for raw in exclude {
|
||||
let pattern = raw.trim();
|
||||
if pattern.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if let Some(anchored) = pattern.strip_prefix('/') {
|
||||
let anchored = anchored.trim_matches('/');
|
||||
if rel_text == anchored || rel_text.starts_with(&format!("{anchored}/")) {
|
||||
return true;
|
||||
}
|
||||
} else if pattern.contains('*') {
|
||||
if wildcard_matches(pattern, &rel_text) || wildcard_matches(pattern, &name) {
|
||||
return true;
|
||||
}
|
||||
} else if components.iter().any(|component| component == pattern) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn path_slash(path: &Path) -> String {
|
||||
path.iter().map(|c| c.to_string_lossy()).collect::<Vec<_>>().join("/")
|
||||
}
|
||||
|
||||
fn wildcard_matches(pattern: &str, text: &str) -> bool {
|
||||
let parts = pattern.split('*').collect::<Vec<_>>();
|
||||
if parts.len() == 1 {
|
||||
return pattern == text;
|
||||
}
|
||||
let mut pos = 0;
|
||||
for (idx, part) in parts.iter().enumerate() {
|
||||
if part.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if idx == 0 && !pattern.starts_with('*') {
|
||||
if !text[pos..].starts_with(part) {
|
||||
return false;
|
||||
}
|
||||
pos += part.len();
|
||||
continue;
|
||||
}
|
||||
let Some(found) = text[pos..].find(part) else {
|
||||
return false;
|
||||
};
|
||||
pos += found + part.len();
|
||||
}
|
||||
pattern.ends_with('*') || parts.last().is_none_or(|last| text.ends_with(last))
|
||||
}
|
||||
|
||||
fn extract_tar(bytes: &[u8], dst: &Path) -> Result<(), String> {
|
||||
std::fs::create_dir_all(dst).map_err(|e| e.to_string())?;
|
||||
tar::Archive::new(Cursor::new(bytes)).unpack(dst).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
fn stream_workspace(ctx: &Ctx, node: ActorAddress, job_id: u64, bytes: &[u8]) {
|
||||
if bytes.is_empty() {
|
||||
let _ = ctx.send(
|
||||
node,
|
||||
NodeJobCommand::WorkspaceChunk {
|
||||
job_id,
|
||||
seq: 0,
|
||||
data: Vec::new(),
|
||||
eof: true,
|
||||
},
|
||||
);
|
||||
return;
|
||||
}
|
||||
let chunks: Vec<&[u8]> = bytes.chunks(CHUNK_SIZE).collect();
|
||||
let total = chunks.len() as u64;
|
||||
for (i, chunk) in chunks.iter().enumerate() {
|
||||
let _ = ctx.send(node, NodeJobCommand::WorkspaceChunk {
|
||||
job_id,
|
||||
seq: i as u64,
|
||||
data: chunk.to_vec(),
|
||||
eof: i as u64 + 1 == total,
|
||||
});
|
||||
}
|
||||
}
|
||||
98
crates/job-runner/src/wire.rs
Normal file
98
crates/job-runner/src/wire.rs
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
//! Orchestrator↔node wire protocol — spec §6 command/event tables, plus the
|
||||
//! bulk-transfer frames that carry workspace/output bytes over the same actor
|
||||
//! plane (the shared iroh endpoint's actor ALPN).
|
||||
//!
|
||||
//! These are `NetworkMessage`s: they cross runtime boundaries (local mailbox or
|
||||
use crate::orchestrator::OrchestratorJobMsg;
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use swactor_transport::{CodecRegistry, JsonCodec, NetworkMessage};
|
||||
|
||||
/// Maximum payload bytes per chunk (workspace/output bytes are streamed).
|
||||
pub const CHUNK_SIZE: usize = 64 * 1024;
|
||||
|
||||
/// Maximum bytes per EDGE_ALPN byte record when shipping bulk tars. Keeping
|
||||
/// records modest bounds the per-write time over a relay so the edge send pump's
|
||||
/// write watchdog never trips on a large workspace/output transfer.
|
||||
pub const EDGE_RECORD_SIZE: usize = 1024 * 1024;
|
||||
|
||||
/// Logical edge-stream id for the orchestrator→worker workspace tar, carried on
|
||||
/// the EDGE_ALPN byte transport instead of the actor plane. Public so the
|
||||
/// substrate-agnostic job-runner crate and the iroh integration layer agree.
|
||||
pub const WORKSPACE_EDGE_ID: u64 = 1;
|
||||
|
||||
/// Logical edge-stream id for the worker→orchestrator outputs tar.
|
||||
pub const OUTPUTS_EDGE_ID: u64 = 2;
|
||||
|
||||
/// Substrate-agnostic bulk byte sink for the EDGE_ALPN path. The integration
|
||||
/// layer (e.g. `job_deploy`) implements this around the real iroh edge handle;
|
||||
/// the job-runner crate stays free of any iroh dependency. When a node actor is
|
||||
/// built without a sink (`None`), it falls back to streaming bytes as actor
|
||||
/// `OutputChunk` messages — the path the in-process tests exercise.
|
||||
pub trait JobEdgeSink: Send + Sync + 'static {
|
||||
/// Enqueue one byte record. Send all records, then drop the sink so the
|
||||
/// underlying transport finishes the stream and the receiver observes
|
||||
/// end-of-stream.
|
||||
fn send_bytes(&self, bytes: Vec<u8>) -> Result<(), String>;
|
||||
}
|
||||
|
||||
/// Orchestrator → node commands. Spec §6 command table + workspace chunks.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum NodeJobCommand {
|
||||
MaterializeWorkspace { job_id: u64 },
|
||||
/// One chunk of the workspace tar stream. `eof` marks the final chunk; the
|
||||
/// node extracts the accumulated tar on the eof chunk.
|
||||
WorkspaceChunk { job_id: u64, seq: u64, data: Vec<u8>, eof: bool },
|
||||
RunSetup { job_id: u64, command: String, env: BTreeMap<String, String> },
|
||||
RunJob { job_id: u64, command: String, env: BTreeMap<String, String> },
|
||||
CollectOutputs { job_id: u64, outputs: Vec<String> },
|
||||
}
|
||||
|
||||
impl NetworkMessage for NodeJobCommand {
|
||||
fn type_tag() -> &'static str {
|
||||
"job_runner::NodeJobCommand"
|
||||
}
|
||||
}
|
||||
|
||||
/// Node → orchestrator events. Spec §6 event table.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum NodeJobEvent {
|
||||
WorkspaceMaterialized { job_id: u64 },
|
||||
SetupCompleted { job_id: u64 },
|
||||
JobExited { job_id: u64, code: i32 },
|
||||
OutputsCollected { job_id: u64 },
|
||||
NodeFault { job_id: u64, reason: String },
|
||||
}
|
||||
|
||||
impl NetworkMessage for NodeJobEvent {
|
||||
fn type_tag() -> &'static str {
|
||||
"job_runner::NodeJobEvent"
|
||||
}
|
||||
}
|
||||
|
||||
/// Node → orchestrator output-byte chunk. Separate from lifecycle events so the
|
||||
/// orchestrator can buffer output bytes independently of the FSM.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct OutputChunk {
|
||||
pub job_id: u64,
|
||||
pub name: String,
|
||||
pub seq: u64,
|
||||
pub data: Vec<u8>,
|
||||
pub eof: bool,
|
||||
}
|
||||
|
||||
impl NetworkMessage for OutputChunk {
|
||||
fn type_tag() -> &'static str {
|
||||
"job_runner::OutputChunk"
|
||||
}
|
||||
}
|
||||
|
||||
/// Register the job wire messages with a codec registry (JSON).
|
||||
pub fn register_job_codecs(registry: &mut CodecRegistry) {
|
||||
registry.register::<NodeJobCommand, JsonCodec<NodeJobCommand>>(JsonCodec::default());
|
||||
registry.register::<NodeJobEvent, JsonCodec<NodeJobEvent>>(JsonCodec::default());
|
||||
registry.register::<OrchestratorJobMsg, JsonCodec<OrchestratorJobMsg>>(JsonCodec::default());
|
||||
registry.register::<OutputChunk, JsonCodec<OutputChunk>>(JsonCodec::default());
|
||||
}
|
||||
405
crates/job-runner/tests/job_plugin.rs
Normal file
405
crates/job-runner/tests/job_plugin.rs
Normal file
|
|
@ -0,0 +1,405 @@
|
|||
//! In-engine proof that the job runner runs *through swactor*: an orchestrator
|
||||
//! FSM actor and a node job actor live in one swactor `Engine` over a Tokio
|
||||
//! substrate; control AND bulk transfer (workspace push / output pull) travel as
|
||||
//! actor messages; `setup`/`run` execute as supervised processes via
|
||||
//! `swactor-process`; the exit code is authoritative. No SSH, no shell-out.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use parking_lot::Mutex;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use swactor::actor::{ActorInterface, Ctx, Message};
|
||||
use swactor::runtime::{RuntimeConfig, RuntimeParts};
|
||||
use swactor_engine::{Engine, TokioBackend, TokioConfig};
|
||||
use swactor_job_runner::{
|
||||
Job, JobDone, JobEdgeSink, JobState, NodeJobActor, NodeJobCommand, NodeJobEvent,
|
||||
OrchestratorJobActor, OrchestratorJobMsg, OutputChunk, Workspace,
|
||||
};
|
||||
|
||||
const POLL: Duration = Duration::from_millis(15);
|
||||
const DEADLINE: Duration = Duration::from_secs(15);
|
||||
|
||||
fn recv_within<T: Message>(inbox: &swactor::runtime::Inbox<T>, deadline: Duration) -> Option<T> {
|
||||
let started = Instant::now();
|
||||
loop {
|
||||
if let Some(v) = inbox.try_recv() {
|
||||
return Some(v);
|
||||
}
|
||||
if started.elapsed() >= deadline {
|
||||
return None;
|
||||
}
|
||||
std::thread::sleep(POLL);
|
||||
}
|
||||
}
|
||||
|
||||
struct CommandSink;
|
||||
|
||||
impl ActorInterface for CommandSink {
|
||||
type Incoming = NodeJobCommand;
|
||||
type Response = ();
|
||||
|
||||
fn handle(&mut self, _ctx: &Ctx, _msg: NodeJobCommand) {}
|
||||
}
|
||||
|
||||
fn tar_file(name: &str, contents: &str) -> Vec<u8> {
|
||||
let mut buf = Vec::new();
|
||||
let mut builder = tar::Builder::new(&mut buf);
|
||||
let mut header = tar::Header::new_gnu();
|
||||
header.set_size(contents.len() as u64);
|
||||
header.set_cksum();
|
||||
let mut reader = std::io::Cursor::new(contents.as_bytes());
|
||||
builder.append_data(&mut header, name, &mut reader).expect("append tar file");
|
||||
builder.finish().expect("finish tar");
|
||||
drop(builder);
|
||||
buf
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn job_runs_through_actor_plane_and_swactor_process() {
|
||||
let ws = tempfile::tempdir().expect("ws tempdir");
|
||||
std::fs::write(ws.path().join("seed.txt"), "seed-value").expect("write seed");
|
||||
let node_workdir = tempfile::tempdir().expect("node workdir tempdir");
|
||||
let landing = tempfile::tempdir().expect("landing tempdir");
|
||||
|
||||
let parts = RuntimeParts::new(RuntimeConfig::default());
|
||||
let runtime = parts.runtime().clone();
|
||||
let sender = runtime.create_sender();
|
||||
let engine = Engine::new(parts, TokioBackend::new(TokioConfig::default()).expect("tokio backend"))
|
||||
.expect("engine");
|
||||
|
||||
let done = runtime.new_inbox::<JobDone>().expect("done inbox");
|
||||
let orch = runtime
|
||||
.spawn(OrchestratorJobActor::new(*done.addr(), landing.path().to_path_buf()))
|
||||
.expect("spawn orchestrator");
|
||||
let node = runtime
|
||||
.spawn(NodeJobActor::new(orch, node_workdir.path().to_path_buf(), sender, 0))
|
||||
.expect("spawn node");
|
||||
|
||||
let job = Job {
|
||||
name: "probe".to_owned(),
|
||||
setup: Some("echo setup-ok > setup_done.txt".to_owned()),
|
||||
run: "echo hello-from-swactor > greeting.txt".to_owned(),
|
||||
workspace: Some(Workspace { workdir: ws.path().to_path_buf(), exclude: vec![] }),
|
||||
outputs: vec![
|
||||
"greeting.txt".to_owned(),
|
||||
"setup_done.txt".to_owned(),
|
||||
"seed.txt".to_owned(),
|
||||
],
|
||||
env: BTreeMap::new(),
|
||||
};
|
||||
|
||||
runtime
|
||||
.send_to(orch, OrchestratorJobMsg::Submit { job, node_actor: node })
|
||||
.expect("submit job");
|
||||
|
||||
let result = recv_within(&done, DEADLINE);
|
||||
drop(engine);
|
||||
let done = result.expect("job did not reach a terminal state within deadline");
|
||||
assert_eq!(done.state, JobState::Completed, "expected COMPLETED, got {:?}", done);
|
||||
assert_eq!(done.exit_code, Some(0), "expected exit code 0");
|
||||
|
||||
let greeting = std::fs::read_to_string(landing.path().join("greeting.txt"))
|
||||
.expect("collected greeting.txt");
|
||||
assert!(greeting.contains("hello-from-swactor"), "greeting content: {greeting}");
|
||||
let setup = std::fs::read_to_string(landing.path().join("setup_done.txt"))
|
||||
.expect("collected setup_done.txt");
|
||||
assert!(setup.contains("setup-ok"), "setup content: {setup}");
|
||||
let seed = std::fs::read_to_string(landing.path().join("seed.txt")).expect("collected seed.txt");
|
||||
assert_eq!(seed, "seed-value", "workspace materialized + collected through swactor");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workspace_excludes_are_not_materialized() {
|
||||
let ws = tempfile::tempdir().expect("ws tempdir");
|
||||
std::fs::write(ws.path().join("keep.txt"), "kept").expect("write keep");
|
||||
std::fs::write(ws.path().join("secret.txt"), "secret").expect("write secret");
|
||||
std::fs::create_dir(ws.path().join("skip_dir")).expect("create skip dir");
|
||||
std::fs::write(ws.path().join("skip_dir/hidden.txt"), "hidden").expect("write hidden");
|
||||
let node_workdir = tempfile::tempdir().expect("node workdir tempdir");
|
||||
let landing = tempfile::tempdir().expect("landing tempdir");
|
||||
|
||||
let parts = RuntimeParts::new(RuntimeConfig::default());
|
||||
let runtime = parts.runtime().clone();
|
||||
let sender = runtime.create_sender();
|
||||
let engine = Engine::new(parts, TokioBackend::new(TokioConfig::default()).expect("tokio backend"))
|
||||
.expect("engine");
|
||||
|
||||
let done = runtime.new_inbox::<JobDone>().expect("done inbox");
|
||||
let orch = runtime
|
||||
.spawn(OrchestratorJobActor::new(*done.addr(), landing.path().to_path_buf()))
|
||||
.expect("spawn orchestrator");
|
||||
let node = runtime
|
||||
.spawn(NodeJobActor::new(orch, node_workdir.path().to_path_buf(), sender, 0))
|
||||
.expect("spawn node");
|
||||
|
||||
let job = Job {
|
||||
name: "exclude-probe".to_owned(),
|
||||
setup: None,
|
||||
run: "test ! -e secret.txt && test ! -e skip_dir/hidden.txt && cat keep.txt > result.txt"
|
||||
.to_owned(),
|
||||
workspace: Some(Workspace {
|
||||
workdir: ws.path().to_path_buf(),
|
||||
exclude: vec!["/secret.txt".to_owned(), "/skip_dir".to_owned()],
|
||||
}),
|
||||
outputs: vec!["result.txt".to_owned()],
|
||||
env: BTreeMap::new(),
|
||||
};
|
||||
|
||||
runtime
|
||||
.send_to(orch, OrchestratorJobMsg::Submit { job, node_actor: node })
|
||||
.expect("submit job");
|
||||
|
||||
let result = recv_within(&done, DEADLINE);
|
||||
drop(engine);
|
||||
let done = result.expect("job did not reach terminal state");
|
||||
assert_eq!(done.state, JobState::Completed);
|
||||
assert_eq!(done.exit_code, Some(0));
|
||||
let result = std::fs::read_to_string(landing.path().join("result.txt"))
|
||||
.expect("collected result.txt");
|
||||
assert_eq!(result, "kept");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nonzero_exit_marks_job_failed_after_best_effort_collect() {
|
||||
let ws = tempfile::tempdir().expect("ws tempdir");
|
||||
let node_workdir = tempfile::tempdir().expect("node workdir tempdir");
|
||||
let landing = tempfile::tempdir().expect("landing tempdir");
|
||||
|
||||
let parts = RuntimeParts::new(RuntimeConfig::default());
|
||||
let runtime = parts.runtime().clone();
|
||||
let sender = runtime.create_sender();
|
||||
let engine = Engine::new(parts, TokioBackend::new(TokioConfig::default()).expect("tokio backend"))
|
||||
.expect("engine");
|
||||
|
||||
let done = runtime.new_inbox::<JobDone>().expect("done inbox");
|
||||
let orch = runtime
|
||||
.spawn(OrchestratorJobActor::new(*done.addr(), landing.path().to_path_buf()))
|
||||
.expect("orch");
|
||||
let node = runtime
|
||||
.spawn(NodeJobActor::new(orch, node_workdir.path().to_path_buf(), sender, 0))
|
||||
.expect("node");
|
||||
|
||||
let job = Job {
|
||||
name: "fail-probe".to_owned(),
|
||||
setup: None,
|
||||
run: "echo partial > partial.txt; exit 3".to_owned(),
|
||||
workspace: Some(Workspace { workdir: ws.path().to_path_buf(), exclude: vec![] }),
|
||||
outputs: vec!["partial.txt".to_owned()],
|
||||
env: BTreeMap::new(),
|
||||
};
|
||||
runtime.send_to(orch, OrchestratorJobMsg::Submit { job, node_actor: node }).expect("submit");
|
||||
|
||||
let result = recv_within(&done, DEADLINE);
|
||||
drop(engine);
|
||||
let done = result.expect("job did not terminate");
|
||||
assert_eq!(done.state, JobState::Failed, "expected FAILED");
|
||||
assert_eq!(done.exit_code, Some(3), "expected exit code 3");
|
||||
let partial = std::fs::read_to_string(landing.path().join("partial.txt"));
|
||||
assert!(partial.is_ok(), "best-effort collect should gather partial.txt");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn job_done_waits_for_output_eof_after_outputs_collected_event() {
|
||||
let landing = tempfile::tempdir().expect("landing tempdir");
|
||||
|
||||
let parts = RuntimeParts::new(RuntimeConfig::default());
|
||||
let runtime = parts.runtime().clone();
|
||||
let engine = Engine::new(parts, TokioBackend::new(TokioConfig::default()).expect("tokio backend"))
|
||||
.expect("engine");
|
||||
|
||||
let done = runtime.new_inbox::<JobDone>().expect("done inbox");
|
||||
let orch = runtime
|
||||
.spawn(OrchestratorJobActor::new(*done.addr(), landing.path().to_path_buf()))
|
||||
.expect("orch");
|
||||
let node = runtime.spawn(CommandSink).expect("node command sink");
|
||||
|
||||
let job = Job {
|
||||
name: "reordered-output".to_owned(),
|
||||
setup: None,
|
||||
run: "ignored".to_owned(),
|
||||
workspace: None,
|
||||
outputs: vec!["greeting.txt".to_owned()],
|
||||
env: BTreeMap::new(),
|
||||
};
|
||||
runtime
|
||||
.send_to(orch, OrchestratorJobMsg::Submit { job, node_actor: node })
|
||||
.expect("submit");
|
||||
runtime
|
||||
.send_to(
|
||||
orch,
|
||||
OrchestratorJobMsg::NodeEvent(NodeJobEvent::JobExited { job_id: 0, code: 0 }),
|
||||
)
|
||||
.expect("job exited");
|
||||
runtime
|
||||
.send_to(
|
||||
orch,
|
||||
OrchestratorJobMsg::NodeEvent(NodeJobEvent::OutputsCollected { job_id: 0 }),
|
||||
)
|
||||
.expect("outputs collected");
|
||||
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
assert!(
|
||||
done.try_recv().is_none(),
|
||||
"job completed before output eof arrived"
|
||||
);
|
||||
|
||||
runtime
|
||||
.send_to(
|
||||
orch,
|
||||
OrchestratorJobMsg::OutputChunk(OutputChunk {
|
||||
job_id: 0,
|
||||
name: "greeting.txt".to_owned(),
|
||||
seq: 0,
|
||||
data: tar_file("greeting.txt", "hello-after-event").to_vec(),
|
||||
eof: true,
|
||||
}),
|
||||
)
|
||||
.expect("output chunk");
|
||||
|
||||
let result = recv_within(&done, DEADLINE);
|
||||
drop(engine);
|
||||
let done = result.expect("job did not complete after output eof");
|
||||
assert_eq!(done.state, JobState::Completed);
|
||||
assert_eq!(done.exit_code, Some(0));
|
||||
let greeting = std::fs::read_to_string(landing.path().join("greeting.txt"))
|
||||
.expect("collected greeting after reordered event");
|
||||
assert_eq!(greeting, "hello-after-event");
|
||||
}
|
||||
|
||||
// ─── Edge-mode (EDGE_ALPN bulk bytes) actor contract ────────────────────────
|
||||
//
|
||||
// The integration layer (job_deploy) drives bulk bytes over EDGE_ALPN and arms
|
||||
// the node actor with a `JobEdgeSink` (outputs) and a workspace-ready flag. These
|
||||
// tests exercise that actor-level contract with in-memory mocks, independent of
|
||||
// iroh, and prove the chunk path is NOT taken when the edge capabilities are set.
|
||||
|
||||
/// Records every byte record shipped through a `JobEdgeSink`.
|
||||
struct RecordingSink(Arc<Mutex<Vec<Vec<u8>>>>);
|
||||
|
||||
impl JobEdgeSink for RecordingSink {
|
||||
fn send_bytes(&self, bytes: Vec<u8>) -> Result<(), String> {
|
||||
self.0.lock().push(bytes);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Collects node→orchestrator lifecycle events for assertions.
|
||||
struct EventSink {
|
||||
events: Arc<Mutex<Vec<NodeJobEvent>>>,
|
||||
}
|
||||
|
||||
impl ActorInterface for EventSink {
|
||||
type Incoming = OrchestratorJobMsg;
|
||||
type Response = ();
|
||||
|
||||
fn handle(&mut self, _ctx: &Ctx, msg: OrchestratorJobMsg) {
|
||||
if let OrchestratorJobMsg::NodeEvent(ev) = msg {
|
||||
self.events.lock().push(ev);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn node_actor_edge_mode_ships_outputs_through_sink() {
|
||||
let workdir = tempfile::tempdir().expect("workdir");
|
||||
std::fs::write(workdir.path().join("out.txt"), "edge-output-bytes").expect("write out");
|
||||
|
||||
let parts = RuntimeParts::new(RuntimeConfig::default());
|
||||
let runtime = parts.runtime().clone();
|
||||
let sender = runtime.create_sender();
|
||||
let engine = Engine::new(parts, TokioBackend::new(TokioConfig::default()).expect("tokio"))
|
||||
.expect("engine");
|
||||
|
||||
let events = Arc::new(Mutex::new(Vec::new()));
|
||||
let orch = runtime.spawn(EventSink { events: events.clone() }).expect("spawn event sink");
|
||||
|
||||
let sent = Arc::new(Mutex::new(Vec::new()));
|
||||
let slot: Arc<Mutex<Option<Box<dyn JobEdgeSink>>>> =
|
||||
Arc::new(Mutex::new(Some(Box::new(RecordingSink(sent.clone())))));
|
||||
|
||||
let node = runtime
|
||||
.spawn(NodeJobActor::new(orch, workdir.path().to_path_buf(), sender, 0).with_output_sink_slot(slot))
|
||||
.expect("spawn node");
|
||||
|
||||
runtime
|
||||
.send_to(
|
||||
node,
|
||||
NodeJobCommand::CollectOutputs { job_id: 0, outputs: vec!["out.txt".to_owned()] },
|
||||
)
|
||||
.expect("collect outputs");
|
||||
|
||||
let started = Instant::now();
|
||||
while started.elapsed() < DEADLINE {
|
||||
if events.lock().iter().any(|ev| {
|
||||
matches!(ev, NodeJobEvent::OutputsCollected { job_id: 0 })
|
||||
}) {
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(POLL);
|
||||
}
|
||||
drop(engine);
|
||||
|
||||
let recorded = sent.lock();
|
||||
assert_eq!(recorded.len(), 1, "edge sink received exactly one byte record");
|
||||
// The record is a tar containing out.txt; extracting it round-trips the bytes.
|
||||
let mut archive = tar::Archive::new(std::io::Cursor::new(&recorded[0]));
|
||||
let mut entries = archive.entries().expect("tar entries");
|
||||
let entry = entries.next().expect("an entry").expect("entry ok");
|
||||
assert_eq!(entry.path().unwrap().to_string_lossy(), "out.txt");
|
||||
|
||||
let collected = events.lock();
|
||||
assert!(
|
||||
collected.iter().any(|ev| matches!(ev, NodeJobEvent::OutputsCollected { job_id: 0 })),
|
||||
"edge collect emitted OutputsCollected, got {:?}", collected
|
||||
);
|
||||
assert!(
|
||||
!collected.iter().any(|ev| matches!(ev, NodeJobEvent::NodeFault { .. })),
|
||||
"no fault expected, got {:?}", collected
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn node_actor_edge_mode_workspace_announces_on_ready_flag() {
|
||||
let workdir = tempfile::tempdir().expect("workdir");
|
||||
|
||||
let parts = RuntimeParts::new(RuntimeConfig::default());
|
||||
let runtime = parts.runtime().clone();
|
||||
let sender = runtime.create_sender();
|
||||
let engine = Engine::new(parts, TokioBackend::new(TokioConfig::default()).expect("tokio"))
|
||||
.expect("engine");
|
||||
|
||||
let events = Arc::new(Mutex::new(Vec::new()));
|
||||
let orch = runtime.spawn(EventSink { events: events.clone() }).expect("spawn event sink");
|
||||
|
||||
let flag = Arc::new(std::sync::atomic::AtomicBool::new(true));
|
||||
let node = runtime
|
||||
.spawn(
|
||||
NodeJobActor::new(orch, workdir.path().to_path_buf(), sender, 0)
|
||||
.with_workspace_ready(flag),
|
||||
)
|
||||
.expect("spawn node");
|
||||
|
||||
runtime
|
||||
.send_to(node, NodeJobCommand::MaterializeWorkspace { job_id: 0 })
|
||||
.expect("materialize");
|
||||
|
||||
let started = Instant::now();
|
||||
while started.elapsed() < DEADLINE {
|
||||
if events.lock().iter().any(|ev| {
|
||||
matches!(ev, NodeJobEvent::WorkspaceMaterialized { job_id: 0 })
|
||||
}) {
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(POLL);
|
||||
}
|
||||
drop(engine);
|
||||
|
||||
let collected = events.lock();
|
||||
assert!(
|
||||
collected
|
||||
.iter()
|
||||
.any(|ev| matches!(ev, NodeJobEvent::WorkspaceMaterialized { job_id: 0 })),
|
||||
"edge workspace emitted WorkspaceMaterialized, got {:?}", collected
|
||||
);
|
||||
}
|
||||
124
crates/job-runner/tests/job_two_runtimes.rs
Normal file
124
crates/job-runner/tests/job_two_runtimes.rs
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
//! Two-runtime proof: the orchestrator FSM actor and the node job actor live on
|
||||
//! **separate swactor runtimes**, meshed by the transport seam (codec encode →
|
||||
//! `TransportRouter` → `Transport` → codec decode → `deliver_raw`) — the same
|
||||
//! seam iroh realizes in production. Control, workspace bytes, output bytes, and
|
||||
//! the supervised-process exit all cross the runtime boundary over the actor
|
||||
//! plane; `setup`/`run` execute via `swactor-process`. No shared filesystem for
|
||||
//! the job, no SSH.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use swactor::actor::{ActorAddress, Message};
|
||||
use swactor::runtime::{Inbox, Runtime, RuntimeConfig, RuntimeParts};
|
||||
use swactor::std::StdExtension;
|
||||
use swactor_engine::{Engine, TokioBackend, TokioConfig};
|
||||
use swactor_transport::{CodecRegistry, CodecRemoteSink, Transport, TransportRouter, WireEnvelope};
|
||||
|
||||
use swactor_job_runner::{
|
||||
register_job_codecs, Job, JobDone, JobState, NodeJobActor, OrchestratorJobActor,
|
||||
OrchestratorJobMsg, Workspace,
|
||||
};
|
||||
use swactor_job_runner::model; // ensure model path compiles; not used directly below
|
||||
|
||||
const POLL: Duration = Duration::from_millis(15);
|
||||
const DEADLINE: Duration = Duration::from_secs(20);
|
||||
|
||||
type BoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
/// Stands in for the iroh transport: carries a `WireEnvelope` from one runtime
|
||||
/// to another, decoding via the shared codec and performing the production
|
||||
/// ingress (`deliver_raw`). This is exactly the seam the iroh driver fills.
|
||||
struct Link {
|
||||
dst: Runtime,
|
||||
codec: Arc<CodecRegistry>,
|
||||
}
|
||||
|
||||
impl Transport for Link {
|
||||
fn send(&self, wire: WireEnvelope) -> Result<(), swactor::Error> {
|
||||
let msg = self.codec.decode(&wire.type_tag, &wire.payload)?;
|
||||
self.dst.deliver_raw(wire.dest, msg)
|
||||
}
|
||||
}
|
||||
|
||||
fn build_runtime(codec: Arc<CodecRegistry>) -> (RuntimeParts, Runtime, Arc<TransportRouter>) {
|
||||
let parts = RuntimeParts::new(RuntimeConfig::default())
|
||||
.with_extension(Arc::new(StdExtension::new()));
|
||||
let rt = parts.runtime().clone();
|
||||
let router = Arc::new(TransportRouter::new());
|
||||
rt.set_remote_sink(Arc::new(CodecRemoteSink::new(codec, router.clone())));
|
||||
(parts, rt, router)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn job_runs_across_two_swactor_runtimes_over_the_actor_plane() {
|
||||
let mut codec = CodecRegistry::new();
|
||||
register_job_codecs(&mut codec);
|
||||
let codec = Arc::new(codec);
|
||||
|
||||
let ws = tempfile::tempdir().expect("ws");
|
||||
std::fs::write(ws.path().join("seed.txt"), "seed-value").expect("seed");
|
||||
let node_workdir = tempfile::tempdir().expect("node workdir");
|
||||
let landing = tempfile::tempdir().expect("landing");
|
||||
|
||||
// Two independent runtimes, each driven by its own engine.
|
||||
let (parts_a, rt_a, router_a) = build_runtime(codec.clone());
|
||||
let (parts_b, rt_b, router_b) = build_runtime(codec.clone());
|
||||
let engine_a = Engine::new(parts_a, TokioBackend::new(TokioConfig::default()).expect("tokio"))
|
||||
.expect("engine A");
|
||||
let engine_b = Engine::new(parts_b, TokioBackend::new(TokioConfig::default()).expect("tokio"))
|
||||
.expect("engine B");
|
||||
|
||||
let done = rt_a.new_inbox::<JobDone>().expect("done inbox");
|
||||
let orch = rt_a
|
||||
.spawn(OrchestratorJobActor::new(*done.addr(), landing.path().to_path_buf()))
|
||||
.expect("spawn orchestrator on A");
|
||||
let node = rt_b
|
||||
.spawn(NodeJobActor::new(orch, node_workdir.path().to_path_buf(), rt_b.create_sender(), 0))
|
||||
.expect("spawn node on B");
|
||||
|
||||
// Cross-runtime routes: A routes the node address → B; B routes the
|
||||
// orchestrator address → A. Each Link delivers to wire.dest on the peer.
|
||||
router_a.add_route(node, Arc::new(Link { dst: rt_b.clone(), codec: codec.clone() }));
|
||||
router_b.add_route(orch, Arc::new(Link { dst: rt_a.clone(), codec: codec.clone() }));
|
||||
|
||||
let job = Job {
|
||||
name: "cross-runtime-probe".to_owned(),
|
||||
setup: Some("echo setup-ok > setup_done.txt".to_owned()),
|
||||
run: "echo hello-across-runtimes > greeting.txt".to_owned(),
|
||||
workspace: Some(Workspace { workdir: ws.path().to_path_buf(), exclude: vec![] }),
|
||||
outputs: vec![
|
||||
"greeting.txt".to_owned(),
|
||||
"setup_done.txt".to_owned(),
|
||||
"seed.txt".to_owned(),
|
||||
],
|
||||
env: std::collections::BTreeMap::new(),
|
||||
};
|
||||
rt_a
|
||||
.send_to(orch, OrchestratorJobMsg::Submit { job, node_actor: node })
|
||||
.expect("submit");
|
||||
|
||||
let started = Instant::now();
|
||||
let mut outcome = None;
|
||||
while started.elapsed() < DEADLINE {
|
||||
if let Some(d) = done.try_recv() {
|
||||
outcome = Some(d);
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(POLL);
|
||||
}
|
||||
drop(engine_a);
|
||||
drop(engine_b);
|
||||
let done = outcome.expect("job did not reach a terminal state across runtimes");
|
||||
assert_eq!(done.state, JobState::Completed, "expected COMPLETED across runtimes, got {:?}", done);
|
||||
assert_eq!(done.exit_code, Some(0));
|
||||
|
||||
let greeting =
|
||||
std::fs::read_to_string(landing.path().join("greeting.txt")).expect("collected greeting");
|
||||
assert!(greeting.contains("hello-across-runtimes"), "greeting: {greeting}");
|
||||
let seed = std::fs::read_to_string(landing.path().join("seed.txt")).expect("collected seed");
|
||||
assert_eq!(seed, "seed-value", "workspace crossed the runtime boundary through swactor");
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn _ensure_paths_compile(_a: ActorAddress, _e: BoxError, _m: model::Job) {}
|
||||
|
|
@ -98,6 +98,16 @@ pub struct PluginNodeHandle {
|
|||
pub provider_process_id: Option<u32>,
|
||||
}
|
||||
|
||||
/// Result of a successful spec-addressed adoption.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct AdoptedNode {
|
||||
pub handle: PluginNodeHandle,
|
||||
/// Stable provider-side address of the resource (docker container name,
|
||||
/// vastai lease label, ...) for snapshot bookkeeping.
|
||||
pub provider_ref: String,
|
||||
}
|
||||
|
||||
|
||||
pub trait PluginObservationSink: Send + Sync {
|
||||
fn observe(&self, observation: PluginObservation);
|
||||
}
|
||||
|
|
@ -135,6 +145,41 @@ pub trait ProvisionPlugin: Send {
|
|||
fn complete_bootstrap(&mut self, handle: &PluginNodeHandle) -> Result<(), String>;
|
||||
|
||||
fn stop_node(&mut self, handle: &PluginNodeHandle) -> Result<(), String>;
|
||||
fn adopt_by_spec(
|
||||
&mut self,
|
||||
_spec: &NodeProvisionSpec,
|
||||
_sink: PluginSink,
|
||||
) -> Result<Option<AdoptedNode>, String> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Stable provider-side address for a spec (docker container name, vastai
|
||||
/// lease label, ...). Derivable without provider state.
|
||||
fn provider_ref_for(&self, _spec: &NodeProvisionSpec) -> String {
|
||||
format!("node-{}-{}", _spec.run_id, _spec.node_id)
|
||||
}
|
||||
|
||||
/// Lists provider resources carrying this daemon's label. Used for orphan
|
||||
/// detection; providers without a label sweep return an empty list.
|
||||
fn list_managed_refs(&self) -> Result<Vec<String>, String> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
|
||||
/// Stops a provider resource addressed by its provision spec. Returns
|
||||
/// `Ok(false)` when nothing matching exists (already stopped is success).
|
||||
fn stop_by_spec(
|
||||
&mut self,
|
||||
_spec: &NodeProvisionSpec,
|
||||
_sink: PluginSink,
|
||||
) -> Result<bool, String> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Releases every owned resource handle without stopping anything, so a
|
||||
/// process exit leaves provider resources running. Callers that want
|
||||
/// teardown must call [`ProvisionPlugin::stop_node`] or
|
||||
/// [`ProvisionPlugin::stop_by_spec`] first.
|
||||
fn detach_all(&mut self) {}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
|
|||
|
|
@ -361,7 +361,9 @@ impl<P: TestablePlugin + 'static> PluginBackendAdapter<P> {
|
|||
|
||||
impl<P: TestablePlugin + 'static> Clone for PluginBackendAdapter<P> {
|
||||
fn clone(&self) -> Self {
|
||||
Self { shared: Arc::clone(&self.shared) }
|
||||
Self {
|
||||
shared: Arc::clone(&self.shared),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -687,18 +689,24 @@ pub fn check_invariants(
|
|||
}
|
||||
}
|
||||
if node.active_bootstrap.is_some() && node.record.bootstrap.is_none() {
|
||||
return Err(format!("node {id:?}: active session without bootstrap facts"));
|
||||
return Err(format!(
|
||||
"node {id:?}: active session without bootstrap facts"
|
||||
));
|
||||
}
|
||||
// Attempt-fact ownership: fixture identities are attempt-encoded,
|
||||
// so a fact from another attempt is detectable here.
|
||||
if let Some(lease) = &node.record.lease
|
||||
&& lease.lease_id.0 != format!("lease-{}", node.attempt.0) {
|
||||
&& lease.lease_id.0 != format!("lease-{}", node.attempt.0)
|
||||
{
|
||||
return Err(format!(
|
||||
"node {id:?}: lease {} leaked from another attempt",
|
||||
lease.lease_id.0
|
||||
));
|
||||
}
|
||||
for session in [node.active_bootstrap, node.record.bootstrap.as_ref().map(|f| f.session_id)]
|
||||
for session in [
|
||||
node.active_bootstrap,
|
||||
node.record.bootstrap.as_ref().map(|f| f.session_id),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
|
|
@ -722,9 +730,12 @@ pub fn check_invariants(
|
|||
if node
|
||||
.record
|
||||
.swactor
|
||||
.as_ref().is_none_or(|swactor| swactor.handed_off_at.is_none())
|
||||
.as_ref()
|
||||
.is_none_or(|swactor| swactor.handed_off_at.is_none())
|
||||
{
|
||||
return Err(format!("node {id:?}: ready without completed swactor handoff"));
|
||||
return Err(format!(
|
||||
"node {id:?}: ready without completed swactor handoff"
|
||||
));
|
||||
}
|
||||
match desired.get(id) {
|
||||
Some(spec) if node.record.desired == *spec => {}
|
||||
|
|
@ -941,9 +952,7 @@ impl<B: HarnessedBackend> Harness<B> {
|
|||
// Resource conservation: convergence owns every live resource.
|
||||
let leaked = self.backend.leaked();
|
||||
if !leaked.is_empty() {
|
||||
self.fail(&format!(
|
||||
"converged machine leaked resources: {leaked:?}"
|
||||
));
|
||||
self.fail(&format!("converged machine leaked resources: {leaked:?}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -965,13 +974,16 @@ impl<B: HarnessedBackend> Harness<B> {
|
|||
session_id: session,
|
||||
swactor_id: SwactorId(format!("sw-{}-{}", id.0, attempt.0)),
|
||||
},
|
||||
BootEvent::Closed => NodeObservation::BootstrapClosed { session_id: session },
|
||||
BootEvent::Closed => NodeObservation::BootstrapClosed {
|
||||
session_id: session,
|
||||
},
|
||||
BootEvent::Failed => NodeObservation::BootstrapFailed {
|
||||
session_id: session,
|
||||
reason: "scripted bootstrap failure".to_owned(),
|
||||
},
|
||||
};
|
||||
self.driver.apply_observation(&id, attempt, observation, self.now);
|
||||
self.driver
|
||||
.apply_observation(&id, attempt, observation, self.now);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1031,8 +1043,6 @@ impl<B: HarnessedBackend> Harness<B> {
|
|||
pub fn state(&self) -> &ClusterState {
|
||||
self.driver.state()
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
impl<B: HarnessedBackend> Drop for Harness<B> {
|
||||
|
|
@ -1126,12 +1136,7 @@ pub fn run_trace_with<B: HarnessedBackend>(
|
|||
fair: bool,
|
||||
order: RunOrder,
|
||||
) -> Harness<B> {
|
||||
let mut harness = Harness::new_ordered(
|
||||
seed,
|
||||
shape(1, vec![group("g0", 1)]),
|
||||
make(),
|
||||
order,
|
||||
);
|
||||
let mut harness = Harness::new_ordered(seed, shape(1, vec![group("g0", 1)]), make(), order);
|
||||
for input in trace {
|
||||
harness.step(input.clone());
|
||||
}
|
||||
|
|
@ -1144,12 +1149,7 @@ pub fn run_trace_with<B: HarnessedBackend>(
|
|||
harness
|
||||
}
|
||||
|
||||
pub fn run_trace(
|
||||
seed: u64,
|
||||
trace: &[Input],
|
||||
fair: bool,
|
||||
order: RunOrder,
|
||||
) -> Harness<FakeBackend> {
|
||||
pub fn run_trace(seed: u64, trace: &[Input], fair: bool, order: RunOrder) -> Harness<FakeBackend> {
|
||||
run_trace_with(FakeBackend::default, seed, trace, fair, order)
|
||||
}
|
||||
|
||||
|
|
@ -1228,7 +1228,11 @@ pub fn assert_trace<B: HarnessedBackend>(
|
|||
|
||||
/// The full battery: adversarial invariants plus fair convergence, over
|
||||
/// a fresh backend per run.
|
||||
pub fn run_trace_battery<B: HarnessedBackend>(make: impl Fn() -> B + Clone, seeds: u64, len: usize) {
|
||||
pub fn run_trace_battery<B: HarnessedBackend>(
|
||||
make: impl Fn() -> B + Clone,
|
||||
seeds: u64,
|
||||
len: usize,
|
||||
) {
|
||||
for seed in 0..seeds {
|
||||
let trace = gen_trace(seed, len);
|
||||
assert_trace(make.clone(), seed, &trace, false);
|
||||
|
|
@ -1293,9 +1297,7 @@ pub fn assert_plugin_contracts<P: TestablePlugin>(plugin: &mut P) {
|
|||
"adoption must not create a second resource"
|
||||
);
|
||||
assert!(
|
||||
plugin
|
||||
.leaked_resources(&[first.id, adopted.id])
|
||||
.is_empty(),
|
||||
plugin.leaked_resources(&[first.id, adopted.id]).is_empty(),
|
||||
"owned resources are not leaks"
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
|
||||
mod common;
|
||||
|
||||
use common::{assert_plugin_contracts, run_trace_battery, FakePlugin, PluginBackendAdapter};
|
||||
use common::{FakePlugin, PluginBackendAdapter, assert_plugin_contracts, run_trace_battery};
|
||||
|
||||
#[test]
|
||||
fn in_memory_plugin_passes_seam_contracts() {
|
||||
|
|
@ -15,11 +15,5 @@ fn in_memory_plugin_passes_seam_contracts() {
|
|||
|
||||
#[test]
|
||||
fn in_memory_plugin_battery_holds_invariants_and_converges() {
|
||||
run_trace_battery(
|
||||
|| PluginBackendAdapter::new(FakePlugin::default()),
|
||||
256,
|
||||
64,
|
||||
);
|
||||
run_trace_battery(|| PluginBackendAdapter::new(FakePlugin::default()), 256, 64);
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -11,13 +11,11 @@ use std::process::{Child, Command};
|
|||
use std::sync::Arc;
|
||||
|
||||
use parking_lot::Mutex;
|
||||
use provisioning::plugin::{
|
||||
NodeProvisionSpec, PluginNodeHandle, PluginSink, ProvisionPlugin,
|
||||
};
|
||||
use provisioning::plugin::{NodeProvisionSpec, PluginNodeHandle, PluginSink, ProvisionPlugin};
|
||||
|
||||
use common::{
|
||||
assert_plugin_contracts, run_trace_battery, AMBIGUOUS_FAULT_MARKER, Fault, TestablePlugin,
|
||||
PluginBackendAdapter,
|
||||
AMBIGUOUS_FAULT_MARKER, Fault, PluginBackendAdapter, TestablePlugin, assert_plugin_contracts,
|
||||
run_trace_battery,
|
||||
};
|
||||
|
||||
struct ProcessPluginState {
|
||||
|
|
@ -164,6 +162,9 @@ fn process_plugin_passes_seam_contracts() {
|
|||
|
||||
#[test]
|
||||
fn process_plugin_battery_holds_invariants_and_converges() {
|
||||
run_trace_battery(|| PluginBackendAdapter::new(ProcessPlugin::default()), 16, 28);
|
||||
run_trace_battery(
|
||||
|| PluginBackendAdapter::new(ProcessPlugin::default()),
|
||||
16,
|
||||
28,
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,8 +7,8 @@ mod common;
|
|||
use std::time::{Duration, UNIX_EPOCH};
|
||||
|
||||
use common::{
|
||||
check_invariants, gen_trace, group, group_with_role, run_trace, sanitized, shape, BootEvent,
|
||||
Harness, Input, RecordingExecutor, Reply, RunOrder,
|
||||
BootEvent, Harness, Input, RecordingExecutor, Reply, RunOrder, check_invariants, gen_trace,
|
||||
group, group_with_role, run_trace, sanitized, shape,
|
||||
};
|
||||
use provisioning::*;
|
||||
|
||||
|
|
@ -27,7 +27,11 @@ fn replayed_traces_are_identical() {
|
|||
|
||||
#[test]
|
||||
fn happy_path_converges() {
|
||||
let mut harness = Harness::new_with_backend(0, shape(1, vec![group("g0", 1)]), common::FakeBackend::default());
|
||||
let mut harness = Harness::new_with_backend(
|
||||
0,
|
||||
shape(1, vec![group("g0", 1)]),
|
||||
common::FakeBackend::default(),
|
||||
);
|
||||
harness.step(Input::Run); // dispatch create lease
|
||||
harness.step(Input::Run); // execute create, dispatch bootstrap start
|
||||
harness.step(Input::Run); // execute bootstrap start, session active
|
||||
|
|
@ -35,14 +39,22 @@ fn happy_path_converges() {
|
|||
harness.step(Input::Boot(BootEvent::Closed));
|
||||
harness.step(Input::Run); // bootstrap convergence accepted
|
||||
assert!(harness.driver.is_converged());
|
||||
assert!(harness.backend.calls().iter().any(|effect| {
|
||||
matches!(effect.command, NodeManagerCommand::CreateLease(_))
|
||||
}));
|
||||
assert!(
|
||||
harness
|
||||
.backend
|
||||
.calls()
|
||||
.iter()
|
||||
.any(|effect| { matches!(effect.command, NodeManagerCommand::CreateLease(_)) })
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ambiguous_create_is_adopted_and_converges() {
|
||||
let mut harness = Harness::new_with_backend(0, shape(1, vec![group("g0", 1)]), common::FakeBackend::default());
|
||||
let mut harness = Harness::new_with_backend(
|
||||
0,
|
||||
shape(1, vec![group("g0", 1)]),
|
||||
common::FakeBackend::default(),
|
||||
);
|
||||
harness.step(Input::Reply(Reply::Ambiguous("create timed out")));
|
||||
harness.step(Input::Run); // create fails ambiguously, backoff starts
|
||||
harness.step(Input::Tick(Duration::from_secs(10))); // retry/adopt
|
||||
|
|
@ -52,7 +64,11 @@ fn ambiguous_create_is_adopted_and_converges() {
|
|||
|
||||
#[test]
|
||||
fn shape_shrink_mid_lifecycle_converges() {
|
||||
let mut harness = Harness::new_with_backend(0, shape(1, vec![group("g0", 2)]), common::FakeBackend::default());
|
||||
let mut harness = Harness::new_with_backend(
|
||||
0,
|
||||
shape(1, vec![group("g0", 2)]),
|
||||
common::FakeBackend::default(),
|
||||
);
|
||||
harness.step(Input::Run);
|
||||
harness.step(Input::Run);
|
||||
harness.step(Input::Boot(BootEvent::Joined));
|
||||
|
|
@ -61,10 +77,12 @@ fn shape_shrink_mid_lifecycle_converges() {
|
|||
harness.fair_tail();
|
||||
assert!(harness.driver.is_converged());
|
||||
assert_eq!(harness.state().nodes.len(), 1);
|
||||
assert!(harness
|
||||
assert!(
|
||||
harness
|
||||
.state()
|
||||
.nodes
|
||||
.contains_key(&LogicalNodeId("g0-0".to_owned())));
|
||||
.contains_key(&LogicalNodeId("g0-0".to_owned()))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -80,7 +98,11 @@ fn same_generation_same_content_is_accepted() {
|
|||
|
||||
#[test]
|
||||
fn deadline_expires_exactly_at_deadline() {
|
||||
let mut harness = Harness::new_with_backend(0, shape(1, vec![group("g0", 1)]), common::FakeBackend::default());
|
||||
let mut harness = Harness::new_with_backend(
|
||||
0,
|
||||
shape(1, vec![group("g0", 1)]),
|
||||
common::FakeBackend::default(),
|
||||
);
|
||||
harness.settle(); // create dispatched at the epoch
|
||||
let timeout = RetryPolicy::default().operation_timeout;
|
||||
let node = harness.state().nodes.values().next().expect("node exists");
|
||||
|
|
@ -90,14 +112,23 @@ fn deadline_expires_exactly_at_deadline() {
|
|||
// One tick before the deadline: still pending, nothing expired.
|
||||
harness.step(Input::Tick(timeout - Duration::from_secs(1)));
|
||||
let node = harness.state().nodes.values().next().expect("node exists");
|
||||
assert!(node.pending.is_some(), "operation expired before its deadline");
|
||||
assert!(
|
||||
node.pending.is_some(),
|
||||
"operation expired before its deadline"
|
||||
);
|
||||
assert_eq!(node.retry.ambiguous_operation, None);
|
||||
|
||||
// Exactly at the deadline: expired, classified ambiguous, never ran.
|
||||
harness.step(Input::Tick(Duration::from_secs(1)));
|
||||
let node = harness.state().nodes.values().next().expect("node exists");
|
||||
assert!(node.pending.is_none(), "operation did not expire at its deadline");
|
||||
assert_eq!(node.retry.ambiguous_operation, Some(OperationKind::CreateLease));
|
||||
assert!(
|
||||
node.pending.is_none(),
|
||||
"operation did not expire at its deadline"
|
||||
);
|
||||
assert_eq!(
|
||||
node.retry.ambiguous_operation,
|
||||
Some(OperationKind::CreateLease)
|
||||
);
|
||||
assert!(
|
||||
harness.backend.calls().is_empty(),
|
||||
"expired operation must not reach the backend"
|
||||
|
|
@ -118,7 +149,10 @@ fn clock_extremes_do_not_panic_or_corrupt_state() {
|
|||
let mut guard = 0;
|
||||
while executor.submitted < 8 {
|
||||
guard += 1;
|
||||
assert!(guard <= 64, "driver stopped making progress at clock extremes");
|
||||
assert!(
|
||||
guard <= 64,
|
||||
"driver stopped making progress at clock extremes"
|
||||
);
|
||||
driver.trigger_if_due(now);
|
||||
driver
|
||||
.drive_until_blocked(now, &mut executor)
|
||||
|
|
@ -128,7 +162,9 @@ fn clock_extremes_do_not_panic_or_corrupt_state() {
|
|||
}
|
||||
check_invariants(driver.state(), &driver.desired().expand().expect("expands"))
|
||||
.unwrap_or_else(|violation| panic!("invariant broken at clock extreme: {violation}"));
|
||||
now = now.checked_add(step).expect("probe clock still representable");
|
||||
now = now
|
||||
.checked_add(step)
|
||||
.expect("probe clock still representable");
|
||||
if let Some(requeue) = driver.requeue_at()
|
||||
&& requeue > now
|
||||
{
|
||||
|
|
@ -160,7 +196,11 @@ fn attempt_allocator_exhaustion_is_reported() {
|
|||
fn latest_desired_wins() {
|
||||
for seed in 0..16 {
|
||||
let trace = sanitized(&gen_trace(seed, 32));
|
||||
let mut harness = Harness::new_with_backend(seed, shape(1, vec![group("g0", 1)]), common::FakeBackend::default());
|
||||
let mut harness = Harness::new_with_backend(
|
||||
seed,
|
||||
shape(1, vec![group("g0", 1)]),
|
||||
common::FakeBackend::default(),
|
||||
);
|
||||
for input in &trace {
|
||||
harness.step(input.clone());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ license = "AGPL-3.0-only"
|
|||
publish = false
|
||||
|
||||
[dependencies]
|
||||
reqwest = { version = "0.12", features = ["json"] }
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
tokio.workspace = true
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ license = "AGPL-3.0-only"
|
|||
|
||||
[dependencies]
|
||||
serde_json = "1"
|
||||
blake3 = "1"
|
||||
swactor = { path = "..", features = ["serde"] }
|
||||
swactor-engine = { path = "../crates/engine" }
|
||||
swactor-transport = { path = "../crates/transport" }
|
||||
|
|
@ -18,7 +17,6 @@ data-plane = { path = "../crates/data-plane" }
|
|||
iroh-driver = { path = "../crates/iroh-driver" }
|
||||
distribution = { path = "../crates/distribution" }
|
||||
iroh = "0.98"
|
||||
tokio.workspace = true
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
parking_lot = "0.12"
|
||||
|
||||
|
|
|
|||
|
|
@ -997,7 +997,7 @@ impl ActorInterface for SupervisorActor {
|
|||
impl SupervisorActor {
|
||||
fn handle_control(&mut self, command: dashboard::control::ControlCommand) {
|
||||
match command {
|
||||
dashboard::control::ControlCommand::Kill { node } => {
|
||||
dashboard::control::ControlCommand::Kill { node, .. } => {
|
||||
match self.manager.find_by_stream_node(&node) {
|
||||
Some(runtime) => {
|
||||
self.emit_event(
|
||||
|
|
@ -1015,7 +1015,7 @@ impl SupervisorActor {
|
|||
None => self.emit_event("control", &node, "kill: unknown node".to_owned()),
|
||||
}
|
||||
}
|
||||
dashboard::control::ControlCommand::Remove { count } => {
|
||||
dashboard::control::ControlCommand::Remove { count, .. } => {
|
||||
let removed = self.slots.len().min(count as usize);
|
||||
if removed == 0 {
|
||||
self.emit_event("control", "", "remove: nothing to remove".to_owned());
|
||||
|
|
@ -1030,7 +1030,7 @@ impl SupervisorActor {
|
|||
eprintln!("demo: remove update_desired failed: {error}");
|
||||
}
|
||||
}
|
||||
dashboard::control::ControlCommand::Provision { count } => {
|
||||
dashboard::control::ControlCommand::Provision { count, .. } => {
|
||||
if count == 0 {
|
||||
return;
|
||||
}
|
||||
|
|
@ -1045,7 +1045,7 @@ impl SupervisorActor {
|
|||
eprintln!("demo: provision update_desired failed: {error}");
|
||||
}
|
||||
}
|
||||
dashboard::control::ControlCommand::EstablishEdge { node } => {
|
||||
dashboard::control::ControlCommand::EstablishEdge { node, .. } => {
|
||||
self.establish_edge(&node);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
6850
xtask/src/main.rs
6850
xtask/src/main.rs
File diff suppressed because it is too large
Load diff
Loading…
Reference in a new issue