diff --git a/.ci.yml b/.ci.yml new file mode 100644 index 0000000..3d30e19 --- /dev/null +++ b/.ci.yml @@ -0,0 +1,16 @@ +pipelines: + ci: + triggers: + - event: push + branches: ["*"] + jobs: + check: + run: echo "HOME=$HOME PATH=$PATH" && . "$HOME/.cargo/env" && which cargo && cargo check --workspace + timeout: 600 + clippy: + run: . "$HOME/.cargo/env" && cargo clippy --workspace --all-targets -- -D warnings + timeout: 600 + test: + needs: [check] + run: . "$HOME/.cargo/env" && cargo xtask test essential + timeout: 900 diff --git a/Cargo.lock b/Cargo.lock index 3065c72..7da30f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -216,6 +216,28 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-lc-rs" +version = "1.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b7b6141e96a8c160799cc2d5adecd5cbbe5054cb8c7c4af53da0f83bb7ad256" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.37.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b092fe214090261288111db7a2b2c2118e5a7f30dc2569f1732c4069a6840549" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "backon" version = "1.6.0" @@ -367,6 +389,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cfg-if" version = "1.0.4" @@ -397,6 +425,21 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901" +[[package]] +name = "ci-relay" +version = "0.1.0" +dependencies = [ + "clap", + "hex", + "hmac", + "iroh", + "serde_json", + "sha2 0.10.9", + "swactor-ci", + "tiny_http", + "tokio", +] + [[package]] name = "ciborium" version = "0.2.2" @@ -464,6 +507,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831" +[[package]] +name = "cmake" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +dependencies = [ + "cc", +] + [[package]] name = "cobs" version = "0.3.0" @@ -479,6 +531,16 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "compact_str" version = "0.8.1" @@ -614,7 +676,7 @@ dependencies = [ "rustc-hash", "serde", "smallvec", - "target-lexicon 0.13.4", + "target-lexicon 0.13.5", ] [[package]] @@ -661,7 +723,7 @@ dependencies = [ "cranelift-codegen", "log", "smallvec", - "target-lexicon 0.13.4", + "target-lexicon 0.13.5", ] [[package]] @@ -678,7 +740,7 @@ checksum = "b8dee82f3f1f2c4cba9177f1cc5e350fe98764379bcd29340caa7b01f85076c7" dependencies = [ "cranelift-codegen", "libc", - "target-lexicon 0.13.4", + "target-lexicon 0.13.5", ] [[package]] @@ -1063,6 +1125,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "crypto-common 0.1.7", + "subtle", ] [[package]] @@ -1150,7 +1213,7 @@ name = "docker-tests" version = "0.1.0" dependencies = [ "distribution", - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", ] @@ -1164,6 +1227,12 @@ dependencies = [ "litrs", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.20" @@ -1383,10 +1452,16 @@ dependencies = [ ] [[package]] -name = "futures" -version = "0.3.31" +name = "fs_extra" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" dependencies = [ "futures-channel", "futures-core", @@ -1412,9 +1487,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", "futures-sink", @@ -1422,15 +1497,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" [[package]] name = "futures-executor" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" dependencies = [ "futures-core", "futures-task", @@ -1439,9 +1514,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" [[package]] name = "futures-lite" @@ -1458,9 +1533,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", @@ -1469,21 +1544,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" [[package]] name = "futures-task" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-util" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-channel", "futures-core", @@ -1493,7 +1568,6 @@ dependencies = [ "futures-task", "memchr", "pin-project-lite", - "pin-utils", "slab", ] @@ -1578,10 +1652,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "wasip2", "wasip3", + "wasm-bindgen", ] [[package]] @@ -1704,6 +1780,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hickory-proto" version = "0.25.2" @@ -1757,6 +1839,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "http" version = "1.4.0" @@ -2153,7 +2244,7 @@ dependencies = [ "pkcs8 0.11.0-rc.11", "portmapper", "rand", - "reqwest", + "reqwest 0.12.28", "rustc-hash", "rustls", "rustls-pki-types", @@ -2308,7 +2399,7 @@ dependencies = [ "pkarr", "postcard", "rand", - "reqwest", + "reqwest 0.12.28", "rustls", "rustls-pki-types", "serde", @@ -2396,6 +2487,28 @@ dependencies = [ "cc", ] +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + [[package]] name = "jobserver" version = "0.1.34" @@ -2480,6 +2593,20 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" +[[package]] +name = "local-runner" +version = "0.1.0" +dependencies = [ + "clap", + "ctrlc", + "iroh", + "runtime-dashboard", + "serde_json", + "swactor", + "swactor-ci", + "tokio", +] + [[package]] name = "lock_api" version = "0.4.14" @@ -3139,9 +3266,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkarr" -version = "5.0.2" +version = "5.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d346b545765a0ef58b6a7e160e17ddaa7427f439b7b9a287df6c88c9e04bf2" +checksum = "2f950360d31be432c0c9467fba5024a94f55128e7f32bc9d32db140369f24c77" dependencies = [ "async-compat", "base32", @@ -3152,11 +3279,11 @@ dependencies = [ "ed25519-dalek 3.0.0-pre.1", "futures-buffered", "futures-lite", - "getrandom 0.3.4", + "getrandom 0.4.1", "log", "lru 0.16.3", "ntimestamp", - "reqwest", + "reqwest 0.13.2", "self_cell", "serde", "sha1_smol", @@ -3510,6 +3637,7 @@ version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" dependencies = [ + "aws-lc-rs", "bytes", "getrandom 0.3.4", "lru-slab", @@ -3753,6 +3881,41 @@ dependencies = [ "webpki-roots 1.0.6", ] +[[package]] +name = "reqwest" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "resolv-conf" version = "0.7.6" @@ -3787,6 +3950,7 @@ dependencies = [ "serde", "serde_json", "swactor", + "swactor-ci", "tiny_http", "tracing", "tracing-subscriber", @@ -3845,6 +4009,7 @@ version = "0.23.36" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -3854,6 +4019,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pki-types" version = "1.14.0" @@ -3864,12 +4041,40 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.103.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -4058,6 +4263,19 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "sha1_smol" version = "1.0.1" @@ -4187,6 +4405,7 @@ dependencies = [ "serde_json", "simulation", "swactor", + "swactor-ci", "tiny_http", "toml", ] @@ -4384,6 +4603,21 @@ dependencies = [ "wat", ] +[[package]] +name = "swactor-ci" +version = "0.1.0" +dependencies = [ + "hex", + "hmac", + "serde", + "serde_json", + "serde_yaml", + "sha2 0.10.9", + "swactor", + "tiny_http", + "ureq", +] + [[package]] name = "swactor-crypto-wasm" version = "0.1.0" @@ -4423,9 +4657,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.115" +version = "2.0.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e614ed320ac28113fa64972c4262d5dbc89deacdfd00c34a3e4cea073243c12" +checksum = "3df424c70518695237746f84cede799c9c58fcb37450d7b23716568cc8bc69cb" dependencies = [ "proc-macro2", "quote", @@ -4487,9 +4721,9 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "target-lexicon" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1dd07eb858a2067e2f3c7155d54e929265c264e6f37efe3ee7a8d1b5a1dd0ba" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" [[package]] name = "tempfile" @@ -4947,9 +5181,9 @@ checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" [[package]] name = "unicode-ident" -version = "1.0.23" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "537dd038a89878be9b64dd4bd1b260315c1bb94f4d784956b81e27a088d9a09e" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-segmentation" @@ -4992,6 +5226,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0" @@ -5377,7 +5617,7 @@ dependencies = [ "serde_json", "smallvec", "sptr", - "target-lexicon 0.13.4", + "target-lexicon 0.13.5", "trait-variant", "wasm-encoder 0.221.3", "wasmparser 0.221.3", @@ -5466,7 +5706,7 @@ dependencies = [ "log", "object 0.36.7", "smallvec", - "target-lexicon 0.13.4", + "target-lexicon 0.13.5", "thiserror 1.0.69", "wasmparser 0.221.3", "wasmtime-environ", @@ -5493,7 +5733,7 @@ dependencies = [ "serde", "serde_derive", "smallvec", - "target-lexicon 0.13.4", + "target-lexicon 0.13.5", "wasm-encoder 0.221.3", "wasmparser 0.221.3", "wasmprinter", @@ -5574,7 +5814,7 @@ dependencies = [ "cranelift-codegen", "gimli", "object 0.36.7", - "target-lexicon 0.13.4", + "target-lexicon 0.13.5", "wasmparser 0.221.3", "wasmtime-cranelift", "wasmtime-environ", @@ -5645,6 +5885,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "0.26.11" @@ -5711,7 +5960,7 @@ dependencies = [ "gimli", "regalloc2", "smallvec", - "target-lexicon 0.13.4", + "target-lexicon 0.13.5", "thiserror 1.0.69", "wasmparser 0.221.3", "wasmtime-cranelift", @@ -5830,6 +6079,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -5875,6 +6133,21 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -5932,6 +6205,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -5950,6 +6229,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -5968,6 +6253,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -5998,6 +6289,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -6016,6 +6313,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -6034,6 +6337,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -6052,6 +6361,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" diff --git a/Cargo.toml b/Cargo.toml index 589030e..3fb1f43 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,22 @@ [workspace] -members = [".", "crates/python", "crates/wasm", "crates/bin-runner", "crates/simulation", "crates/runtime-dashboard", "crates/distribution", "crates/std", "crates/datastore", "crates/shared-types", "crates/crypto-wasm", "tests/docker", "xtask"] +members = [ + ".", + "crates/python", + "crates/wasm", + "crates/bin-runner", + "crates/simulation", + "crates/runtime-dashboard", + "crates/distribution", + "crates/std", + "crates/datastore", + "crates/shared-types", + "crates/crypto-wasm", + "tests/docker", + "crates/ci", + "crates/local-runner", + "crates/ci-relay", + "xtask", +] exclude = ["tools/depgraph"] [package] @@ -9,8 +26,8 @@ edition = "2024" autobenches = false [profile.bench] -debug = true -strip = false +debug = true +strip = false [lib] crate-type = ["rlib"] @@ -21,7 +38,8 @@ getrandom = ["dep:getrandom"] serde = ["dep:serde"] tracing = ["dep:tracing"] no_random = [] # compile without access to a source of randomness -transport = [] # transport-agnostic messaging (no mandatory deps; codec is user-provided) +transport = [ +] # transport-agnostic messaging (no mandatory deps; codec is user-provided) wasm = ["no_random", "dep:web-time"] # browser/wasm32 target support [dependencies] diff --git a/crates/ci-relay/Cargo.toml b/crates/ci-relay/Cargo.toml new file mode 100644 index 0000000..215dbcd --- /dev/null +++ b/crates/ci-relay/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "ci-relay" +version = "0.1.0" +edition = "2024" + +[[bin]] +name = "ci-relay" +path = "src/main.rs" + +[dependencies] +swactor-ci = { path = "../ci", features = ["local"] } +iroh = "0.96" +tokio = { version = "1", features = ["rt-multi-thread"] } +tiny_http = "0.12" +serde_json = "1" +hmac = "0.12" +sha2 = "0.10" +hex = "0.4" +clap = { version = "4", features = ["derive"] } diff --git a/crates/ci-relay/src/main.rs b/crates/ci-relay/src/main.rs new file mode 100644 index 0000000..1271b43 --- /dev/null +++ b/crates/ci-relay/src/main.rs @@ -0,0 +1,243 @@ +//! ci-relay — webhook relay for VPS side. +//! +//! Receives Forgejo webhook POSTs over HTTP, then forwards the parsed +//! `WebhookEvent` payloads to the Thinkpad local-runner over iroh. + +use std::sync::Arc; +use std::time::Duration; + +use clap::Parser; +use iroh::{Endpoint, RelayMode}; +use tokio::sync::Mutex as TokioMutex; + +use swactor_ci::webhook_server::parse_webhook_json; +use swactor_ci::{EventType, WebhookEvent}; + +/// ALPN protocol identifier for CI relay traffic over iroh. +const ALPN: &[u8] = b"swactor/ci/1"; + +/// Wire tag for WebhookEvent messages. +const WEBHOOK_TAG: &str = "ci::WebhookEvent"; + +#[derive(Parser)] +#[command(name = "ci-relay", about = "Webhook relay: Forgejo → iroh → local-runner")] +struct Args { + /// HTTP port for receiving Forgejo webhooks. + #[arg(long, default_value = "8787")] + port: u16, + + /// Webhook secret for HMAC-SHA256 verification (empty to skip). + #[arg(long, default_value = "")] + secret: String, +} + +fn main() { + let args = Args::parse(); + + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("failed to build tokio runtime"); + + let endpoint = rt.block_on(async { + Endpoint::builder() + .alpns(vec![ALPN.to_vec()]) + .relay_mode(RelayMode::Default) + .bind() + .await + .expect("failed to bind iroh endpoint") + }); + + let node_id = endpoint.id(); + eprintln!("ci-relay started"); + eprintln!(" Iroh Node ID: {node_id}"); + eprintln!(" Webhook HTTP: http://0.0.0.0:{}", args.port); + eprintln!(); + eprintln!("Waiting for runner to connect..."); + + // Shared state: the active connection from the Thinkpad runner. + let connection: Arc>> = + Arc::new(TokioMutex::new(None)); + + // Spawn a task that accepts inbound iroh connections from the runner. + { + let endpoint = endpoint.clone(); + let connection = Arc::clone(&connection); + rt.spawn(async move { + loop { + match endpoint.accept().await { + Some(incoming) => match incoming.await { + Ok(conn) => { + let remote = conn.remote_id(); + eprintln!("Runner connected: {remote}"); + *connection.lock().await = Some(conn); + } + Err(e) => { + eprintln!("iroh accept error: {e}"); + } + }, + None => { + eprintln!("iroh endpoint closed"); + break; + } + } + } + }); + } + + // Run the HTTP webhook listener on a standard thread (blocking). + let secret = args.secret.clone(); + let server = tiny_http::Server::http(format!("0.0.0.0:{}", args.port)) + .expect("failed to start HTTP server"); + + eprintln!("Listening for webhooks..."); + + for mut request in server.incoming_requests() { + let response = handle_webhook(&mut request, &secret, &connection, &rt); + let _ = request.respond(response); + } +} + +/// Handle an incoming webhook HTTP request. +/// +/// Parses and verifies the webhook, then forwards the event over iroh. +fn handle_webhook( + request: &mut tiny_http::Request, + secret: &str, + connection: &Arc>>, + rt: &tokio::runtime::Runtime, +) -> tiny_http::Response>> { + use hmac::{Hmac, Mac}; + use sha2::Sha256; + + if request.method() != &tiny_http::Method::Post { + return tiny_http::Response::from_string("method not allowed").with_status_code(405); + } + + // Read body. + let mut body = String::new(); + if let Err(e) = std::io::Read::read_to_string(&mut request.as_reader(), &mut body) { + eprintln!("webhook: failed to read body: {e}"); + return tiny_http::Response::from_string("bad request").with_status_code(400); + } + + // Verify HMAC-SHA256 signature if secret is non-empty. + if !secret.is_empty() { + let sig_header = request + .headers() + .iter() + .find(|h| h.field.equiv("X-Forgejo-Signature")) + .map(|h| h.value.as_str().to_string()); + + match sig_header { + Some(sig_hex) => { + type HmacSha256 = Hmac; + let mut mac = + HmacSha256::new_from_slice(secret.as_bytes()).expect("HMAC key creation"); + hmac::Mac::update(&mut mac, body.as_bytes()); + let expected = hex::encode(mac.finalize().into_bytes()); + if sig_hex != expected { + eprintln!("webhook: signature mismatch"); + return tiny_http::Response::from_string("unauthorized").with_status_code(401); + } + } + None => { + eprintln!("webhook: missing signature header"); + return tiny_http::Response::from_string("unauthorized").with_status_code(401); + } + } + } + + // Determine event type from Forgejo header. + let event_header = request + .headers() + .iter() + .find(|h| h.field.equiv("X-Forgejo-Event")) + .map(|h| h.value.as_str().to_string()) + .unwrap_or_default(); + + let event_type = match event_header.as_str() { + "push" => EventType::Push, + "create" => EventType::Tag, + "pull_request" => EventType::Merge, + other => { + eprintln!("webhook: ignoring event type '{other}'"); + return tiny_http::Response::from_string("ignored").with_status_code(200); + } + }; + + // Parse JSON body. + let json: serde_json::Value = match serde_json::from_str(&body) { + Ok(v) => v, + Err(e) => { + eprintln!("webhook: failed to parse JSON: {e}"); + return tiny_http::Response::from_string("bad json").with_status_code(400); + } + }; + + let webhook_event = match parse_webhook_json(&json, event_type) { + Some(e) => e, + None => { + eprintln!("webhook: could not extract fields from JSON"); + return tiny_http::Response::from_string("bad payload").with_status_code(400); + } + }; + + eprintln!( + "webhook: {} {} on {}/{}", + webhook_event.commit_sha.get(..8).unwrap_or(&webhook_event.commit_sha), + webhook_event.branch, + webhook_event.repo_owner, + webhook_event.repo_name, + ); + + // Forward over iroh. + match forward_event(&webhook_event, connection, rt) { + Ok(()) => { + eprintln!(" → forwarded to runner"); + tiny_http::Response::from_string("ok").with_status_code(200) + } + Err(e) => { + eprintln!(" → forward failed: {e}"); + tiny_http::Response::from_string("relay error").with_status_code(502) + } + } +} + +/// Serialize and send a WebhookEvent over the iroh connection. +fn forward_event( + event: &WebhookEvent, + connection: &Arc>>, + rt: &tokio::runtime::Runtime, +) -> Result<(), Box> { + let payload = serde_json::to_vec(event)?; + + rt.block_on(async { + let guard = connection.lock().await; + let conn = guard.as_ref().ok_or("no runner connected")?; + + let mut send = conn.open_uni().await?; + write_tagged_message(&mut send, WEBHOOK_TAG.as_bytes(), &payload).await?; + send.finish()?; + + // Wait briefly for the stream to flush. + tokio::time::sleep(Duration::from_millis(50)).await; + + Ok(()) + }) +} + +/// Write a tagged message to a QUIC send stream. +/// +/// Frame format: `[4B tag_len][tag_bytes][payload_bytes]` +async fn write_tagged_message( + send: &mut iroh::endpoint::SendStream, + tag: &[u8], + payload: &[u8], +) -> Result<(), Box> { + let tag_len = (tag.len() as u32).to_be_bytes(); + send.write_all(&tag_len).await?; + send.write_all(tag).await?; + send.write_all(payload).await?; + Ok(()) +} diff --git a/crates/ci/Cargo.toml b/crates/ci/Cargo.toml new file mode 100644 index 0000000..5a28b8e --- /dev/null +++ b/crates/ci/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "swactor-ci" +version = "0.1.0" +edition = "2024" + +[dependencies] +swactor = { path = "../..", features = ["serde"] } +serde = { version = "1", features = ["derive"] } +serde_yaml = "0.9" +serde_json = "1" + +# Local runner library dependencies +tiny_http = { version = "0.12", optional = true } +ureq = { version = "2", features = ["json"], optional = true } +hmac = { version = "0.12", optional = true } +sha2 = { version = "0.10", optional = true } +hex = { version = "0.4", optional = true } + +[features] +default = [] +local = [ + "dep:tiny_http", + "dep:ureq", + "dep:hmac", + "dep:sha2", + "dep:hex", +] + +[dev-dependencies] diff --git a/crates/ci/src/coordinator.rs b/crates/ci/src/coordinator.rs new file mode 100644 index 0000000..1e94aac --- /dev/null +++ b/crates/ci/src/coordinator.rs @@ -0,0 +1,430 @@ +//! Coordinator actor: central brain of the CI system. +//! +//! Receives webhook events, manages pipeline lifecycles, dispatches jobs +//! to the Provisioner and RunnerSupervisor actors. + +use std::collections::HashMap; + +use swactor::actor::{ActorAddress, ActorInterface, Ctx}; + +use crate::pipeline::PipelineExecution; +use crate::yaml::{self, CiYaml}; +use crate::{ + CiConfig, JobComplete, JobId, JobProgress, JobStatus, PipelineId, ProvisionRequest, + ProvisionResponse, StatusUpdate, TerminateRequest, WebhookEvent, +}; + +/// Messages the Coordinator can receive. +#[derive(Debug, Clone)] +pub enum CoordinatorMsg { + /// A webhook event from Forgejo. + Webhook(WebhookEvent), + /// The CI YAML config to use (loaded externally or from the repo). + SetCiYaml(CiYaml), + /// Response from the Provisioner. + ProvisionResponse(ProvisionResponse), + /// Streamed output from a RunnerSupervisor. + JobProgress(JobProgress), + /// Final result from a RunnerSupervisor. + JobComplete(JobComplete), + /// Notification that the provisioner is offline (detected via SWIM). + ProvisionerOffline, + /// Notification that the provisioner is back online. + ProvisionerOnline, +} + +/// The Coordinator actor state. +pub struct Coordinator { + config: CiConfig, + ci_yaml: Option, + pipelines: HashMap, + next_pipeline_id: u64, + provisioner_addr: Option, + provisioner_online: bool, + /// Maps job_id → runner supervisor address. + runner_addrs: HashMap, + /// Captured status updates (for testing/simulation). + status_updates: Vec, + /// Jobs waiting for the provisioner to come online. + queued_provisions: Vec, +} + +impl Coordinator { + pub fn new(config: CiConfig) -> Self { + Self { + config, + ci_yaml: None, + pipelines: HashMap::new(), + next_pipeline_id: 1, + provisioner_addr: None, + provisioner_online: false, + runner_addrs: HashMap::new(), + status_updates: Vec::new(), + queued_provisions: Vec::new(), + } + } + + pub fn with_provisioner(mut self, addr: ActorAddress) -> Self { + self.provisioner_addr = Some(addr); + self.provisioner_online = true; + self + } + + pub fn with_ci_yaml(mut self, yaml: CiYaml) -> Self { + self.ci_yaml = Some(yaml); + self + } + + pub fn pipelines(&self) -> &HashMap { + &self.pipelines + } + + pub fn status_updates(&self) -> &[StatusUpdate] { + &self.status_updates + } + + fn handle_webhook(&mut self, ctx: &Ctx, event: WebhookEvent) { + let ci = match &self.ci_yaml { + Some(ci) => ci.clone(), + None => return, + }; + + let matched = yaml::matching_pipelines(&ci, &event); + for pipeline_name in matched { + let pipeline_def = &ci.pipelines[&pipeline_name]; + let pipeline_id = PipelineId(self.next_pipeline_id); + self.next_pipeline_id += 1; + + // Build job definitions. + let job_defs: Vec<_> = pipeline_def + .jobs + .iter() + .map(|(name, def)| yaml::to_job_definition(name, def)) + .collect(); + + let pipeline = PipelineExecution::new( + pipeline_id, + pipeline_name.clone(), + event.repo_owner.clone(), + event.repo_name.clone(), + event.commit_sha.clone(), + event.branch.clone(), + job_defs, + ); + + // Set pending status on Forgejo. + self.emit_status_update(StatusUpdate { + repo_owner: event.repo_owner.clone(), + repo_name: event.repo_name.clone(), + commit_sha: event.commit_sha.clone(), + state: "pending".into(), + context: format!("ci/{pipeline_name}"), + description: format!("Pipeline '{pipeline_name}' is pending"), + target_url: None, + }); + + self.pipelines.insert(pipeline_id, pipeline); + + // Start eligible jobs. + self.advance_pipeline(ctx, pipeline_id); + } + } + + fn advance_pipeline(&mut self, ctx: &Ctx, pipeline_id: PipelineId) { + let pipeline = match self.pipelines.get(&pipeline_id) { + Some(p) => p, + None => return, + }; + + // If pipeline is already terminal, emit final status. + if pipeline.status.is_terminal() { + let update = StatusUpdate { + repo_owner: pipeline.repo_owner.clone(), + repo_name: pipeline.repo_name.clone(), + commit_sha: pipeline.commit_sha.clone(), + state: pipeline.status.forgejo_state().into(), + context: format!("ci/{}", pipeline.pipeline_name), + description: format!( + "Pipeline '{}' {}", + pipeline.pipeline_name, + pipeline.status.forgejo_state() + ), + target_url: None, + }; + self.emit_status_update(update); + return; + } + + let eligible = pipeline.eligible_jobs(); + let repo_owner = pipeline.repo_owner.clone(); + let repo_name = pipeline.repo_name.clone(); + let commit_sha = pipeline.commit_sha.clone(); + + for job_name in eligible { + let job_id = JobId { + pipeline_id, + job_name: job_name.clone(), + }; + + let spec = { + let pipeline = self.pipelines.get(&pipeline_id).unwrap(); + let job = &pipeline.jobs[&job_name]; + crate::InstanceSpec { + docker_required: job.definition.docker, + ..Default::default() + } + }; + + if self.provisioner_online { + // Request provisioning. + if let Some(prov_addr) = self.provisioner_addr { + let request = ProvisionRequest { + job_id: job_id.clone(), + instance_spec: spec, + }; + let _ = ctx.send(prov_addr, crate::provisioner::ProvisionerMsg::Provision(request)); + } + if let Some(pipeline) = self.pipelines.get_mut(&pipeline_id) { + if let Some(job) = pipeline.jobs.get_mut(&job_name) { + job.status = JobStatus::Provisioning; + } + } + } else { + // Queue for later. + let request = ProvisionRequest { + job_id: job_id.clone(), + instance_spec: spec, + }; + self.queued_provisions.push(request); + if let Some(pipeline) = self.pipelines.get_mut(&pipeline_id) { + if let Some(job) = pipeline.jobs.get_mut(&job_name) { + job.status = JobStatus::WaitingForProvisioner; + } + } + } + + // Emit per-job status. + self.emit_status_update(StatusUpdate { + repo_owner: repo_owner.clone(), + repo_name: repo_name.clone(), + commit_sha: commit_sha.clone(), + state: "pending".into(), + context: format!("ci/{job_name}"), + description: format!("Job '{job_name}' is provisioning"), + target_url: None, + }); + } + } + + fn handle_provision_response(&mut self, ctx: &Ctx, response: ProvisionResponse) { + let pipeline_id = response.job_id.pipeline_id; + let job_name = response.job_id.job_name.clone(); + + match response.result { + Ok(instance) => { + // Store instance_id for cleanup. + if let Some(pipeline) = self.pipelines.get_mut(&pipeline_id) { + if let Some(job) = pipeline.jobs.get_mut(&job_name) { + job.instance_id = Some(instance.instance_id.clone()); + job.status = JobStatus::Running; + } + } + + // Spawn a RunnerSupervisor for this job. + let pipeline = &self.pipelines[&pipeline_id]; + let job = &pipeline.jobs[&job_name]; + + let start_job = crate::StartJob { + job_id: response.job_id.clone(), + instance: instance.clone(), + repo_url: format!( + "{}/{}/{}", + self.config.forgejo_url, pipeline.repo_owner, pipeline.repo_name + ), + commit_sha: pipeline.commit_sha.clone(), + job_def: job.definition.clone(), + }; + + let runner = crate::runner::RunnerSupervisor::new( + ctx.self_addr(), + start_job, + ); + + match ctx.spawn(runner) { + Ok(runner_addr) => { + self.runner_addrs.insert(response.job_id, runner_addr); + } + Err(_) => { + // Failed to spawn runner — mark job as failed. + if let Some(pipeline) = self.pipelines.get_mut(&pipeline_id) { + pipeline.set_job_status( + &job_name, + JobStatus::Failed { + reason: "failed to spawn runner".into(), + }, + ); + } + self.advance_pipeline(ctx, pipeline_id); + } + } + } + Err(err) => { + if let Some(pipeline) = self.pipelines.get_mut(&pipeline_id) { + pipeline.set_job_status( + &job_name, + JobStatus::Failed { + reason: err.to_string(), + }, + ); + } + self.advance_pipeline(ctx, pipeline_id); + } + } + } + + fn handle_job_complete(&mut self, ctx: &Ctx, complete: JobComplete) { + let pipeline_id = complete.job_id.pipeline_id; + let job_name = complete.job_id.job_name.clone(); + + // Send terminate request for the instance. + if let Some(pipeline) = self.pipelines.get(&pipeline_id) { + if let Some(job) = pipeline.jobs.get(&job_name) { + if let Some(ref instance_id) = job.instance_id { + let terminate = TerminateRequest { + job_id: complete.job_id.clone(), + instance_id: instance_id.clone(), + }; + if let Some(prov_addr) = self.provisioner_addr { + let _ = ctx.send( + prov_addr, + crate::provisioner::ProvisionerMsg::Terminate(terminate), + ); + } + } + } + } + + // Update job status. + let status = match complete.result { + Ok(_) => JobStatus::Passed, + Err(ref failure) => JobStatus::Failed { + reason: failure.to_string(), + }, + }; + + let (repo_owner, repo_name, commit_sha) = { + let pipeline = match self.pipelines.get(&pipeline_id) { + Some(p) => p, + None => return, + }; + ( + pipeline.repo_owner.clone(), + pipeline.repo_name.clone(), + pipeline.commit_sha.clone(), + ) + }; + + // Emit per-job final status. + self.emit_status_update(StatusUpdate { + repo_owner, + repo_name, + commit_sha, + state: match &status { + JobStatus::Passed => "success".into(), + _ => "failure".into(), + }, + context: format!("ci/{job_name}"), + description: format!("Job '{job_name}' completed"), + target_url: None, + }); + + if let Some(pipeline) = self.pipelines.get_mut(&pipeline_id) { + pipeline.set_job_status(&job_name, status); + } + + // Remove runner address. + self.runner_addrs.remove(&complete.job_id); + + // Advance pipeline to schedule downstream jobs. + self.advance_pipeline(ctx, pipeline_id); + } + + fn handle_provisioner_offline(&mut self) { + self.provisioner_online = false; + + // Mark all provisioning jobs as waiting. + for pipeline in self.pipelines.values_mut() { + for job in pipeline.jobs.values_mut() { + if job.status == JobStatus::Provisioning { + job.status = JobStatus::WaitingForProvisioner; + } + } + } + } + + fn handle_provisioner_online(&mut self, ctx: &Ctx) { + self.provisioner_online = true; + + // Flush queued provision requests. + let queued = std::mem::take(&mut self.queued_provisions); + for request in queued { + if let Some(prov_addr) = self.provisioner_addr { + let _ = ctx.send(prov_addr, crate::provisioner::ProvisionerMsg::Provision(request)); + } + } + + // Re-advance pipelines that have waiting jobs. + let pipeline_ids: Vec = self.pipelines.keys().copied().collect(); + for pid in pipeline_ids { + // Move waiting jobs back to provisioning. + if let Some(pipeline) = self.pipelines.get_mut(&pid) { + let waiting_jobs: Vec = pipeline + .jobs + .iter() + .filter(|(_, j)| j.status == JobStatus::WaitingForProvisioner) + .map(|(name, _)| name.clone()) + .collect(); + + for job_name in waiting_jobs { + if let Some(job) = pipeline.jobs.get_mut(&job_name) { + job.status = JobStatus::Pending; + } + } + } + self.advance_pipeline(ctx, pid); + } + } + + fn emit_status_update(&mut self, update: StatusUpdate) { + self.status_updates.push(update); + } +} + +impl ActorInterface for Coordinator { + type Incoming = CoordinatorMsg; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, msg: CoordinatorMsg) { + match msg { + CoordinatorMsg::Webhook(event) => self.handle_webhook(ctx, event), + CoordinatorMsg::SetCiYaml(yaml) => { + self.ci_yaml = Some(yaml); + } + CoordinatorMsg::ProvisionResponse(resp) => { + self.handle_provision_response(ctx, resp); + } + CoordinatorMsg::JobProgress(progress) => { + if let Some(pipeline) = self.pipelines.get_mut(&progress.job_id.pipeline_id) { + if let Some(job) = pipeline.jobs.get_mut(&progress.job_id.job_name) { + job.output_lines.push(progress.output_line); + } + } + } + CoordinatorMsg::JobComplete(complete) => { + self.handle_job_complete(ctx, complete); + } + CoordinatorMsg::ProvisionerOffline => self.handle_provisioner_offline(), + CoordinatorMsg::ProvisionerOnline => self.handle_provisioner_online(ctx), + } + } +} diff --git a/crates/ci/src/lib.rs b/crates/ci/src/lib.rs new file mode 100644 index 0000000..5a0f2ad --- /dev/null +++ b/crates/ci/src/lib.rs @@ -0,0 +1,315 @@ +pub mod coordinator; +pub mod local_coordinator; +pub mod local_runner; +pub mod pipeline; +pub mod provisioner; +pub mod runner; +pub mod status_reporter; +pub mod webhook_server; +pub mod yaml; + +use std::collections::HashMap; +use std::fmt; +use std::net::IpAddr; + +use serde::{Deserialize, Serialize}; + +// ─── Core Identifiers ─────────────────────────────────────────────────────── + +/// Unique identifier for a pipeline execution. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct PipelineId(pub u64); + +impl fmt::Display for PipelineId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "pipeline-{}", self.0) + } +} + +/// Unique identifier for a job within a pipeline. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct JobId { + pub pipeline_id: PipelineId, + pub job_name: String, +} + +impl fmt::Display for JobId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}/{}", self.pipeline_id, self.job_name) + } +} + +// ─── Job Status ───────────────────────────────────────────────────────────── + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum JobStatus { + Pending, + WaitingForProvisioner, + Provisioning, + Running, + Passed, + Failed { reason: String }, + Skipped, + Interrupted, +} + +impl JobStatus { + pub fn is_terminal(&self) -> bool { + matches!( + self, + JobStatus::Passed | JobStatus::Failed { .. } | JobStatus::Skipped | JobStatus::Interrupted + ) + } +} + +// ─── Pipeline Status ──────────────────────────────────────────────────────── + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum PipelineStatus { + Pending, + Running, + Passed, + Failed, + Error { reason: String }, +} + +impl PipelineStatus { + pub fn is_terminal(&self) -> bool { + matches!( + self, + PipelineStatus::Passed | PipelineStatus::Failed | PipelineStatus::Error { .. } + ) + } + + /// Convert to Forgejo commit status string. + pub fn forgejo_state(&self) -> &'static str { + match self { + PipelineStatus::Pending => "pending", + PipelineStatus::Running => "pending", + PipelineStatus::Passed => "success", + PipelineStatus::Failed => "failure", + PipelineStatus::Error { .. } => "error", + } + } +} + +// ─── Instance Types ───────────────────────────────────────────────────────── + +/// Specification for a spot instance. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InstanceSpec { + pub min_cpus: u32, + pub min_ram_mb: u32, + pub min_disk_gb: u32, + pub docker_required: bool, + pub region_preferences: Vec, +} + +impl Default for InstanceSpec { + fn default() -> Self { + Self { + min_cpus: 2, + min_ram_mb: 2048, + min_disk_gb: 20, + docker_required: false, + region_preferences: Vec::new(), + } + } +} + +/// Connection details for a provisioned spot instance. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InstanceReady { + pub instance_id: String, + pub ip: IpAddr, + pub ssh_port: u16, + pub ssh_host_key: String, +} + +// ─── Provisioner ↔ Coordinator Messages ───────────────────────────────────── + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ProvisionRequest { + pub job_id: JobId, + pub instance_spec: InstanceSpec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ProvisionResponse { + pub job_id: JobId, + pub result: Result, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum ProvisionError { + NoCapacity, + ProviderError(String), + Timeout, + ProvisionerOffline, +} + +impl fmt::Display for ProvisionError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ProvisionError::NoCapacity => write!(f, "no capacity available"), + ProvisionError::ProviderError(msg) => write!(f, "provider error: {msg}"), + ProvisionError::Timeout => write!(f, "provisioning timed out"), + ProvisionError::ProvisionerOffline => write!(f, "provisioner is offline"), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TerminateRequest { + pub job_id: JobId, + pub instance_id: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TerminateAck { + pub job_id: JobId, +} + +// ─── Coordinator ↔ RunnerSupervisor Messages ──────────────────────────────── + +/// Sent from Coordinator to RunnerSupervisor to begin a job. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StartJob { + pub job_id: JobId, + pub instance: InstanceReady, + pub repo_url: String, + pub commit_sha: String, + pub job_def: JobDefinition, +} + +/// Streamed output from RunnerSupervisor back to Coordinator. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct JobProgress { + pub job_id: JobId, + pub output_line: String, +} + +/// Final result from RunnerSupervisor. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct JobComplete { + pub job_id: JobId, + pub result: Result, + pub artifacts: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct JobSuccess; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum JobFailure { + CommandFailed { exit_code: i32, last_lines: Vec }, + SshError(String), + ExecError(String), + Timeout, + Interrupted, +} + +impl fmt::Display for JobFailure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + JobFailure::CommandFailed { exit_code, .. } => { + write!(f, "command exited with code {exit_code}") + } + JobFailure::SshError(msg) => write!(f, "SSH error: {msg}"), + JobFailure::ExecError(msg) => write!(f, "exec error: {msg}"), + JobFailure::Timeout => write!(f, "job timed out"), + JobFailure::Interrupted => write!(f, "spot instance interrupted"), + } + } +} + +// ─── Job Definition ───────────────────────────────────────────────────────── + +/// A parsed job from the .ci.yml file. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct JobDefinition { + pub name: String, + pub run: Vec, + pub needs: Vec, + pub timeout_secs: u64, + pub docker: bool, + pub artifacts: Vec, + pub env: HashMap, +} + +// ─── Webhook Types ────────────────────────────────────────────────────────── + +/// Parsed webhook event from Forgejo. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WebhookEvent { + pub event_type: EventType, + pub repo_owner: String, + pub repo_name: String, + pub branch: String, + pub commit_sha: String, + pub tag: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum EventType { + Push, + Tag, + Merge, +} + +// ─── Forgejo Status Updates ───────────────────────────────────────────────── + +/// A commit status update to send to Forgejo's API. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StatusUpdate { + pub repo_owner: String, + pub repo_name: String, + pub commit_sha: String, + pub state: String, + pub context: String, + pub description: String, + pub target_url: Option, +} + +// ─── Coordinator Config ───────────────────────────────────────────────────── + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CiConfig { + pub webhook_port: u16, + pub webhook_secret: String, + pub forgejo_url: String, + pub forgejo_token: String, + pub data_dir: String, +} + +impl Default for CiConfig { + fn default() -> Self { + Self { + webhook_port: 8787, + webhook_secret: String::new(), + forgejo_url: String::new(), + forgejo_token: String::new(), + data_dir: "./ci-data".into(), + } + } +} + +// ─── Local CI Types ────────────────────────────────────────────────────────── + +/// Job execution request for local runner (no InstanceReady needed). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LocalStartJob { + pub job_id: JobId, + pub work_dir: String, + pub job_def: JobDefinition, + pub env_overrides: HashMap, +} + +/// Configuration for the local CI runner. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LocalCiConfig { + pub ci: CiConfig, + pub repo_url: String, + pub work_dir: String, + pub ci_yaml_path: String, +} diff --git a/crates/ci/src/local_coordinator.rs b/crates/ci/src/local_coordinator.rs new file mode 100644 index 0000000..2858662 --- /dev/null +++ b/crates/ci/src/local_coordinator.rs @@ -0,0 +1,565 @@ +//! LocalCoordinator actor: single-machine CI brain. +//! +//! Receives webhook events, queues pipelines, executes jobs one at a time +//! directly on the host. Supports branch-level supersede for queued pipelines. + +use std::collections::{HashMap, VecDeque}; +use std::process::Command; +use std::sync::{Arc, Mutex}; + +use swactor::actor::{ActorAddress, ActorInterface, Ctx}; + +use crate::pipeline::PipelineExecution; +use crate::status_reporter::{JobOutput, StatusReporterMsg}; +use crate::yaml::{self, CiYaml}; +use crate::{ + JobComplete, JobId, JobProgress, JobStatus, LocalCiConfig, LocalStartJob, PipelineId, + PipelineStatus, StatusUpdate, WebhookEvent, +}; + +/// Messages the LocalCoordinator can receive. +#[derive(Debug, Clone)] +pub enum LocalCoordinatorMsg { + Webhook(WebhookEvent), + SetCiYaml(CiYaml), + JobProgress(JobProgress), + JobComplete(JobComplete), + GitReady { + pipeline_id: PipelineId, + job_name: String, + work_dir: String, + }, +} + +/// Lightweight snapshot of coordinator state (no dashboard dependency). +/// The binary layer converts this to `CiSnapshot` for the dashboard. +#[derive(Debug, Clone, Default)] +pub struct LocalCiSnapshot { + pub active_pipelines: Vec, + pub recent_pipelines: Vec, + pub has_running_job: bool, +} + +/// The LocalCoordinator actor state. +pub struct LocalCoordinator { + config: LocalCiConfig, + ci_yaml: Option, + pipelines: HashMap, + next_pipeline_id: u64, + /// FIFO queue of pipeline IDs awaiting execution. + queue: VecDeque, + /// The pipeline currently being executed. + active_pipeline: Option, + /// At most one running job (job_id, runner actor address). + running_job: Option<(JobId, ActorAddress)>, + /// Status reporter actor address. + status_reporter_addr: Option, + /// Bounded ring of finished pipelines. + completed: VecDeque, + /// Shared snapshot for external consumers (e.g. dashboard binary). + ci_snapshot: Arc>, +} + +impl LocalCoordinator { + pub fn new(config: LocalCiConfig) -> Self { + Self { + config, + ci_yaml: None, + pipelines: HashMap::new(), + next_pipeline_id: 1, + queue: VecDeque::new(), + active_pipeline: None, + running_job: None, + status_reporter_addr: None, + completed: VecDeque::new(), + ci_snapshot: Arc::new(Mutex::new(LocalCiSnapshot::default())), + } + } + + pub fn with_status_reporter(mut self, addr: ActorAddress) -> Self { + self.status_reporter_addr = Some(addr); + self + } + + pub fn with_ci_yaml(mut self, yaml: CiYaml) -> Self { + self.ci_yaml = Some(yaml); + self + } + + pub fn ci_snapshot(&self) -> Arc> { + Arc::clone(&self.ci_snapshot) + } + + pub fn pipelines(&self) -> &HashMap { + &self.pipelines + } + + pub fn completed(&self) -> &VecDeque { + &self.completed + } + + pub fn queue(&self) -> &VecDeque { + &self.queue + } + + pub fn active_pipeline(&self) -> Option { + self.active_pipeline + } + + pub fn running_job(&self) -> Option<&(JobId, ActorAddress)> { + self.running_job.as_ref() + } + + fn handle_webhook(&mut self, ctx: &Ctx, event: WebhookEvent) { + let ci = match &self.ci_yaml { + Some(ci) => ci.clone(), + None => return, + }; + + let matched = yaml::matching_pipelines(&ci, &event); + for pipeline_name in matched { + let pipeline_def = &ci.pipelines[&pipeline_name]; + let pipeline_id = PipelineId(self.next_pipeline_id); + self.next_pipeline_id += 1; + + let job_defs: Vec<_> = pipeline_def + .jobs + .iter() + .map(|(name, def)| yaml::to_job_definition(name, def)) + .collect(); + + let pipeline = PipelineExecution::new( + pipeline_id, + pipeline_name.clone(), + event.repo_owner.clone(), + event.repo_name.clone(), + event.commit_sha.clone(), + event.branch.clone(), + job_defs, + ); + + self.emit_status( + ctx, + StatusUpdate { + repo_owner: event.repo_owner.clone(), + repo_name: event.repo_name.clone(), + commit_sha: event.commit_sha.clone(), + state: "pending".into(), + context: format!("ci/{pipeline_name}"), + description: format!("Pipeline '{pipeline_name}' is pending"), + target_url: None, + }, + ); + + self.pipelines.insert(pipeline_id, pipeline); + self.enqueue_pipeline(pipeline_id, &event.branch); + } + + self.try_schedule_next(ctx); + } + + /// Enqueue a pipeline, superseding any queued pipeline for the same branch. + fn enqueue_pipeline(&mut self, pipeline_id: PipelineId, branch: &str) { + // Scan queue for entry with same branch (not the active pipeline). + let supersede_idx = self.queue.iter().position(|&qid| { + self.pipelines + .get(&qid) + .map(|p| p.branch == branch) + .unwrap_or(false) + }); + + if let Some(idx) = supersede_idx { + let old_id = self.queue[idx]; + // Mark old pipeline as superseded. + if let Some(old_pipeline) = self.pipelines.get_mut(&old_id) { + old_pipeline.status = PipelineStatus::Error { + reason: "superseded".into(), + }; + // Mark all pending jobs as skipped. + let job_names: Vec = old_pipeline.jobs.keys().cloned().collect(); + for name in job_names { + if old_pipeline.jobs[&name].status == JobStatus::Pending { + old_pipeline.set_job_status(&name, JobStatus::Skipped); + } + } + } + // Archive the superseded pipeline. + if let Some(old_pipeline) = self.pipelines.remove(&old_id) { + self.archive_pipeline(old_pipeline); + } + // Replace queue entry. + self.queue[idx] = pipeline_id; + } else { + self.queue.push_back(pipeline_id); + } + } + + /// Core scheduling: one job at a time. + fn try_schedule_next(&mut self, ctx: &Ctx) { + // If a job is already running, nothing to do. + if self.running_job.is_some() { + return; + } + + // If we have an active pipeline, try to find eligible jobs. + if let Some(active_id) = self.active_pipeline { + if let Some(pipeline) = self.pipelines.get(&active_id) { + let eligible = pipeline.eligible_jobs(); + if !eligible.is_empty() { + let job_name = eligible[0].clone(); + self.start_job(ctx, active_id, &job_name); + return; + } + + // No eligible jobs — check if pipeline is terminal. + if pipeline.status.is_terminal() { + let pipeline = self.pipelines.remove(&active_id).unwrap(); + self.emit_status( + ctx, + StatusUpdate { + repo_owner: pipeline.repo_owner.clone(), + repo_name: pipeline.repo_name.clone(), + commit_sha: pipeline.commit_sha.clone(), + state: pipeline.status.forgejo_state().into(), + context: format!("ci/{}", pipeline.pipeline_name), + description: format!( + "Pipeline '{}' {}", + pipeline.pipeline_name, + pipeline.status.forgejo_state() + ), + target_url: None, + }, + ); + self.emit_pipeline_comment(ctx, &pipeline); + self.archive_pipeline(pipeline); + self.active_pipeline = None; + // Recurse to pick next from queue. + self.try_schedule_next(ctx); + return; + } + } + // Pipeline exists but no eligible jobs and not terminal — waiting for running job. + return; + } + + // No active pipeline — pop from queue. + if let Some(next_id) = self.queue.pop_front() { + self.active_pipeline = Some(next_id); + self.try_schedule_next(ctx); + } + } + + fn start_job(&mut self, ctx: &Ctx, pipeline_id: PipelineId, job_name: &str) { + let pipeline = match self.pipelines.get_mut(&pipeline_id) { + Some(p) => p, + None => return, + }; + let job = match pipeline.jobs.get_mut(job_name) { + Some(j) => j, + None => return, + }; + + job.status = JobStatus::Running; + + let work_dir = format!( + "{}/pipeline-{}", + self.config.work_dir, pipeline_id.0 + ); + + // Build CI env overrides. + let mut env_overrides = HashMap::new(); + env_overrides.insert("CI".into(), "true".into()); + env_overrides.insert("CI_COMMIT_SHA".into(), pipeline.commit_sha.clone()); + env_overrides.insert("CI_BRANCH".into(), pipeline.branch.clone()); + env_overrides.insert("CI_PIPELINE_ID".into(), pipeline_id.0.to_string()); + env_overrides.insert("CI_JOB_NAME".into(), job_name.to_string()); + + let start_job = LocalStartJob { + job_id: job.job_id.clone(), + work_dir: work_dir.clone(), + job_def: job.definition.clone(), + env_overrides, + }; + + // Perform git checkout inline (blocks this worker, acceptable for local runner). + let sha = pipeline.commit_sha.clone(); + let repo_url = self.config.repo_url.clone(); + let git_ok = self.git_checkout(&repo_url, &sha, &work_dir); + + if !git_ok { + if let Some(pipeline) = self.pipelines.get_mut(&pipeline_id) { + pipeline.set_job_status( + job_name, + JobStatus::Failed { + reason: "git checkout failed".into(), + }, + ); + } + self.try_schedule_next(ctx); + return; + } + + // Emit per-job running status. + if let Some(pipeline) = self.pipelines.get(&pipeline_id) { + self.emit_status( + ctx, + StatusUpdate { + repo_owner: pipeline.repo_owner.clone(), + repo_name: pipeline.repo_name.clone(), + commit_sha: pipeline.commit_sha.clone(), + state: "pending".into(), + context: format!("ci/{job_name}"), + description: format!("Job '{job_name}' is running"), + target_url: None, + }, + ); + } + + // Spawn LocalRunner actor. + let runner = + crate::local_runner::LocalRunner::new(ctx.self_addr(), start_job); + + match ctx.spawn(runner) { + Ok(runner_addr) => { + let job_id = JobId { + pipeline_id, + job_name: job_name.to_string(), + }; + self.running_job = Some((job_id, runner_addr)); + } + Err(_) => { + if let Some(pipeline) = self.pipelines.get_mut(&pipeline_id) { + pipeline.set_job_status( + job_name, + JobStatus::Failed { + reason: "failed to spawn runner".into(), + }, + ); + } + self.try_schedule_next(ctx); + } + } + } + + fn git_checkout(&self, repo_url: &str, sha: &str, work_dir: &str) -> bool { + let path = std::path::Path::new(work_dir); + if path.join(".git").exists() { + // Already cloned — fetch and checkout. + let fetch = Command::new("git") + .args(["fetch", "origin"]) + .current_dir(work_dir) + .output(); + if fetch.is_err() || !fetch.unwrap().status.success() { + return false; + } + let checkout = Command::new("git") + .args(["checkout", sha]) + .current_dir(work_dir) + .output(); + checkout.map(|o| o.status.success()).unwrap_or(false) + } else { + // Fresh clone. + if let Some(parent) = path.parent() { + let _ = std::fs::create_dir_all(parent); + } + let clone = Command::new("git") + .args(["clone", repo_url, work_dir]) + .output(); + if clone.is_err() || !clone.as_ref().unwrap().status.success() { + return false; + } + let checkout = Command::new("git") + .args(["checkout", sha]) + .current_dir(work_dir) + .output(); + checkout.map(|o| o.status.success()).unwrap_or(false) + } + } + + fn handle_job_complete(&mut self, ctx: &Ctx, complete: JobComplete) { + let pipeline_id = complete.job_id.pipeline_id; + let job_name = complete.job_id.job_name.clone(); + + let status = match complete.result { + Ok(_) => JobStatus::Passed, + Err(ref failure) => JobStatus::Failed { + reason: failure.to_string(), + }, + }; + + // Build a description with output tail for failures. + let description = match &status { + JobStatus::Passed => format!("Job '{job_name}' passed"), + JobStatus::Failed { reason } => { + let output_tail = self + .pipelines + .get(&pipeline_id) + .and_then(|p| p.jobs.get(&job_name)) + .map(|j| { + let lines: Vec<&str> = j + .output_lines + .iter() + .rev() + .take(10) + .map(|s| s.as_str()) + .collect(); + lines.into_iter().rev().collect::>().join("\n") + }) + .unwrap_or_default(); + + let mut desc = format!("Job '{job_name}' failed: {reason}"); + if !output_tail.is_empty() { + desc.push_str("\n"); + desc.push_str(&output_tail); + } + // Cap at ~250 chars for the status description field. + if desc.len() > 250 { + desc.truncate(247); + desc.push_str("..."); + } + desc + } + _ => format!("Job '{job_name}' completed"), + }; + + // Emit per-job final status. + if let Some(pipeline) = self.pipelines.get(&pipeline_id) { + self.emit_status( + ctx, + StatusUpdate { + repo_owner: pipeline.repo_owner.clone(), + repo_name: pipeline.repo_name.clone(), + commit_sha: pipeline.commit_sha.clone(), + state: match &status { + JobStatus::Passed => "success".into(), + _ => "failure".into(), + }, + context: format!("ci/{job_name}"), + description, + target_url: None, + }, + ); + } + + if let Some(pipeline) = self.pipelines.get_mut(&pipeline_id) { + pipeline.set_job_status(&job_name, status); + } + + // Clear running job. + self.running_job = None; + + // Schedule next. + self.try_schedule_next(ctx); + } + + fn emit_status(&self, ctx: &Ctx, update: StatusUpdate) { + if let Some(reporter_addr) = self.status_reporter_addr { + let _ = ctx.send( + reporter_addr, + StatusReporterMsg::Report { + update, + forgejo_url: self.config.ci.forgejo_url.clone(), + forgejo_token: self.config.ci.forgejo_token.clone(), + }, + ); + } + } + + fn emit_pipeline_comment(&self, ctx: &Ctx, pipeline: &crate::pipeline::PipelineExecution) { + let reporter_addr = match self.status_reporter_addr { + Some(addr) => addr, + None => return, + }; + + let job_outputs: Vec = pipeline + .jobs + .values() + .map(|job| { + let passed = job.status == JobStatus::Passed; + let failure_reason = match &job.status { + JobStatus::Failed { reason } => Some(reason.clone()), + _ => None, + }; + JobOutput { + job_name: job.definition.name.clone(), + passed, + failure_reason, + output_lines: job.output_lines.clone(), + } + }) + .collect(); + + let _ = ctx.send( + reporter_addr, + StatusReporterMsg::PostPipelineComment { + repo_owner: pipeline.repo_owner.clone(), + repo_name: pipeline.repo_name.clone(), + commit_sha: pipeline.commit_sha.clone(), + branch: pipeline.branch.clone(), + pipeline_name: pipeline.pipeline_name.clone(), + pipeline_state: pipeline.status.forgejo_state().to_string(), + job_outputs, + forgejo_url: self.config.ci.forgejo_url.clone(), + forgejo_token: self.config.ci.forgejo_token.clone(), + }, + ); + } + + fn archive_pipeline(&mut self, pipeline: PipelineExecution) { + self.completed.push_back(pipeline); + if self.completed.len() > 50 { + self.completed.pop_front(); + } + } + + fn update_snapshot(&self) { + let active: Vec = self.pipelines.values().cloned().collect(); + let recent: Vec = self + .completed + .iter() + .rev() + .take(20) + .cloned() + .collect(); + + if let Ok(mut snap) = self.ci_snapshot.lock() { + snap.active_pipelines = active; + snap.recent_pipelines = recent; + snap.has_running_job = self.running_job.is_some(); + } + } +} + +impl ActorInterface for LocalCoordinator { + type Incoming = LocalCoordinatorMsg; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, msg: LocalCoordinatorMsg) { + match msg { + LocalCoordinatorMsg::Webhook(event) => self.handle_webhook(ctx, event), + LocalCoordinatorMsg::SetCiYaml(yaml) => { + self.ci_yaml = Some(yaml); + } + LocalCoordinatorMsg::JobProgress(progress) => { + if let Some(pipeline) = self.pipelines.get_mut(&progress.job_id.pipeline_id) { + if let Some(job) = pipeline.jobs.get_mut(&progress.job_id.job_name) { + job.output_lines.push(progress.output_line); + } + } + } + LocalCoordinatorMsg::JobComplete(complete) => { + self.handle_job_complete(ctx, complete); + } + LocalCoordinatorMsg::GitReady { + pipeline_id, + job_name, + work_dir, + } => { + // Git ready is used in the async variant; for now handled inline in start_job. + let _ = (pipeline_id, job_name, work_dir); + } + } + + self.update_snapshot(); + } +} diff --git a/crates/ci/src/local_runner.rs b/crates/ci/src/local_runner.rs new file mode 100644 index 0000000..0cd0b7f --- /dev/null +++ b/crates/ci/src/local_runner.rs @@ -0,0 +1,304 @@ +//! LocalRunner actor: executes job commands directly on the host via shell. +//! +//! Short-lived actor, one per job. Spawned by LocalCoordinator when a job +//! is ready to execute. + +use std::io::BufRead; +use std::process::{Child, Command, Stdio}; +use std::sync::{Arc, Mutex}; + +use swactor::actor::{ActorAddress, ActorInterface, Ctx}; + +use crate::local_coordinator::LocalCoordinatorMsg; +use crate::{JobComplete, JobFailure, JobProgress, JobSuccess, LocalStartJob}; + +/// Messages the LocalRunner can receive. +#[derive(Debug, Clone)] +pub enum LocalRunnerMsg { + /// Begin executing the job (sent to self in on_start). + Execute, + /// Simulated: job completed (for testing without real shell). + SimComplete(Result<(), String>), +} + +/// LocalRunner actor state. +pub struct LocalRunner { + coordinator_addr: ActorAddress, + start_job: LocalStartJob, +} + +impl LocalRunner { + pub fn new(coordinator_addr: ActorAddress, start_job: LocalStartJob) -> Self { + Self { + coordinator_addr, + start_job, + } + } + + /// Execute all commands in the job definition, streaming output back. + fn execute(&self, ctx: &Ctx) { + let job_id = &self.start_job.job_id; + let work_dir = &self.start_job.work_dir; + let timeout_secs = self.start_job.job_def.timeout_secs; + + eprintln!( + "[runner] job {}/{} starting ({} commands, timeout {}s, workdir {})", + job_id.pipeline_id.0, + job_id.job_name, + self.start_job.job_def.run.len(), + timeout_secs, + work_dir, + ); + + for cmd_str in &self.start_job.job_def.run { + eprintln!("[runner] exec: {cmd_str}"); + + // Send progress: command being run. + let _ = ctx.send( + self.coordinator_addr, + LocalCoordinatorMsg::JobProgress(JobProgress { + job_id: job_id.clone(), + output_line: format!("$ {cmd_str}"), + }), + ); + + let child_result = Command::new("sh") + .arg("-c") + .arg(cmd_str) + .current_dir(work_dir) + .envs(&self.start_job.env_overrides) + .envs(&self.start_job.job_def.env) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn(); + + let mut child = match child_result { + Ok(c) => c, + Err(e) => { + eprintln!("[runner] spawn failed: {e}"); + let _ = ctx.send( + self.coordinator_addr, + LocalCoordinatorMsg::JobComplete(JobComplete { + job_id: job_id.clone(), + result: Err(JobFailure::ExecError(e.to_string())), + artifacts: Vec::new(), + }), + ); + ctx.stop_self(); + return; + } + }; + + // Timeout: spawn a thread that kills the child after timeout_secs. + let child_id = child.id(); + let kill_flag = Arc::new(Mutex::new(false)); + let kill_flag_clone = Arc::clone(&kill_flag); + let (done_tx, done_rx) = std::sync::mpsc::channel::<()>(); + let timeout_handle = std::thread::spawn(move || { + if done_rx + .recv_timeout(std::time::Duration::from_secs(timeout_secs)) + .is_err() + { + *kill_flag_clone.lock().unwrap() = true; + // Actually kill the child process so the pipe readers unblock. + let _ = std::process::Command::new("kill") + .args(["-9", &child_id.to_string()]) + .status(); + } + }); + + // Read stdout and stderr concurrently to avoid pipe-buffer deadlock. + let (last_lines, timed_out) = + drain_child_output(&mut child, job_id, ctx, self.coordinator_addr, &kill_flag); + + let status = child.wait(); + + // Signal timeout thread that we're done. + let _ = done_tx.send(()); + let _ = timeout_handle.join(); + + if timed_out || *kill_flag.lock().unwrap() { + eprintln!("[runner] command timed out after {timeout_secs}s"); + let _ = ctx.send( + self.coordinator_addr, + LocalCoordinatorMsg::JobComplete(JobComplete { + job_id: job_id.clone(), + result: Err(JobFailure::Timeout), + artifacts: Vec::new(), + }), + ); + ctx.stop_self(); + return; + } + + match status { + Ok(exit) if exit.success() => { + eprintln!("[runner] command succeeded"); + } + Ok(exit) => { + let exit_code = exit.code().unwrap_or(-1); + eprintln!("[runner] command failed (exit {exit_code})"); + for line in &last_lines { + eprintln!("[runner] {line}"); + } + let _ = ctx.send( + self.coordinator_addr, + LocalCoordinatorMsg::JobComplete(JobComplete { + job_id: job_id.clone(), + result: Err(JobFailure::CommandFailed { + exit_code, + last_lines, + }), + artifacts: Vec::new(), + }), + ); + ctx.stop_self(); + return; + } + Err(e) => { + eprintln!("[runner] wait failed: {e}"); + let _ = ctx.send( + self.coordinator_addr, + LocalCoordinatorMsg::JobComplete(JobComplete { + job_id: job_id.clone(), + result: Err(JobFailure::ExecError(e.to_string())), + artifacts: Vec::new(), + }), + ); + ctx.stop_self(); + return; + } + } + } + + eprintln!( + "[runner] job {}/{} passed", + job_id.pipeline_id.0, job_id.job_name + ); + let _ = ctx.send( + self.coordinator_addr, + LocalCoordinatorMsg::JobComplete(JobComplete { + job_id: job_id.clone(), + result: Ok(JobSuccess), + artifacts: Vec::new(), + }), + ); + ctx.stop_self(); + } +} + +/// Drain stdout and stderr from a child process concurrently. +/// +/// Spawns a background thread for stderr so that both pipes are consumed +/// in parallel, preventing the classic pipe-buffer deadlock where the child +/// blocks writing to a full stderr while the parent blocks reading stdout. +/// +/// Returns (last_lines, timed_out). +fn drain_child_output( + child: &mut Child, + job_id: &crate::JobId, + ctx: &Ctx, + coordinator_addr: ActorAddress, + kill_flag: &Arc>, +) -> (Vec, bool) { + let stdout = child.stdout.take(); + let stderr = child.stderr.take(); + + // Collect stderr on a background thread. + let stderr_job_id = job_id.clone(); + let stderr_kill = Arc::clone(kill_flag); + let stderr_handle = std::thread::spawn(move || { + let mut lines = Vec::new(); + if let Some(stderr) = stderr { + let reader = std::io::BufReader::new(stderr); + for line in reader.lines() { + if *stderr_kill.lock().unwrap() { + break; + } + if let Ok(line) = line { + lines.push(line); + } + } + } + lines + }); + + // Read stdout on the current thread, streaming progress. + let mut last_lines: Vec = Vec::new(); + if let Some(stdout) = stdout { + let reader = std::io::BufReader::new(stdout); + for line in reader.lines() { + if let Ok(line) = line { + let _ = ctx.send( + coordinator_addr, + LocalCoordinatorMsg::JobProgress(JobProgress { + job_id: job_id.clone(), + output_line: line.clone(), + }), + ); + last_lines.push(line); + if last_lines.len() > 50 { + last_lines.remove(0); + } + } + } + } + + // Join stderr thread and stream its lines as progress. + let timed_out = *kill_flag.lock().unwrap(); + let stderr_lines = stderr_handle.join().unwrap_or_default(); + for line in &stderr_lines { + let _ = ctx.send( + coordinator_addr, + LocalCoordinatorMsg::JobProgress(JobProgress { + job_id: stderr_job_id.clone(), + output_line: format!("[stderr] {line}"), + }), + ); + } + + // Merge stderr into last_lines tail. + for line in stderr_lines { + last_lines.push(line); + if last_lines.len() > 50 { + last_lines.remove(0); + } + } + + (last_lines, timed_out) +} + +impl ActorInterface for LocalRunner { + type Incoming = LocalRunnerMsg; + type Response = (); + + fn on_start(&mut self, ctx: &Ctx) { + let _ = ctx.send(ctx.self_addr(), LocalRunnerMsg::Execute); + } + + fn handle(&mut self, ctx: &Ctx, msg: LocalRunnerMsg) { + match msg { + LocalRunnerMsg::Execute => { + self.execute(ctx); + } + LocalRunnerMsg::SimComplete(result) => { + let complete = JobComplete { + job_id: self.start_job.job_id.clone(), + result: match result { + Ok(()) => Ok(JobSuccess), + Err(msg) => Err(JobFailure::CommandFailed { + exit_code: 1, + last_lines: vec![msg], + }), + }, + artifacts: Vec::new(), + }; + let _ = ctx.send( + self.coordinator_addr, + LocalCoordinatorMsg::JobComplete(complete), + ); + ctx.stop_self(); + } + } + } +} diff --git a/crates/ci/src/pipeline.rs b/crates/ci/src/pipeline.rs new file mode 100644 index 0000000..20f241e --- /dev/null +++ b/crates/ci/src/pipeline.rs @@ -0,0 +1,391 @@ +//! Pipeline resolution, DAG execution logic, and job ordering. + +use std::collections::{HashMap, HashSet, VecDeque}; + +use crate::{JobDefinition, JobId, JobStatus, PipelineId, PipelineStatus}; + +/// A pipeline execution: tracks the DAG of jobs and their statuses. +#[derive(Debug, Clone)] +pub struct PipelineExecution { + pub pipeline_id: PipelineId, + pub pipeline_name: String, + pub repo_owner: String, + pub repo_name: String, + pub commit_sha: String, + pub branch: String, + pub status: PipelineStatus, + pub jobs: HashMap, +} + +/// State of a single job within a pipeline execution. +#[derive(Debug, Clone)] +pub struct JobExecution { + pub job_id: JobId, + pub definition: JobDefinition, + pub status: JobStatus, + pub output_lines: Vec, + pub instance_id: Option, +} + +impl PipelineExecution { + /// Create a new pipeline execution from a set of job definitions. + pub fn new( + pipeline_id: PipelineId, + pipeline_name: String, + repo_owner: String, + repo_name: String, + commit_sha: String, + branch: String, + job_defs: Vec, + ) -> Self { + let mut jobs = HashMap::new(); + for def in job_defs { + let job_id = JobId { + pipeline_id, + job_name: def.name.clone(), + }; + jobs.insert( + def.name.clone(), + JobExecution { + job_id, + definition: def, + status: JobStatus::Pending, + output_lines: Vec::new(), + instance_id: None, + }, + ); + } + Self { + pipeline_id, + pipeline_name, + repo_owner, + repo_name, + commit_sha, + branch, + status: PipelineStatus::Pending, + jobs, + } + } + + /// Return job names that are eligible for execution: Pending with all needs satisfied. + pub fn eligible_jobs(&self) -> Vec { + self.jobs + .values() + .filter(|job| { + job.status == JobStatus::Pending + && job.definition.needs.iter().all(|dep| { + self.jobs + .get(dep) + .map(|d| d.status == JobStatus::Passed) + .unwrap_or(false) + }) + }) + .map(|job| job.definition.name.clone()) + .collect() + } + + /// Mark a job as a given status. If a job fails, propagate skip to dependents. + pub fn set_job_status(&mut self, job_name: &str, status: JobStatus) { + if let Some(job) = self.jobs.get_mut(job_name) { + job.status = status.clone(); + } + + // If failure or interruption, skip all transitive dependents. + if matches!( + status, + JobStatus::Failed { .. } | JobStatus::Interrupted + ) { + let to_skip = self.transitive_dependents(job_name); + for dep_name in to_skip { + if let Some(dep_job) = self.jobs.get_mut(&dep_name) { + if dep_job.status == JobStatus::Pending { + dep_job.status = JobStatus::Skipped; + } + } + } + } + + // Update pipeline status. + self.update_pipeline_status(); + } + + /// Compute the overall pipeline status from individual job statuses. + fn update_pipeline_status(&mut self) { + let all_terminal = self.jobs.values().all(|j| j.status.is_terminal()); + let any_running = self.jobs.values().any(|j| { + matches!( + j.status, + JobStatus::Running | JobStatus::Provisioning | JobStatus::WaitingForProvisioner + ) + }); + let any_failed = self.jobs.values().any(|j| { + matches!( + j.status, + JobStatus::Failed { .. } | JobStatus::Interrupted + ) + }); + + if all_terminal { + self.status = if any_failed { + PipelineStatus::Failed + } else { + PipelineStatus::Passed + }; + } else if any_running + || self + .jobs + .values() + .any(|j| j.status == JobStatus::Passed) + { + self.status = PipelineStatus::Running; + } + } + + /// Find all transitive dependents of a job (jobs that directly or indirectly need it). + fn transitive_dependents(&self, job_name: &str) -> Vec { + let mut result = Vec::new(); + let mut queue: VecDeque<&str> = VecDeque::new(); + queue.push_back(job_name); + let mut visited = HashSet::new(); + + while let Some(current) = queue.pop_front() { + for (name, job) in &self.jobs { + if job.definition.needs.iter().any(|n| n == current) && visited.insert(name.clone()) + { + result.push(name.clone()); + queue.push_back(name.as_str()); + } + } + } + result + } +} + +// ─── DAG Validation ───────────────────────────────────────────────────────── + +/// Topological sort of job definitions. Returns ordered job names or an error if cyclic. +pub fn topological_sort(jobs: &HashMap) -> Result, DagError> { + let mut in_degree: HashMap<&str, usize> = HashMap::new(); + let mut dependents: HashMap<&str, Vec<&str>> = HashMap::new(); + + for (name, def) in jobs { + in_degree.entry(name.as_str()).or_insert(0); + for dep in &def.needs { + if !jobs.contains_key(dep) { + return Err(DagError::MissingDependency { + job: name.clone(), + dependency: dep.clone(), + }); + } + dependents.entry(dep.as_str()).or_default().push(name.as_str()); + *in_degree.entry(name.as_str()).or_insert(0) += 1; + } + } + + let mut queue: VecDeque<&str> = in_degree + .iter() + .filter(|(_, deg)| **deg == 0) + .map(|(&name, _)| name) + .collect(); + + // Sort the initial queue for deterministic ordering. + let mut sorted_queue: Vec<&str> = queue.drain(..).collect(); + sorted_queue.sort(); + queue.extend(sorted_queue); + + let mut result = Vec::new(); + while let Some(node) = queue.pop_front() { + result.push(node.to_string()); + if let Some(deps) = dependents.get(node) { + let mut next_nodes = Vec::new(); + for &dep in deps { + if let Some(deg) = in_degree.get_mut(dep) { + *deg -= 1; + if *deg == 0 { + next_nodes.push(dep); + } + } + } + next_nodes.sort(); + queue.extend(next_nodes); + } + } + + if result.len() != jobs.len() { + return Err(DagError::Cycle); + } + Ok(result) +} + +#[derive(Debug, Clone)] +pub enum DagError { + Cycle, + MissingDependency { job: String, dependency: String }, +} + +impl std::fmt::Display for DagError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + DagError::Cycle => write!(f, "job dependency cycle detected"), + DagError::MissingDependency { job, dependency } => { + write!(f, "job '{job}' depends on unknown job '{dependency}'") + } + } + } +} + +impl std::error::Error for DagError {} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + use crate::JobDefinition; + + fn make_job(name: &str, needs: &[&str]) -> JobDefinition { + JobDefinition { + name: name.into(), + run: vec!["echo test".into()], + needs: needs.iter().map(|s| s.to_string()).collect(), + timeout_secs: 300, + docker: false, + artifacts: Vec::new(), + env: HashMap::new(), + } + } + + #[test] + fn topological_sort_linear_chain() { + let mut jobs = HashMap::new(); + jobs.insert("a".into(), make_job("a", &[])); + jobs.insert("b".into(), make_job("b", &["a"])); + jobs.insert("c".into(), make_job("c", &["b"])); + + let order = topological_sort(&jobs).unwrap(); + assert_eq!(order, vec!["a", "b", "c"]); + } + + #[test] + fn topological_sort_diamond() { + let mut jobs = HashMap::new(); + jobs.insert("a".into(), make_job("a", &[])); + jobs.insert("b".into(), make_job("b", &["a"])); + jobs.insert("c".into(), make_job("c", &["a"])); + jobs.insert("d".into(), make_job("d", &["b", "c"])); + + let order = topological_sort(&jobs).unwrap(); + let pos = |name: &str| order.iter().position(|n| n == name).unwrap(); + assert!(pos("a") < pos("b")); + assert!(pos("a") < pos("c")); + assert!(pos("b") < pos("d")); + assert!(pos("c") < pos("d")); + } + + #[test] + fn topological_sort_detects_cycle() { + let mut jobs = HashMap::new(); + jobs.insert("a".into(), make_job("a", &["b"])); + jobs.insert("b".into(), make_job("b", &["a"])); + + let result = topological_sort(&jobs); + assert!(matches!(result, Err(DagError::Cycle))); + } + + #[test] + fn topological_sort_independent_jobs() { + let mut jobs = HashMap::new(); + jobs.insert("a".into(), make_job("a", &[])); + jobs.insert("b".into(), make_job("b", &[])); + jobs.insert("c".into(), make_job("c", &[])); + + let order = topological_sort(&jobs).unwrap(); + // All jobs present, order is alphabetical for independent nodes + assert_eq!(order.len(), 3); + assert_eq!(order, vec!["a", "b", "c"]); + } + + #[test] + fn pipeline_eligible_jobs_respects_dag() { + let jobs = vec![ + make_job("fmt", &[]), + make_job("clippy", &[]), + make_job("test", &["fmt", "clippy"]), + ]; + + let mut pipeline = PipelineExecution::new( + PipelineId(1), + "check".into(), + "user".into(), + "repo".into(), + "abc123".into(), + "main".into(), + jobs, + ); + + // Initially, fmt and clippy are eligible + let mut eligible = pipeline.eligible_jobs(); + eligible.sort(); + assert_eq!(eligible, vec!["clippy", "fmt"]); + + // After fmt passes, test is still not eligible (clippy pending) + pipeline.set_job_status("fmt", JobStatus::Passed); + let eligible = pipeline.eligible_jobs(); + assert_eq!(eligible, vec!["clippy"]); + + // After clippy passes, test becomes eligible + pipeline.set_job_status("clippy", JobStatus::Passed); + let eligible = pipeline.eligible_jobs(); + assert_eq!(eligible, vec!["test"]); + } + + #[test] + fn pipeline_failure_skips_dependents() { + let jobs = vec![ + make_job("fmt", &[]), + make_job("test", &["fmt"]), + make_job("bench", &["test"]), + ]; + + let mut pipeline = PipelineExecution::new( + PipelineId(1), + "check".into(), + "user".into(), + "repo".into(), + "abc123".into(), + "main".into(), + jobs, + ); + + // fmt fails → test and bench should be skipped + pipeline.set_job_status( + "fmt", + JobStatus::Failed { + reason: "formatting error".into(), + }, + ); + + assert_eq!(pipeline.jobs["test"].status, JobStatus::Skipped); + assert_eq!(pipeline.jobs["bench"].status, JobStatus::Skipped); + assert_eq!(pipeline.status, PipelineStatus::Failed); + } + + #[test] + fn pipeline_all_pass_yields_passed() { + let jobs = vec![make_job("a", &[]), make_job("b", &["a"])]; + + let mut pipeline = PipelineExecution::new( + PipelineId(1), + "check".into(), + "user".into(), + "repo".into(), + "abc123".into(), + "main".into(), + jobs, + ); + + pipeline.set_job_status("a", JobStatus::Passed); + pipeline.set_job_status("b", JobStatus::Passed); + assert_eq!(pipeline.status, PipelineStatus::Passed); + } +} diff --git a/crates/ci/src/provisioner.rs b/crates/ci/src/provisioner.rs new file mode 100644 index 0000000..26b007c --- /dev/null +++ b/crates/ci/src/provisioner.rs @@ -0,0 +1,86 @@ +//! Provisioner actor: manages spot instance lifecycle via pluggable provider scripts. +//! +//! In real deployment, runs on the developer's laptop and calls cloud provider APIs. +//! In simulation, provisions are driven by the simulation harness. + +use swactor::actor::{ActorAddress, ActorInterface, Ctx}; + +use crate::coordinator::CoordinatorMsg; +use crate::{ + InstanceReady, ProvisionError, ProvisionRequest, ProvisionResponse, TerminateRequest, +}; + +/// Messages the Provisioner can receive. +#[derive(Debug, Clone)] +pub enum ProvisionerMsg { + /// Request to provision a new spot instance. + Provision(ProvisionRequest), + /// Request to terminate a spot instance. + Terminate(TerminateRequest), + /// Simulated: provisioning result delivered asynchronously. + SimProvisionResult { + request: ProvisionRequest, + result: Result, + }, +} + +/// Provisioner actor state. +/// +/// In real deployment, this would invoke provider scripts. +/// In simulation, the sim harness controls provision outcomes. +pub struct Provisioner { + coordinator_addr: ActorAddress, + /// Active instances tracked for cleanup. + active_instances: Vec, +} + +impl Provisioner { + pub fn new(coordinator_addr: ActorAddress) -> Self { + Self { + coordinator_addr, + active_instances: Vec::new(), + } + } + + pub fn active_instances(&self) -> &[String] { + &self.active_instances + } +} + +impl ActorInterface for Provisioner { + type Incoming = ProvisionerMsg; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, msg: ProvisionerMsg) { + match msg { + ProvisionerMsg::Provision(_request) => { + // In real deployment: invoke provider script, await result. + // In simulation: the sim harness sends SimProvisionResult. + } + ProvisionerMsg::Terminate(request) => { + self.active_instances.retain(|id| id != &request.instance_id); + // In real deployment: invoke provider destroy script. + // In simulation: just track the termination. + let _ = ctx.send( + self.coordinator_addr, + CoordinatorMsg::ProvisionResponse(ProvisionResponse { + job_id: request.job_id.clone(), + result: Err(ProvisionError::NoCapacity), // placeholder, terminate doesn't need response + }), + ); + } + ProvisionerMsg::SimProvisionResult { request, result } => { + if let Ok(ref instance) = result { + self.active_instances.push(instance.instance_id.clone()); + } + let _ = ctx.send( + self.coordinator_addr, + CoordinatorMsg::ProvisionResponse(ProvisionResponse { + job_id: request.job_id, + result, + }), + ); + } + } + } +} diff --git a/crates/ci/src/runner.rs b/crates/ci/src/runner.rs new file mode 100644 index 0000000..99c0357 --- /dev/null +++ b/crates/ci/src/runner.rs @@ -0,0 +1,84 @@ +//! RunnerSupervisor actor: manages SSH session and job execution on a spot instance. +//! +//! Spawned per-job by the Coordinator. Owns the connection to the spot instance. + +use swactor::actor::{ActorAddress, ActorInterface, Ctx}; + +use crate::coordinator::CoordinatorMsg; +use crate::{JobComplete, JobFailure, JobProgress, JobSuccess, StartJob}; + +/// Messages the RunnerSupervisor can receive. +#[derive(Debug, Clone)] +pub enum RunnerMsg { + /// Begin executing the job (sent immediately after spawn via on_start). + Execute, + /// Simulated: job command output line. + OutputLine(String), + /// Simulated: job completed successfully. + SimComplete(Result<(), String>), +} + +/// RunnerSupervisor actor state. +/// +/// In real deployment, this would manage an SSH connection. +/// In simulation, job execution is driven by external messages. +pub struct RunnerSupervisor { + coordinator_addr: ActorAddress, + start_job: StartJob, +} + +impl RunnerSupervisor { + pub fn new(coordinator_addr: ActorAddress, start_job: StartJob) -> Self { + Self { + coordinator_addr, + start_job, + } + } +} + +impl ActorInterface for RunnerSupervisor { + type Incoming = RunnerMsg; + type Response = (); + + fn on_start(&mut self, ctx: &Ctx) { + // In simulation, the sim harness will send SimComplete messages. + // In real deployment, this would initiate SSH connection + command execution. + let _ = ctx.send(ctx.self_addr(), RunnerMsg::Execute); + } + + fn handle(&mut self, ctx: &Ctx, msg: RunnerMsg) { + match msg { + RunnerMsg::Execute => { + // In real mode, we'd SSH into the instance and run commands. + // In simulation, this is a no-op; SimComplete drives completion. + } + RunnerMsg::OutputLine(line) => { + let _ = ctx.send( + self.coordinator_addr, + CoordinatorMsg::JobProgress(JobProgress { + job_id: self.start_job.job_id.clone(), + output_line: line, + }), + ); + } + RunnerMsg::SimComplete(result) => { + let complete = JobComplete { + job_id: self.start_job.job_id.clone(), + result: match result { + Ok(()) => Ok(JobSuccess), + Err(msg) => Err(JobFailure::CommandFailed { + exit_code: 1, + last_lines: vec![msg], + }), + }, + artifacts: Vec::new(), + }; + let _ = ctx.send( + self.coordinator_addr, + CoordinatorMsg::JobComplete(complete), + ); + ctx.stop_self(); + } + } + } +} diff --git a/crates/ci/src/status_reporter.rs b/crates/ci/src/status_reporter.rs new file mode 100644 index 0000000..35f08e6 --- /dev/null +++ b/crates/ci/src/status_reporter.rs @@ -0,0 +1,327 @@ +//! StatusReporter actor: fire-and-forget Forgejo commit status updates. +//! +//! Receives status update messages and POSTs them to the Forgejo API. +//! Can also post pipeline summary comments to PRs. + +use swactor::actor::{ActorInterface, Ctx}; + +use crate::StatusUpdate; + +/// Captured output for a single job, used to build PR comments. +#[derive(Debug, Clone)] +pub struct JobOutput { + pub job_name: String, + pub passed: bool, + pub failure_reason: Option, + pub output_lines: Vec, +} + +/// Messages the StatusReporter can receive. +#[derive(Debug, Clone)] +pub enum StatusReporterMsg { + Report { + update: StatusUpdate, + forgejo_url: String, + forgejo_token: String, + }, + PostPipelineComment { + repo_owner: String, + repo_name: String, + commit_sha: String, + branch: String, + pipeline_name: String, + pipeline_state: String, + job_outputs: Vec, + forgejo_url: String, + forgejo_token: String, + }, +} + +/// StatusReporter actor state. +pub struct StatusReporter; + +impl StatusReporter { + pub fn new() -> Self { + Self + } + + #[cfg(feature = "local")] + fn post_status(update: &StatusUpdate, forgejo_url: &str, forgejo_token: &str) { + let url = format!( + "{}/api/v1/repos/{}/{}/statuses/{}", + forgejo_url.trim_end_matches('/'), + update.repo_owner, + update.repo_name, + update.commit_sha, + ); + + let mut body = serde_json::json!({ + "state": update.state, + "context": update.context, + "description": update.description, + }); + + if let Some(ref target_url) = update.target_url { + body["target_url"] = serde_json::Value::String(target_url.clone()); + } + + let result = ureq::post(&url) + .set("Authorization", &format!("token {forgejo_token}")) + .set("Content-Type", "application/json") + .send_string(&body.to_string()); + + if let Err(e) = result { + eprintln!("StatusReporter: failed to post status to {url}: {e}"); + } + } + + #[cfg(feature = "local")] + fn find_pr_for_branch( + forgejo_url: &str, + forgejo_token: &str, + repo_owner: &str, + repo_name: &str, + branch: &str, + ) -> Option { + let url = format!( + "{}/api/v1/repos/{}/{}/pulls?state=open&limit=50", + forgejo_url.trim_end_matches('/'), + repo_owner, + repo_name, + ); + + let response = ureq::get(&url) + .set("Authorization", &format!("token {forgejo_token}")) + .call(); + + let response = match response { + Ok(r) => r, + Err(e) => { + eprintln!("StatusReporter: failed to list PRs: {e}"); + return None; + } + }; + + let body: serde_json::Value = match response.into_json() { + Ok(v) => v, + Err(e) => { + eprintln!("StatusReporter: failed to parse PR list: {e}"); + return None; + } + }; + + let prs = body.as_array()?; + for pr in prs { + let head_ref = pr.get("head")?.get("ref")?.as_str()?; + if head_ref == branch { + return pr.get("number")?.as_u64(); + } + } + None + } + + #[cfg(feature = "local")] + fn post_pr_comment( + forgejo_url: &str, + forgejo_token: &str, + repo_owner: &str, + repo_name: &str, + pr_number: u64, + body_text: &str, + ) -> Option { + let url = format!( + "{}/api/v1/repos/{}/{}/issues/{}/comments", + forgejo_url.trim_end_matches('/'), + repo_owner, + repo_name, + pr_number, + ); + + let body = serde_json::json!({ + "body": body_text, + }); + + let result = ureq::post(&url) + .set("Authorization", &format!("token {forgejo_token}")) + .set("Content-Type", "application/json") + .send_string(&body.to_string()); + + match result { + Ok(response) => { + let json: serde_json::Value = response.into_json().ok()?; + json.get("html_url")?.as_str().map(|s| s.to_string()) + } + Err(e) => { + eprintln!("StatusReporter: failed to post PR comment: {e}"); + None + } + } + } + + #[cfg(feature = "local")] + fn build_pipeline_comment( + pipeline_name: &str, + pipeline_state: &str, + commit_sha: &str, + job_outputs: &[JobOutput], + ) -> String { + let mut md = format!("## Pipeline `{pipeline_name}` — {pipeline_state}\n\n"); + let short_sha = if commit_sha.len() > 7 { + &commit_sha[..7] + } else { + commit_sha + }; + md.push_str(&format!("Commit: `{short_sha}`\n\n")); + + for job in job_outputs { + let status_label = if job.passed { + "passed".to_string() + } else { + match &job.failure_reason { + Some(reason) => format!("failed: {reason}"), + None => "failed".to_string(), + } + }; + + md.push_str(&format!( + "
\n{} — {}\n\n", + job.job_name, status_label + )); + + let max_lines = 100; + let total = job.output_lines.len(); + let lines: &[String] = if total > max_lines { + md.push_str(&format!("_Showing last {max_lines} of {total} lines_\n\n")); + &job.output_lines[total - max_lines..] + } else { + &job.output_lines + }; + + md.push_str("```\n"); + for line in lines { + md.push_str(line); + md.push('\n'); + } + md.push_str("```\n\n
\n\n"); + } + + md + } + + #[cfg(feature = "local")] + fn handle_pipeline_comment( + repo_owner: &str, + repo_name: &str, + commit_sha: &str, + branch: &str, + pipeline_name: &str, + pipeline_state: &str, + job_outputs: &[JobOutput], + forgejo_url: &str, + forgejo_token: &str, + ) { + let pr_number = match Self::find_pr_for_branch( + forgejo_url, + forgejo_token, + repo_owner, + repo_name, + branch, + ) { + Some(n) => n, + None => { + eprintln!( + "StatusReporter: no open PR for branch '{branch}', skipping comment" + ); + return; + } + }; + + let comment_body = + Self::build_pipeline_comment(pipeline_name, pipeline_state, commit_sha, job_outputs); + + let comment_url = Self::post_pr_comment( + forgejo_url, + forgejo_token, + repo_owner, + repo_name, + pr_number, + &comment_body, + ); + + // Re-post pipeline status with target_url pointing to the comment. + if let Some(ref url) = comment_url { + let update = StatusUpdate { + repo_owner: repo_owner.to_string(), + repo_name: repo_name.to_string(), + commit_sha: commit_sha.to_string(), + state: pipeline_state.to_string(), + context: format!("ci/{pipeline_name}"), + description: format!("Pipeline '{pipeline_name}' {pipeline_state}"), + target_url: Some(url.clone()), + }; + Self::post_status(&update, forgejo_url, forgejo_token); + } + } +} + +impl ActorInterface for StatusReporter { + type Incoming = StatusReporterMsg; + type Response = (); + + fn handle(&mut self, _ctx: &Ctx, msg: StatusReporterMsg) { + match msg { + StatusReporterMsg::Report { + update, + forgejo_url, + forgejo_token, + } => { + #[cfg(feature = "local")] + Self::post_status(&update, &forgejo_url, &forgejo_token); + + #[cfg(not(feature = "local"))] + { + let _ = (update, forgejo_url, forgejo_token); + } + } + StatusReporterMsg::PostPipelineComment { + repo_owner, + repo_name, + commit_sha, + branch, + pipeline_name, + pipeline_state, + job_outputs, + forgejo_url, + forgejo_token, + } => { + #[cfg(feature = "local")] + Self::handle_pipeline_comment( + &repo_owner, + &repo_name, + &commit_sha, + &branch, + &pipeline_name, + &pipeline_state, + &job_outputs, + &forgejo_url, + &forgejo_token, + ); + + #[cfg(not(feature = "local"))] + { + let _ = ( + repo_owner, + repo_name, + commit_sha, + branch, + pipeline_name, + pipeline_state, + job_outputs, + forgejo_url, + forgejo_token, + ); + } + } + } + } +} diff --git a/crates/ci/src/webhook_server.rs b/crates/ci/src/webhook_server.rs new file mode 100644 index 0000000..f636ebd --- /dev/null +++ b/crates/ci/src/webhook_server.rs @@ -0,0 +1,188 @@ +//! Webhook HTTP listener: receives Forgejo webhook POSTs and forwards +//! them to the LocalCoordinator actor. +//! +//! Runs as a standard thread (not an actor) using `tiny_http`. + +use crate::{EventType, WebhookEvent}; + +/// Start the webhook listener in a new thread. +/// +/// Returns a join handle for the listener thread. +#[cfg(feature = "local")] +pub fn start_webhook_listener( + port: u16, + secret: String, + runtime: std::sync::Arc, + coordinator_addr: swactor::actor::ActorAddress, +) -> std::thread::JoinHandle<()> { + std::thread::Builder::new() + .name("webhook-listener".into()) + .spawn(move || { + let server = tiny_http::Server::http(format!("0.0.0.0:{port}")) + .expect("failed to start webhook server"); + + eprintln!("Webhook listener on http://0.0.0.0:{port}"); + + for mut request in server.incoming_requests() { + let response = handle_request(&mut request, &secret, &runtime, coordinator_addr); + let _ = request.respond(response); + } + }) + .expect("failed to spawn webhook listener thread") +} + +#[cfg(feature = "local")] +fn handle_request( + request: &mut tiny_http::Request, + secret: &str, + runtime: &std::sync::Arc, + coordinator_addr: swactor::actor::ActorAddress, +) -> tiny_http::Response>> { + use hmac::{Hmac, Mac}; + use sha2::Sha256; + use crate::local_coordinator::LocalCoordinatorMsg; + + // Only accept POST. + if request.method() != &tiny_http::Method::Post { + return tiny_http::Response::from_string("method not allowed") + .with_status_code(405); + } + + // Read body. + let mut body = String::new(); + if let Err(e) = std::io::Read::read_to_string(&mut request.as_reader(), &mut body) { + eprintln!("webhook: failed to read body: {e}"); + return tiny_http::Response::from_string("bad request") + .with_status_code(400); + } + + // Verify HMAC-SHA256 signature if secret is non-empty. + if !secret.is_empty() { + let sig_header = request + .headers() + .iter() + .find(|h| h.field.equiv("X-Forgejo-Signature")) + .map(|h| h.value.as_str().to_string()); + + match sig_header { + Some(sig_hex) => { + type HmacSha256 = Hmac; + let mut mac = HmacSha256::new_from_slice(secret.as_bytes()) + .expect("HMAC key creation"); + hmac::Mac::update(&mut mac, body.as_bytes()); + let expected = hex::encode(mac.finalize().into_bytes()); + if sig_hex != expected { + eprintln!("webhook: signature mismatch"); + return tiny_http::Response::from_string("unauthorized") + .with_status_code(401); + } + } + None => { + eprintln!("webhook: missing signature header"); + return tiny_http::Response::from_string("unauthorized") + .with_status_code(401); + } + } + } + + // Determine event type from Forgejo header. + let event_header = request + .headers() + .iter() + .find(|h| h.field.equiv("X-Forgejo-Event")) + .map(|h| h.value.as_str().to_string()) + .unwrap_or_default(); + + let event_type = match event_header.as_str() { + "push" => EventType::Push, + "create" => EventType::Tag, + "pull_request" => EventType::Merge, + other => { + eprintln!("webhook: ignoring event type '{other}'"); + return tiny_http::Response::from_string("ignored").with_status_code(200); + } + }; + + // Parse JSON body to extract fields. + let json: serde_json::Value = match serde_json::from_str(&body) { + Ok(v) => v, + Err(e) => { + eprintln!("webhook: failed to parse JSON: {e}"); + return tiny_http::Response::from_string("bad json").with_status_code(400); + } + }; + + let webhook_event = match parse_webhook_json(&json, event_type) { + Some(e) => e, + None => { + eprintln!("webhook: could not extract webhook fields from JSON"); + return tiny_http::Response::from_string("bad payload").with_status_code(400); + } + }; + + // Send to coordinator. + let _ = runtime.send_to(coordinator_addr, LocalCoordinatorMsg::Webhook(webhook_event)); + + tiny_http::Response::from_string("ok").with_status_code(200) +} + +/// Parse a Forgejo webhook JSON payload into a WebhookEvent. +pub fn parse_webhook_json(json: &serde_json::Value, event_type: EventType) -> Option { + let repo = json.get("repository")?; + let repo_owner = repo + .get("owner") + .and_then(|o| o.get("login")) + .or_else(|| repo.get("owner").and_then(|o| o.get("username"))) + .and_then(|v| v.as_str())? + .to_string(); + let repo_name = repo.get("name").and_then(|v| v.as_str())?.to_string(); + + let (branch, commit_sha, tag) = match event_type { + EventType::Push => { + let reference = json.get("ref").and_then(|v| v.as_str()).unwrap_or(""); + let branch = reference.strip_prefix("refs/heads/").unwrap_or(reference); + let sha = json + .get("after") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + (branch.to_string(), sha, None) + } + EventType::Tag => { + let reference = json.get("ref").and_then(|v| v.as_str()).unwrap_or(""); + let tag_name = reference.strip_prefix("refs/tags/").unwrap_or(reference); + let sha = json + .get("sha") + .or_else(|| json.get("after")) + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + (String::new(), sha, Some(tag_name.to_string())) + } + EventType::Merge => { + let pr = json.get("pull_request")?; + let branch = pr + .get("head") + .and_then(|h| h.get("ref")) + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + let sha = pr + .get("head") + .and_then(|h| h.get("sha")) + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + (branch, sha, None) + } + }; + + Some(WebhookEvent { + event_type, + repo_owner, + repo_name, + branch, + commit_sha, + tag, + }) +} diff --git a/crates/ci/src/yaml.rs b/crates/ci/src/yaml.rs new file mode 100644 index 0000000..2bf5df4 --- /dev/null +++ b/crates/ci/src/yaml.rs @@ -0,0 +1,433 @@ +//! Parser for `.ci.yml` pipeline configuration files. + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +use crate::{EventType, JobDefinition, WebhookEvent}; + +/// Root of a `.ci.yml` file. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CiYaml { + pub pipelines: HashMap, +} + +/// A single pipeline definition. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PipelineDef { + pub triggers: Vec, + pub jobs: HashMap, +} + +/// A trigger condition. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TriggerDef { + pub event: TriggerEvent, + #[serde(default)] + pub branches: Vec, + #[serde(default)] + pub exclude: Vec, + #[serde(default)] + pub pattern: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum TriggerEvent { + Push, + Tag, + Merge, +} + +/// A job definition in YAML form. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct JobDef { + pub run: RunCommand, + #[serde(default)] + pub needs: Vec, + #[serde(default)] + pub timeout: Option, + #[serde(default)] + pub docker: bool, + #[serde(default)] + pub artifacts: Vec, + #[serde(default)] + pub env: HashMap, +} + +/// `run` can be a single string or an array of strings. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum RunCommand { + Single(String), + Multiple(Vec), +} + +impl RunCommand { + pub fn into_vec(self) -> Vec { + match self { + RunCommand::Single(s) => vec![s], + RunCommand::Multiple(v) => v, + } + } +} + +// ─── Parsing ──────────────────────────────────────────────────────────────── + +/// Parse a `.ci.yml` string into a `CiYaml`. +pub fn parse_ci_yaml(input: &str) -> Result { + let ci: CiYaml = serde_yaml::from_str(input).map_err(ParseError::Yaml)?; + + // Validate: check for unknown job references in `needs` + for (pipeline_name, pipeline) in &ci.pipelines { + let job_names: Vec<&str> = pipeline.jobs.keys().map(|s| s.as_str()).collect(); + for (job_name, job) in &pipeline.jobs { + for dep in &job.needs { + if !job_names.contains(&dep.as_str()) { + return Err(ParseError::UnknownDependency { + pipeline: pipeline_name.clone(), + job: job_name.clone(), + dependency: dep.clone(), + }); + } + } + } + } + + Ok(ci) +} + +/// Convert a YAML `JobDef` to the runtime `JobDefinition`. +pub fn to_job_definition(name: &str, def: &JobDef) -> JobDefinition { + JobDefinition { + name: name.to_string(), + run: def.run.clone().into_vec(), + needs: def.needs.clone(), + timeout_secs: def.timeout.unwrap_or(300), + docker: def.docker, + artifacts: def.artifacts.clone(), + env: def.env.clone(), + } +} + +// ─── Trigger Matching ─────────────────────────────────────────────────────── + +/// Returns the names of pipelines whose triggers match the given webhook event. +pub fn matching_pipelines(ci: &CiYaml, event: &WebhookEvent) -> Vec { + ci.pipelines + .iter() + .filter(|(_, pipeline)| pipeline.triggers.iter().any(|t| trigger_matches(t, event))) + .map(|(name, _)| name.clone()) + .collect() +} + +/// Check whether a single trigger matches a webhook event. +fn trigger_matches(trigger: &TriggerDef, event: &WebhookEvent) -> bool { + // Event type must match. + let event_matches = match (&trigger.event, &event.event_type) { + (TriggerEvent::Push, EventType::Push) => true, + (TriggerEvent::Tag, EventType::Tag) => true, + (TriggerEvent::Merge, EventType::Merge) => true, + _ => false, + }; + if !event_matches { + return false; + } + + // For tag events, check pattern. + if trigger.event == TriggerEvent::Tag { + if let Some(ref pattern) = trigger.pattern { + return glob_matches(pattern, event.tag.as_deref().unwrap_or("")); + } + return true; + } + + // For push/merge, check branch filters. + let branch = &event.branch; + + // If exclude patterns are specified and branch matches any, reject. + if trigger.exclude.iter().any(|pat| glob_matches(pat, branch)) { + return false; + } + + // If branch patterns are specified, at least one must match. + if trigger.branches.is_empty() { + return true; + } + trigger.branches.iter().any(|pat| glob_matches(pat, branch)) +} + +/// Simple glob matching supporting `*` (any chars) and `?` (one char). +pub fn glob_matches(pattern: &str, text: &str) -> bool { + glob_matches_inner(pattern.as_bytes(), text.as_bytes()) +} + +fn glob_matches_inner(pat: &[u8], text: &[u8]) -> bool { + match (pat.first(), text.first()) { + (None, None) => true, + (Some(b'*'), _) => { + // '*' matches zero or more characters + glob_matches_inner(&pat[1..], text) + || (!text.is_empty() && glob_matches_inner(pat, &text[1..])) + } + (Some(b'?'), Some(_)) => glob_matches_inner(&pat[1..], &text[1..]), + (Some(a), Some(b)) if a == b => glob_matches_inner(&pat[1..], &text[1..]), + _ => false, + } +} + +// ─── Errors ───────────────────────────────────────────────────────────────── + +#[derive(Debug)] +pub enum ParseError { + Yaml(serde_yaml::Error), + UnknownDependency { + pipeline: String, + job: String, + dependency: String, + }, +} + +impl std::fmt::Display for ParseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ParseError::Yaml(e) => write!(f, "YAML parse error: {e}"), + ParseError::UnknownDependency { + pipeline, + job, + dependency, + } => write!( + f, + "pipeline '{pipeline}', job '{job}': unknown dependency '{dependency}'" + ), + } + } +} + +impl std::error::Error for ParseError {} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_minimal_ci_yaml() { + let yaml = r#" +pipelines: + check: + triggers: + - event: push + branches: ["*"] + jobs: + test: + run: cargo test +"#; + let ci = parse_ci_yaml(yaml).unwrap(); + assert_eq!(ci.pipelines.len(), 1); + assert!(ci.pipelines.contains_key("check")); + let check = &ci.pipelines["check"]; + assert_eq!(check.jobs.len(), 1); + assert!(check.jobs.contains_key("test")); + } + + #[test] + fn parse_full_ci_yaml() { + let yaml = r#" +pipelines: + check: + triggers: + - event: push + branches: ["*"] + exclude: ["master"] + jobs: + fmt: + run: cargo fmt -- --check + clippy: + run: cargo clippy --all-features -- -D warnings + test: + needs: [fmt, clippy] + run: cargo test + full: + triggers: + - event: push + branches: ["master"] + jobs: + test: + run: cargo test --all-features + timeout: 600 + bench: + needs: [test] + run: cargo bench -- --output-format json + artifacts: ["target/criterion/**"] + docker: + needs: [test] + run: cargo test -p docker-tests -- --ignored + docker: true + release: + triggers: + - event: tag + pattern: "v*" + jobs: + build: + run: cargo build --release + artifacts: ["target/release/swactor-node"] +"#; + let ci = parse_ci_yaml(yaml).unwrap(); + assert_eq!(ci.pipelines.len(), 3); + assert!(ci.pipelines.contains_key("check")); + assert!(ci.pipelines.contains_key("full")); + assert!(ci.pipelines.contains_key("release")); + + let full = &ci.pipelines["full"]; + assert_eq!(full.jobs.len(), 3); + assert_eq!(full.jobs["bench"].needs, vec!["test"]); + assert!(full.jobs["docker"].docker); + } + + #[test] + fn unknown_dependency_rejected() { + let yaml = r#" +pipelines: + check: + triggers: + - event: push + branches: ["*"] + jobs: + test: + needs: [nonexistent] + run: cargo test +"#; + let result = parse_ci_yaml(yaml); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!( + matches!(err, ParseError::UnknownDependency { .. }), + "expected UnknownDependency, got: {err}" + ); + } + + #[test] + fn trigger_matching_push_branch() { + let yaml = r#" +pipelines: + check: + triggers: + - event: push + branches: ["*"] + exclude: ["master"] + jobs: + test: + run: cargo test + full: + triggers: + - event: push + branches: ["master"] + jobs: + test: + run: cargo test +"#; + let ci = parse_ci_yaml(yaml).unwrap(); + + // Push to feature branch → matches "check" only + let event = WebhookEvent { + event_type: EventType::Push, + repo_owner: "user".into(), + repo_name: "repo".into(), + branch: "feature-x".into(), + commit_sha: "abc123".into(), + tag: None, + }; + let mut matched = matching_pipelines(&ci, &event); + matched.sort(); + assert_eq!(matched, vec!["check"]); + + // Push to master → matches "full" only (check excludes master) + let event = WebhookEvent { + event_type: EventType::Push, + repo_owner: "user".into(), + repo_name: "repo".into(), + branch: "master".into(), + commit_sha: "abc123".into(), + tag: None, + }; + let mut matched = matching_pipelines(&ci, &event); + matched.sort(); + assert_eq!(matched, vec!["full"]); + } + + #[test] + fn trigger_matching_tag() { + let yaml = r#" +pipelines: + release: + triggers: + - event: tag + pattern: "v*" + jobs: + build: + run: cargo build --release +"#; + let ci = parse_ci_yaml(yaml).unwrap(); + + let event = WebhookEvent { + event_type: EventType::Tag, + repo_owner: "user".into(), + repo_name: "repo".into(), + branch: "master".into(), + commit_sha: "abc123".into(), + tag: Some("v1.0.0".into()), + }; + let matched = matching_pipelines(&ci, &event); + assert_eq!(matched, vec!["release"]); + + // Non-matching tag + let event = WebhookEvent { + event_type: EventType::Tag, + repo_owner: "user".into(), + repo_name: "repo".into(), + branch: "master".into(), + commit_sha: "abc123".into(), + tag: Some("nightly-1".into()), + }; + let matched = matching_pipelines(&ci, &event); + assert!(matched.is_empty()); + } + + #[test] + fn glob_matching() { + assert!(glob_matches("*", "anything")); + assert!(glob_matches("v*", "v1.0.0")); + assert!(!glob_matches("v*", "nightly")); + assert!(glob_matches("feature-?", "feature-x")); + assert!(!glob_matches("feature-?", "feature-xy")); + assert!(glob_matches("master", "master")); + assert!(!glob_matches("master", "main")); + } + + #[test] + fn run_command_single_and_multiple() { + let yaml = r#" +pipelines: + check: + triggers: + - event: push + branches: ["*"] + jobs: + single: + run: cargo test + multi: + run: + - cargo fmt -- --check + - cargo test +"#; + let ci = parse_ci_yaml(yaml).unwrap(); + let check = &ci.pipelines["check"]; + + let single = to_job_definition("single", &check.jobs["single"]); + assert_eq!(single.run, vec!["cargo test"]); + + let multi = to_job_definition("multi", &check.jobs["multi"]); + assert_eq!(multi.run, vec!["cargo fmt -- --check", "cargo test"]); + } + +} diff --git a/crates/datastore/src/api.rs b/crates/datastore/src/api.rs index d86a493..ceb05e0 100644 --- a/crates/datastore/src/api.rs +++ b/crates/datastore/src/api.rs @@ -473,6 +473,7 @@ fn handle_data(request: tiny_http::Request, url: &str, state: &ApiState) { state.metrics.record_get(&content_hash.to_hex()); + let (entry, manifest) = match poll_response(&inbox, POLL_TIMEOUT) { Some(DatastoreResponse::GetOk { entry, manifest }) => (entry, manifest), Some(DatastoreResponse::NotFound) => { @@ -831,6 +832,7 @@ fn try_remote_get( state.metrics.end_transfer(&hash_hex); + if !all_ok { continue; } diff --git a/crates/datastore/src/types.rs b/crates/datastore/src/types.rs index 55a98a0..6a3f66f 100644 --- a/crates/datastore/src/types.rs +++ b/crates/datastore/src/types.rs @@ -10,6 +10,7 @@ use serde::{Deserialize, Serialize}; use distribution::types::NodeId; +// Re-export the shared ContentHash — single canonical type across the crate ecosystem. pub use shared_types::ContentHash; // ─── ObjectEntry ──────────────────────────────────────────────────────────── diff --git a/crates/local-runner/Cargo.toml b/crates/local-runner/Cargo.toml new file mode 100644 index 0000000..5164e74 --- /dev/null +++ b/crates/local-runner/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "local-runner" +version = "0.1.0" +edition = "2024" + +[[bin]] +name = "local-runner" +path = "src/main.rs" + +[dependencies] +swactor = { path = "../..", features = ["serde"] } +swactor-ci = { path = "../ci", features = ["local"] } +runtime-dashboard = { path = "../runtime-dashboard", features = ["ci"] } +clap = { version = "4", features = ["derive"] } +ctrlc = "3" +iroh = "0.96" +tokio = { version = "1", features = ["rt-multi-thread"] } +serde_json = "1" diff --git a/crates/local-runner/src/main.rs b/crates/local-runner/src/main.rs new file mode 100644 index 0000000..4519663 --- /dev/null +++ b/crates/local-runner/src/main.rs @@ -0,0 +1,374 @@ +//! local-runner — single-machine CI runner for the Thinkpad. +//! +//! Receives Forgejo webhooks, queues pipelines, and executes jobs +//! one at a time for benchmark isolation. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::Duration; + +use clap::Parser; + +use swactor::actor::ActorAddress; +use swactor::config::RuntimeConfig; +use swactor::runtime::Runtime; + +use runtime_dashboard::ci_collector::{ + CiSnapshot, CiStatsProvider, JobSnapshot, PipelineSnapshot, ProvisionerStatus, +}; + +use swactor_ci::local_coordinator::{LocalCiSnapshot, LocalCoordinator, LocalCoordinatorMsg}; +use swactor_ci::pipeline::PipelineExecution; +use swactor_ci::status_reporter::StatusReporter; +use swactor_ci::webhook_server; +use swactor_ci::yaml; +use swactor_ci::{CiConfig, LocalCiConfig}; + +#[derive(Parser)] +#[command(name = "local-runner", about = "Swactor local CI runner")] +struct Args { + /// Webhook listen port. + #[arg(long, default_value = "8787")] + port: u16, + + /// Forgejo instance URL. + #[arg(long, default_value = "")] + forgejo_url: String, + + /// Forgejo API token. + #[arg(long, default_value = "")] + forgejo_token: String, + + /// Webhook secret for HMAC verification (empty to skip). + #[arg(long, default_value = "")] + secret: String, + + /// Path to .ci.yml file. + #[arg(long, default_value = ".ci.yml")] + yaml: String, + + /// Base directory for git checkouts. + #[arg(long, default_value = "./ci-work")] + work_dir: String, + + /// Git clone URL for the repository. + #[arg(long, default_value = "")] + repo_url: String, + + /// Dashboard HTTP port (omit to disable). + #[arg(long)] + dashboard_port: Option, + + /// Iroh Node ID of the ci-relay on the VPS (hex). + /// When set, webhooks arrive via iroh instead of HTTP. + #[arg(long)] + relay_node_id: Option, +} + +/// Bridge from LocalCiSnapshot to CiSnapshot for the dashboard. +struct LocalCiSnapshotProvider { + snapshot: Arc>, +} + +impl CiStatsProvider for LocalCiSnapshotProvider { + fn snapshot(&self) -> CiSnapshot { + let local = self.snapshot.lock().unwrap().clone(); + CiSnapshot { + active_pipelines: local + .active_pipelines + .iter() + .map(pipeline_to_dashboard) + .collect(), + recent_pipelines: local + .recent_pipelines + .iter() + .map(pipeline_to_dashboard) + .collect(), + provisioner_status: ProvisionerStatus::Online, + active_instances: if local.has_running_job { 1 } else { 0 }, + } + } +} + +fn pipeline_to_dashboard(p: &PipelineExecution) -> PipelineSnapshot { + PipelineSnapshot { + pipeline_id: p.pipeline_id, + pipeline_name: p.pipeline_name.clone(), + repo_owner: p.repo_owner.clone(), + repo_name: p.repo_name.clone(), + commit_sha: p.commit_sha.clone(), + branch: p.branch.clone(), + status: p.status.clone(), + jobs: p + .jobs + .values() + .map(|j| JobSnapshot { + job_id: j.job_id.clone(), + job_name: j.definition.name.clone(), + status: j.status.clone(), + output_line_count: j.output_lines.len(), + }) + .collect(), + } +} + +fn main() { + let args = Args::parse(); + let stop = Arc::new(AtomicBool::new(false)); + + // Signal handler. + { + let stop = Arc::clone(&stop); + ctrlc::set_handler(move || { + stop.store(true, Ordering::Relaxed); + }) + .expect("failed to set signal handler"); + } + + // Optionally start dashboard. + let dash = args.dashboard_port.map(|port| { + let d = runtime_dashboard::start_dashboard(runtime_dashboard::DashboardConfig { + port, + ..Default::default() + }); + d.install_tracing(); + d + }); + + // Create 2-thread runtime. + let num_threads = 2; + let collector = runtime_dashboard::collector::StatsCollector::new(num_threads); + let mut rt = Runtime::new(RuntimeConfig { + num_threads, + max_actors: 256, + channel_buffer_size: 2000, + ..Default::default() + }); + rt.set_stats_hook(collector.clone()); + + // Build config. + let ci_config = CiConfig { + webhook_port: args.port, + webhook_secret: args.secret.clone(), + forgejo_url: args.forgejo_url.clone(), + forgejo_token: args.forgejo_token.clone(), + data_dir: args.work_dir.clone(), + }; + let local_config = LocalCiConfig { + ci: ci_config, + repo_url: args.repo_url.clone(), + work_dir: args.work_dir.clone(), + ci_yaml_path: args.yaml.clone(), + }; + + // Spawn StatusReporter. + let reporter_addr = rt + .spawn(StatusReporter::new()) + .expect("failed to spawn StatusReporter"); + + // Spawn LocalCoordinator. + let coordinator = LocalCoordinator::new(local_config).with_status_reporter(reporter_addr); + let ci_snapshot = coordinator.ci_snapshot(); + let coordinator_addr = rt + .spawn(coordinator) + .expect("failed to spawn LocalCoordinator"); + + // Load CI YAML from disk. + let yaml_content = std::fs::read_to_string(&args.yaml) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", args.yaml)); + let ci_yaml = yaml::parse_ci_yaml(&yaml_content) + .unwrap_or_else(|e| panic!("failed to parse CI YAML: {e}")); + + // Start runtime. + let handle = rt.run().expect("failed to start runtime"); + + // Send CiYaml to coordinator. + let _ = handle + .runtime + .send_to(coordinator_addr, LocalCoordinatorMsg::SetCiYaml(ci_yaml)); + + // Wire dashboard. + if let Some(ref d) = dash { + d.set_runtime(Arc::clone(&handle.runtime), collector); + let provider = Arc::new(LocalCiSnapshotProvider { + snapshot: ci_snapshot, + }); + d.set_ci(provider); + } + + // Start webhook source: iroh relay or HTTP listener. + if let Some(ref relay_id_hex) = args.relay_node_id { + start_iroh_receiver( + relay_id_hex, + Arc::clone(&handle.runtime), + coordinator_addr, + Arc::clone(&stop), + ); + } else { + let _webhook_handle = webhook_server::start_webhook_listener( + args.port, + args.secret, + Arc::clone(&handle.runtime), + coordinator_addr, + ); + } + + eprintln!("Local CI runner started"); + if args.relay_node_id.is_some() { + eprintln!(" Webhook: via iroh relay"); + } else { + eprintln!(" Webhook: http://0.0.0.0:{}", args.port); + } + eprintln!(" YAML: {}", args.yaml); + eprintln!(" Workdir: {}", args.work_dir); + if let Some(port) = args.dashboard_port { + eprintln!(" Dashboard: http://0.0.0.0:{port}"); + } + + // Main loop — just wait for ctrlc. + while !stop.load(Ordering::Relaxed) { + thread::sleep(Duration::from_millis(100)); + } + + eprintln!("\nShutting down..."); + handle.shutdown(); + if let Some(d) = dash { + d.shutdown(); + } + handle.join(); +} + +// ─── Iroh Webhook Receiver ────────────────────────────────────────────────── + +/// ALPN protocol identifier — must match ci-relay. +const CI_ALPN: &[u8] = b"swactor/ci/1"; + +/// Connect to the VPS ci-relay via iroh and receive WebhookEvents. +/// +/// Runs in a background thread with its own tokio runtime. +fn start_iroh_receiver( + relay_id_hex: &str, + swactor_rt: Arc, + coordinator_addr: ActorAddress, + stop: Arc, +) { + let relay_key: iroh::PublicKey = relay_id_hex + .parse() + .unwrap_or_else(|e| panic!("invalid relay node ID '{relay_id_hex}': {e}")); + + thread::Builder::new() + .name("iroh-receiver".into()) + .spawn(move || { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("failed to build tokio runtime for iroh receiver"); + + rt.block_on(async move { + let endpoint = iroh::Endpoint::builder() + .alpns(vec![CI_ALPN.to_vec()]) + .relay_mode(iroh::RelayMode::Default) + .bind() + .await + .expect("failed to bind iroh endpoint"); + + eprintln!(" Iroh local ID: {}", endpoint.id()); + + // Outer reconnection loop: reconnect when the connection drops. + while !stop.load(Ordering::Relaxed) { + eprintln!(" Connecting to relay {relay_key}..."); + + let conn = match endpoint.connect(relay_key, CI_ALPN).await { + Ok(c) => c, + Err(e) => { + eprintln!("iroh: connect failed: {e}, retrying in 5s..."); + tokio::time::sleep(Duration::from_secs(5)).await; + continue; + } + }; + + eprintln!(" Connected to relay!"); + + // Receive loop: the relay opens uni streams to send us events. + while !stop.load(Ordering::Relaxed) { + match tokio::time::timeout(Duration::from_secs(1), conn.accept_uni()).await + { + Ok(Ok(mut recv)) => { + match read_tagged_message(&mut recv).await { + Ok((tag, payload)) => { + if tag == "ci::WebhookEvent" { + match serde_json::from_slice::< + swactor_ci::WebhookEvent, + >( + &payload + ) { + Ok(event) => { + eprintln!( + "iroh: received webhook {} on {}", + event + .commit_sha + .get(..8) + .unwrap_or(&event.commit_sha), + event.branch, + ); + let _ = swactor_rt.send_to( + coordinator_addr, + LocalCoordinatorMsg::Webhook(event), + ); + } + Err(e) => { + eprintln!( + "iroh: failed to deserialize event: {e}" + ) + } + } + } else { + eprintln!("iroh: unknown tag '{tag}', ignoring"); + } + } + Err(e) => { + eprintln!("iroh: read error: {e}"); + break; + } + } + } + Ok(Err(e)) => { + eprintln!("iroh: connection lost: {e}, reconnecting..."); + break; + } + Err(_) => { + // 1s poll timeout — just loop and check stop flag. + } + } + } + } + + endpoint.close().await; + }); + }) + .expect("failed to spawn iroh-receiver thread"); +} + +/// Read a tagged message from a QUIC recv stream. +/// +/// Frame format: `[4B tag_len][tag_bytes][payload_bytes]` +async fn read_tagged_message( + recv: &mut iroh::endpoint::RecvStream, +) -> Result<(String, Vec), Box> { + let mut tag_len_buf = [0u8; 4]; + recv.read_exact(&mut tag_len_buf).await?; + let tag_len = u32::from_be_bytes(tag_len_buf) as usize; + + if tag_len > 1024 { + return Err("tag too large".into()); + } + + let mut tag_buf = vec![0u8; tag_len]; + recv.read_exact(&mut tag_buf).await?; + let tag = String::from_utf8(tag_buf)?; + + let payload = recv.read_to_end(64 * 1024).await?; + + Ok((tag, payload)) +} diff --git a/crates/runtime-dashboard/Cargo.toml b/crates/runtime-dashboard/Cargo.toml index 76fde7d..4556bd8 100644 --- a/crates/runtime-dashboard/Cargo.toml +++ b/crates/runtime-dashboard/Cargo.toml @@ -17,6 +17,7 @@ distribution = { path = "../distribution", optional = true } clap = { version = "4", features = ["derive"], optional = true } ctrlc = "3" iroh = { version = "0.96", optional = true } +swactor-ci = { path = "../ci", optional = true } [features] default = ["distribution"] @@ -25,6 +26,7 @@ distribution = ["dep:distribution"] node = ["distribution", "dep:clap", "swactor/transport", "tcp"] tcp = ["distribution/tcp"] iroh = ["distribution/iroh", "dep:iroh"] +ci = ["dep:swactor-ci"] [[bin]] name = "swactor-tui" diff --git a/crates/runtime-dashboard/src/ci_collector.rs b/crates/runtime-dashboard/src/ci_collector.rs new file mode 100644 index 0000000..85b321e --- /dev/null +++ b/crates/runtime-dashboard/src/ci_collector.rs @@ -0,0 +1,210 @@ +//! CI Dashboard extension: provides HTTP API endpoints and stats for CI pipelines. + +use serde::{Deserialize, Serialize}; + +use swactor_ci::{JobId, JobStatus, PipelineId, PipelineStatus}; + +// ─── Stats Provider ───────────────────────────────────────────────────────── + +/// Trait for providing CI snapshot data to the dashboard. +pub trait CiStatsProvider: Send + Sync { + fn snapshot(&self) -> CiSnapshot; +} + +/// Point-in-time snapshot of CI system state. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CiSnapshot { + pub active_pipelines: Vec, + pub recent_pipelines: Vec, + pub provisioner_status: ProvisionerStatus, + pub active_instances: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum ProvisionerStatus { + Online, + Offline, + Unknown, +} + +/// Snapshot of a single pipeline execution. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PipelineSnapshot { + pub pipeline_id: PipelineId, + pub pipeline_name: String, + pub repo_owner: String, + pub repo_name: String, + pub commit_sha: String, + pub branch: String, + pub status: PipelineStatus, + pub jobs: Vec, +} + +/// Snapshot of a single job within a pipeline. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct JobSnapshot { + pub job_id: JobId, + pub job_name: String, + pub status: JobStatus, + pub output_line_count: usize, +} + +// ─── HTTP API Responses ───────────────────────────────────────────────────── + +/// Response for GET /api/ci/pipelines +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PipelineListResponse { + pub pipelines: Vec, +} + +/// Response for GET /api/ci/pipelines/{id} +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PipelineDetailResponse { + pub pipeline: PipelineSnapshot, +} + +/// Response for GET /api/ci/status +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SystemStatusResponse { + pub provisioner_status: ProvisionerStatus, + pub active_pipelines: usize, + pub active_instances: usize, +} + +/// Response for GET /api/ci/pipelines/{id}/jobs/{job_id}/log +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct JobLogResponse { + pub job_id: JobId, + pub lines: Vec, +} + +// ─── Route Matching ───────────────────────────────────────────────────────── + +/// Parsed API route for the CI dashboard. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CiRoute { + ListPipelines, + PipelineDetail { id: u64 }, + JobLog { pipeline_id: u64, job_name: String }, + Artifact { job_id: String, path: String }, + SystemStatus, + NotFound, +} + +/// Parse a request path into a CiRoute. +pub fn parse_route(path: &str) -> CiRoute { + let parts: Vec<&str> = path.trim_start_matches('/').split('/').collect(); + match parts.as_slice() { + ["api", "ci", "pipelines"] => CiRoute::ListPipelines, + ["api", "ci", "pipelines", id] => { + if let Ok(id) = id.parse() { + CiRoute::PipelineDetail { id } + } else { + CiRoute::NotFound + } + } + ["api", "ci", "pipelines", id, "jobs", job_name, "log"] => { + if let Ok(pipeline_id) = id.parse() { + CiRoute::JobLog { + pipeline_id, + job_name: job_name.to_string(), + } + } else { + CiRoute::NotFound + } + } + ["api", "ci", "artifacts", job_id, rest @ ..] if !rest.is_empty() => CiRoute::Artifact { + job_id: job_id.to_string(), + path: rest.join("/"), + }, + ["api", "ci", "status"] => CiRoute::SystemStatus, + _ => CiRoute::NotFound, + } +} + +/// Render a CiSnapshot into a JSON response for the given route. +pub fn handle_route(route: &CiRoute, snapshot: &CiSnapshot) -> Option { + match route { + CiRoute::ListPipelines => { + let resp = PipelineListResponse { + pipelines: snapshot + .active_pipelines + .iter() + .chain(snapshot.recent_pipelines.iter()) + .cloned() + .collect(), + }; + serde_json::to_string(&resp).ok() + } + CiRoute::PipelineDetail { id } => { + let pipeline = snapshot + .active_pipelines + .iter() + .chain(snapshot.recent_pipelines.iter()) + .find(|p| p.pipeline_id.0 == *id)?; + let resp = PipelineDetailResponse { + pipeline: pipeline.clone(), + }; + serde_json::to_string(&resp).ok() + } + CiRoute::SystemStatus => { + let resp = SystemStatusResponse { + provisioner_status: snapshot.provisioner_status.clone(), + active_pipelines: snapshot.active_pipelines.len(), + active_instances: snapshot.active_instances, + }; + serde_json::to_string(&resp).ok() + } + CiRoute::JobLog { .. } | CiRoute::Artifact { .. } => { + // These require access to stored data beyond the snapshot. + None + } + CiRoute::NotFound => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn route_parsing() { + assert_eq!(parse_route("/api/ci/pipelines"), CiRoute::ListPipelines); + assert_eq!( + parse_route("/api/ci/pipelines/42"), + CiRoute::PipelineDetail { id: 42 } + ); + assert_eq!( + parse_route("/api/ci/pipelines/1/jobs/test/log"), + CiRoute::JobLog { + pipeline_id: 1, + job_name: "test".into() + } + ); + assert_eq!( + parse_route("/api/ci/artifacts/job-1/target/release/bin"), + CiRoute::Artifact { + job_id: "job-1".into(), + path: "target/release/bin".into() + } + ); + assert_eq!(parse_route("/api/ci/status"), CiRoute::SystemStatus); + assert_eq!(parse_route("/api/ci/unknown"), CiRoute::NotFound); + } + + #[test] + fn handle_system_status() { + let snapshot = CiSnapshot { + active_pipelines: vec![], + recent_pipelines: vec![], + provisioner_status: ProvisionerStatus::Online, + active_instances: 2, + }; + + let route = CiRoute::SystemStatus; + let json = handle_route(&route, &snapshot).unwrap(); + let resp: SystemStatusResponse = serde_json::from_str(&json).unwrap(); + assert_eq!(resp.provisioner_status, ProvisionerStatus::Online); + assert_eq!(resp.active_instances, 2); + } +} diff --git a/crates/runtime-dashboard/src/lib.rs b/crates/runtime-dashboard/src/lib.rs index 1db0871..bc0b5a2 100644 --- a/crates/runtime-dashboard/src/lib.rs +++ b/crates/runtime-dashboard/src/lib.rs @@ -23,6 +23,9 @@ pub mod distribution_collector; mod datastore_html; pub mod datastore_collector; +#[cfg(feature = "ci")] +pub mod ci_collector; + use std::io; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; @@ -95,6 +98,8 @@ pub struct DashboardHandle { #[cfg(feature = "distribution")] distribution: Arc>>>, datastore: Arc>>>, + #[cfg(feature = "ci")] + ci: Arc>>>, } impl DashboardHandle { @@ -132,6 +137,12 @@ impl DashboardHandle { *self.datastore.lock().unwrap() = Some(provider); } + /// Attach a CI stats provider, enabling the `/api/ci/*` endpoints. + #[cfg(feature = "ci")] + pub fn set_ci(&self, provider: Arc) { + *self.ci.lock().unwrap() = Some(provider); + } + /// Access the time-series history store (for TUI sparklines, etc.). pub fn history(&self) -> &Arc { &self.history @@ -190,6 +201,10 @@ pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle { let datastore: Arc>>> = Arc::new(Mutex::new(None)); + #[cfg(feature = "ci")] + let ci: Arc>>> = + Arc::new(Mutex::new(None)); + server::spawn_http_server( Arc::clone(&store), Arc::clone(&runtime), @@ -200,6 +215,8 @@ pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle { #[cfg(feature = "distribution")] Arc::clone(&distribution), Arc::clone(&datastore), + #[cfg(feature = "ci")] + Arc::clone(&ci), ); // Start stats recorder thread when recording is enabled @@ -243,6 +260,8 @@ pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle { #[cfg(feature = "distribution")] distribution, datastore, + #[cfg(feature = "ci")] + ci, } } diff --git a/crates/runtime-dashboard/src/server.rs b/crates/runtime-dashboard/src/server.rs index ee95b19..d7d369d 100644 --- a/crates/runtime-dashboard/src/server.rs +++ b/crates/runtime-dashboard/src/server.rs @@ -27,6 +27,9 @@ use crate::distribution_html::DISTRIBUTION_HTML; use crate::datastore_collector::DatastoreStatsProvider; use crate::datastore_html::DATASTORE_HTML; +#[cfg(feature = "ci")] +use crate::ci_collector::CiStatsProvider; + /// Format a server-sent event. fn format_sse(event: &str, data: &str) -> Vec { format!("event: {event}\ndata: {data}\n\n").into_bytes() @@ -132,6 +135,8 @@ pub(crate) fn spawn_http_server( #[cfg(feature = "distribution")] distribution: Arc>>>, datastore: Arc>>>, + #[cfg(feature = "ci")] + ci: Arc>>>, ) { let addr = format!("0.0.0.0:{port}"); let server = tiny_http::Server::http(&addr).expect("failed to bind HTTP server"); @@ -149,6 +154,8 @@ pub(crate) fn spawn_http_server( #[cfg(feature = "distribution")] let distribution = Arc::clone(&distribution); let datastore = Arc::clone(&datastore); + #[cfg(feature = "ci")] + let ci = Arc::clone(&ci); thread::spawn(move || { loop { let request = match server.recv() { @@ -176,6 +183,8 @@ pub(crate) fn spawn_http_server( #[cfg(feature = "distribution")] Arc::clone(&distribution), Arc::clone(&datastore), + #[cfg(feature = "ci")] + Arc::clone(&ci), ); } "/api/stats" => { @@ -220,6 +229,10 @@ pub(crate) fn spawn_http_server( "/api/logs" => { handle_logs_api(request, &url, Arc::clone(&store)); } + #[cfg(feature = "ci")] + _ if path.starts_with("/api/ci/") => { + handle_ci_api(request, path, Arc::clone(&ci)); + } _ if path.starts_with("/actor/") => { let hex = &path[7..]; // strip "/actor/" respond_actor_detail(request, hex); @@ -253,6 +266,8 @@ fn handle_live_sse( #[cfg(feature = "distribution")] distribution: Arc>>>, datastore: Arc>>>, + #[cfg(feature = "ci")] + ci: Arc>>>, ) { let (tx, rx) = mpsc::channel::>(); let response = make_sse_response(rx); @@ -335,6 +350,20 @@ fn handle_live_sse( } } + // Send CI snapshot if provider is attached + #[cfg(feature = "ci")] + { + let maybe_ci = ci.lock().unwrap().clone(); + if let Some(provider) = maybe_ci { + let snapshot = provider.snapshot(); + if let Ok(json) = serde_json::to_string(&snapshot) { + if tx.send(format_sse("ci", &json)).is_err() { + return; + } + } + } + } + // Send new activity events let (batch, new_cursor) = store.read_from(cursor); if !batch.is_empty() { @@ -469,6 +498,35 @@ fn handle_datastore_api( let _ = request.respond(response); } +#[cfg(feature = "ci")] +fn handle_ci_api( + request: tiny_http::Request, + path: &str, + ci: Arc>>>, +) { + use crate::ci_collector; + + let route = ci_collector::parse_route(path); + let json = match ci.lock().unwrap().as_ref() { + Some(provider) => { + let snapshot = provider.snapshot(); + ci_collector::handle_route(&route, &snapshot) + .unwrap_or_else(|| r#"{"error":"not found"}"#.to_string()) + } + None => serde_json::json!({ + "error": "CI provider not attached" + }) + .to_string(), + }; + + let response = tiny_http::Response::from_string(json).with_header( + "Content-Type: application/json" + .parse::() + .unwrap(), + ); + let _ = request.respond(response); +} + fn handle_topology_api( request: tiny_http::Request, runtime: Arc>>>, diff --git a/crates/simulation/Cargo.toml b/crates/simulation/Cargo.toml index 93c8b35..6f77f91 100644 --- a/crates/simulation/Cargo.toml +++ b/crates/simulation/Cargo.toml @@ -7,6 +7,7 @@ edition = "2024" default = [] gossip = ["dep:log"] dashboard = ["gossip", "dep:tiny_http", "dep:toml"] +ci = ["dep:swactor-ci"] [dependencies] distribution = { path = "../distribution" } @@ -17,9 +18,11 @@ getrandom = "0.2" log = { version = "0.4", optional = true } tiny_http = { version = "0.12", optional = true } toml = { version = "0.8", optional = true } +swactor-ci = { path = "../ci", optional = true } [dev-dependencies] -simulation = { path = ".", features = ["gossip"] } +simulation = { path = ".", features = ["gossip", "ci"] } +swactor-ci = { path = "../ci" } [[example]] name = "gossip_sim" diff --git a/crates/simulation/src/ci/local_sim.rs b/crates/simulation/src/ci/local_sim.rs new file mode 100644 index 0000000..e97bcd1 --- /dev/null +++ b/crates/simulation/src/ci/local_sim.rs @@ -0,0 +1,555 @@ +//! Local CI simulation: deterministic, round-based execution of the local +//! coordinator's queue + scheduling logic. +//! +//! No actors, no IO. Models the one-at-a-time scheduling with supersede. +//! Follows the same pattern as `sim.rs`. + +use std::collections::{HashMap, VecDeque}; + +use swactor_ci::pipeline::PipelineExecution; +use swactor_ci::yaml::{self, CiYaml}; +use swactor_ci::{JobId, JobStatus, PipelineId, PipelineStatus, StatusUpdate, WebhookEvent}; + +// ─── Simulation Config ────────────────────────────────────────────────────── + +/// Configuration for a local CI simulation run. +#[derive(Debug, Clone)] +pub struct LocalSimConfig { + pub name: String, + pub num_rounds: usize, + pub ci_yaml: String, + pub webhook_schedule: Vec<(usize, WebhookEvent)>, + /// Rounds a job takes to execute. + pub job_duration: usize, + /// Force specific jobs to fail: (round, job_name_substring). + pub job_failure_schedule: Vec<(usize, String)>, +} + +impl Default for LocalSimConfig { + fn default() -> Self { + Self { + name: "local-sim".into(), + num_rounds: 50, + ci_yaml: String::new(), + webhook_schedule: Vec::new(), + job_duration: 3, + job_failure_schedule: Vec::new(), + } + } +} + +// ─── Simulation Trace ─────────────────────────────────────────────────────── + +#[derive(Debug, Clone)] +pub enum LocalSimEvent { + WebhookReceived { commit_sha: String }, + PipelineCreated { pipeline_id: PipelineId, name: String }, + PipelineSuperseded { pipeline_id: PipelineId }, + PipelineCompleted { pipeline_id: PipelineId, status: PipelineStatus }, + JobStarted { job_id: JobId }, + JobCompleted { job_id: JobId, passed: bool }, + JobSkipped { job_id: JobId }, +} + +#[derive(Debug, Clone)] +pub struct LocalSimSnapshot { + pub queued_pipelines: usize, + pub active_pipeline: Option, + pub running_job: Option, + pub completed_pipelines: usize, +} + +#[derive(Debug, Clone)] +pub struct LocalSimTrace { + pub name: String, + pub events: Vec<(usize, LocalSimEvent)>, + pub snapshots: Vec, + pub status_updates: Vec, + pub num_rounds: usize, + pub final_pipelines: Vec, +} + +// ─── Simulation State ─────────────────────────────────────────────────────── + +struct RunningJob { + job_id: JobId, + started_round: usize, +} + +/// Run a local CI simulation and return the trace. +pub fn run_simulation(config: LocalSimConfig) -> LocalSimTrace { + let ci_yaml: CiYaml = + yaml::parse_ci_yaml(&config.ci_yaml).expect("LocalSimConfig.ci_yaml must be valid YAML"); + + let mut events: Vec<(usize, LocalSimEvent)> = Vec::new(); + let mut snapshots: Vec = Vec::new(); + let mut all_status_updates: Vec = Vec::new(); + + // Coordinator state. + let mut pipelines: HashMap = HashMap::new(); + let mut next_pipeline_id: u64 = 1; + let mut queue: VecDeque = VecDeque::new(); + let mut active_pipeline: Option = None; + let mut running_job: Option = None; + let mut completed_pipelines: Vec = Vec::new(); + + for round in 1..=config.num_rounds { + // 1. Inject webhook events for this round. + for (sched_round, event) in &config.webhook_schedule { + if *sched_round == round { + events.push(( + round, + LocalSimEvent::WebhookReceived { + commit_sha: event.commit_sha.clone(), + }, + )); + + let matched = yaml::matching_pipelines(&ci_yaml, event); + for pipeline_name in matched { + let pipeline_id = PipelineId(next_pipeline_id); + next_pipeline_id += 1; + + let pipeline_def = &ci_yaml.pipelines[&pipeline_name]; + let job_defs: Vec<_> = pipeline_def + .jobs + .iter() + .map(|(name, def)| yaml::to_job_definition(name, def)) + .collect(); + + let pipeline = PipelineExecution::new( + pipeline_id, + pipeline_name.clone(), + event.repo_owner.clone(), + event.repo_name.clone(), + event.commit_sha.clone(), + event.branch.clone(), + job_defs, + ); + + events.push(( + round, + LocalSimEvent::PipelineCreated { + pipeline_id, + name: pipeline_name.clone(), + }, + )); + + all_status_updates.push(StatusUpdate { + repo_owner: event.repo_owner.clone(), + repo_name: event.repo_name.clone(), + commit_sha: event.commit_sha.clone(), + state: "pending".into(), + context: format!("ci/{pipeline_name}"), + description: format!("Pipeline '{pipeline_name}' is pending"), + target_url: None, + }); + + pipelines.insert(pipeline_id, pipeline); + + // Enqueue with supersede logic. + let supersede_idx = queue.iter().position(|&qid| { + pipelines + .get(&qid) + .map(|p| p.branch == event.branch) + .unwrap_or(false) + }); + + if let Some(idx) = supersede_idx { + let old_id = queue[idx]; + if let Some(old_pipeline) = pipelines.get_mut(&old_id) { + old_pipeline.status = PipelineStatus::Error { + reason: "superseded".into(), + }; + let job_names: Vec = + old_pipeline.jobs.keys().cloned().collect(); + for name in job_names { + if old_pipeline.jobs[&name].status == JobStatus::Pending { + old_pipeline.set_job_status(&name, JobStatus::Skipped); + } + } + } + events.push(( + round, + LocalSimEvent::PipelineSuperseded { + pipeline_id: old_id, + }, + )); + if let Some(old_pipeline) = pipelines.remove(&old_id) { + // Emit terminal status for superseded pipeline. + all_status_updates.push(StatusUpdate { + repo_owner: old_pipeline.repo_owner.clone(), + repo_name: old_pipeline.repo_name.clone(), + commit_sha: old_pipeline.commit_sha.clone(), + state: "error".into(), + context: format!("ci/{}", old_pipeline.pipeline_name), + description: "superseded".into(), + target_url: None, + }); + completed_pipelines.push(old_pipeline); + } + queue[idx] = pipeline_id; + } else { + queue.push_back(pipeline_id); + } + } + } + } + + // 2. Complete running job if it has reached duration. + if let Some(ref rj) = running_job { + if round - rj.started_round >= config.job_duration { + let job_id = rj.job_id.clone(); + + let should_fail = config + .job_failure_schedule + .iter() + .any(|(r, name_sub)| *r <= round && job_id.job_name.contains(name_sub.as_str())); + + let passed = !should_fail; + + if passed { + if let Some(pipeline) = pipelines.get_mut(&job_id.pipeline_id) { + pipeline.set_job_status(&job_id.job_name, JobStatus::Passed); + } + } else { + if let Some(pipeline) = pipelines.get_mut(&job_id.pipeline_id) { + pipeline.set_job_status( + &job_id.job_name, + JobStatus::Failed { + reason: "command failed".into(), + }, + ); + } + } + + events.push(( + round, + LocalSimEvent::JobCompleted { + job_id: job_id.clone(), + passed, + }, + )); + + // Emit skipped events for any jobs that were skipped due to failure. + if !passed { + if let Some(pipeline) = pipelines.get(&job_id.pipeline_id) { + for (_name, job) in &pipeline.jobs { + if job.status == JobStatus::Skipped { + events.push(( + round, + LocalSimEvent::JobSkipped { + job_id: job.job_id.clone(), + }, + )); + } + } + } + } + + running_job = None; + } + } + + // 3. Schedule next (one-at-a-time). + schedule_next( + &mut pipelines, + &mut queue, + &mut active_pipeline, + &mut running_job, + &mut completed_pipelines, + &mut events, + &mut all_status_updates, + round, + ); + + // 4. Snapshot. + snapshots.push(LocalSimSnapshot { + queued_pipelines: queue.len(), + active_pipeline, + running_job: running_job.as_ref().map(|rj| rj.job_id.clone()), + completed_pipelines: completed_pipelines.len(), + }); + } + + // Collect remaining active pipelines into final output. + let mut final_pipelines: Vec = pipelines.into_values().collect(); + final_pipelines.extend(completed_pipelines); + + LocalSimTrace { + name: config.name, + events, + snapshots, + status_updates: all_status_updates, + num_rounds: config.num_rounds, + final_pipelines, + } +} + +#[allow(clippy::too_many_arguments)] +fn schedule_next( + pipelines: &mut HashMap, + queue: &mut VecDeque, + active_pipeline: &mut Option, + running_job: &mut Option, + completed_pipelines: &mut Vec, + events: &mut Vec<(usize, LocalSimEvent)>, + status_updates: &mut Vec, + round: usize, +) { + // If a job is running, nothing to do. + if running_job.is_some() { + return; + } + + // If we have an active pipeline, try eligible jobs. + if let Some(active_id) = *active_pipeline { + if let Some(pipeline) = pipelines.get(&active_id) { + let eligible = pipeline.eligible_jobs(); + if !eligible.is_empty() { + let job_name = eligible[0].clone(); + let job_id = JobId { + pipeline_id: active_id, + job_name: job_name.clone(), + }; + + // Mark as running. + if let Some(pipeline) = pipelines.get_mut(&active_id) { + if let Some(job) = pipeline.jobs.get_mut(&job_name) { + job.status = JobStatus::Running; + } + } + + events.push((round, LocalSimEvent::JobStarted { job_id: job_id.clone() })); + + *running_job = Some(RunningJob { + job_id, + started_round: round, + }); + return; + } + + // No eligible jobs — check terminal. + if pipeline.status.is_terminal() { + let pipeline = pipelines.remove(&active_id).unwrap(); + let status = pipeline.status.clone(); + + status_updates.push(StatusUpdate { + repo_owner: pipeline.repo_owner.clone(), + repo_name: pipeline.repo_name.clone(), + commit_sha: pipeline.commit_sha.clone(), + state: pipeline.status.forgejo_state().into(), + context: format!("ci/{}", pipeline.pipeline_name), + description: format!( + "Pipeline '{}' {}", + pipeline.pipeline_name, + pipeline.status.forgejo_state() + ), + target_url: None, + }); + + events.push(( + round, + LocalSimEvent::PipelineCompleted { + pipeline_id: active_id, + status, + }, + )); + + completed_pipelines.push(pipeline); + *active_pipeline = None; + + // Recurse. + schedule_next( + pipelines, + queue, + active_pipeline, + running_job, + completed_pipelines, + events, + status_updates, + round, + ); + return; + } + } + // Pipeline exists but no eligible jobs and not terminal — waiting. + return; + } + + // No active pipeline — pop from queue. + if let Some(next_id) = queue.pop_front() { + *active_pipeline = Some(next_id); + schedule_next( + pipelines, + queue, + active_pipeline, + running_job, + completed_pipelines, + events, + status_updates, + round, + ); + } +} + +// ─── Properties ───────────────────────────────────────────────────────────── + +/// At most one job running in any snapshot. +pub fn check_one_at_a_time(trace: &LocalSimTrace) -> bool { + trace + .snapshots + .iter() + .all(|s| s.running_job.is_some() as usize <= 1) +} + +/// Superseded pipelines never have a Running job. +pub fn check_superseded_no_running(trace: &LocalSimTrace) -> bool { + let superseded: Vec = trace + .events + .iter() + .filter_map(|(_, e)| match e { + LocalSimEvent::PipelineSuperseded { pipeline_id } => Some(*pipeline_id), + _ => None, + }) + .collect(); + + let started_jobs: Vec<&JobId> = trace + .events + .iter() + .filter_map(|(_, e)| match e { + LocalSimEvent::JobStarted { job_id } => Some(job_id), + _ => None, + }) + .collect(); + + for pid in &superseded { + if started_jobs + .iter() + .any(|jid| jid.pipeline_id == *pid) + { + return false; + } + } + true +} + +/// All non-superseded pipelines reach terminal status. +pub fn check_termination(trace: &LocalSimTrace) -> bool { + let superseded: Vec = trace + .events + .iter() + .filter_map(|(_, e)| match e { + LocalSimEvent::PipelineSuperseded { pipeline_id } => Some(*pipeline_id), + _ => None, + }) + .collect(); + + for pipeline in &trace.final_pipelines { + if superseded.contains(&pipeline.pipeline_id) { + continue; + } + if !pipeline.status.is_terminal() { + return false; + } + } + true +} + +/// Within a pipeline, jobs respect dependency order. +pub fn check_dag_ordering(trace: &LocalSimTrace) -> bool { + let mut started: HashMap<(u64, &str), usize> = HashMap::new(); + let mut completed: HashMap<(u64, &str), usize> = HashMap::new(); + + for (round, event) in &trace.events { + match event { + LocalSimEvent::JobStarted { job_id } => { + started.insert( + (job_id.pipeline_id.0, job_id.job_name.as_str()), + *round, + ); + } + LocalSimEvent::JobCompleted { job_id, .. } => { + completed.insert( + (job_id.pipeline_id.0, job_id.job_name.as_str()), + *round, + ); + } + _ => {} + } + } + + for pipeline in &trace.final_pipelines { + for (name, job) in &pipeline.jobs { + if let Some(&start_round) = started.get(&(pipeline.pipeline_id.0, name.as_str())) { + for dep in &job.definition.needs { + if let Some(&dep_complete_round) = + completed.get(&(pipeline.pipeline_id.0, dep.as_str())) + { + if dep_complete_round > start_round { + return false; + } + } + } + } + } + } + true +} + +/// Different branches execute in queue order (FIFO). +pub fn check_fifo_order(trace: &LocalSimTrace) -> bool { + // Collect pipeline creation order and first job start per pipeline. + let mut creation_order: Vec = Vec::new(); + let mut first_start: HashMap = HashMap::new(); + + for (round, event) in &trace.events { + if let LocalSimEvent::PipelineCreated { pipeline_id, .. } = event { + creation_order.push(*pipeline_id); + } + if let LocalSimEvent::JobStarted { job_id } = event { + first_start + .entry(job_id.pipeline_id) + .or_insert(*round); + } + } + + // For each pair of pipelines created in order, if both started, the earlier-created + // one should have started no later. + for i in 0..creation_order.len() { + for j in (i + 1)..creation_order.len() { + let pid_a = creation_order[i]; + let pid_b = creation_order[j]; + if let (Some(&start_a), Some(&start_b)) = + (first_start.get(&pid_a), first_start.get(&pid_b)) + { + if start_a > start_b { + return false; + } + } + } + } + true +} + +/// Every webhook produces a terminal status (success/failure/error). +pub fn check_all_webhooks_terminate(trace: &LocalSimTrace) -> bool { + let webhook_commits: Vec<&str> = trace + .events + .iter() + .filter_map(|(_, e)| match e { + LocalSimEvent::WebhookReceived { commit_sha } => Some(commit_sha.as_str()), + _ => None, + }) + .collect(); + + for sha in webhook_commits { + let has_terminal = trace.status_updates.iter().any(|u| { + u.commit_sha == sha + && (u.state == "success" || u.state == "failure" || u.state == "error") + }); + if !has_terminal { + return false; + } + } + true +} diff --git a/crates/simulation/src/ci/mod.rs b/crates/simulation/src/ci/mod.rs new file mode 100644 index 0000000..90e3a11 --- /dev/null +++ b/crates/simulation/src/ci/mod.rs @@ -0,0 +1,2 @@ +pub mod local_sim; +pub mod sim; diff --git a/crates/simulation/src/ci/sim.rs b/crates/simulation/src/ci/sim.rs new file mode 100644 index 0000000..3187db3 --- /dev/null +++ b/crates/simulation/src/ci/sim.rs @@ -0,0 +1,612 @@ +//! CI protocol simulation: deterministic, round-based execution of the CI pipeline +//! lifecycle without real IO (no SSH, no cloud API, no HTTP). +//! +//! Follows the same pattern as `crates/simulation/src/distribution/sim.rs`: +//! configure → run rounds → collect trace → analyze properties. + +use std::collections::HashMap; + +use swactor_ci::pipeline::PipelineExecution; +use swactor_ci::yaml::{self, CiYaml}; +use swactor_ci::{ + JobId, JobStatus, PipelineId, ProvisionRequest, StatusUpdate, WebhookEvent, +}; + +// ─── Simulation Config ────────────────────────────────────────────────────── + +/// Configuration for a CI simulation run. +#[derive(Debug, Clone)] +pub struct CiSimConfig { + pub name: String, + pub num_rounds: usize, + + /// CI YAML to use for all simulated repos. + pub ci_yaml: String, + + /// Webhook events to inject at specific rounds. + pub webhook_schedule: Vec<(usize, WebhookEvent)>, + + /// Rounds of latency for provisioning to complete. + pub provision_latency: usize, + + /// Probability that provisioning fails (0.0-1.0). + pub provision_failure_rate: f64, + + /// Rounds of latency for a job to complete. + pub job_duration: usize, + + /// Force specific jobs to fail: (round, job_name_substring). + pub job_failure_schedule: Vec<(usize, String)>, + + /// Rounds during which the provisioner is offline: (start_round, end_round). + pub provisioner_offline_schedule: Vec<(usize, usize)>, + + /// Round at which a specific job's instance is interrupted. + pub instance_interrupt_schedule: Vec<(usize, String)>, +} + +impl Default for CiSimConfig { + fn default() -> Self { + Self { + name: "ci-sim".into(), + num_rounds: 50, + ci_yaml: String::new(), + webhook_schedule: Vec::new(), + provision_latency: 2, + provision_failure_rate: 0.0, + job_duration: 3, + job_failure_schedule: Vec::new(), + provisioner_offline_schedule: Vec::new(), + instance_interrupt_schedule: Vec::new(), + } + } +} + +// ─── Simulation Trace ─────────────────────────────────────────────────────── + +/// Event recorded during simulation. +#[derive(Debug, Clone)] +pub enum CiSimEvent { + WebhookReceived { commit_sha: String }, + PipelineCreated { pipeline_id: PipelineId, name: String }, + ProvisionRequested { job_id: JobId }, + ProvisionCompleted { job_id: JobId, success: bool }, + JobStarted { job_id: JobId }, + JobCompleted { job_id: JobId, passed: bool }, + JobSkipped { job_id: JobId }, + InstanceTerminated { instance_id: String }, + ProvisionerWentOffline, + ProvisionerCameOnline, + StatusUpdateEmitted(StatusUpdate), +} + +/// Per-round snapshot of simulation state. +#[derive(Debug, Clone)] +pub struct CiSimSnapshot { + pub active_pipelines: usize, + pub completed_pipelines: usize, + pub active_provisions: usize, + pub active_jobs: usize, + pub provisioner_online: bool, + pub active_instances: usize, +} + +/// Complete trace output from a CI simulation. +#[derive(Debug, Clone)] +pub struct CiSimTrace { + pub name: String, + pub events: Vec<(usize, CiSimEvent)>, + pub snapshots: Vec, + pub status_updates: Vec, + pub num_rounds: usize, + /// Final state of all pipelines. + pub final_pipelines: Vec, + /// Instances that were provisioned. + pub provisioned_instances: Vec, + /// Instances that were terminated. + pub terminated_instances: Vec, +} + +// ─── Simulation State ─────────────────────────────────────────────────────── + +/// Tracks an in-flight provision request. +struct PendingProvision { + request: ProvisionRequest, + started_round: usize, +} + +/// Tracks an in-flight job execution. +struct RunningJob { + job_id: JobId, + started_round: usize, + instance_id: String, +} + +/// Run a CI simulation and return the trace. +pub fn run_simulation(config: CiSimConfig) -> CiSimTrace { + let ci_yaml: CiYaml = yaml::parse_ci_yaml(&config.ci_yaml) + .expect("CiSimConfig.ci_yaml must be valid YAML"); + + let mut events: Vec<(usize, CiSimEvent)> = Vec::new(); + let mut snapshots: Vec = Vec::new(); + + // Coordinator state (simulated directly, not as actor). + let mut pipelines: HashMap = HashMap::new(); + let mut next_pipeline_id: u64 = 1; + let mut all_status_updates: Vec = Vec::new(); + + // Provisioner state. + let mut provisioner_online = true; + let mut pending_provisions: Vec = Vec::new(); + let mut queued_provisions: Vec = Vec::new(); + let mut provisioned_instances: Vec = Vec::new(); + let mut terminated_instances: Vec = Vec::new(); + let mut active_instances: Vec = Vec::new(); + let mut next_instance_id: u64 = 1; + + // Runner state. + let mut running_jobs: Vec = Vec::new(); + + // Simple deterministic "RNG" for provision failure decisions. + let mut rng_counter: u64 = 0x853c49e6748fea9b; + let mut det_random = || -> f64 { + rng_counter = rng_counter.wrapping_mul(6364136223846793005).wrapping_add(1); + (rng_counter >> 33) as f64 / (u32::MAX as f64) + }; + + for round in 1..=config.num_rounds { + // 1. Apply provisioner online/offline schedule. + let should_be_offline = config + .provisioner_offline_schedule + .iter() + .any(|(start, end)| round >= *start && round <= *end); + + if should_be_offline && provisioner_online { + provisioner_online = false; + events.push((round, CiSimEvent::ProvisionerWentOffline)); + + // Move pending provisions to queue. + for pending in pending_provisions.drain(..) { + queued_provisions.push(pending.request); + } + } else if !should_be_offline && !provisioner_online { + provisioner_online = true; + events.push((round, CiSimEvent::ProvisionerCameOnline)); + + // Flush queued provisions. + for request in queued_provisions.drain(..) { + pending_provisions.push(PendingProvision { + request, + started_round: round, + }); + } + + // Also resubmit any WaitingForProvisioner jobs. + let mut resubmits = Vec::new(); + for pipeline in pipelines.values_mut() { + for job in pipeline.jobs.values_mut() { + if job.status == JobStatus::WaitingForProvisioner { + job.status = JobStatus::Provisioning; + resubmits.push(ProvisionRequest { + job_id: job.job_id.clone(), + instance_spec: swactor_ci::InstanceSpec { + docker_required: job.definition.docker, + ..Default::default() + }, + }); + } + } + } + for request in resubmits { + events.push((round, CiSimEvent::ProvisionRequested { job_id: request.job_id.clone() })); + pending_provisions.push(PendingProvision { + request, + started_round: round, + }); + } + } + + // 2. Inject webhook events for this round. + for (sched_round, event) in &config.webhook_schedule { + if *sched_round == round { + events.push(( + round, + CiSimEvent::WebhookReceived { + commit_sha: event.commit_sha.clone(), + }, + )); + + let matched = yaml::matching_pipelines(&ci_yaml, event); + for pipeline_name in matched { + let pipeline_id = PipelineId(next_pipeline_id); + next_pipeline_id += 1; + + let pipeline_def = &ci_yaml.pipelines[&pipeline_name]; + let job_defs: Vec<_> = pipeline_def + .jobs + .iter() + .map(|(name, def)| yaml::to_job_definition(name, def)) + .collect(); + + let pipeline = PipelineExecution::new( + pipeline_id, + pipeline_name.clone(), + event.repo_owner.clone(), + event.repo_name.clone(), + event.commit_sha.clone(), + event.branch.clone(), + job_defs, + ); + + events.push(( + round, + CiSimEvent::PipelineCreated { + pipeline_id, + name: pipeline_name.clone(), + }, + )); + + // Emit pending status. + all_status_updates.push(StatusUpdate { + repo_owner: event.repo_owner.clone(), + repo_name: event.repo_name.clone(), + commit_sha: event.commit_sha.clone(), + state: "pending".into(), + context: format!("ci/{pipeline_name}"), + description: format!("Pipeline '{pipeline_name}' is pending"), + target_url: None, + }); + + pipelines.insert(pipeline_id, pipeline); + } + } + } + + // 3. Complete provisions that have reached latency. + let mut completed_provisions = Vec::new(); + pending_provisions.retain(|pending| { + if round - pending.started_round >= config.provision_latency { + completed_provisions.push(pending.request.clone()); + false + } else { + true + } + }); + + for request in completed_provisions { + let should_fail = det_random() < config.provision_failure_rate; + + if should_fail { + events.push(( + round, + CiSimEvent::ProvisionCompleted { + job_id: request.job_id.clone(), + success: false, + }, + )); + + if let Some(pipeline) = pipelines.get_mut(&request.job_id.pipeline_id) { + pipeline.set_job_status( + &request.job_id.job_name, + JobStatus::Failed { + reason: "provision failed".into(), + }, + ); + } + } else { + let instance_id = format!("instance-{next_instance_id}"); + next_instance_id += 1; + provisioned_instances.push(instance_id.clone()); + active_instances.push(instance_id.clone()); + + events.push(( + round, + CiSimEvent::ProvisionCompleted { + job_id: request.job_id.clone(), + success: true, + }, + )); + + // Mark job as running and record instance. + if let Some(pipeline) = pipelines.get_mut(&request.job_id.pipeline_id) { + if let Some(job) = pipeline.jobs.get_mut(&request.job_id.job_name) { + job.status = JobStatus::Running; + job.instance_id = Some(instance_id.clone()); + } + } + + events.push(( + round, + CiSimEvent::JobStarted { + job_id: request.job_id.clone(), + }, + )); + + running_jobs.push(RunningJob { + job_id: request.job_id, + started_round: round, + instance_id, + }); + } + } + + // 4. Apply instance interruptions. + for (interrupt_round, job_name_sub) in &config.instance_interrupt_schedule { + if *interrupt_round == round { + running_jobs.retain(|rj| { + if rj.job_id.job_name.contains(job_name_sub.as_str()) { + // Instance interrupted. + if let Some(pipeline) = pipelines.get_mut(&rj.job_id.pipeline_id) { + pipeline.set_job_status(&rj.job_id.job_name, JobStatus::Interrupted); + } + events.push(( + round, + CiSimEvent::JobCompleted { + job_id: rj.job_id.clone(), + passed: false, + }, + )); + // Terminate the instance. + active_instances.retain(|id| id != &rj.instance_id); + terminated_instances.push(rj.instance_id.clone()); + events.push(( + round, + CiSimEvent::InstanceTerminated { + instance_id: rj.instance_id.clone(), + }, + )); + false + } else { + true + } + }); + } + } + + // 5. Complete jobs that have reached duration. + let mut newly_completed = Vec::new(); + running_jobs.retain(|rj| { + if round - rj.started_round >= config.job_duration { + newly_completed.push((rj.job_id.clone(), rj.instance_id.clone())); + false + } else { + true + } + }); + + for (job_id, instance_id) in newly_completed { + // Check if this job should fail per the schedule. + let should_fail = config + .job_failure_schedule + .iter() + .any(|(r, name_sub)| *r <= round && job_id.job_name.contains(name_sub.as_str())); + + let passed = !should_fail; + + if passed { + if let Some(pipeline) = pipelines.get_mut(&job_id.pipeline_id) { + pipeline.set_job_status(&job_id.job_name, JobStatus::Passed); + } + } else { + if let Some(pipeline) = pipelines.get_mut(&job_id.pipeline_id) { + pipeline.set_job_status( + &job_id.job_name, + JobStatus::Failed { + reason: "command failed".into(), + }, + ); + } + } + + events.push(( + round, + CiSimEvent::JobCompleted { + job_id: job_id.clone(), + passed, + }, + )); + + // Terminate instance. + active_instances.retain(|id| id != &instance_id); + terminated_instances.push(instance_id.clone()); + events.push(( + round, + CiSimEvent::InstanceTerminated { + instance_id: instance_id.clone(), + }, + )); + } + + // 6. Advance all pipelines: schedule newly-eligible jobs. + let pipeline_ids: Vec = pipelines.keys().copied().collect(); + for pid in pipeline_ids { + let eligible = pipelines[&pid].eligible_jobs(); + for job_name in eligible { + let job_id = JobId { + pipeline_id: pid, + job_name: job_name.clone(), + }; + let docker_required = pipelines[&pid].jobs[&job_name].definition.docker; + + let request = ProvisionRequest { + job_id: job_id.clone(), + instance_spec: swactor_ci::InstanceSpec { + docker_required, + ..Default::default() + }, + }; + + if provisioner_online { + if let Some(pipeline) = pipelines.get_mut(&pid) { + if let Some(job) = pipeline.jobs.get_mut(&job_name) { + job.status = JobStatus::Provisioning; + } + } + events.push(( + round, + CiSimEvent::ProvisionRequested { job_id }, + )); + pending_provisions.push(PendingProvision { + request, + started_round: round, + }); + } else { + if let Some(pipeline) = pipelines.get_mut(&pid) { + if let Some(job) = pipeline.jobs.get_mut(&job_name) { + job.status = JobStatus::WaitingForProvisioner; + } + } + queued_provisions.push(request); + } + } + + // Emit final status updates for terminal pipelines. + if let Some(pipeline) = pipelines.get(&pid) { + if pipeline.status.is_terminal() { + // Check if we already emitted a terminal status for this pipeline. + let context = format!("ci/{}", pipeline.pipeline_name); + let already_emitted = all_status_updates.iter().any(|u| { + u.context == context + && u.commit_sha == pipeline.commit_sha + && (u.state == "success" || u.state == "failure" || u.state == "error") + }); + if !already_emitted { + all_status_updates.push(StatusUpdate { + repo_owner: pipeline.repo_owner.clone(), + repo_name: pipeline.repo_name.clone(), + commit_sha: pipeline.commit_sha.clone(), + state: pipeline.status.forgejo_state().into(), + context, + description: format!( + "Pipeline '{}' {}", + pipeline.pipeline_name, + pipeline.status.forgejo_state() + ), + target_url: None, + }); + + // Emit per-job skipped events. + for (_name, job) in &pipeline.jobs { + if job.status == JobStatus::Skipped { + events.push(( + round, + CiSimEvent::JobSkipped { + job_id: job.job_id.clone(), + }, + )); + } + } + } + } + } + } + + // 7. Snapshot. + let completed_count = pipelines.values().filter(|p| p.status.is_terminal()).count(); + snapshots.push(CiSimSnapshot { + active_pipelines: pipelines.len() - completed_count, + completed_pipelines: completed_count, + active_provisions: pending_provisions.len(), + active_jobs: running_jobs.len(), + provisioner_online, + active_instances: active_instances.len(), + }); + } + + CiSimTrace { + name: config.name, + events, + snapshots, + status_updates: all_status_updates, + num_rounds: config.num_rounds, + final_pipelines: pipelines.into_values().collect(), + provisioned_instances, + terminated_instances, + } +} + +// ─── Properties ───────────────────────────────────────────────────────────── + +/// Every webhook eventually produces a terminal Forgejo status (success/failure/error). +pub fn check_all_webhooks_terminate(trace: &CiSimTrace) -> bool { + let webhook_commits: Vec<&str> = trace + .events + .iter() + .filter_map(|(_, e)| match e { + CiSimEvent::WebhookReceived { commit_sha } => Some(commit_sha.as_str()), + _ => None, + }) + .collect(); + + for sha in webhook_commits { + let has_terminal = trace.status_updates.iter().any(|u| { + u.commit_sha == sha && (u.state == "success" || u.state == "failure" || u.state == "error") + }); + if !has_terminal { + return false; + } + } + true +} + +/// Job DAG ordering is always respected: no job runs before its `needs`. +pub fn check_dag_ordering(trace: &CiSimTrace) -> bool { + // Build a map of (pipeline_id, job_name) → round when started. + let mut started: HashMap<(u64, &str), usize> = HashMap::new(); + let mut completed: HashMap<(u64, &str), usize> = HashMap::new(); + + for (round, event) in &trace.events { + match event { + CiSimEvent::JobStarted { job_id } => { + started.insert( + (job_id.pipeline_id.0, job_id.job_name.as_str()), + *round, + ); + } + CiSimEvent::JobCompleted { job_id, .. } => { + completed.insert( + (job_id.pipeline_id.0, job_id.job_name.as_str()), + *round, + ); + } + _ => {} + } + } + + // For each pipeline, check that if job B needs job A, then A completed before B started. + for pipeline in &trace.final_pipelines { + for (name, job) in &pipeline.jobs { + if let Some(&start_round) = started.get(&(pipeline.pipeline_id.0, name.as_str())) { + for dep in &job.definition.needs { + if let Some(&dep_complete_round) = + completed.get(&(pipeline.pipeline_id.0, dep.as_str())) + { + if dep_complete_round > start_round { + return false; + } + } + } + } + } + } + true +} + +/// Every provisioned instance is eventually terminated (no resource leaks). +pub fn check_no_instance_leaks(trace: &CiSimTrace) -> bool { + // Every instance that was provisioned should also be terminated. + for instance_id in &trace.provisioned_instances { + if !trace.terminated_instances.contains(instance_id) { + return false; + } + } + true +} + +/// Coordinator state is bounded: active pipeline count doesn't grow unboundedly. +pub fn check_bounded_state(trace: &CiSimTrace, max_active: usize) -> bool { + trace + .snapshots + .iter() + .all(|s| s.active_pipelines <= max_active) +} diff --git a/crates/simulation/src/lib.rs b/crates/simulation/src/lib.rs index fcb4c4d..c7712b4 100644 --- a/crates/simulation/src/lib.rs +++ b/crates/simulation/src/lib.rs @@ -9,3 +9,6 @@ pub mod gossip; #[cfg(feature = "dashboard")] pub mod dashboard; + +#[cfg(feature = "ci")] +pub mod ci; diff --git a/crates/simulation/tests/ci_properties.rs b/crates/simulation/tests/ci_properties.rs new file mode 100644 index 0000000..31d162c --- /dev/null +++ b/crates/simulation/tests/ci_properties.rs @@ -0,0 +1,275 @@ +//! Property-based tests for the CI simulation. +//! +//! These verify invariants that should hold across all possible simulation configurations. + +use swactor_ci::{EventType, WebhookEvent}; +use simulation::ci::sim::{self, CiSimConfig}; + +fn simple_yaml() -> String { + r#" +pipelines: + check: + triggers: + - event: push + branches: ["*"] + jobs: + fmt: + run: cargo fmt -- --check + test: + needs: [fmt] + run: cargo test +"# + .into() +} + +fn push(branch: &str, sha: &str) -> WebhookEvent { + WebhookEvent { + event_type: EventType::Push, + repo_owner: "user".into(), + repo_name: "repo".into(), + branch: branch.into(), + commit_sha: sha.into(), + tag: None, + } +} + +// ─── Property: Every webhook produces a terminal status ───────────────────── + +#[test] +fn property_all_webhooks_terminate_single() { + let config = CiSimConfig { + name: "prop-single".into(), + num_rounds: 40, + ci_yaml: simple_yaml(), + webhook_schedule: vec![(1, push("main", "sha-1"))], + provision_latency: 1, + job_duration: 2, + ..Default::default() + }; + let trace = sim::run_simulation(config); + assert!( + sim::check_all_webhooks_terminate(&trace), + "single webhook must terminate" + ); +} + +#[test] +fn property_all_webhooks_terminate_burst() { + // Burst of webhooks all at once. + let config = CiSimConfig { + name: "prop-burst".into(), + num_rounds: 60, + ci_yaml: simple_yaml(), + webhook_schedule: (1..=5) + .map(|i| (1, push("main", &format!("sha-burst-{i}")))) + .collect(), + provision_latency: 1, + job_duration: 2, + ..Default::default() + }; + let trace = sim::run_simulation(config); + assert!( + sim::check_all_webhooks_terminate(&trace), + "burst of 5 webhooks must all terminate" + ); +} + +#[test] +fn property_all_webhooks_terminate_staggered() { + // Webhooks spread across rounds. + let config = CiSimConfig { + name: "prop-staggered".into(), + num_rounds: 60, + ci_yaml: simple_yaml(), + webhook_schedule: vec![ + (1, push("main", "sha-s1")), + (5, push("main", "sha-s2")), + (10, push("main", "sha-s3")), + (15, push("main", "sha-s4")), + ], + provision_latency: 2, + job_duration: 3, + ..Default::default() + }; + let trace = sim::run_simulation(config); + assert!( + sim::check_all_webhooks_terminate(&trace), + "staggered webhooks must all terminate" + ); +} + +// ─── Property: DAG ordering is always respected ───────────────────────────── + +#[test] +fn property_dag_ordering_always_respected() { + // Deep chain: a → b → c → d + let yaml = r#" +pipelines: + deep: + triggers: + - event: push + branches: ["*"] + jobs: + a: + run: echo a + b: + needs: [a] + run: echo b + c: + needs: [b] + run: echo c + d: + needs: [c] + run: echo d +"#; + let config = CiSimConfig { + name: "prop-dag-deep".into(), + num_rounds: 40, + ci_yaml: yaml.into(), + webhook_schedule: vec![(1, push("main", "sha-dag"))], + provision_latency: 1, + job_duration: 2, + ..Default::default() + }; + let trace = sim::run_simulation(config); + assert!( + sim::check_dag_ordering(&trace), + "deep DAG ordering must be respected" + ); +} + +#[test] +fn property_dag_ordering_diamond() { + let yaml = r#" +pipelines: + diamond: + triggers: + - event: push + branches: ["*"] + jobs: + root: + run: echo root + left: + needs: [root] + run: echo left + right: + needs: [root] + run: echo right + merge: + needs: [left, right] + run: echo merge +"#; + let config = CiSimConfig { + name: "prop-dag-diamond".into(), + num_rounds: 40, + ci_yaml: yaml.into(), + webhook_schedule: vec![(1, push("main", "sha-diamond"))], + provision_latency: 1, + job_duration: 2, + ..Default::default() + }; + let trace = sim::run_simulation(config); + assert!( + sim::check_dag_ordering(&trace), + "diamond DAG ordering must be respected" + ); +} + +// ─── Property: No instance leaks ──────────────────────────────────────────── + +#[test] +fn property_no_instance_leaks_under_failures() { + let config = CiSimConfig { + name: "prop-no-leaks-fail".into(), + num_rounds: 40, + ci_yaml: simple_yaml(), + webhook_schedule: vec![ + (1, push("main", "sha-leak1")), + (3, push("main", "sha-leak2")), + ], + provision_latency: 1, + job_duration: 2, + job_failure_schedule: vec![(0, "fmt".into())], + ..Default::default() + }; + let trace = sim::run_simulation(config); + assert!( + sim::check_no_instance_leaks(&trace), + "no instance leaks even when jobs fail" + ); +} + +#[test] +fn property_no_instance_leaks_under_interruption() { + let yaml = r#" +pipelines: + check: + triggers: + - event: push + branches: ["*"] + jobs: + test: + run: cargo test +"#; + let config = CiSimConfig { + name: "prop-no-leaks-interrupt".into(), + num_rounds: 40, + ci_yaml: yaml.into(), + webhook_schedule: vec![(1, push("main", "sha-int"))], + provision_latency: 1, + job_duration: 5, + instance_interrupt_schedule: vec![(4, "test".into())], + ..Default::default() + }; + let trace = sim::run_simulation(config); + assert!( + sim::check_no_instance_leaks(&trace), + "interrupted instances must be terminated" + ); +} + +// ─── Property: Bounded state ──────────────────────────────────────────────── + +#[test] +fn property_bounded_state_under_rapid_pushes() { + let config = CiSimConfig { + name: "prop-bounded".into(), + num_rounds: 100, + ci_yaml: simple_yaml(), + webhook_schedule: (1..=20) + .map(|i| (i, push("main", &format!("sha-rapid-{i}")))) + .collect(), + provision_latency: 1, + job_duration: 2, + ..Default::default() + }; + let trace = sim::run_simulation(config); + + // With 20 pushes, each triggering 1 pipeline with 2 jobs, we should never + // have more than 20 active pipelines at once (and in practice much fewer). + assert!( + sim::check_bounded_state(&trace, 20), + "active pipelines should be bounded" + ); +} + +// ─── Property: Provisioner offline doesn't lose work ──────────────────────── + +#[test] +fn property_provisioner_offline_eventually_resolves() { + let config = CiSimConfig { + name: "prop-offline-resolve".into(), + num_rounds: 60, + ci_yaml: simple_yaml(), + webhook_schedule: vec![(3, push("main", "sha-offline"))], + provision_latency: 1, + job_duration: 2, + provisioner_offline_schedule: vec![(1, 15)], + ..Default::default() + }; + let trace = sim::run_simulation(config); + assert!( + sim::check_all_webhooks_terminate(&trace), + "webhooks during provisioner outage must still terminate" + ); +} diff --git a/crates/simulation/tests/ci_scenarios.rs b/crates/simulation/tests/ci_scenarios.rs new file mode 100644 index 0000000..33317eb --- /dev/null +++ b/crates/simulation/tests/ci_scenarios.rs @@ -0,0 +1,368 @@ +//! Scenario tests for the CI simulation. +//! +//! Each test tells a story: set up a scenario, run the simulation, verify outcomes. + +use swactor_ci::{EventType, WebhookEvent}; +use simulation::ci::sim::{self, CiSimConfig, CiSimEvent}; + +fn basic_ci_yaml() -> String { + r#" +pipelines: + check: + triggers: + - event: push + branches: ["*"] + exclude: ["master"] + jobs: + fmt: + run: cargo fmt -- --check + clippy: + run: cargo clippy + test: + needs: [fmt, clippy] + run: cargo test + full: + triggers: + - event: push + branches: ["master"] + jobs: + test: + run: cargo test --all-features + timeout: 600 + bench: + needs: [test] + run: cargo bench +"# + .into() +} + +fn push_event(branch: &str, sha: &str) -> WebhookEvent { + WebhookEvent { + event_type: EventType::Push, + repo_owner: "user".into(), + repo_name: "repo".into(), + branch: branch.into(), + commit_sha: sha.into(), + tag: None, + } +} + +// ─── Scenario: Single push triggers correct pipeline ──────────────────────── + +#[test] +fn single_push_to_feature_branch_triggers_check_pipeline() { + let config = CiSimConfig { + name: "single-push-feature".into(), + num_rounds: 30, + ci_yaml: basic_ci_yaml(), + webhook_schedule: vec![(1, push_event("feature-x", "abc123"))], + provision_latency: 1, + job_duration: 2, + ..Default::default() + }; + + let trace = sim::run_simulation(config); + + // A pipeline was created. + let pipeline_created = trace + .events + .iter() + .filter(|(_, e)| matches!(e, CiSimEvent::PipelineCreated { .. })) + .count(); + assert_eq!(pipeline_created, 1, "exactly one pipeline should be created"); + + // The pipeline name should be "check" (not "full", since branch is not master). + let check_created = trace.events.iter().any(|(_, e)| { + matches!(e, CiSimEvent::PipelineCreated { name, .. } if name == "check") + }); + assert!(check_created, "pipeline 'check' should be created"); + + // All jobs eventually complete (fmt, clippy, test). + let completed_jobs: Vec<_> = trace + .events + .iter() + .filter_map(|(_, e)| match e { + CiSimEvent::JobCompleted { job_id, passed } => Some((job_id.job_name.clone(), *passed)), + _ => None, + }) + .collect(); + + assert_eq!(completed_jobs.len(), 3, "all 3 jobs should complete"); + assert!( + completed_jobs.iter().all(|(_, passed)| *passed), + "all jobs should pass" + ); + + // A terminal Forgejo status is emitted. + assert!( + sim::check_all_webhooks_terminate(&trace), + "webhook should produce terminal status" + ); +} + +// ─── Scenario: Push to master triggers full pipeline ──────────────────────── + +#[test] +fn push_to_master_triggers_full_pipeline() { + let config = CiSimConfig { + name: "push-master".into(), + num_rounds: 30, + ci_yaml: basic_ci_yaml(), + webhook_schedule: vec![(1, push_event("master", "def456"))], + provision_latency: 1, + job_duration: 2, + ..Default::default() + }; + + let trace = sim::run_simulation(config); + + let full_created = trace.events.iter().any(|(_, e)| { + matches!(e, CiSimEvent::PipelineCreated { name, .. } if name == "full") + }); + assert!(full_created, "pipeline 'full' should be created"); + + // Both test and bench should eventually complete. + let completed: Vec = trace + .events + .iter() + .filter_map(|(_, e)| match e { + CiSimEvent::JobCompleted { job_id, .. } => Some(job_id.job_name.clone()), + _ => None, + }) + .collect(); + + assert!(completed.contains(&"test".to_string()), "test should complete"); + assert!(completed.contains(&"bench".to_string()), "bench should complete"); +} + +// ─── Scenario: Jobs execute in DAG order ──────────────────────────────────── + +#[test] +fn jobs_execute_in_dag_order() { + let config = CiSimConfig { + name: "dag-order".into(), + num_rounds: 30, + ci_yaml: basic_ci_yaml(), + webhook_schedule: vec![(1, push_event("feature-y", "aaa111"))], + provision_latency: 1, + job_duration: 2, + ..Default::default() + }; + + let trace = sim::run_simulation(config); + + // test should start after fmt and clippy complete. + assert!( + sim::check_dag_ordering(&trace), + "DAG ordering must be respected" + ); +} + +// ─── Scenario: Job failure skips dependents ───────────────────────────────── + +#[test] +fn job_failure_skips_downstream_dependents() { + let config = CiSimConfig { + name: "failure-skip".into(), + num_rounds: 30, + ci_yaml: basic_ci_yaml(), + webhook_schedule: vec![(1, push_event("feature-z", "bbb222"))], + provision_latency: 1, + job_duration: 2, + // fmt will fail, so test (which needs fmt) should be skipped. + job_failure_schedule: vec![(0, "fmt".into())], + ..Default::default() + }; + + let trace = sim::run_simulation(config); + + // test should be skipped. + let test_skipped = trace.events.iter().any(|(_, e)| { + matches!(e, CiSimEvent::JobSkipped { job_id } if job_id.job_name == "test") + }); + assert!(test_skipped, "test job should be skipped when fmt fails"); + + // Pipeline should be marked as failed. + let pipeline_failed = trace.status_updates.iter().any(|u| { + u.context == "ci/check" && u.state == "failure" + }); + assert!(pipeline_failed, "pipeline should report failure status"); +} + +// ─── Scenario: Parallel pushes execute independently ──────────────────────── + +#[test] +fn parallel_pushes_execute_independently() { + let config = CiSimConfig { + name: "parallel-pushes".into(), + num_rounds: 40, + ci_yaml: basic_ci_yaml(), + webhook_schedule: vec![ + (1, push_event("feature-a", "ccc333")), + (1, push_event("feature-b", "ddd444")), + ], + provision_latency: 1, + job_duration: 2, + ..Default::default() + }; + + let trace = sim::run_simulation(config); + + // Two pipelines should be created. + let pipeline_count = trace + .events + .iter() + .filter(|(_, e)| matches!(e, CiSimEvent::PipelineCreated { .. })) + .count(); + assert_eq!(pipeline_count, 2, "two pipelines should be created"); + + // Both should have terminal status. + assert!( + sim::check_all_webhooks_terminate(&trace), + "both webhooks should produce terminal statuses" + ); +} + +// ─── Scenario: Provisioner goes offline, jobs queue and resume ────────────── + +#[test] +fn provisioner_offline_queues_then_resumes() { + let config = CiSimConfig { + name: "provisioner-offline".into(), + num_rounds: 50, + ci_yaml: basic_ci_yaml(), + // Push at round 3, provisioner offline rounds 1-10. + webhook_schedule: vec![(3, push_event("feature-q", "eee555"))], + provision_latency: 1, + job_duration: 2, + provisioner_offline_schedule: vec![(1, 10)], + ..Default::default() + }; + + let trace = sim::run_simulation(config); + + // Provisioner went offline and came back. + let went_offline = trace + .events + .iter() + .any(|(_, e)| matches!(e, CiSimEvent::ProvisionerWentOffline)); + let came_online = trace + .events + .iter() + .any(|(_, e)| matches!(e, CiSimEvent::ProvisionerCameOnline)); + assert!(went_offline, "provisioner should go offline"); + assert!(came_online, "provisioner should come back online"); + + // Despite the outage, all jobs should eventually complete. + assert!( + sim::check_all_webhooks_terminate(&trace), + "pipeline should complete after provisioner returns" + ); +} + +// ─── Scenario: Spot instance interrupted mid-job ──────────────────────────── + +#[test] +fn spot_instance_interruption_reports_failure() { + // Use a simpler pipeline so we have a clear target to interrupt. + let yaml = r#" +pipelines: + check: + triggers: + - event: push + branches: ["*"] + jobs: + test: + run: cargo test +"#; + let config = CiSimConfig { + name: "spot-interrupt".into(), + num_rounds: 30, + ci_yaml: yaml.into(), + webhook_schedule: vec![(1, push_event("main", "fff666"))], + provision_latency: 1, + job_duration: 5, + // Interrupt the test job at round 4 (while it's still running). + instance_interrupt_schedule: vec![(4, "test".into())], + ..Default::default() + }; + + let trace = sim::run_simulation(config); + + // The pipeline should be terminal (failed due to interruption). + let has_failure_status = trace + .status_updates + .iter() + .any(|u| u.state == "failure" || u.state == "error"); + assert!( + has_failure_status, + "interrupted job should produce a failure status" + ); + + // The instance should be terminated. + assert!( + sim::check_no_instance_leaks(&trace), + "interrupted instance should be terminated" + ); +} + +// ─── Scenario: All jobs pass → pipeline success ───────────────────────────── + +#[test] +fn all_jobs_pass_marks_pipeline_success() { + let yaml = r#" +pipelines: + check: + triggers: + - event: push + branches: ["*"] + jobs: + lint: + run: cargo clippy + test: + needs: [lint] + run: cargo test +"#; + let config = CiSimConfig { + name: "all-pass".into(), + num_rounds: 30, + ci_yaml: yaml.into(), + webhook_schedule: vec![(1, push_event("main", "ggg777"))], + provision_latency: 1, + job_duration: 2, + ..Default::default() + }; + + let trace = sim::run_simulation(config); + + let has_success = trace + .status_updates + .iter() + .any(|u| u.state == "success" && u.context == "ci/check"); + assert!(has_success, "pipeline should be marked as success"); +} + +// ─── Scenario: Every provisioned instance is terminated ───────────────────── + +#[test] +fn no_instance_resource_leaks() { + let config = CiSimConfig { + name: "no-leaks".into(), + num_rounds: 40, + ci_yaml: basic_ci_yaml(), + webhook_schedule: vec![ + (1, push_event("feature-1", "h1")), + (5, push_event("feature-2", "h2")), + ], + provision_latency: 1, + job_duration: 2, + ..Default::default() + }; + + let trace = sim::run_simulation(config); + + assert!( + sim::check_no_instance_leaks(&trace), + "all provisioned instances must be terminated" + ); +} diff --git a/crates/simulation/tests/local_ci_properties.rs b/crates/simulation/tests/local_ci_properties.rs new file mode 100644 index 0000000..c5dea8f --- /dev/null +++ b/crates/simulation/tests/local_ci_properties.rs @@ -0,0 +1,256 @@ +//! Property-based tests for the local CI simulation. +//! +//! These verify invariants that should hold across all possible simulation configurations. + +use simulation::ci::local_sim::{self, LocalSimConfig}; +use swactor_ci::{EventType, WebhookEvent}; + +fn simple_yaml() -> String { + r#" +pipelines: + check: + triggers: + - event: push + branches: ["*"] + jobs: + fmt: + run: cargo fmt -- --check + test: + needs: [fmt] + run: cargo test +"# + .into() +} + +fn push(branch: &str, sha: &str) -> WebhookEvent { + WebhookEvent { + event_type: EventType::Push, + repo_owner: "user".into(), + repo_name: "repo".into(), + branch: branch.into(), + commit_sha: sha.into(), + tag: None, + } +} + +// ─── Property: One-at-a-time ──────────────────────────────────────────────── + +#[test] +fn property_one_at_a_time_single_push() { + let config = LocalSimConfig { + name: "prop-1at1-single".into(), + num_rounds: 30, + ci_yaml: simple_yaml(), + webhook_schedule: vec![(1, push("main", "sha-1"))], + job_duration: 2, + ..Default::default() + }; + let trace = local_sim::run_simulation(config); + assert!( + local_sim::check_one_at_a_time(&trace), + "at most one job running at a time" + ); +} + +#[test] +fn property_one_at_a_time_burst() { + let config = LocalSimConfig { + name: "prop-1at1-burst".into(), + num_rounds: 100, + ci_yaml: simple_yaml(), + webhook_schedule: (1..=10) + .map(|i| (1, push(&format!("branch-{i}"), &format!("sha-{i}")))) + .collect(), + job_duration: 2, + ..Default::default() + }; + let trace = local_sim::run_simulation(config); + assert!( + local_sim::check_one_at_a_time(&trace), + "at most one job running at a time under burst" + ); +} + +#[test] +fn property_one_at_a_time_staggered() { + let config = LocalSimConfig { + name: "prop-1at1-stagger".into(), + num_rounds: 80, + ci_yaml: simple_yaml(), + webhook_schedule: vec![ + (1, push("a", "sha-a")), + (3, push("b", "sha-b")), + (5, push("c", "sha-c")), + (7, push("d", "sha-d")), + ], + job_duration: 3, + ..Default::default() + }; + let trace = local_sim::run_simulation(config); + assert!( + local_sim::check_one_at_a_time(&trace), + "at most one job running at a time under stagger" + ); +} + +// ─── Property: Supersede correctness ──────────────────────────────────────── + +#[test] +fn property_superseded_pipelines_never_run() { + let config = LocalSimConfig { + name: "prop-supersede".into(), + num_rounds: 60, + ci_yaml: simple_yaml(), + // Same branch, rapid pushes while jobs are long. + webhook_schedule: (1..=8) + .map(|i| (i, push("feature", &format!("sha-ss-{i}")))) + .collect(), + job_duration: 4, + ..Default::default() + }; + let trace = local_sim::run_simulation(config); + assert!( + local_sim::check_superseded_no_running(&trace), + "superseded pipelines should never have a running job" + ); +} + +// ─── Property: Termination ────────────────────────────────────────────────── + +#[test] +fn property_all_non_superseded_terminate() { + let config = LocalSimConfig { + name: "prop-terminate".into(), + num_rounds: 100, + ci_yaml: simple_yaml(), + webhook_schedule: vec![ + (1, push("a", "sha-t1")), + (2, push("b", "sha-t2")), + (3, push("a", "sha-t3")), + (10, push("c", "sha-t4")), + ], + job_duration: 3, + ..Default::default() + }; + let trace = local_sim::run_simulation(config); + assert!( + local_sim::check_termination(&trace), + "all non-superseded pipelines must reach terminal status" + ); +} + +#[test] +fn property_all_webhooks_terminate() { + let config = LocalSimConfig { + name: "prop-wh-terminate".into(), + num_rounds: 100, + ci_yaml: simple_yaml(), + webhook_schedule: (1..=5) + .map(|i| (i * 3, push("main", &format!("sha-wh-{i}")))) + .collect(), + job_duration: 2, + ..Default::default() + }; + let trace = local_sim::run_simulation(config); + assert!( + local_sim::check_all_webhooks_terminate(&trace), + "every webhook must produce a terminal status" + ); +} + +// ─── Property: DAG ordering ───────────────────────────────────────────────── + +#[test] +fn property_dag_ordering_deep_chain() { + let yaml = r#" +pipelines: + deep: + triggers: + - event: push + branches: ["*"] + jobs: + a: + run: echo a + b: + needs: [a] + run: echo b + c: + needs: [b] + run: echo c + d: + needs: [c] + run: echo d +"#; + let config = LocalSimConfig { + name: "prop-dag-deep".into(), + num_rounds: 50, + ci_yaml: yaml.into(), + webhook_schedule: vec![(1, push("main", "sha-dag"))], + job_duration: 2, + ..Default::default() + }; + let trace = local_sim::run_simulation(config); + assert!( + local_sim::check_dag_ordering(&trace), + "deep DAG ordering must be respected" + ); +} + +#[test] +fn property_dag_ordering_diamond() { + let yaml = r#" +pipelines: + diamond: + triggers: + - event: push + branches: ["*"] + jobs: + root: + run: echo root + left: + needs: [root] + run: echo left + right: + needs: [root] + run: echo right + merge: + needs: [left, right] + run: echo merge +"#; + let config = LocalSimConfig { + name: "prop-dag-diamond".into(), + num_rounds: 50, + ci_yaml: yaml.into(), + webhook_schedule: vec![(1, push("main", "sha-dia"))], + job_duration: 2, + ..Default::default() + }; + let trace = local_sim::run_simulation(config); + assert!( + local_sim::check_dag_ordering(&trace), + "diamond DAG ordering must be respected" + ); +} + +// ─── Property: FIFO across branches ───────────────────────────────────────── + +#[test] +fn property_fifo_across_branches() { + let config = LocalSimConfig { + name: "prop-fifo".into(), + num_rounds: 80, + ci_yaml: simple_yaml(), + webhook_schedule: vec![ + (1, push("a", "sha-f1")), + (2, push("b", "sha-f2")), + (3, push("c", "sha-f3")), + ], + job_duration: 3, + ..Default::default() + }; + let trace = local_sim::run_simulation(config); + assert!( + local_sim::check_fifo_order(&trace), + "branches must execute in FIFO queue order" + ); +} diff --git a/crates/simulation/tests/local_ci_scenarios.rs b/crates/simulation/tests/local_ci_scenarios.rs new file mode 100644 index 0000000..cdc93d0 --- /dev/null +++ b/crates/simulation/tests/local_ci_scenarios.rs @@ -0,0 +1,324 @@ +//! Scenario tests for the local CI simulation. +//! +//! Each test tells a story: set up a scenario, run the simulation, verify outcomes. + +use simulation::ci::local_sim::{self, LocalSimConfig, LocalSimEvent}; +use swactor_ci::{EventType, WebhookEvent}; + +fn basic_ci_yaml() -> String { + r#" +pipelines: + check: + triggers: + - event: push + branches: ["*"] + exclude: ["master"] + jobs: + fmt: + run: cargo fmt -- --check + clippy: + run: cargo clippy + test: + needs: [fmt, clippy] + run: cargo test + full: + triggers: + - event: push + branches: ["master"] + jobs: + test: + run: cargo test --all-features + timeout: 600 + bench: + needs: [test] + run: cargo bench +"# + .into() +} + +fn push_event(branch: &str, sha: &str) -> WebhookEvent { + WebhookEvent { + event_type: EventType::Push, + repo_owner: "user".into(), + repo_name: "repo".into(), + branch: branch.into(), + commit_sha: sha.into(), + tag: None, + } +} + +// ─── Scenario: Single push → single job → passes ──────────────────────────── + +#[test] +fn single_push_runs_and_completes() { + let yaml = r#" +pipelines: + check: + triggers: + - event: push + branches: ["*"] + jobs: + test: + run: cargo test +"#; + let config = LocalSimConfig { + name: "single-push".into(), + num_rounds: 20, + ci_yaml: yaml.into(), + webhook_schedule: vec![(1, push_event("main", "sha1"))], + job_duration: 2, + ..Default::default() + }; + + let trace = local_sim::run_simulation(config); + + let created = trace + .events + .iter() + .filter(|(_, e)| matches!(e, LocalSimEvent::PipelineCreated { .. })) + .count(); + assert_eq!(created, 1); + + let completed = trace + .events + .iter() + .filter(|(_, e)| matches!(e, LocalSimEvent::JobCompleted { passed: true, .. })) + .count(); + assert_eq!(completed, 1); + + assert!(local_sim::check_all_webhooks_terminate(&trace)); +} + +// ─── Scenario: Push A, push A again while queued → only latest runs ───────── + +#[test] +fn supersede_queued_same_branch() { + let yaml = r#" +pipelines: + check: + triggers: + - event: push + branches: ["*"] + jobs: + test: + run: cargo test +"#; + let config = LocalSimConfig { + name: "supersede-queued".into(), + num_rounds: 30, + ci_yaml: yaml.into(), + // Push A at round 1 occupies the runner. Push A' at round 2 queues. + // Push A'' at round 3 should supersede A'. + webhook_schedule: vec![ + (1, push_event("feature", "sha-a1")), + (2, push_event("feature", "sha-a2")), + (3, push_event("feature", "sha-a3")), + ], + job_duration: 5, + ..Default::default() + }; + + let trace = local_sim::run_simulation(config); + + // sha-a2 should be superseded. + let superseded = trace + .events + .iter() + .filter(|(_, e)| matches!(e, LocalSimEvent::PipelineSuperseded { .. })) + .count(); + assert!(superseded >= 1, "at least one pipeline should be superseded"); + + // sha-a2 should have an "error" status (superseded). + let a2_error = trace + .status_updates + .iter() + .any(|u| u.commit_sha == "sha-a2" && u.state == "error"); + assert!(a2_error, "superseded pipeline should report error status"); + + // sha-a1 and sha-a3 should both reach terminal status. + let a1_terminal = trace + .status_updates + .iter() + .any(|u| u.commit_sha == "sha-a1" && (u.state == "success" || u.state == "failure")); + let a3_terminal = trace + .status_updates + .iter() + .any(|u| u.commit_sha == "sha-a3" && (u.state == "success" || u.state == "failure")); + assert!(a1_terminal, "first push should complete"); + assert!(a3_terminal, "latest push should complete"); + + assert!(local_sim::check_superseded_no_running(&trace)); +} + +// ─── Scenario: Push A, push A while running → running completes, new queues ─ + +#[test] +fn push_while_running_does_not_supersede_active() { + let yaml = r#" +pipelines: + check: + triggers: + - event: push + branches: ["*"] + jobs: + test: + run: cargo test +"#; + let config = LocalSimConfig { + name: "no-supersede-active".into(), + num_rounds: 30, + ci_yaml: yaml.into(), + // Push A at round 1 starts running immediately. + // Push A' at round 2 should queue (not cancel the running job). + webhook_schedule: vec![ + (1, push_event("feature", "sha-run1")), + (2, push_event("feature", "sha-run2")), + ], + job_duration: 4, + ..Default::default() + }; + + let trace = local_sim::run_simulation(config); + + // Both should reach terminal status. + let run1_terminal = trace + .status_updates + .iter() + .any(|u| u.commit_sha == "sha-run1" && u.state == "success"); + let run2_terminal = trace + .status_updates + .iter() + .any(|u| u.commit_sha == "sha-run2" && u.state == "success"); + + assert!(run1_terminal, "running pipeline should complete normally"); + assert!(run2_terminal, "queued pipeline should run after"); + + assert!(local_sim::check_one_at_a_time(&trace)); +} + +// ─── Scenario: Push A, push B → both run in FIFO order ───────────────────── + +#[test] +fn different_branches_run_fifo() { + let yaml = r#" +pipelines: + check: + triggers: + - event: push + branches: ["*"] + jobs: + test: + run: cargo test +"#; + let config = LocalSimConfig { + name: "fifo-branches".into(), + num_rounds: 30, + ci_yaml: yaml.into(), + webhook_schedule: vec![ + (1, push_event("feature-a", "sha-fa")), + (1, push_event("feature-b", "sha-fb")), + ], + job_duration: 3, + ..Default::default() + }; + + let trace = local_sim::run_simulation(config); + + // Both should complete. + assert!(local_sim::check_all_webhooks_terminate(&trace)); + + // FIFO order respected. + assert!(local_sim::check_fifo_order(&trace)); + + // One at a time. + assert!(local_sim::check_one_at_a_time(&trace)); +} + +// ─── Scenario: Job failure → dependents skipped → next pipeline starts ────── + +#[test] +fn job_failure_skips_dependents_and_advances() { + let config = LocalSimConfig { + name: "failure-skip-advance".into(), + num_rounds: 40, + ci_yaml: basic_ci_yaml(), + webhook_schedule: vec![ + (1, push_event("feature-x", "sha-fail")), + (2, push_event("feature-y", "sha-next")), + ], + job_duration: 2, + // fmt fails. + job_failure_schedule: vec![(0, "fmt".into())], + ..Default::default() + }; + + let trace = local_sim::run_simulation(config); + + // test should be skipped in feature-x pipeline (needs fmt which fails). + let test_skipped = trace.events.iter().any(|(_, e)| { + matches!(e, LocalSimEvent::JobSkipped { job_id } if job_id.job_name == "test") + }); + assert!(test_skipped, "test should be skipped when fmt fails"); + + // Both pipelines should reach terminal. + assert!(local_sim::check_all_webhooks_terminate(&trace)); + + // One at a time. + assert!(local_sim::check_one_at_a_time(&trace)); +} + +// ─── Scenario: Diamond DAG → jobs serialize respecting deps ───────────────── + +#[test] +fn diamond_dag_serialized_with_deps() { + let yaml = r#" +pipelines: + diamond: + triggers: + - event: push + branches: ["*"] + jobs: + root: + run: echo root + left: + needs: [root] + run: echo left + right: + needs: [root] + run: echo right + merge: + needs: [left, right] + run: echo merge +"#; + let config = LocalSimConfig { + name: "diamond-dag".into(), + num_rounds: 50, + ci_yaml: yaml.into(), + webhook_schedule: vec![(1, push_event("main", "sha-diamond"))], + job_duration: 2, + ..Default::default() + }; + + let trace = local_sim::run_simulation(config); + + // All 4 jobs should complete. + let completed = trace + .events + .iter() + .filter(|(_, e)| matches!(e, LocalSimEvent::JobCompleted { .. })) + .count(); + assert_eq!(completed, 4, "all 4 diamond jobs should complete"); + + // DAG ordering respected. + assert!(local_sim::check_dag_ordering(&trace)); + + // One at a time (serial). + assert!(local_sim::check_one_at_a_time(&trace)); + + // Pipeline should succeed. + let success = trace + .status_updates + .iter() + .any(|u| u.state == "success" && u.context == "ci/diamond"); + assert!(success, "diamond pipeline should succeed"); +} diff --git a/docs/development_history/ci/CI_DEPLOYMENT.md b/docs/development_history/ci/CI_DEPLOYMENT.md new file mode 100644 index 0000000..82be2ff --- /dev/null +++ b/docs/development_history/ci/CI_DEPLOYMENT.md @@ -0,0 +1,140 @@ +# CI Pipeline Deployment — Development History + +> Covers the first real deployment of the CI pipeline: Forgejo (VPS) → ci-relay +> (iroh) → local-runner (Thinkpad). Verified end-to-end with a smoke-test +> pipeline that reports status back to Forgejo. +> +> *Branch: `spot-instance`* + +--- + +## What Was Done + +### Deployed Components + +| Component | Machine | How | +|-----------|---------|-----| +| `.ci.yml` | Repo root | Smoke pipeline: `echo "CI is alive"` on push to `*` | +| `ci-relay` | VPS | Release binary, systemd service | +| `local-runner` | Runner host | Release binary, started via nohup | +| Forgejo webhook | VPS (Docker) | Hook #1, fires on push to relay's HTTP listener | + +### Deployment Steps + +1. **Created `.ci.yml`** — minimal smoke pipeline (`echo "CI is alive"`) +2. **Generated webhook secret** — `openssl rand -hex 32` → `~/.ssh/forgejo.ci-webhook-secret` +3. **Built release binaries** — `cargo build --release -p ci-relay -p local-runner` +4. **Distributed binaries** — `scp` to VPS (`docean:`) and Thinkpad (`thinkpad:`) +5. **Deployed ci-relay as systemd service** on VPS: + - Service file: `/etc/systemd/system/ci-relay.service` + - Iroh Node ID: `` +6. **Started local-runner on Thinkpad** — connects to relay via iroh, confirmed "Connected to relay!" +7. **Configured Forgejo**: + - Added `[webhook] ALLOWED_HOST_LIST = loopback,` to `app.ini` (Forgejo blocks private IPs by default) + - Restarted Forgejo container + - Created webhook via API targeting `http://:8787` + - **Fixed UFW firewall** — Docker bridge traffic to port 8787 was blocked by default DROP policy; added a UFW rule allowing the Docker subnet +8. **Verified end-to-end** — pushed commit, Forgejo shows green check: + - `ci/hello`: success — "Job 'hello' completed" + - `ci/smoke`: success — "Pipeline 'smoke' success" + +### Issue Encountered: UFW Blocking Docker Bridge + +The plan assumed Docker bridge traffic (`172.17.0.1`) would reach the host's port 8787 unimpeded. UFW's default INPUT policy is DROP, which blocks this. The fix was a single firewall rule allowing the Docker subnet. + +### Credentials & Secrets + +| File | Purpose | Location | +|------|---------|----------| +| Forgejo API token | CI status reporting | Spot instance + Thinkpad | +| HMAC webhook secret | Webhook signature verification | Spot instance + Thinkpad | + +Secrets are stored outside the repo. The webhook secret is embedded in the systemd service `ExecStart` line on the VPS. To rotate it: update the service file, restart ci-relay, update Forgejo webhook config. + +### Connection Details + +- **ci-relay** listens on HTTP (webhooks) + iroh (runner connection) +- **local-runner** connects outbound to relay's iroh Node ID (NAT-friendly) +- **Status reports** go directly from runner → Forgejo API over HTTPS (no relay) + +--- + +## Next Step: Real CI Jobs + +The smoke-test pipeline proves the plumbing works. The next step is replacing `echo "CI is alive"` with actual CI jobs in `.ci.yml`. + +Candidates for the first real pipeline: + +1. **`cargo check`** — fast compilation check, catches most errors +2. **`cargo test`** — full test suite (simulation tests can be slow) +3. **`cargo clippy`** — lint pass +4. **Benchmark runs** — the whole reason for running CI on the Thinkpad (consistent hardware) + +Things to consider: + +- **Rust toolchain on Thinkpad**: `local-runner` shells out to run jobs, so the Thinkpad needs `rustup`/`cargo` installed and on PATH +- **Build cache**: consecutive runs in separate `pipeline-N` dirs won't share a target directory. Consider a shared `CARGO_TARGET_DIR` or `sccache` for faster builds +- **Job timeouts**: no timeout mechanism exists yet; a hung `cargo build` would block the single-threaded job queue forever +- **Multiple jobs**: `.ci.yml` supports multiple jobs per pipeline, but they run sequentially. Could add `cargo check` as a fast gate before `cargo test` +- **Branch filtering**: currently triggers on `*` — may want to restrict benchmarks to `master` only + +### Suggested `.ci.yml` Evolution + +```yaml +pipelines: + check: + triggers: + - event: push + branches: ["*"] + jobs: + check: + run: cargo check --workspace + test: + run: cargo test --workspace + clippy: + run: cargo clippy --workspace -- -D warnings + + bench: + triggers: + - event: push + branches: ["master"] + jobs: + bench: + run: cargo bench --workspace +``` + +--- + +## Operational Notes + +### Restarting ci-relay (VPS) + +```bash +ssh +systemctl restart ci-relay +journalctl -u ci-relay -f +``` + +### Restarting local-runner (runner host) + +```bash +ssh +pkill local-runner +nohup ~/local-runner \ + --relay-node-id \ + --forgejo-url https://zachery.lol/code \ + --forgejo-token "$(cat )" \ + --yaml ~/.ci.yml \ + --work-dir ~/ci-work \ + --repo-url https://zachery.lol/code/zacheryasc/swactor.git \ + > ~/local-runner.log 2>&1 & +``` + +### Checking webhook deliveries + +```bash +# Forgejo webhook UI: Settings → Webhooks → Hook #1 → Recent Deliveries +# Or test delivery via API: +curl -X POST "https://zachery.lol/code/api/v1/repos/zacheryasc/swactor/hooks/1/tests" \ + -H "Authorization: token " +``` diff --git a/docs/development_history/ci/CI_OUTPUT_IN_FORGEJO.md b/docs/development_history/ci/CI_OUTPUT_IN_FORGEJO.md new file mode 100644 index 0000000..c1c24b3 --- /dev/null +++ b/docs/development_history/ci/CI_OUTPUT_IN_FORGEJO.md @@ -0,0 +1,139 @@ +# CI Output Visible in Forgejo — Development History + +> Added two mechanisms so CI results are visible directly in the Forgejo web +> UI without SSH-ing into the runner: **enhanced commit status descriptions** +> and **PR comments** with full job output. +> +> *Branch: `spot-instance`* + +--- + +## Problem + +The CI pipeline worked end-to-end but job output was only visible in the +runner's stderr log on the Thinkpad. To see why clippy failed, you had to +`ssh thinkpad 'tail ~/local-runner.log'`. Forgejo's commit status descriptions +just said "Job 'clippy' completed" with no output. + +Forgejo lacks GitHub's Checks API (no annotations, no log viewer), so we use +two complementary approaches. + +## What Was Done + +### 1. Enhanced Commit Status Descriptions + +On job completion, the status description now includes: + +- **On success**: `"Job 'check' passed"` +- **On failure**: `"Job 'clippy' failed: command exited with code 101\n[stderr] error: you should consider..."` — last ~10 lines of output, capped at 250 characters. + +This is visible directly on the PR page and commit page in Forgejo without +clicking anything. + +### 2. PR Comments with Full Output + +When a pipeline reaches terminal state, the StatusReporter: + +1. Queries `GET /repos/{owner}/{repo}/pulls?state=open` to find the PR for the branch +2. Builds a markdown comment with `
` sections per job (up to 100 lines each) +3. Posts it via `POST /repos/{owner}/{repo}/issues/{pr_number}/comments` +4. Re-posts the pipeline commit status with `target_url` pointing to the comment + +Example comment format: + +```markdown +## Pipeline `ci` — failure + +Commit: `5b7ae7b` + +
+clippy — failed: command exited with code 101 + +_Showing last 100 of 523 lines_ + +\``` +error[E0599]: ... +\``` + +
+ +
+check — passed + +\``` +$ cargo check --workspace + Compiling ... +\``` + +
+``` + +### 3. `target_url` on Commit Statuses + +Added `target_url: Option` to `StatusUpdate`. When a PR comment is +successfully posted, the pipeline's commit status badge links directly to that +comment. Clicking the status badge on the PR page jumps to the output. + +## Files Changed + +| File | Change | +|------|--------| +| `crates/ci/src/lib.rs` | Added `target_url: Option` to `StatusUpdate` | +| `crates/ci/src/status_reporter.rs` | Added `JobOutput`, `PostPipelineComment`, `find_pr_for_branch()`, `post_pr_comment()`, `build_pipeline_comment()`, `handle_pipeline_comment()` | +| `crates/ci/src/local_coordinator.rs` | Enhanced `handle_job_complete()` descriptions; added `emit_pipeline_comment()`, called from `try_schedule_next()` | +| `crates/ci/src/coordinator.rs` | Mechanical `target_url: None` at 4 sites | +| `crates/simulation/src/ci/local_sim.rs` | Mechanical `target_url: None` at 3 sites | +| `crates/simulation/src/ci/sim.rs` | Mechanical `target_url: None` at 2 sites | +| `crates/ci/Cargo.toml` | Added `features = ["json"]` to `ureq` for `into_json()` | + +## Deployment & Verification + +Built and deployed updated `local-runner` to the Thinkpad, pushed to the +`spot-instance` branch (which has PR #42 open), and observed: + +**Working:** + +- Commit statuses show descriptive output. The clippy failure status reads: + `Job 'clippy' failed: command exited with code 101` followed by the tail of + the clippy output, truncated at 250 chars. +- Passed jobs show `"Job 'check' passed"` / `"Job 'test' passed"`. +- Pipeline-level status correctly reports `ci/ci → failure`. + +**Blocked on token scope:** + +- PR comment posting returned HTTP 403. The Forgejo API token has + `write:repository` scope (sufficient for commit statuses) but needs + `write:issue` scope to post comments on PRs/issues. +- The code degrades gracefully: logs the error, skips the comment, posts the + pipeline status without `target_url`. + +## TODO + +- [ ] Regenerate Forgejo API token with `write:issue` scope to enable PR comments +- [ ] After token update, re-deploy and verify the comment + `target_url` flow end-to-end + +## Edge Cases Handled + +| Case | Behavior | +|------|----------| +| No open PR for branch | Comment silently skipped, status posted without `target_url` | +| API failures (403, network) | Logged via `eprintln!`, degrades gracefully | +| Long output | Capped at last 100 lines per job in PR comment, with `_Showing last N of M lines_` note | +| Long description | Capped at 250 chars for commit status description field | +| All HTTP code | Gated behind `#[cfg(feature = "local")]` — simulation builds unaffected | + +## Architecture Note + +All new HTTP calls (PR listing, comment posting) happen in the StatusReporter +actor, which is fire-and-forget. The LocalCoordinator never blocks on HTTP. +The flow is: + +``` +LocalCoordinator StatusReporter + | | + |-- emit_status(StatusUpdate) ----->|-- POST /statuses/{sha} + | | + |-- PostPipelineComment ----------->|-- GET /pulls?state=open + | |-- POST /issues/{n}/comments + | |-- POST /statuses/{sha} (with target_url) +``` diff --git a/docs/development_history/ci/CI_RELAY.md b/docs/development_history/ci/CI_RELAY.md new file mode 100644 index 0000000..d4edd7c --- /dev/null +++ b/docs/development_history/ci/CI_RELAY.md @@ -0,0 +1,520 @@ +# CI Webhook Relay via Iroh — Development History + +> Covers the implementation of `ci-relay` and the iroh webhook receiver in +> `local-runner`, enabling Forgejo webhooks to reach a NAT'd CI runner via +> iroh's QUIC transport with automatic NAT traversal. +> +> ~3 files created · ~2 files modified · ~350 insertions +> +> *Branch: `spot-instance`* + +--- + +## Table of Contents + +1. [Problem & Motivation](#1-problem--motivation) +2. [Architecture](#2-architecture) +3. [What Was Built](#3-what-was-built) +4. [ci-relay Binary](#4-ci-relay-binary) +5. [local-runner Iroh Receiver](#5-local-runner-iroh-receiver) +6. [Wire Protocol](#6-wire-protocol) +7. [Connection Flow](#7-connection-flow) +8. [Design Decisions & Tradeoffs](#8-design-decisions--tradeoffs) +9. [Manual Testing Guide](#9-manual-testing-guide) +10. [Known Gaps & Future Improvements](#10-known-gaps--future-improvements) + +--- + +## 1. Problem & Motivation + +The CI runner (`local-runner`) was designed for same-LAN usage: Forgejo sends +webhooks over HTTP to the runner's listen port. In the real deployment: + +- **Forgejo** runs on a VPS (`zachery.lol` / `139.59.195.69`) +- **CI runner** runs on a Thinkpad at home (`192.168.1.102`), behind NAT + +The VPS cannot reach the Thinkpad directly — no inbound port is open, no +static IP, no UPnP. Traditional solutions (SSH reverse tunnel, VPN, port +forwarding on router) all require ongoing configuration and are fragile. + +iroh is already integrated in swactor's distribution layer (`iroh_driver.rs`) +for SWIM protocol traffic. It provides QUIC connections with automatic NAT +traversal via relay servers — exactly what's needed to bridge the webhook gap. + +### Why Not Just SSH Tunnel? + +An SSH tunnel (`ssh -R 8787:localhost:8787 zachery.lol`) would work, but: + +- Tunnels drop on network changes (laptop suspend, WiFi roaming) +- Requires autossh or systemd to keep alive +- Another moving part to debug when CI stops working +- Doesn't reuse any existing infrastructure + +iroh handles reconnection, relay fallback, and NAT traversal automatically. +The implementation reuses the same tagged-message-over-QUIC-stream pattern +already proven in `iroh_driver.rs`. + +--- + +## 2. Architecture + +``` +┌─────────────────────────────┐ ┌──────────────────────────────────┐ +│ VPS (zachery.lol) │ │ Thinkpad (192.168.1.102) │ +│ │ iroh │ │ +│ Forgejo ──webhook──► Relay ├───────►│ local-runner │ +│ :8787 │ QUIC │ (coordinator, runner, reporter) │ +│ │ │ │ +└─────────────────────────────┘ └──────────────────────────────────┘ +``` + +**VPS side** — `ci-relay` binary: +- HTTP listener receives webhook POSTs from Forgejo (localhost only) +- iroh endpoint accepts the runner's inbound connection +- Forwards parsed `WebhookEvent` payloads over iroh uni streams + +**Thinkpad side** — `local-runner` with `--relay-node-id`: +- Connects to the VPS relay's iroh endpoint on startup +- Receives `WebhookEvent` over iroh uni streams +- Feeds events into `LocalCoordinator` via existing `Webhook` message +- Status updates go directly Thinkpad → Forgejo API over HTTPS (no relay needed) + +The relay is intentionally minimal — it's a bridge, not a CI component. All CI +logic stays in `local-runner`. + +--- + +## 3. What Was Built + +| Component | Location | Nature | +|-----------|----------|--------| +| ci-relay binary | `crates/ci-relay/Cargo.toml`, `src/main.rs` | **New** — VPS webhook relay | +| Iroh receiver | `crates/local-runner/src/main.rs` | **Modified** — iroh webhook source | +| Dependencies | `crates/local-runner/Cargo.toml` | **Modified** — added iroh, tokio, serde_json | +| Workspace | `Cargo.toml` | **Modified** — added ci-relay to members | + +--- + +## 4. ci-relay Binary + +### `crates/ci-relay/src/main.rs` + +The relay runs two subsystems on a single process: + +1. **iroh acceptor** (tokio task): accepts inbound connections from the runner, + caches the most recent one in `Arc>>` +2. **HTTP listener** (main thread, blocking `tiny_http`): receives Forgejo + webhook POSTs, verifies HMAC, parses event, forwards over iroh + +### Webhook Handling + +Reuses the same verification and parsing logic as `webhook_server.rs`: + +- HMAC-SHA256 verification via `X-Forgejo-Signature` header (skippable with empty secret) +- Event type from `X-Forgejo-Event` header: `push` → `Push`, `create` → `Tag`, `pull_request` → `Merge` +- JSON parsing via `parse_webhook_json()` (re-exported from `swactor-ci`) + +The relay uses `parse_webhook_json` directly rather than duplicating parsing +logic. This keeps webhook interpretation consistent between HTTP and iroh paths. + +### Forwarding + +On webhook receipt, the relay: +1. Serializes the `WebhookEvent` to JSON +2. Opens a unidirectional QUIC stream on the cached connection +3. Writes the tagged message (`ci::WebhookEvent` tag + JSON payload) +4. Finishes the stream + +If no runner is connected, the relay returns HTTP 502 to Forgejo. Forgejo will +retry the webhook per its configured retry policy. + +### CLI + +``` +ci-relay [OPTIONS] + +Options: + --port HTTP port for Forgejo webhooks [default: 8787] + --secret HMAC-SHA256 secret [default: "" (no verification)] +``` + +On startup, the relay prints its iroh Node ID — this is the value the runner +needs for `--relay-node-id`. + +--- + +## 5. local-runner Iroh Receiver + +### New CLI Flag + +``` +--relay-node-id Iroh Node ID of the VPS ci-relay +``` + +When `--relay-node-id` is provided: +- The HTTP webhook listener is **not started** (no port conflict, no exposure) +- An `iroh-receiver` thread starts instead + +When omitted, behavior is unchanged — the HTTP listener starts on `--port` +as before. + +### `start_iroh_receiver()` + +Spawns a dedicated thread (`iroh-receiver`) with its own single-threaded tokio +runtime: + +1. Creates an iroh `Endpoint` with ALPN `b"swactor/ci/1"` +2. Connects to the relay's `PublicKey` (parsed from the hex flag) +3. Enters a receive loop: + - `conn.accept_uni()` with 1-second timeout + - On stream: reads tagged message, deserializes `WebhookEvent` + - Sends `LocalCoordinatorMsg::Webhook(event)` to the coordinator via the swactor runtime + - On timeout: checks the `stop` flag (for graceful shutdown via Ctrl-C) + - On connection error: breaks and exits + +The thread respects the same `AtomicBool` stop flag as the main loop, so +Ctrl-C cleanly shuts down both the swactor runtime and the iroh connection. + +--- + +## 6. Wire Protocol + +### ALPN + +```rust +const CI_ALPN: &[u8] = b"swactor/ci/1"; +``` + +Distinct from SWIM traffic (`b"swactor/swim/1"`). This allows both protocols +to coexist on the same iroh endpoint in the future if needed. + +### Frame Format + +Same tagged-message format as `iroh_driver.rs`: + +``` +[4 bytes: tag_len (big-endian u32)] +[tag_len bytes: tag string] +[remaining bytes: payload] +``` + +For webhook events: +- Tag: `"ci::WebhookEvent"` (17 bytes) +- Payload: JSON-serialized `WebhookEvent` + +### Transport + +Each webhook is one unidirectional QUIC stream. The relay opens the stream, +writes the tagged message, and finishes. The runner reads the message and the +stream closes. No persistent framing or multiplexing needed — QUIC streams +are lightweight. + +--- + +## 7. Connection Flow + +``` +1. VPS starts ci-relay + → iroh Endpoint binds + → prints Node ID (ed25519 public key, hex) + → HTTP listener starts on --port + → waits for runner connection + +2. Thinkpad starts local-runner --relay-node-id + → iroh Endpoint binds + → connects to relay's PublicKey + → iroh handles NAT traversal (direct or via relay server) + → relay logs "Runner connected: " + +3. Forgejo sends webhook POST to localhost:8787 on VPS + → relay verifies HMAC, parses event + → relay opens uni stream on cached connection + → writes tagged WebhookEvent + → runner receives, deserializes, dispatches to coordinator + +4. Coordinator triggers pipeline + → StatusReporter posts status to Forgejo API directly + (Thinkpad → zachery.lol over HTTPS, no relay involvement) +``` + +The iroh connection is initiated by the runner (outbound from NAT), so no port +forwarding is needed. iroh's relay servers handle the initial rendezvous, then +attempt direct QUIC hole-punching for subsequent traffic. + +--- + +## 8. Design Decisions & Tradeoffs + +### 8.1 Separate Binary vs. Library Module + +**Choice**: `ci-relay` is a standalone binary, not a module in `swactor-ci`. + +**Why**: The relay runs on the VPS, which doesn't need swactor's runtime, +actors, or any CI execution logic. A small binary with minimal dependencies +deploys easily. It only depends on `swactor-ci` for `parse_webhook_json` and +the `WebhookEvent`/`EventType` types. + +**Tradeoff**: Two binaries to build and deploy instead of one. Acceptable +given they run on different machines. + +### 8.2 Runner Connects to Relay (Not Vice Versa) + +**Choice**: The runner initiates the iroh connection to the relay. + +**Why**: The runner is behind NAT. iroh can traverse NAT for established +connections, but the initial rendezvous requires at least one side to be +reachable. The VPS relay has a public IP and gets a stable relay URL from iroh's +infrastructure. The runner connects outbound, which always works regardless of +NAT type. + +### 8.3 Single Cached Connection (Not Connection Pool) + +**Choice**: The relay caches exactly one runner connection in +`Arc>>`. + +**Why**: There's one runner. If a new connection arrives (e.g., runner +restarts), it replaces the old one. No pool management needed. + +**Tradeoff**: If multiple runners were needed, this would need a map. For +single-runner use, the simplicity is worth it. + +### 8.4 Own Tokio Runtime Per Thread + +**Choice**: The iroh-receiver thread creates its own single-threaded tokio +runtime rather than sharing the swactor runtime or the main thread's runtime. + +**Why**: swactor's runtime is not tokio — it's a custom actor scheduler. The +iroh receiver needs async for QUIC operations. A dedicated single-threaded +runtime keeps the iroh I/O isolated from actor scheduling. Same pattern as +`IrohDriver` in the distribution layer (which owns a multi-thread runtime). + +### 8.5 HTTP 502 When No Runner Connected + +**Choice**: If Forgejo sends a webhook but no runner is connected, the relay +returns HTTP 502 (Bad Gateway). + +**Why**: 502 tells Forgejo the upstream is unavailable. Forgejo will retry +the webhook according to its retry policy. This is better than 200 (silently +dropping) or 500 (suggesting a relay bug). When the runner reconnects, the +next webhook will succeed. + +--- + +## 9. Manual Testing Guide + +### Prerequisites + +Build both binaries: + +```bash +cargo build -p ci-relay -p local-runner +``` + +### 9.1 Local Smoke Test (Single Machine) + +This tests the full relay path without needing two machines or Forgejo. + +**Terminal 1 — Start the relay:** + +```bash +./target/debug/ci-relay --port 9787 +``` + +Output: +``` +ci-relay started + Iroh Node ID: + Webhook HTTP: http://0.0.0.0:9787 + +Waiting for runner to connect... +Listening for webhooks... +``` + +Copy the Node ID. + +**Terminal 2 — Start the runner:** + +You need a `.ci.yml` file. Create a minimal one: + +```yaml +# /tmp/test-ci.yml +pipelines: + test: + triggers: + - event: push + branches: ["*"] + jobs: + hello: + run: echo "hello from CI" +``` + +Then start: + +```bash +./target/debug/local-runner \ + --relay-node-id \ + --yaml /tmp/test-ci.yml \ + --work-dir /tmp/ci-work-test +``` + +You should see: +``` + Iroh local ID: + Connecting to relay ... + Connected to relay! +Local CI runner started + Webhook: via iroh relay +``` + +And in Terminal 1: +``` +Runner connected: +``` + +**Terminal 3 — Send a fake webhook:** + +```bash +curl -X POST http://localhost:9787 \ + -H "Content-Type: application/json" \ + -H "X-Forgejo-Event: push" \ + -d '{ + "ref": "refs/heads/main", + "after": "abc123def456789012345678901234567890abcd", + "repository": { + "name": "test-repo", + "owner": { "login": "testuser" } + } + }' +``` + +Expected output: + +- **curl** returns: `ok` +- **Terminal 1** (relay): + ``` + webhook: abc123de main on testuser/test-repo + → forwarded to runner + ``` +- **Terminal 2** (runner): + ``` + iroh: received webhook abc123de on main + ``` + +The runner will also try to post status to Forgejo and log URL errors (since +we didn't pass `--forgejo-url`) — that's expected and confirms the event +reached the coordinator. + +### 9.2 HMAC Verification Test + +Start the relay with a secret: + +```bash +./target/debug/ci-relay --port 9787 --secret mysecret +``` + +**Without signature — should be rejected (401):** + +```bash +curl -v -X POST http://localhost:9787 \ + -H "X-Forgejo-Event: push" \ + -d '{"ref":"refs/heads/main","after":"abc123","repository":{"name":"r","owner":{"login":"u"}}}' +``` + +**With correct signature:** + +```bash +# Compute HMAC-SHA256 +BODY='{"ref":"refs/heads/main","after":"abc123","repository":{"name":"r","owner":{"login":"u"}}}' +SIG=$(echo -n "$BODY" | openssl dgst -sha256 -hmac "mysecret" | awk '{print $2}') + +curl -X POST http://localhost:9787 \ + -H "X-Forgejo-Event: push" \ + -H "X-Forgejo-Signature: $SIG" \ + -d "$BODY" +``` + +Should return `ok` and forward to the runner. + +### 9.3 Runner Reconnection Test + +1. Start relay and runner as in 9.1 +2. Kill the runner (Ctrl-C in Terminal 2) +3. Restart the runner with the same `--relay-node-id` +4. The relay should log `Runner connected: ` again +5. Send another webhook — it should flow through + +### 9.4 No Runner Connected Test + +1. Start the relay only (no runner) +2. Send a webhook via curl +3. Should get HTTP 502 and relay logs: `forward failed: no runner connected` + +### 9.5 Full End-to-End with Forgejo + +For a real deployment: + +**On VPS:** + +```bash +./ci-relay --port 8787 --secret +``` + +**On Thinkpad:** + +```bash +./local-runner \ + --relay-node-id \ + --forgejo-url https://zachery.lol \ + --forgejo-token \ + --yaml .ci.yml \ + --work-dir ~/ci-work \ + --repo-url https://zachery.lol//.git +``` + +**In Forgejo (repo settings → Webhooks):** + +- Target URL: `http://localhost:8787` +- Secret: `` +- Events: Push, Create (tags), Pull Request + +Push a commit and watch: +1. Relay logs the webhook and forwards it +2. Runner logs the received event and starts a pipeline +3. Forgejo shows commit status checks (pending → success/failure) + +### 9.6 Inspecting Iroh Connectivity + +Both binaries print their iroh Node ID on startup. To verify they're using +relay servers (expected when both are behind NAT or on different networks), +look for connection timing: + +- **Fast connection (~1-3s)**: direct QUIC hole-punch succeeded +- **Slower connection (~5-10s)**: using iroh relay server fallback + +If connection hangs indefinitely, check that both machines have internet +access and can reach iroh's relay servers (`https://relay.iroh.network`). + +--- + +## 10. Known Gaps & Future Improvements + +| Gap | Effort | Impact | Notes | +|-----|--------|--------|-------| +| Reconnection on runner side | Small | High | If the iroh connection drops mid-operation, the runner currently exits the receive loop. Should retry with backoff. | +| Multiple runner support | Medium | Medium | Relay caches one connection. For running CI on multiple machines, need a connection map keyed by runner identity. | +| Health check / heartbeat | Small | Medium | Neither side detects a silently dead connection until the next webhook. A periodic ping would surface stale connections faster. | +| Relay authentication | Small | Medium | Any iroh endpoint can connect to the relay. Should verify the runner's public key against an allowlist. | +| Binary size | Small | Low | ci-relay pulls in `swactor-ci` (which includes all CI types). A slimmer dependency with just `WebhookEvent` + `parse_webhook_json` would reduce the VPS binary. | +| Logging | Small | Low | Both binaries use `eprintln!`. Structured logging (tracing) would help in production. | + +--- + +## Files Created/Modified + +| Action | File | Purpose | +|--------|------|---------| +| Created | `crates/ci-relay/Cargo.toml` | Relay binary manifest | +| Created | `crates/ci-relay/src/main.rs` | Webhook relay: HTTP → iroh | +| Modified | `crates/local-runner/Cargo.toml` | Added iroh, tokio, serde_json deps | +| Modified | `crates/local-runner/src/main.rs` | Added `--relay-node-id` flag and iroh receiver | +| Modified | `Cargo.toml` (workspace root) | Added ci-relay to workspace members |