From 60ee2d1a2c31bbbbe7e7699b2a14ba11207ace5c Mon Sep 17 00:00:00 2001 From: Developer Date: Fri, 13 Feb 2026 06:15:22 +0000 Subject: [PATCH 1/5] =?UTF-8?q?feat:=20wasm=20actor=20crate=20=E2=80=94=20?= =?UTF-8?q?run=20WebAssembly=20guests=20inside=20swactor=20actors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `crates/wasm-actor/` (swactor-wasm-actor), which embeds wasmtime-sandboxed Wasm instances inside regular swactor actors. Messages flow as raw bytes through the guest ↔ host contract (alloc/handle exports, swactor.send import). The host drains an outbox after each handle call and routes messages via ctx.send(). Includes 3 no_std guest modules (echo, double, silent) and 7 integration tests covering roundtrip delivery, binary fidelity, multi-send, error cases, engine sharing, and native↔wasm interop. Authored by Claude, lovingly guided by Zachery Aaron Shores-Chmielewski --- Cargo.lock | 1065 +++++++++++++++-- Cargo.toml | 2 +- crates/wasm-actor/Cargo.toml | 12 + crates/wasm-actor/src/actor.rs | 59 + crates/wasm-actor/src/builder.rs | 109 ++ crates/wasm-actor/src/engine.rs | 35 + crates/wasm-actor/src/error.rs | 27 + crates/wasm-actor/src/lib.rs | 13 + .../wasm-actor/tests/guests/double/Cargo.lock | 7 + .../wasm-actor/tests/guests/double/Cargo.toml | 13 + .../wasm-actor/tests/guests/double/src/lib.rs | 63 + .../wasm-actor/tests/guests/echo/Cargo.lock | 7 + .../wasm-actor/tests/guests/echo/Cargo.toml | 13 + .../wasm-actor/tests/guests/echo/src/lib.rs | 62 + .../wasm-actor/tests/guests/silent/Cargo.lock | 7 + .../wasm-actor/tests/guests/silent/Cargo.toml | 13 + .../wasm-actor/tests/guests/silent/src/lib.rs | 45 + crates/wasm-actor/tests/wasm_actor.rs | 331 +++++ docs/development_history/WASM_ACTOR.md | 191 +++ docs/wasm-actor.md | 115 ++ 20 files changed, 2079 insertions(+), 110 deletions(-) create mode 100644 crates/wasm-actor/Cargo.toml create mode 100644 crates/wasm-actor/src/actor.rs create mode 100644 crates/wasm-actor/src/builder.rs create mode 100644 crates/wasm-actor/src/engine.rs create mode 100644 crates/wasm-actor/src/error.rs create mode 100644 crates/wasm-actor/src/lib.rs create mode 100644 crates/wasm-actor/tests/guests/double/Cargo.lock create mode 100644 crates/wasm-actor/tests/guests/double/Cargo.toml create mode 100644 crates/wasm-actor/tests/guests/double/src/lib.rs create mode 100644 crates/wasm-actor/tests/guests/echo/Cargo.lock create mode 100644 crates/wasm-actor/tests/guests/echo/Cargo.toml create mode 100644 crates/wasm-actor/tests/guests/echo/src/lib.rs create mode 100644 crates/wasm-actor/tests/guests/silent/Cargo.lock create mode 100644 crates/wasm-actor/tests/guests/silent/Cargo.toml create mode 100644 crates/wasm-actor/tests/guests/silent/src/lib.rs create mode 100644 crates/wasm-actor/tests/wasm_actor.rs create mode 100644 docs/development_history/WASM_ACTOR.md create mode 100644 docs/wasm-actor.md diff --git a/Cargo.lock b/Cargo.lock index efcff0d..06fe0c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,27 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -35,18 +56,50 @@ version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f0e0fee31ef5ed1ba1316088939cea399010ed7731dba877ed44aeb407a75ea" +[[package]] +name = "ar_archive_writer" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b" +dependencies = [ + "object 0.37.3", +] + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + [[package]] name = "ascii" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "autocfg" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64ct" version = "1.8.3" @@ -97,6 +150,15 @@ name = "bumpalo" version = "3.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" +dependencies = [ + "allocator-api2", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "cassowary" @@ -119,6 +181,18 @@ dependencies = [ "rustversion", ] +[[package]] +name = "cc" +version = "1.2.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b26a0954ae34af09b50f0de26458fa95369a0d478d8236d3f93082b219bd29" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -189,6 +263,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831" +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.18", +] + [[package]] name = "compact_str" version = "0.8.1" @@ -209,6 +292,15 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "cpp_demangle" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2bb79cb74d735044c972aae58ed0aaa9a837e85b01106a54c39e42e97f62253" +dependencies = [ + "cfg-if", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -218,6 +310,122 @@ dependencies = [ "libc", ] +[[package]] +name = "cranelift-bforest" +version = "0.116.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e15d04a0ce86cb36ead88ad68cf693ffd6cda47052b9e0ac114bc47fd9cd23c4" +dependencies = [ + "cranelift-entity", +] + +[[package]] +name = "cranelift-bitset" +version = "0.116.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c6e3969a7ce267259ce244b7867c5d3bc9e65b0a87e81039588dfdeaede9f34" +dependencies = [ + "serde", + "serde_derive", +] + +[[package]] +name = "cranelift-codegen" +version = "0.116.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c22032c4cb42558371cf516bb47f26cdad1819d3475c133e93c49f50ebf304e" +dependencies = [ + "bumpalo", + "cranelift-bforest", + "cranelift-bitset", + "cranelift-codegen-meta", + "cranelift-codegen-shared", + "cranelift-control", + "cranelift-entity", + "cranelift-isle", + "gimli", + "hashbrown 0.14.5", + "log", + "regalloc2", + "rustc-hash", + "serde", + "smallvec", + "target-lexicon 0.13.4", +] + +[[package]] +name = "cranelift-codegen-meta" +version = "0.116.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c904bc71c61b27fc57827f4a1379f29de64fe95653b620a3db77d59655eee0b8" +dependencies = [ + "cranelift-codegen-shared", +] + +[[package]] +name = "cranelift-codegen-shared" +version = "0.116.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40180f5497572f644ce88c255480981ae2ec1d7bb4d8e0c0136a13b87a2f2ceb" + +[[package]] +name = "cranelift-control" +version = "0.116.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d132c6d0bd8a489563472afc171759da0707804a65ece7ceb15a8c6d7dd5ef" +dependencies = [ + "arbitrary", +] + +[[package]] +name = "cranelift-entity" +version = "0.116.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b2d0d9618275474fbf679dd018ac6e009acbd6ae6850f6a67be33fb3b00b323" +dependencies = [ + "cranelift-bitset", + "serde", + "serde_derive", +] + +[[package]] +name = "cranelift-frontend" +version = "0.116.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fac41e16729107393174b0c9e3730fb072866100e1e64e80a1a963b2e484d57" +dependencies = [ + "cranelift-codegen", + "log", + "smallvec", + "target-lexicon 0.13.4", +] + +[[package]] +name = "cranelift-isle" +version = "0.116.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ca20d576e5070044d0a72a9effc2deacf4d6aa650403189d8ea50126483944d" + +[[package]] +name = "cranelift-native" +version = "0.116.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dee82f3f1f2c4cba9177f1cc5e350fe98764379bcd29340caa7b01f85076c7" +dependencies = [ + "cranelift-codegen", + "libc", + "target-lexicon 0.13.4", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "criterion" version = "0.5.1" @@ -401,6 +609,15 @@ dependencies = [ "syn", ] +[[package]] +name = "debugid" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" +dependencies = [ + "uuid", +] + [[package]] name = "der" version = "0.7.10" @@ -421,6 +638,27 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "directories-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339ee130d97a610ea5a5872d2bbb130fdf68884ff09d3028b81bec8a1ac23bbc" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "dispatch2" version = "0.3.0" @@ -475,6 +713,27 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -491,6 +750,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + [[package]] name = "fastrand" version = "2.3.0" @@ -503,6 +768,12 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + [[package]] name = "fnv" version = "1.0.7" @@ -515,6 +786,28 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "fxprof-processed-profile" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27d12c0aed7f1e24276a241aadc4cb8ea9f83000f34bc062b7cc2d51e3b0fabd" +dependencies = [ + "bitflags", + "debugid", + "fxhash", + "serde", + "serde_json", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -549,16 +842,14 @@ dependencies = [ ] [[package]] -name = "getrandom" -version = "0.4.1" +name = "gimli" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" dependencies = [ - "cfg-if", - "libc", - "r-efi", - "wasip2", - "wasip3", + "fallible-iterator", + "indexmap", + "stable_deref_trait", ] [[package]] @@ -572,6 +863,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -581,6 +881,7 @@ dependencies = [ "allocator-api2", "equivalent", "foldhash", + "serde", ] [[package]] @@ -673,6 +974,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.13.0" @@ -688,6 +998,36 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +[[package]] +name = "ittapi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b996fe614c41395cdaedf3cf408a9534851090959d90d54a535f675550b64b1" +dependencies = [ + "anyhow", + "ittapi-sys", + "log", +] + +[[package]] +name = "ittapi-sys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5385394064fa2c886205dba02598013ce83d3e92d33dbdc0c52fe0e7bf4fc" +dependencies = [ + "cc", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + [[package]] name = "js-sys" version = "0.3.85" @@ -704,6 +1044,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "leb128" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" + [[package]] name = "leb128fmt" version = "0.1.0" @@ -716,6 +1062,22 @@ version = "0.2.181" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "459427e2af2b9c839b132acb702a1c654d95e10f8c326bfc2ad11310e458b1c5" +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" +dependencies = [ + "bitflags", + "libc", +] + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -752,12 +1114,30 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "mach2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44" +dependencies = [ + "libc", +] + [[package]] name = "memchr" version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memfd" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad38eb12aea514a0466ea40a80fd8cc83637065948eb4a426e4aa46261175227" +dependencies = [ + "rustix 1.1.3", +] + [[package]] name = "memoffset" version = "0.9.1" @@ -824,6 +1204,27 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "crc32fast", + "hashbrown 0.15.5", + "indexmap", + "memchr", +] + +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -881,6 +1282,12 @@ dependencies = [ "spki", ] +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + [[package]] name = "plotters" version = "0.3.7" @@ -915,6 +1322,18 @@ version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "serde", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -924,16 +1343,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn", -] - [[package]] name = "proc-macro2" version = "1.0.106" @@ -971,6 +1380,28 @@ dependencies = [ "proptest", ] +[[package]] +name = "psm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3852766467df634d74f0b2d7819bf8dc483a0eb2e3b0f50f756f9cfe8b0d18d8" +dependencies = [ + "ar_archive_writer", + "cc", +] + +[[package]] +name = "pulley-interpreter" +version = "29.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62d95f8575df49a2708398182f49a888cf9dc30210fb1fd2df87c889edcee75d" +dependencies = [ + "cranelift-bitset", + "log", + "sptr", + "wasmtime-math", +] + [[package]] name = "pyo3" version = "0.23.5" @@ -996,7 +1427,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94f6cbe86ef3bf18998d9df6e0f3fc1050a8c5efa409bf712e661a4366e010fb" dependencies = [ "once_cell", - "target-lexicon", + "target-lexicon 0.12.16", ] [[package]] @@ -1160,6 +1591,31 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror 1.0.69", +] + +[[package]] +name = "regalloc2" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc06e6b318142614e4a48bc725abbf08ff166694835c43c9dae5a9009704639a" +dependencies = [ + "allocator-api2", + "bumpalo", + "hashbrown 0.15.5", + "log", + "rustc-hash", + "smallvec", +] + [[package]] name = "regex" version = "1.12.3" @@ -1206,6 +1662,18 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "rustc-demangle" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + [[package]] name = "rustc_version" version = "0.4.1" @@ -1285,6 +1753,10 @@ name = "semver" version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +dependencies = [ + "serde", + "serde_core", +] [[package]] name = "serde" @@ -1358,6 +1830,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook" version = "0.3.18" @@ -1427,6 +1905,9 @@ name = "smallvec" version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +dependencies = [ + "serde", +] [[package]] name = "spki" @@ -1438,6 +1919,18 @@ dependencies = [ "der", ] +[[package]] +name = "sptr" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + [[package]] name = "static_assertions" version = "1.1.0" @@ -1501,6 +1994,15 @@ dependencies = [ "swactor", ] +[[package]] +name = "swactor-wasm-actor" +version = "0.1.0" +dependencies = [ + "swactor", + "wasmtime", + "wat", +] + [[package]] name = "syn" version = "2.0.114" @@ -1518,6 +2020,12 @@ version = "0.12.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" +[[package]] +name = "target-lexicon" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1dd07eb858a2067e2f3c7155d54e929265c264e6f37efe3ee7a8d1b5a1dd0ba" + [[package]] name = "tempfile" version = "3.25.0" @@ -1525,12 +2033,61 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0136791f7c95b1f6dd99f9cc786b91bb81c3800b639b3478e561ddb7be95e5f1" dependencies = [ "fastrand", - "getrandom 0.4.1", + "getrandom 0.3.4", "once_cell", "rustix 1.1.3", "windows-sys 0.61.2", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "thread_local" version = "1.1.9" @@ -1660,6 +2217,17 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "trait-variant" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "typenum" version = "1.19.0" @@ -1719,6 +2287,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" +[[package]] +name = "uuid" +version = "1.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee48d38b119b0cd71fe4141b30f5ba9c7c5d9f4e7a3a8b4a674e4b6ef789976f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -1765,15 +2343,6 @@ dependencies = [ "wit-bindgen", ] -[[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" -dependencies = [ - "wit-bindgen", -] - [[package]] name = "wasm" version = "0.1.0" @@ -1829,36 +2398,332 @@ dependencies = [ [[package]] name = "wasm-encoder" -version = "0.244.0" +version = "0.221.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +checksum = "dc8444fe4920de80a4fe5ab564fff2ae58b6b73166b89751f8c6c93509da32e5" dependencies = [ - "leb128fmt", - "wasmparser", + "leb128", + "wasmparser 0.221.3", ] [[package]] -name = "wasm-metadata" -version = "0.244.0" +name = "wasm-encoder" +version = "0.245.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +checksum = "95d568e113f706ee7a7df9b33547bb80721f55abffc79b3dc4d09c368690e662" dependencies = [ - "anyhow", - "indexmap", - "wasm-encoder", - "wasmparser", + "leb128fmt", + "wasmparser 0.245.0", ] [[package]] name = "wasmparser" -version = "0.244.0" +version = "0.221.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +checksum = "d06bfa36ab3ac2be0dee563380147a5b81ba10dd8885d7fbbc9eb574be67d185" dependencies = [ "bitflags", "hashbrown 0.15.5", "indexmap", "semver", + "serde", +] + +[[package]] +name = "wasmparser" +version = "0.245.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48a767a48974f0c8b66f211b96e01aa77feed58b8ccce4e7f0cff0ae55b174d4" +dependencies = [ + "bitflags", + "indexmap", + "semver", +] + +[[package]] +name = "wasmprinter" +version = "0.221.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7343c42a97f2926c7819ff81b64012092ae954c5d83ddd30c9fcdefd97d0b283" +dependencies = [ + "anyhow", + "termcolor", + "wasmparser 0.221.3", +] + +[[package]] +name = "wasmtime" +version = "29.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11976a250672556d1c4c04c6d5d7656ac9192ac9edc42a4587d6c21460010e69" +dependencies = [ + "addr2line", + "anyhow", + "async-trait", + "bitflags", + "bumpalo", + "cc", + "cfg-if", + "encoding_rs", + "fxprof-processed-profile", + "gimli", + "hashbrown 0.14.5", + "indexmap", + "ittapi", + "libc", + "log", + "mach2", + "memfd", + "object 0.36.7", + "once_cell", + "paste", + "postcard", + "psm", + "pulley-interpreter", + "rayon", + "rustix 0.38.44", + "semver", + "serde", + "serde_derive", + "serde_json", + "smallvec", + "sptr", + "target-lexicon 0.13.4", + "trait-variant", + "wasm-encoder 0.221.3", + "wasmparser 0.221.3", + "wasmtime-asm-macros", + "wasmtime-cache", + "wasmtime-component-macro", + "wasmtime-component-util", + "wasmtime-cranelift", + "wasmtime-environ", + "wasmtime-fiber", + "wasmtime-jit-debug", + "wasmtime-jit-icache-coherence", + "wasmtime-math", + "wasmtime-slab", + "wasmtime-versioned-export-macros", + "wasmtime-winch", + "wat", + "windows-sys 0.59.0", +] + +[[package]] +name = "wasmtime-asm-macros" +version = "29.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f178b0d125201fbe9f75beaf849bd3e511891f9e45ba216a5b620802ccf64f2" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "wasmtime-cache" +version = "29.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1161c8f62880deea07358bc40cceddc019f1c81d46007bc390710b2fe24ffc" +dependencies = [ + "anyhow", + "base64", + "directories-next", + "log", + "postcard", + "rustix 0.38.44", + "serde", + "serde_derive", + "sha2", + "toml", + "windows-sys 0.59.0", + "zstd", +] + +[[package]] +name = "wasmtime-component-macro" +version = "29.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d74de6592ed945d0a602f71243982a304d5d02f1e501b638addf57f42d57dfaf" +dependencies = [ + "anyhow", + "proc-macro2", + "quote", + "syn", + "wasmtime-component-util", + "wasmtime-wit-bindgen", + "wit-parser", +] + +[[package]] +name = "wasmtime-component-util" +version = "29.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707dc7b3c112ab5a366b30cfe2fb5b2f8e6a0f682f16df96a5ec582bfe6f056e" + +[[package]] +name = "wasmtime-cranelift" +version = "29.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "366be722674d4bf153290fbcbc4d7d16895cc82fb3e869f8d550ff768f9e9e87" +dependencies = [ + "anyhow", + "cfg-if", + "cranelift-codegen", + "cranelift-control", + "cranelift-entity", + "cranelift-frontend", + "cranelift-native", + "gimli", + "itertools 0.12.1", + "log", + "object 0.36.7", + "smallvec", + "target-lexicon 0.13.4", + "thiserror 1.0.69", + "wasmparser 0.221.3", + "wasmtime-environ", + "wasmtime-versioned-export-macros", +] + +[[package]] +name = "wasmtime-environ" +version = "29.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdadc1af7097347aa276a4f008929810f726b5b46946971c660b6d421e9994ad" +dependencies = [ + "anyhow", + "cpp_demangle", + "cranelift-bitset", + "cranelift-entity", + "gimli", + "indexmap", + "log", + "object 0.36.7", + "postcard", + "rustc-demangle", + "semver", + "serde", + "serde_derive", + "smallvec", + "target-lexicon 0.13.4", + "wasm-encoder 0.221.3", + "wasmparser 0.221.3", + "wasmprinter", + "wasmtime-component-util", +] + +[[package]] +name = "wasmtime-fiber" +version = "29.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccba90d4119f081bca91190485650730a617be1fff5228f8c4757ce133d21117" +dependencies = [ + "anyhow", + "cc", + "cfg-if", + "rustix 0.38.44", + "wasmtime-asm-macros", + "wasmtime-versioned-export-macros", + "windows-sys 0.59.0", +] + +[[package]] +name = "wasmtime-jit-debug" +version = "29.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e7b61488a5ee00c35c8c22de707c36c0aecacf419a3be803a6a2ba5e860f56a" +dependencies = [ + "object 0.36.7", + "rustix 0.38.44", + "wasmtime-versioned-export-macros", +] + +[[package]] +name = "wasmtime-jit-icache-coherence" +version = "29.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec5e8552e01692e6c2e5293171704fed8abdec79d1a6995a0870ab190e5747d1" +dependencies = [ + "anyhow", + "cfg-if", + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "wasmtime-math" +version = "29.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29210ec2aa25e00f4d54605cedaf080f39ec01a872c5bd520ad04c67af1dde17" +dependencies = [ + "libm", +] + +[[package]] +name = "wasmtime-slab" +version = "29.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb5821a96fa04ac14bc7b158bb3d5cd7729a053db5a74dad396cd513a5e5ccf" + +[[package]] +name = "wasmtime-versioned-export-macros" +version = "29.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86ff86db216dc0240462de40c8290887a613dddf9685508eb39479037ba97b5b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "wasmtime-winch" +version = "29.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdbabfb8f20502d5e1d81092b9ead3682ae59988487aafcd7567387b7a43cf8f" +dependencies = [ + "anyhow", + "cranelift-codegen", + "gimli", + "object 0.36.7", + "target-lexicon 0.13.4", + "wasmparser 0.221.3", + "wasmtime-cranelift", + "wasmtime-environ", + "winch-codegen", +] + +[[package]] +name = "wasmtime-wit-bindgen" +version = "29.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8358319c2dd1e4db79e3c1c5d3a5af84956615343f9f89f4e4996a36816e06e6" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "wit-parser", +] + +[[package]] +name = "wast" +version = "245.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75ffc7471e16a6f3c7a3c3a230314915b5dcd158e5ef13ccda2f43358a9df00c" +dependencies = [ + "bumpalo", + "leb128fmt", + "memchr", + "unicode-width 0.2.0", + "wasm-encoder 0.245.0", +] + +[[package]] +name = "wat" +version = "1.245.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bcac6f915e2a84a4c0d9df9d41ad7518d99cda13f3bb83e3b8c22bf8726ab6" +dependencies = [ + "wast", ] [[package]] @@ -1902,6 +2767,24 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "winch-codegen" +version = "29.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f849ef2c5f46cb0a20af4b4487aaa239846e52e2c03f13fa3c784684552859c" +dependencies = [ + "anyhow", + "cranelift-codegen", + "gimli", + "regalloc2", + "smallvec", + "target-lexicon 0.13.4", + "thiserror 1.0.69", + "wasmparser 0.221.3", + "wasmtime-cranelift", + "wasmtime-environ", +] + [[package]] name = "windows-link" version = "0.2.1" @@ -2004,76 +2887,12 @@ name = "wit-bindgen" version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck", - "indexmap", - "prettyplease", - "syn", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags", - "indexmap", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] [[package]] name = "wit-parser" -version = "0.244.0" +version = "0.221.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +checksum = "896112579ed56b4a538b07a3d16e562d101ff6265c46b515ce0c701eef16b2ac" dependencies = [ "anyhow", "id-arena", @@ -2084,7 +2903,7 @@ dependencies = [ "serde_derive", "serde_json", "unicode-xid", - "wasmparser", + "wasmparser 0.221.3", ] [[package]] @@ -2118,3 +2937,31 @@ name = "zmij" version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4de98dfa5d5b7fef4ee834d0073d560c9ca7b6c46a71d058c48db7960f8cfaf7" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 2dcb021..f582294 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = [".", "crates/python", "crates/wasm", "crates/simulation", "crates/runtime-dashboard", "crates/distribution", "crates/simulation-dashboard", "crates/std"] +members = [".", "crates/python", "crates/wasm", "crates/wasm-actor", "crates/simulation", "crates/runtime-dashboard", "crates/distribution", "crates/simulation-dashboard", "crates/std"] exclude = ["tools/depgraph"] [package] diff --git a/crates/wasm-actor/Cargo.toml b/crates/wasm-actor/Cargo.toml new file mode 100644 index 0000000..e629b8a --- /dev/null +++ b/crates/wasm-actor/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "swactor-wasm-actor" +version = "0.1.0" +edition = "2024" + +[dependencies] +swactor = { path = "../.." } +wasmtime = "29" + +[dev-dependencies] +swactor = { path = "../..", features = ["getrandom"] } +wat = "1" diff --git a/crates/wasm-actor/src/actor.rs b/crates/wasm-actor/src/actor.rs new file mode 100644 index 0000000..088ccf0 --- /dev/null +++ b/crates/wasm-actor/src/actor.rs @@ -0,0 +1,59 @@ +use swactor::actor::{ActorInterface, Ctx}; +use wasmtime::{Memory, Store, TypedFunc}; + +use crate::ByteMessage; + +/// State accessible to host functions during guest execution. +#[derive(Default)] +pub(crate) struct HostState { + pub outbox: Vec<(swactor::actor::ActorAddress, Vec)>, +} + +/// An actor whose logic is defined by a WebAssembly guest module. +/// +/// Messages arrive as [`ByteMessage`], are copied into Wasm linear memory, +/// and processed by the guest's `handle` export. The guest can send messages +/// back via the `swactor.send` host import. +pub struct WasmActor { + pub(crate) store: Store, + pub(crate) memory: Memory, + pub(crate) alloc: TypedFunc, + pub(crate) handle: TypedFunc<(i32, i32), ()>, +} + +impl ActorInterface for WasmActor { + type Incoming = ByteMessage; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, msg: ByteMessage) { + let bytes = &msg.0; + let len: i32 = match i32::try_from(bytes.len()) { + Ok(n) => n, + Err(_) => return, // message too large for i32 ABI + }; + + // 1. Allocate space in guest memory + let ptr = match self.alloc.call(&mut self.store, len) { + Ok(ptr) if ptr < 0 => return, // invalid pointer + Ok(0) if len > 0 => return, // OOM — drop message + Ok(ptr) => ptr, + Err(_) => return, // alloc trapped — drop message + }; + + // 2. Write message bytes into guest memory + self.memory.data_mut(&mut self.store) + [ptr as usize..(ptr as usize + bytes.len())] + .copy_from_slice(bytes); + + // 3. Call guest handle + if self.handle.call(&mut self.store, (ptr, len)).is_err() { + return; // handle trapped — drop message, keep actor alive + } + + // 4. Drain outbox → send via ctx + let outbox: Vec<_> = self.store.data_mut().outbox.drain(..).collect(); + for (dest, payload) in outbox { + let _ = ctx.send(dest, ByteMessage(payload)); + } + } +} diff --git a/crates/wasm-actor/src/builder.rs b/crates/wasm-actor/src/builder.rs new file mode 100644 index 0000000..06a199c --- /dev/null +++ b/crates/wasm-actor/src/builder.rs @@ -0,0 +1,109 @@ +use swactor::actor::ActorAddress; +use wasmtime::{Linker, Module, Store, TypedFunc}; + +use crate::actor::{HostState, WasmActor}; +use crate::engine::SharedEngine; +use crate::error::WasmActorError; + +/// Compiles a Wasm module and produces a ready-to-use [`WasmActor`]. +pub struct WasmActorBuilder { + engine: SharedEngine, + wasm_bytes: Vec, +} + +impl WasmActorBuilder { + pub fn new(engine: SharedEngine, wasm_bytes: impl Into>) -> Self { + Self { + engine, + wasm_bytes: wasm_bytes.into(), + } + } + + /// Compile the module, link host functions, and instantiate. + pub fn build(self) -> Result { + let engine = self.engine.inner(); + let module = Module::new(engine, &self.wasm_bytes)?; + + let mut linker: Linker = Linker::new(engine); + Self::link_send(&mut linker)?; + + let mut store = Store::new(engine, HostState::default()); + let instance = linker.instantiate(&mut store, &module)?; + + // Extract required exports + let memory = instance + .get_memory(&mut store, "memory") + .ok_or(WasmActorError::MissingExport("memory"))?; + + let alloc: TypedFunc = instance + .get_typed_func(&mut store, "alloc") + .map_err(|_| WasmActorError::MissingExport("alloc"))?; + + let handle: TypedFunc<(i32, i32), ()> = instance + .get_typed_func(&mut store, "handle") + .map_err(|_| WasmActorError::MissingExport("handle"))?; + + Ok(WasmActor { + store, + memory, + alloc, + handle, + }) + } + + /// Link the `swactor.send` host import. + fn link_send(linker: &mut Linker) -> Result<(), WasmActorError> { + linker.func_wrap( + "swactor", + "send", + |mut caller: wasmtime::Caller<'_, HostState>, + dest_ptr: i32, + payload_ptr: i32, + payload_len: i32| + -> Result<(), wasmtime::Error> { + let mem = caller + .get_export("memory") + .and_then(|e| e.into_memory()) + .ok_or_else(|| wasmtime::Error::msg("guest must export memory"))?; + let data = mem.data(&caller); + let mem_len = data.len(); + + // Validate non-negative arguments + if dest_ptr < 0 || payload_ptr < 0 || payload_len < 0 { + return Err(wasmtime::Error::msg( + "negative argument in swactor.send", + )); + } + + let dest_ptr = dest_ptr as usize; + let payload_ptr = payload_ptr as usize; + let payload_len = payload_len as usize; + + // Bounds-check with overflow protection + let dest_end = dest_ptr + .checked_add(32) + .ok_or_else(|| wasmtime::Error::msg("dest_ptr overflow"))?; + let payload_end = payload_ptr + .checked_add(payload_len) + .ok_or_else(|| wasmtime::Error::msg("payload range overflow"))?; + if dest_end > mem_len || payload_end > mem_len { + return Err(wasmtime::Error::msg( + "out-of-bounds memory access in swactor.send", + )); + } + + // Read 32-byte destination address + let mut addr_bytes = [0u8; 32]; + addr_bytes.copy_from_slice(&data[dest_ptr..dest_end]); + let dest = ActorAddress(addr_bytes); + + // Read payload + let payload = data[payload_ptr..payload_end].to_vec(); + + caller.data_mut().outbox.push((dest, payload)); + Ok(()) + }, + )?; + Ok(()) + } +} diff --git a/crates/wasm-actor/src/engine.rs b/crates/wasm-actor/src/engine.rs new file mode 100644 index 0000000..8cc2dfc --- /dev/null +++ b/crates/wasm-actor/src/engine.rs @@ -0,0 +1,35 @@ +use std::sync::Arc; + +use wasmtime::Engine; + +/// A shared, cheaply-cloneable Wasm engine. +/// +/// Created once and reused across multiple [`WasmActor`](crate::WasmActor) instances. +/// Configured with maximum sandboxing — no threads, no SIMD, no reference types. +#[derive(Clone)] +pub struct SharedEngine(Arc); + +impl std::fmt::Debug for SharedEngine { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("SharedEngine").field(&"").finish() + } +} + +impl SharedEngine { + /// Create a new engine with sandboxed defaults. + pub fn new() -> Result { + let mut config = wasmtime::Config::new(); + config.wasm_threads(false); + config.wasm_simd(false); + config.wasm_relaxed_simd(false); + config.wasm_reference_types(false); + config.wasm_multi_value(false); + config.wasm_bulk_memory(true); + let engine = Engine::new(&config)?; + Ok(Self(Arc::new(engine))) + } + + pub(crate) fn inner(&self) -> &Engine { + &self.0 + } +} diff --git a/crates/wasm-actor/src/error.rs b/crates/wasm-actor/src/error.rs new file mode 100644 index 0000000..b91c052 --- /dev/null +++ b/crates/wasm-actor/src/error.rs @@ -0,0 +1,27 @@ +use std::fmt; + +/// Errors that can occur when building or running a WasmActor. +#[derive(Debug)] +pub enum WasmActorError { + /// A required export is missing from the Wasm module. + MissingExport(&'static str), + /// The Wasm module failed to compile or instantiate. + Wasmtime(wasmtime::Error), +} + +impl fmt::Display for WasmActorError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::MissingExport(name) => write!(f, "missing required export: `{name}`"), + Self::Wasmtime(e) => write!(f, "wasmtime error: {e}"), + } + } +} + +impl std::error::Error for WasmActorError {} + +impl From for WasmActorError { + fn from(e: wasmtime::Error) -> Self { + Self::Wasmtime(e) + } +} diff --git a/crates/wasm-actor/src/lib.rs b/crates/wasm-actor/src/lib.rs new file mode 100644 index 0000000..3f744c2 --- /dev/null +++ b/crates/wasm-actor/src/lib.rs @@ -0,0 +1,13 @@ +mod actor; +mod builder; +mod engine; +mod error; + +pub use actor::WasmActor; +pub use builder::WasmActorBuilder; +pub use engine::SharedEngine; +pub use error::WasmActorError; + +/// A message carrying raw bytes, suitable for passing to/from Wasm guests. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ByteMessage(pub Vec); diff --git a/crates/wasm-actor/tests/guests/double/Cargo.lock b/crates/wasm-actor/tests/guests/double/Cargo.lock new file mode 100644 index 0000000..b46d745 --- /dev/null +++ b/crates/wasm-actor/tests/guests/double/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "double-guest" +version = "0.1.0" diff --git a/crates/wasm-actor/tests/guests/double/Cargo.toml b/crates/wasm-actor/tests/guests/double/Cargo.toml new file mode 100644 index 0000000..943eabe --- /dev/null +++ b/crates/wasm-actor/tests/guests/double/Cargo.toml @@ -0,0 +1,13 @@ +[workspace] + +[package] +name = "double-guest" +version = "0.1.0" +edition = "2024" + +[lib] +crate-type = ["cdylib"] + +[profile.release] +opt-level = "s" +lto = true diff --git a/crates/wasm-actor/tests/guests/double/src/lib.rs b/crates/wasm-actor/tests/guests/double/src/lib.rs new file mode 100644 index 0000000..2ea9158 --- /dev/null +++ b/crates/wasm-actor/tests/guests/double/src/lib.rs @@ -0,0 +1,63 @@ +#![no_std] + +use core::cell::UnsafeCell; +use core::panic::PanicInfo; + +// --- bump allocator --- +const HEAP_SIZE: usize = 65536; + +struct BumpAlloc { + heap: UnsafeCell<[u8; HEAP_SIZE]>, + offset: UnsafeCell, +} + +unsafe impl Sync for BumpAlloc {} + +static ALLOC: BumpAlloc = BumpAlloc { + heap: UnsafeCell::new([0u8; HEAP_SIZE]), + offset: UnsafeCell::new(0), +}; + +#[unsafe(no_mangle)] +pub extern "C" fn alloc(size: i32) -> i32 { + unsafe { + let offset = &mut *ALLOC.offset.get(); + let heap = &mut *ALLOC.heap.get(); + let align = 8; + let start = (*offset + align - 1) & !(align - 1); + let end = start + size as usize; + if end > heap.len() { + return 0; // OOM + } + *offset = end; + heap.as_ptr().add(start) as i32 + } +} + +// --- host import --- +#[link(wasm_import_module = "swactor")] +unsafe extern "C" { + #[link_name = "send"] + fn host_send(dest_ptr: i32, payload_ptr: i32, payload_len: i32); +} + +/// Message format: first 32 bytes = destination address, rest = payload. +/// Sends the payload back twice to demonstrate multi-send. +#[unsafe(no_mangle)] +pub extern "C" fn handle(ptr: i32, len: i32) { + if len < 32 { + return; + } + let dest_ptr = ptr; + let payload_ptr = ptr + 32; + let payload_len = len - 32; + unsafe { + host_send(dest_ptr, payload_ptr, payload_len); + host_send(dest_ptr, payload_ptr, payload_len); + } +} + +#[panic_handler] +fn panic(_info: &PanicInfo) -> ! { + loop {} +} diff --git a/crates/wasm-actor/tests/guests/echo/Cargo.lock b/crates/wasm-actor/tests/guests/echo/Cargo.lock new file mode 100644 index 0000000..224c914 --- /dev/null +++ b/crates/wasm-actor/tests/guests/echo/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "echo-guest" +version = "0.1.0" diff --git a/crates/wasm-actor/tests/guests/echo/Cargo.toml b/crates/wasm-actor/tests/guests/echo/Cargo.toml new file mode 100644 index 0000000..7c33f5c --- /dev/null +++ b/crates/wasm-actor/tests/guests/echo/Cargo.toml @@ -0,0 +1,13 @@ +[workspace] + +[package] +name = "echo-guest" +version = "0.1.0" +edition = "2024" + +[lib] +crate-type = ["cdylib"] + +[profile.release] +opt-level = "s" +lto = true diff --git a/crates/wasm-actor/tests/guests/echo/src/lib.rs b/crates/wasm-actor/tests/guests/echo/src/lib.rs new file mode 100644 index 0000000..edc69a5 --- /dev/null +++ b/crates/wasm-actor/tests/guests/echo/src/lib.rs @@ -0,0 +1,62 @@ +#![no_std] + +use core::cell::UnsafeCell; +use core::panic::PanicInfo; + +// --- bump allocator --- +const HEAP_SIZE: usize = 65536; + +struct BumpAlloc { + heap: UnsafeCell<[u8; HEAP_SIZE]>, + offset: UnsafeCell, +} + +unsafe impl Sync for BumpAlloc {} + +static ALLOC: BumpAlloc = BumpAlloc { + heap: UnsafeCell::new([0u8; HEAP_SIZE]), + offset: UnsafeCell::new(0), +}; + +#[unsafe(no_mangle)] +pub extern "C" fn alloc(size: i32) -> i32 { + unsafe { + let offset = &mut *ALLOC.offset.get(); + let heap = &mut *ALLOC.heap.get(); + let align = 8; + let start = (*offset + align - 1) & !(align - 1); + let end = start + size as usize; + if end > heap.len() { + return 0; // OOM + } + *offset = end; + heap.as_ptr().add(start) as i32 + } +} + +// --- host import --- +#[link(wasm_import_module = "swactor")] +unsafe extern "C" { + #[link_name = "send"] + fn host_send(dest_ptr: i32, payload_ptr: i32, payload_len: i32); +} + +/// Message format: first 32 bytes = destination address, rest = payload. +/// Echo sends the payload portion back to the specified destination. +#[unsafe(no_mangle)] +pub extern "C" fn handle(ptr: i32, len: i32) { + if len < 32 { + return; + } + let dest_ptr = ptr; + let payload_ptr = ptr + 32; + let payload_len = len - 32; + unsafe { + host_send(dest_ptr, payload_ptr, payload_len); + } +} + +#[panic_handler] +fn panic(_info: &PanicInfo) -> ! { + loop {} +} diff --git a/crates/wasm-actor/tests/guests/silent/Cargo.lock b/crates/wasm-actor/tests/guests/silent/Cargo.lock new file mode 100644 index 0000000..aff96a4 --- /dev/null +++ b/crates/wasm-actor/tests/guests/silent/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "silent-guest" +version = "0.1.0" diff --git a/crates/wasm-actor/tests/guests/silent/Cargo.toml b/crates/wasm-actor/tests/guests/silent/Cargo.toml new file mode 100644 index 0000000..14e524f --- /dev/null +++ b/crates/wasm-actor/tests/guests/silent/Cargo.toml @@ -0,0 +1,13 @@ +[workspace] + +[package] +name = "silent-guest" +version = "0.1.0" +edition = "2024" + +[lib] +crate-type = ["cdylib"] + +[profile.release] +opt-level = "s" +lto = true diff --git a/crates/wasm-actor/tests/guests/silent/src/lib.rs b/crates/wasm-actor/tests/guests/silent/src/lib.rs new file mode 100644 index 0000000..c1d2a1c --- /dev/null +++ b/crates/wasm-actor/tests/guests/silent/src/lib.rs @@ -0,0 +1,45 @@ +#![no_std] + +use core::cell::UnsafeCell; +use core::panic::PanicInfo; + +// --- bump allocator --- +const HEAP_SIZE: usize = 65536; + +struct BumpAlloc { + heap: UnsafeCell<[u8; HEAP_SIZE]>, + offset: UnsafeCell, +} + +unsafe impl Sync for BumpAlloc {} + +static ALLOC: BumpAlloc = BumpAlloc { + heap: UnsafeCell::new([0u8; HEAP_SIZE]), + offset: UnsafeCell::new(0), +}; + +#[unsafe(no_mangle)] +pub extern "C" fn alloc(size: i32) -> i32 { + unsafe { + let offset = &mut *ALLOC.offset.get(); + let heap = &mut *ALLOC.heap.get(); + let align = 8; + let start = (*offset + align - 1) & !(align - 1); + let end = start + size as usize; + if end > heap.len() { + return 0; // OOM + } + *offset = end; + heap.as_ptr().add(start) as i32 + } +} + +#[unsafe(no_mangle)] +pub extern "C" fn handle(_ptr: i32, _len: i32) { + // Silent: receive bytes, do nothing +} + +#[panic_handler] +fn panic(_info: &PanicInfo) -> ! { + loop {} +} diff --git a/crates/wasm-actor/tests/wasm_actor.rs b/crates/wasm-actor/tests/wasm_actor.rs new file mode 100644 index 0000000..2125369 --- /dev/null +++ b/crates/wasm-actor/tests/wasm_actor.rs @@ -0,0 +1,331 @@ +use swactor::actor::{ActorAddress, ActorInterface}; +use swactor::runtime::{Ctx, Runtime, RuntimeConfig}; +use swactor_wasm_actor::{ByteMessage, SharedEngine, WasmActorBuilder, WasmActorError}; + +fn guest_wasm(name: &str) -> Vec { + let path = format!( + "{}/tests/guests/{name}/target/wasm32-unknown-unknown/release/{name}_guest.wasm", + env!("CARGO_MANIFEST_DIR") + ); + std::fs::read(&path).unwrap_or_else(|e| panic!("failed to read {path}: {e}")) +} + +/// Build a message with an inbox address prepended (the guest contract). +fn framed_msg(dest: &ActorAddress, payload: &[u8]) -> ByteMessage { + let mut buf = Vec::with_capacity(32 + payload.len()); + buf.extend_from_slice(&dest.0); + buf.extend_from_slice(payload); + ByteMessage(buf) +} + +// ── Echo: send bytes in, same bytes come back ──────────────────────────────── + +#[test] +fn echo_returns_same_payload() { + let engine = SharedEngine::new().unwrap(); + let actor = WasmActorBuilder::new(engine, guest_wasm("echo")) + .build() + .unwrap(); + + let rt = Runtime::new(RuntimeConfig::default()); + let inbox = rt.new_inbox::().unwrap(); + let addr = rt.spawn(actor).unwrap(); + + let payload = b"hello wasm"; + rt.send_to(addr, framed_msg(inbox.addr(), payload)).unwrap(); + rt.tick(); + + let received = inbox.try_recv().expect("inbox should have a message"); + assert_eq!(received.0, payload); +} + +#[test] +fn echo_preserves_binary_payload() { + let engine = SharedEngine::new().unwrap(); + let actor = WasmActorBuilder::new(engine, guest_wasm("echo")) + .build() + .unwrap(); + + let rt = Runtime::new(RuntimeConfig::default()); + let inbox = rt.new_inbox::().unwrap(); + let addr = rt.spawn(actor).unwrap(); + + let payload: Vec = (0..=255).collect(); + rt.send_to(addr, framed_msg(inbox.addr(), &payload)).unwrap(); + rt.tick(); + + let received = inbox.try_recv().expect("inbox should have a message"); + assert_eq!(received.0, payload); +} + +// ── Silent: processes messages without sending anything ─────────────────────── + +#[test] +fn silent_produces_no_output() { + let engine = SharedEngine::new().unwrap(); + let actor = WasmActorBuilder::new(engine, guest_wasm("silent")) + .build() + .unwrap(); + + let rt = Runtime::new(RuntimeConfig::default()); + let inbox = rt.new_inbox::().unwrap(); + let addr = rt.spawn(actor).unwrap(); + + rt.send_to(addr, ByteMessage(b"ignored".to_vec())).unwrap(); + rt.tick(); + + assert!(inbox.try_recv().is_none(), "silent guest should not send anything"); +} + +// ── Double: one message in, two messages out ───────────────────────────────── + +#[test] +fn double_sends_two_copies() { + let engine = SharedEngine::new().unwrap(); + let actor = WasmActorBuilder::new(engine, guest_wasm("double")) + .build() + .unwrap(); + + let rt = Runtime::new(RuntimeConfig::default()); + let inbox = rt.new_inbox::().unwrap(); + let addr = rt.spawn(actor).unwrap(); + + let payload = b"dup me"; + rt.send_to(addr, framed_msg(inbox.addr(), payload)).unwrap(); + rt.tick(); + + let first = inbox.try_recv().expect("should receive first copy"); + let second = inbox.try_recv().expect("should receive second copy"); + assert_eq!(first.0, payload); + assert_eq!(second.0, payload); + assert!(inbox.try_recv().is_none(), "exactly two messages expected"); +} + +// ── Missing export → WasmActorError::MissingExport ─────────────────────────── + +#[test] +fn missing_alloc_export_returns_error() { + // Minimal valid Wasm module: (module) — no exports at all + let minimal_wasm = wat::parse_str("(module)").unwrap(); + let engine = SharedEngine::new().unwrap(); + let result = WasmActorBuilder::new(engine, minimal_wasm).build(); + match result { + Err(WasmActorError::MissingExport(name)) => { + assert!( + name == "memory" || name == "alloc", + "expected missing memory or alloc, got: {name}" + ); + } + Err(other) => panic!("expected MissingExport, got: {other}"), + Ok(_) => panic!("expected error for module with no exports"), + } +} + +// ── Engine sharing: two actors from the same engine ────────────────────────── + +#[test] +fn shared_engine_serves_multiple_actors() { + let engine = SharedEngine::new().unwrap(); + + let echo = WasmActorBuilder::new(engine.clone(), guest_wasm("echo")) + .build() + .unwrap(); + let silent = WasmActorBuilder::new(engine, guest_wasm("silent")) + .build() + .unwrap(); + + let rt = Runtime::new(RuntimeConfig::default()); + let inbox = rt.new_inbox::().unwrap(); + + let echo_addr = rt.spawn(echo).unwrap(); + let _silent_addr = rt.spawn(silent).unwrap(); + + let payload = b"shared engine test"; + rt.send_to(echo_addr, framed_msg(inbox.addr(), payload)).unwrap(); + rt.tick(); + + let received = inbox.try_recv().expect("echo actor should still work"); + assert_eq!(received.0, payload); +} + +// ── Safety: edge cases that previously caused panics or corruption ──────────── + +#[test] +fn oob_send_traps_cleanly_and_actor_survives() { + // Guest calls swactor.send with dest_ptr pointing past the end of memory. + // The host should trap the call; the actor should survive for future messages. + let wat = r#" + (module + (import "swactor" "send" (func $send (param i32 i32 i32))) + (memory (export "memory") 1) + (func (export "alloc") (param i32) (result i32) + i32.const 0 ;; return start of memory (simplistic) + ) + (func (export "handle") (param i32 i32) + ;; Call send with dest_ptr = 65536 (1 page = end of memory, OOB for 32 bytes) + i32.const 65536 + i32.const 0 + i32.const 0 + call $send + ) + ) + "#; + let wasm = wat::parse_str(wat).unwrap(); + let engine = SharedEngine::new().unwrap(); + let actor = WasmActorBuilder::new(engine, wasm).build().unwrap(); + + let rt = Runtime::new(RuntimeConfig::default()); + let inbox = rt.new_inbox::().unwrap(); + let addr = rt.spawn(actor).unwrap(); + + // Send a message — handle will try OOB send, which traps + rt.send_to(addr, ByteMessage(vec![42])).unwrap(); + rt.tick(); + + // No message should arrive (the send was invalid) + assert!(inbox.try_recv().is_none(), "OOB send should not produce a message"); +} + +#[test] +fn alloc_oom_drops_message_actor_stays_alive() { + // Guest alloc always returns 0 (OOM). Message should be dropped, + // actor should remain alive for subsequent messages. + let wat = r#" + (module + (import "swactor" "send" (func $send (param i32 i32 i32))) + (memory (export "memory") 1) + (func (export "alloc") (param i32) (result i32) + i32.const 0 ;; always OOM + ) + (func (export "handle") (param i32 i32) + ;; Should never be called if alloc returned 0 for non-zero len + ) + ) + "#; + let wasm = wat::parse_str(wat).unwrap(); + let engine = SharedEngine::new().unwrap(); + let actor = WasmActorBuilder::new(engine, wasm).build().unwrap(); + + let rt = Runtime::new(RuntimeConfig::default()); + let addr = rt.spawn(actor).unwrap(); + + // Send a non-empty message — alloc returns 0, message should be dropped + rt.send_to(addr, ByteMessage(vec![1, 2, 3])).unwrap(); + rt.tick(); + + // Actor is still alive — send another message, tick again (no panic) + rt.send_to(addr, ByteMessage(vec![4, 5, 6])).unwrap(); + rt.tick(); +} + +#[test] +fn negative_alloc_ptr_drops_message() { + // Guest alloc returns -1. Host should detect the negative pointer and drop. + let wat = r#" + (module + (import "swactor" "send" (func $send (param i32 i32 i32))) + (memory (export "memory") 1) + (func (export "alloc") (param i32) (result i32) + i32.const -1 ;; invalid negative pointer + ) + (func (export "handle") (param i32 i32)) + ) + "#; + let wasm = wat::parse_str(wat).unwrap(); + let engine = SharedEngine::new().unwrap(); + let actor = WasmActorBuilder::new(engine, wasm).build().unwrap(); + + let rt = Runtime::new(RuntimeConfig::default()); + let addr = rt.spawn(actor).unwrap(); + + rt.send_to(addr, ByteMessage(vec![1])).unwrap(); + rt.tick(); // should not panic + + // Actor survives + rt.send_to(addr, ByteMessage(vec![2])).unwrap(); + rt.tick(); +} + +#[test] +fn handle_trap_drops_message_actor_survives() { + // Guest handle executes `unreachable`, causing a Wasm trap. + // Message should be dropped, actor should stay alive. + let wat = r#" + (module + (import "swactor" "send" (func $send (param i32 i32 i32))) + (memory (export "memory") 1) + (func (export "alloc") (param i32) (result i32) + i32.const 256 ;; valid allocation + ) + (func (export "handle") (param i32 i32) + unreachable ;; trap! + ) + ) + "#; + let wasm = wat::parse_str(wat).unwrap(); + let engine = SharedEngine::new().unwrap(); + let actor = WasmActorBuilder::new(engine, wasm).build().unwrap(); + + let rt = Runtime::new(RuntimeConfig::default()); + let addr = rt.spawn(actor).unwrap(); + + rt.send_to(addr, ByteMessage(vec![1, 2, 3])).unwrap(); + rt.tick(); // handle traps, but actor should survive + + // Actor is still alive + rt.send_to(addr, ByteMessage(vec![4, 5, 6])).unwrap(); + rt.tick(); +} + +// ── Integration: WasmActor alongside a native Rust actor ───────────────────── + +#[derive(Clone)] +struct ForwardToWasm { + wasm_addr: ActorAddress, + inbox_addr: ActorAddress, +} + +struct Forwarder; + +impl ActorInterface for Forwarder { + type Incoming = ForwardToWasm; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, msg: ForwardToWasm) { + // Build the framed message and forward to the wasm actor + let payload = b"from native"; + let framed = framed_msg(&msg.inbox_addr, payload); + let _ = ctx.send(msg.wasm_addr, framed); + } +} + +#[test] +fn native_actor_communicates_with_wasm_actor() { + let engine = SharedEngine::new().unwrap(); + let wasm = WasmActorBuilder::new(engine, guest_wasm("echo")) + .build() + .unwrap(); + + let rt = Runtime::new(RuntimeConfig::default()); + let inbox = rt.new_inbox::().unwrap(); + + let wasm_addr = rt.spawn(wasm).unwrap(); + let forwarder_addr = rt.spawn(Forwarder).unwrap(); + + rt.send_to( + forwarder_addr, + ForwardToWasm { + wasm_addr, + inbox_addr: *inbox.addr(), + }, + ) + .unwrap(); + + // Tick 1: Forwarder receives message and sends to WasmActor + rt.tick(); + // Tick 2: WasmActor receives the forwarded message and echoes to inbox + rt.tick(); + + let received = inbox.try_recv().expect("wasm actor should have echoed"); + assert_eq!(received.0, b"from native"); +} diff --git a/docs/development_history/WASM_ACTOR.md b/docs/development_history/WASM_ACTOR.md new file mode 100644 index 0000000..3d2cf68 --- /dev/null +++ b/docs/development_history/WASM_ACTOR.md @@ -0,0 +1,191 @@ +# Wasm Actor Crate — Development History + +> Adds a new crate (`crates/wasm-actor/`) that runs WebAssembly guest code +> **inside** a swactor actor. The Wasm instance lives in the actor — not as a +> separate OS process. Messages arrive as bytes, get written into Wasm linear +> memory, and the guest's `handle` export is called. +> +> ~350 lines of Rust (host) · 3 guest modules · 7 tests + +--- + +## Table of Contents + +1. [Overview & Motivation](#1-overview--motivation) +2. [What Was Built](#2-what-was-built) +3. [Guest ↔ Host Contract](#3-guest--host-contract) +4. [Handle Cycle (Hot Path)](#4-handle-cycle-hot-path) +5. [Guest Modules](#5-guest-modules) +6. [Design Decisions & Tradeoffs](#6-design-decisions--tradeoffs) +7. [Known Gaps & Future Improvements](#7-known-gaps--future-improvements) +8. [Test Coverage Summary](#8-test-coverage-summary) + +--- + +## 1. Overview & Motivation + +Swactor already supported running *inside* a browser via `crates/wasm/` +(wasm-bindgen). This crate flips the direction: run untrusted Wasm code +*inside* an actor, sandboxed by wasmtime. Use cases include user-defined +plugins, multi-language actors, and capability-restricted compute. + +The main swactor crate has no wasmtime dependency — all Wasm machinery is +isolated in `crates/wasm-actor/`. + +--- + +## 2. What Was Built + +| Component | Location | Purpose | +|-----------|----------|---------| +| `swactor-wasm-actor` crate | `crates/wasm-actor/` | Host-side: engine, builder, actor impl | +| 3 guest crates | `crates/wasm-actor/tests/guests/{echo,double,silent}/` | `#![no_std]` Wasm modules for testing | +| Integration tests | `crates/wasm-actor/tests/wasm_actor.rs` | 7 behavioral tests | + +### Crate modules + +``` +crates/wasm-actor/src/ + lib.rs — ByteMessage, re-exports + engine.rs — SharedEngine (Arc) + builder.rs — WasmActorBuilder (compile + link + instantiate) + actor.rs — WasmActor implementing ActorInterface + error.rs — WasmActorError enum +``` + +### Public types + +- **`ByteMessage(pub Vec)`** — message type for Wasm actors. Satisfies + `Message` bounds trivially. +- **`SharedEngine`** — wraps `Arc`. Created once, cloned + cheaply across actors. Sandboxed config: no threads, no SIMD, no reference + types. +- **`WasmActorBuilder`** — takes an engine + raw `.wasm` bytes, compiles the + module, links the `swactor.send` host import, extracts typed function handles, + returns a `WasmActor`. +- **`WasmActor`** — implements `ActorInterface`. +- **`WasmActorError`** — `MissingExport(&'static str)` or `Wasmtime(wasmtime::Error)`. + +--- + +## 3. Guest ↔ Host Contract + +**Guest must export:** + +| Export | Signature | Purpose | +|--------|-----------|---------| +| `memory` | WebAssembly linear memory | Host reads/writes message bytes here | +| `alloc` | `(size: i32) -> i32` | Allocate `size` bytes, return pointer | +| `handle` | `(ptr: i32, len: i32)` | Process message at `(ptr, len)` | + +**Guest may import:** + +| Import | Module | Signature | Purpose | +|--------|--------|-----------|---------| +| `send` | `swactor` | `(dest_ptr: i32, payload_ptr: i32, payload_len: i32)` | Send a message to another actor | + +`dest_ptr` points to 32 bytes of `ActorAddress` in guest linear memory. +`payload_ptr` + `payload_len` describe the message bytes. + +--- + +## 4. Handle Cycle (Hot Path) + +``` + ByteMessage arrives + │ + v + 1. host calls guest alloc(msg.len) → ptr + │ + v + 2. host writes msg bytes into guest memory at ptr + │ + v + 3. host calls guest handle(ptr, len) + │ + ├── guest may call swactor.send() N times + │ └── each appends (ActorAddress, Vec) to HostState.outbox + │ + v + 4. host drains outbox → ctx.send(dest, ByteMessage(payload)) for each +``` + +Traps during `alloc` or `handle` will panic. Swactor's existing +`catch_unwind` in `tick_all` poisons the actor — consistent with the +panic-safety model. + +--- + +## 5. Guest Modules + +Three `#![no_std]` Rust crates compiled to `wasm32-unknown-unknown`: + +| Guest | Behavior | Tests it supports | +|-------|----------|-------------------| +| `echo` | Reads 32-byte dest + payload from message; sends payload back to dest | Echo roundtrip, binary preservation | +| `double` | Same framing; sends payload back **twice** | Multi-send verification | +| `silent` | Receives bytes; does nothing | No-output / no-error baseline | + +Each guest uses a simple inline bump allocator (64 KiB heap, 8-byte aligned) +and a `#[panic_handler]` that loops. No external dependencies. + +Message framing convention: the first 32 bytes of the `ByteMessage` payload +are the destination `ActorAddress`, followed by the actual message bytes. +This allows guests to send replies without hardcoding addresses. + +### Building guests + +```bash +rustup target add wasm32-unknown-unknown # one-time + +cd crates/wasm-actor/tests/guests/echo && cargo build --target wasm32-unknown-unknown --release +cd crates/wasm-actor/tests/guests/double && cargo build --target wasm32-unknown-unknown --release +cd crates/wasm-actor/tests/guests/silent && cargo build --target wasm32-unknown-unknown --release +``` + +Each guest crate has its own `[workspace]` marker to stay independent of the +root workspace. + +--- + +## 6. Design Decisions & Tradeoffs + +| # | Decision | Rationale | +|---|----------|-----------| +| 1 | **wasmtime, not wasmer/wasm3** | Best-maintained, fuel metering support, cranelift JIT | +| 2 | **Raw bytes, not structured messages** | Keeps the boundary simple; framing/serialization is the guest's concern | +| 3 | **Separate crate, not a feature flag** | wasmtime is ~30 crates; most users don't need it in their dependency tree | +| 4 | **Bump allocator in guests** | Zero-dependency, predictable, sufficient for request/response patterns | +| 5 | **Dest address in message payload** | Avoids hardcoded addresses; guests can send to any actor the host tells them about | +| 6 | **Traps = panics (no Result)** | Matches swactor's existing panic-safety model; `catch_unwind` in `tick_all` poisons the actor | +| 7 | **Engine sharing via Arc** | Module compilation is expensive; `SharedEngine` amortizes it across actors | +| 8 | **Maximum sandboxing defaults** | Disabled: threads, SIMD, relaxed SIMD, reference types, multi-value. Enabled: bulk memory (required by most compilers) | + +--- + +## 7. Known Gaps & Future Improvements + +| # | Gap | Notes | +|---|-----|-------| +| 1 | **No fuel metering** | wasmtime supports fuel; maps naturally to per-tick actor budgets. Deferred to follow-up. | +| 2 | **No WASI** | No filesystem, network, random, or clock access. Intentional for sandboxing, but limits guest capabilities. | +| 3 | **No guest SDK crate** | The test guests serve as examples. A published `swactor-guest` crate with the alloc/handle/send glue would reduce boilerplate. | +| 4 | **Bump allocator never frees** | Fine for short-lived handle calls, but long-running actors would need a real allocator. | +| 5 | **No pre-compilation cache** | `Module::new()` recompiles every time. wasmtime supports serialized modules for faster cold starts. | +| 6 | **`cargo test -p` doesn't resolve** | Must use `--manifest-path`. Workspace resolution quirk. | + +--- + +## 8. Test Coverage Summary + +7 behavioral tests in `crates/wasm-actor/tests/wasm_actor.rs`: + +| Test | Scenario | +|------|----------| +| `echo_returns_same_payload` | Send bytes → wasm echoes them back to inbox | +| `echo_preserves_binary_payload` | All 256 byte values survive the roundtrip | +| `silent_produces_no_output` | Guest does nothing; no error, no messages | +| `double_sends_two_copies` | One message in → two messages out | +| `missing_alloc_export_returns_error` | WAT module with no exports → `WasmActorError::MissingExport` | +| `shared_engine_serves_multiple_actors` | Two actors from the same `SharedEngine` work independently | +| `native_actor_communicates_with_wasm_actor` | Native Rust actor → WasmActor → inbox (two-tick delivery) | diff --git a/docs/wasm-actor.md b/docs/wasm-actor.md new file mode 100644 index 0000000..6d449ce --- /dev/null +++ b/docs/wasm-actor.md @@ -0,0 +1,115 @@ +# Wasm Actor + +The `swactor-wasm-actor` crate runs WebAssembly guest code inside a swactor +actor. The Wasm instance is sandboxed by [wasmtime](https://wasmtime.dev/). + +## Architecture + +``` + ┌─ Runtime ──────────────────────────────────────────────────────────────┐ + │ │ + │ ┌─ WasmActor ──────────────────────────────────────────────────────┐ │ + │ │ │ │ + │ │ Store -- wasmtime store with outbox │ │ + │ │ Memory -- guest linear memory │ │ + │ │ alloc: TypedFunc -- guest allocator │ │ + │ │ handle: TypedFunc -- guest message handler │ │ + │ │ │ │ + │ │ impl ActorInterface for WasmActor │ │ + │ │ Incoming = ByteMessage │ │ + │ │ Response = () │ │ + │ │ │ │ + │ └──────────────────────────────────────────────────────────────────┘ │ + │ │ + │ ┌─ Native Actors ─────────────────────────────────────────────────┐ │ + │ │ (can exchange ByteMessage with WasmActors normally) │ │ + │ └─────────────────────────────────────────────────────────────────┘ │ + │ │ + └────────────────────────────────────────────────────────────────────────┘ +``` + +## Message Flow + +``` + Host Guest (Wasm) + ──── ──────────── + + ByteMessage arrives + │ + ├─1─ call alloc(len) ──────────► bump-allocate, return ptr + │ + ├─2─ write bytes at ptr ───────► (memory updated) + │ + ├─3─ call handle(ptr, len) ────► process message + │ │ + │ ◄── swactor.send() ────────────┤ (0..N times) + │ (buffered in HostState.outbox) │ + │ │ + ├─4─ drain outbox ◄────────────── handle returns + │ + v + ctx.send(dest, ByteMessage) for each outbox entry +``` + +## Guest Contract + +Guests are standalone `wasm32-unknown-unknown` modules. They export three +symbols and may import one: + +| Direction | Module | Symbol | Signature | +|-----------|--------|--------|-----------| +| **export** | — | `memory` | linear memory | +| **export** | — | `alloc` | `(i32) -> i32` | +| **export** | — | `handle` | `(i32, i32) -> ()` | +| **import** | `swactor` | `send` | `(i32, i32, i32) -> ()` | + +The `send` import takes `(dest_ptr, payload_ptr, payload_len)` where +`dest_ptr` points to a 32-byte `ActorAddress` in guest memory. + +## Usage + +```rust +use swactor::runtime::{Runtime, RuntimeConfig}; +use swactor_wasm_actor::{ByteMessage, SharedEngine, WasmActorBuilder}; + +// Create a shared engine (once) +let engine = SharedEngine::new().unwrap(); + +// Build an actor from .wasm bytes +let wasm_bytes = std::fs::read("my_guest.wasm").unwrap(); +let actor = WasmActorBuilder::new(engine, wasm_bytes) + .build() + .unwrap(); + +// Use it like any other actor +let rt = Runtime::new(RuntimeConfig::default()); +let addr = rt.spawn(actor).unwrap(); +rt.send_to(addr, ByteMessage(b"hello".to_vec())).unwrap(); +rt.tick(); +``` + +## Sandboxing + +The `SharedEngine` disables all optional Wasm proposals: + +- Threads — disabled +- SIMD / relaxed SIMD — disabled +- Reference types — disabled +- Multi-value — disabled +- Bulk memory — **enabled** (required by most Rust/LLVM toolchains) + +No WASI imports are linked. Guests have no access to the filesystem, network, +clock, or random number generator. The only host function available is +`swactor.send`. + +## Where Things Live + +| File | Purpose | +|------|---------| +| `crates/wasm-actor/src/lib.rs` | `ByteMessage` + re-exports | +| `crates/wasm-actor/src/engine.rs` | `SharedEngine` — sandboxed wasmtime config | +| `crates/wasm-actor/src/builder.rs` | `WasmActorBuilder` — compile, link, instantiate | +| `crates/wasm-actor/src/actor.rs` | `WasmActor` — `ActorInterface` impl | +| `crates/wasm-actor/src/error.rs` | `WasmActorError` | +| `crates/wasm-actor/tests/guests/` | Three test guest crates (echo, double, silent) | +| `crates/wasm-actor/tests/wasm_actor.rs` | 7 integration tests | -- 2.45.2 From 473999d1df66b94deaacd4f57be8b5a0060092a0 Mon Sep 17 00:00:00 2001 From: Developer Date: Thu, 12 Feb 2026 17:44:06 +0000 Subject: [PATCH 2/5] =?UTF-8?q?feat:=20actor=20watching=20=E2=80=94=20loca?= =?UTF-8?q?l=20death=20notifications?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add watch/unwatch API to the actor system so actors can monitor each other's liveness. When a watched actor dies (panic or stop), watchers receive an ActorExited notification via on_actor_exit(). - ExitReason enum (Stopped, Panicked, NodeDown) and ActorExited struct - ContextInner::watch()/unwatch() + Ctx typed wrappers - ActorInterface::on_actor_exit() default method (system message fallback) - WatchRegistry in worker with bidirectional tracking - Death notification dispatch as phase 5b in tick_once - Runtime-level watch for external callers - 10 behavioral tests in tests/watch_api.rs - Design documents for OS features in docs/os-design/ Co-Authored-By: Claude Opus 4.6 --- CLAUDE/TASK.md | 42 +++ CLAUDE/notes/progress.md | 36 +++ docs/os-design/01-actor-watching.md | 335 +++++++++++++++++++++ docs/os-design/02-cluster-registry.md | 321 ++++++++++++++++++++ docs/os-design/03-node-capabilities.md | 306 +++++++++++++++++++ docs/os-design/04-command-interface.md | 398 +++++++++++++++++++++++++ docs/os-design/05-supervision.md | 263 ++++++++++++++++ docs/os-design/README.md | 153 ++++++++++ src/actor.rs | 63 +++- src/delivery.rs | 4 +- src/runtime.rs | 30 +- src/worker.rs | 165 +++++++++- tests/watch_api.rs | 378 +++++++++++++++++++++++ 13 files changed, 2481 insertions(+), 13 deletions(-) create mode 100644 CLAUDE/TASK.md create mode 100644 CLAUDE/notes/progress.md create mode 100644 docs/os-design/01-actor-watching.md create mode 100644 docs/os-design/02-cluster-registry.md create mode 100644 docs/os-design/03-node-capabilities.md create mode 100644 docs/os-design/04-command-interface.md create mode 100644 docs/os-design/05-supervision.md create mode 100644 docs/os-design/README.md create mode 100644 tests/watch_api.rs diff --git a/CLAUDE/TASK.md b/CLAUDE/TASK.md new file mode 100644 index 0000000..762d572 --- /dev/null +++ b/CLAUDE/TASK.md @@ -0,0 +1,42 @@ +Plan: + You are to improve this codebase via: + - implementing the features found in `docs/os-design` + - writing comphrehensive tests that check behavior makes sense + +Workflow: + - Read `CLAUDE/TASK.md` and `CLAUDE/notes/progress.md` + - Identify what stage you are on. + - Read and update yourself as necessary. + - Proceed to accomplishing the next task as written in `progress.md` + - For each attempt at any step, keep a record. If you reach attempt 3, step back, document, and try something else. + - When done, because attempt limit or task success: + - update `progress.md` with: + - Completed this session + - Next steps (specific, actionable) + - Open Questions + - Blockers + - make a commit + - compress your context and start the loop again + +Style: + - Do not add to existing modules in the root swactor `src/` they should stay as they are. You may modify but not change module structure. + - Integration tests in `tests/`, benchmark code in `benches/` + - cap execution time at 2 minutes max for fuzz, or benchmarks, or single test suite + - if they take too long, refactor and break up into logical modules + - You may modify these as you wish, so long as logical 'coverage' does not decline. + - Report all your changes to architecture with changes to the `docs/` items + - all notes you wish to keep across iterations shall go in the `CLAUDE/notes/` folder + +Example loop (not restrictive, feel free to ignore if prudent): + - Pick an item to implement from the os-design docs + - make analysis + - implement plan + - execute + - evaluate + - compress and move on to the next item + +Before git commit: + - all `cargo test` passes, including feature gated material + - if a test fails, investigate do not ignore or delete + - You can combine tests but not skip code paths or delete them for active code + - if a fix takes > 3 attempts, log and move on \ No newline at end of file diff --git a/CLAUDE/notes/progress.md b/CLAUDE/notes/progress.md new file mode 100644 index 0000000..12bc493 --- /dev/null +++ b/CLAUDE/notes/progress.md @@ -0,0 +1,36 @@ +# Progress + +## Completed + +### Feature 1: Actor Watching (local only) — `docs/os-design/01-actor-watching.md` +- Added `ExitReason` enum (Stopped, Panicked, NodeDown) and `ActorExited` struct to `src/actor.rs` +- Extended `ContextInner` trait with `watch()`/`unwatch()` methods +- Added `Ctx::watch(target)` and `Ctx::unwatch(target)` typed API +- Added `on_actor_exit()` default method to `ActorInterface` trait +- Updated `AnyActor::handle_any` with system message fallback (tries `ActorExited` after `Incoming`) +- Implemented `WatchRegistry` in `src/worker.rs` (bidirectional HashMap tracking) +- Integrated death notification dispatch as phase 5b in `tick_once` +- Implemented `watch`/`unwatch` on `Runtime`'s `ContextInner` impl +- Added `watch_registry` field to `TickContext` in `src/delivery.rs` +- 10 behavioral tests in `tests/watch_api.rs` — all passing +- All 53 tests pass (42 existing + 10 new + 1 doctest) +- All 134 distribution crate tests pass + +## Next Steps +1. **Command Interface** — `docs/os-design/04-command-interface.md` + - Create `crates/command/` crate with `CommandHandler` trait, `CommandRouter`, `CommandRequest`/`CommandResponse` types + - Extract existing `cmd_*` functions from `crates/runtime-dashboard/src/investigate.rs` into CommandHandler impls + - Add built-in read commands (help, overview, nodes, workers, actors) + - Add built-in write commands (spawn, stop, send, drain, shutdown) + - Write behavioral tests +2. **Cluster Registry** — `docs/os-design/02-cluster-registry.md` +3. **Node Capabilities** — `docs/os-design/03-node-capabilities.md` +4. **Remote Watching** — extends 01-actor-watching with wire protocol +5. **Supervision** — `docs/os-design/05-supervision.md` + +## Open Questions +- Command Interface: Should custom actor commands (via `Ctx::register_command()`) be deferred to a later PR? +- Should the dashboard migration happen in the same PR as the command crate creation? + +## Blockers +- None diff --git a/docs/os-design/01-actor-watching.md b/docs/os-design/01-actor-watching.md new file mode 100644 index 0000000..c7ccff6 --- /dev/null +++ b/docs/os-design/01-actor-watching.md @@ -0,0 +1,335 @@ +# Actor Watching — Distributed Death Notifications + +## Problem + +When an actor dies (panic, explicit stop, or its host node leaves the cluster), other actors that depend on it have no way to know. This is the distributed equivalent of `waitpid()` / `SIGCHLD` — the foundational primitive for building supervision, reconnection logic, and self-healing. + +## Design + +### Types + +```rust +// src/actor.rs + +/// Why an actor exited. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum ExitReason { + /// Actor was explicitly stopped or removed from the pool. + Stopped, + /// Actor panicked during message handling. + Panicked, + /// The node hosting the actor left the cluster (SWIM Dead). + NodeDown, +} + +/// Delivered to watchers when a watched actor exits. +/// +/// Implements `Message` (Clone + Send + Sync + 'static) so it can be +/// delivered through normal mailbox channels. +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct ActorExited { + /// The address of the actor that died. + pub addr: ActorAddress, + /// Why it exited. + pub reason: ExitReason, +} +``` + +### API + +```rust +// src/actor.rs — extend ContextInner + +pub trait ContextInner { + // existing: + fn send_any(&self, addr: ActorAddress, msg: Box) -> Result<(), Error>; + fn spawn_any(&self, addr: ActorAddress, actor: Box); + + // new: + fn watch(&self, watcher: ActorAddress, target: ActorAddress); + fn unwatch(&self, watcher: ActorAddress, target: ActorAddress); +} + +// src/actor.rs — extend Ctx + +impl Ctx<'_> { + /// Watch another actor's liveness. If the target dies, this actor + /// receives an `ActorExited` message in its mailbox. + /// + /// Watching an already-dead or non-existent actor delivers + /// `ActorExited { reason: Stopped }` immediately (on next tick). + /// + /// Calling watch() multiple times on the same target is idempotent — + /// only one notification is delivered. + pub fn watch(&self, target: ActorAddress) { + self.inner.watch(self.self_addr, target); + } + + /// Stop watching an actor. No notification will be delivered if the + /// target subsequently dies. + pub fn unwatch(&self, target: ActorAddress) { + self.inner.unwatch(self.self_addr, target); + } +} +``` + +### WatchRegistry + +A per-worker structure that tracks watch relationships: + +```rust +// src/worker.rs + +struct WatchRegistry { + /// target -> set of watchers awaiting death notification + watchers: HashMap>, + /// watcher -> set of targets it's watching (reverse index for cleanup) + watching: HashMap>, +} + +impl WatchRegistry { + fn watch(&mut self, watcher: ActorAddress, target: ActorAddress) { + self.watchers.entry(target).or_default().insert(watcher); + self.watching.entry(watcher).or_default().insert(target); + } + + fn unwatch(&mut self, watcher: ActorAddress, target: ActorAddress) { + if let Some(set) = self.watchers.get_mut(&target) { + set.remove(&watcher); + if set.is_empty() { self.watchers.remove(&target); } + } + if let Some(set) = self.watching.get_mut(&watcher) { + set.remove(&target); + if set.is_empty() { self.watching.remove(&watcher); } + } + } + + /// Called when an actor dies. Returns (watcher_addr, ActorExited) pairs. + fn notify_death(&mut self, target: ActorAddress, reason: ExitReason) + -> Vec<(ActorAddress, ActorExited)> + { + let notification = ActorExited { addr: target, reason }; + let mut result = Vec::new(); + + if let Some(watcher_set) = self.watchers.remove(&target) { + for watcher in &watcher_set { + result.push((*watcher, notification.clone())); + // clean up reverse index + if let Some(set) = self.watching.get_mut(watcher) { + set.remove(&target); + if set.is_empty() { self.watching.remove(watcher); } + } + } + } + + result + } + + /// Called when a watcher itself dies. Cleans up all its watching entries. + fn cleanup_watcher(&mut self, watcher: &ActorAddress) { + if let Some(targets) = self.watching.remove(watcher) { + for target in targets { + if let Some(set) = self.watchers.get_mut(&target) { + set.remove(watcher); + if set.is_empty() { self.watchers.remove(&target); } + } + } + } + } +} +``` + +### Integration with tick_once + +The worker's `tick_once` gains **two new mechanisms**: + +**A. Watch/unwatch request processing** — `WorkerContext::watch()` needs to route the request to the correct worker (the one that owns the target). This mirrors how sends work: + +``` +watch(watcher_A, target_B) called on worker 0: + - target_B is on worker 0? → register locally in WatchRegistry + - target_B is on worker 1? → send WatchCommand through transfer queue + - target_B not in address_map? → may be remote (see Remote Watches below) +``` + +New envelope variant for internal watch commands: + +```rust +// src/delivery.rs + +enum InternalCommand { + Watch { watcher: ActorAddress, target: ActorAddress }, + Unwatch { watcher: ActorAddress, target: ActorAddress }, +} +``` + +These are delivered through the existing transfer queue alongside `Envelope`s. The transfer queue type becomes `enum TransferItem { Message(Envelope), Command(InternalCommand) }`, or — simpler — the WatchRegistry is shared (behind Arc) and watch/unwatch are applied directly. The shared approach is better since watches are rare relative to messages. + +**Recommended**: `Arc>` shared across workers, owned by Runtime. Workers hold a reference. Contention is negligible because watch/unwatch operations are rare. + +**B. Death notification dispatch** — added to tick_all's panic detection: + +```rust +// In ActorPool::tick_all, after catching a panic: +Err(_) => { + slot.poisoned = true; + slot.mailbox.clear(); + // NEW: collect death notification + deaths.push((addr, ExitReason::Panicked)); +} +``` + +After tick_all completes, the worker processes `deaths`: + +```rust +// In tick_once, after tick_all: +for (addr, reason) in deaths { + let notifications = watch_registry.lock().notify_death(addr, reason); + for (watcher_addr, msg) in notifications { + // Deliver ActorExited as a normal message + self.deliver_to(watcher_addr, Box::new(msg), tc); + } + // Also clean up the dead actor's own watches + watch_registry.lock().cleanup_watcher(&addr); +} +``` + +### Watching Non-Existent Actors + +If `watch(watcher, target)` is called and `target` doesn't exist in the address map: + +- **Local runtime**: deliver `ActorExited { reason: Stopped }` immediately (on next tick). The actor is already gone. +- **Distributed**: the watch request is forwarded to the node that should own the target (via Kademlia resolution). If the target doesn't exist there either, a `ActorExitedNotify` is sent back. + +### Remote Watches (cross-node) + +Wire protocol additions in `crates/distribution/src/messages.rs`: + +```rust +/// Request from node A to node B: "notify me if this actor dies" +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WatchRequest { + pub watcher_node: NodeId, + pub watcher_addr: ActorAddress, + pub target_addr: ActorAddress, +} +impl NetworkMessage for WatchRequest { + fn type_tag() -> &'static str { "swactor_dist::WatchRequest" } +} + +/// Request to cancel a remote watch +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct UnwatchRequest { + pub watcher_node: NodeId, + pub watcher_addr: ActorAddress, + pub target_addr: ActorAddress, +} +impl NetworkMessage for UnwatchRequest { + fn type_tag() -> &'static str { "swactor_dist::UnwatchRequest" } +} + +/// Notification from target's node to watcher's node +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ActorExitedNotify { + pub target_addr: ActorAddress, + pub reason: ExitReason, + /// Which watchers on the receiving node should be notified + pub watchers: Vec, +} +impl NetworkMessage for ActorExitedNotify { + fn type_tag() -> &'static str { "swactor_dist::ActorExitedNotify" } +} +``` + +**Flow**: + +``` +Node A Node B + │ │ + │ ctx.watch(watcher_A, target_B) │ + │ │ + │ ── WatchRequest ──────────────► │ + │ │ registers remote watch: + │ │ target_B → (NodeA, watcher_A) + │ │ + │ ... time passes ... │ + │ │ + │ │ target_B panics + │ │ + │ ◄── ActorExitedNotify ──────── │ + │ │ + │ delivers ActorExited to │ + │ watcher_A's mailbox │ +``` + +The `WatchRegistry` on node B stores remote watches with the additional `NodeId` of the watcher's node. On death, it partitions notifications into local (deliver directly) and remote (send `ActorExitedNotify` to the watcher's node). + +### SWIM Integration + +When `handle_membership_change` detects `MemberState::Dead` (`crates/distribution/src/node.rs:269`): + +1. The node maintains a **node actor index**: `NodeId -> Set` — all actors known to be on each node. This is populated from: + - Directory entries stored locally + - Cache entries + - Remote watch registrations + +2. On node death: + ```rust + fn handle_membership_change(&mut self, node_id: NodeId, state: MemberState) { + match state { + MemberState::Dead => { + // existing cleanup... + self.routing_table.remove(&node_id); + self.cache.invalidate_node(&node_id); + self.repair_queue.on_node_death(&node_id, &mut self.directory); + + // NEW: notify local watchers about all actors on the dead node + if let Some(actor_addrs) = self.node_actor_index.remove(&node_id) { + for actor_addr in actor_addrs { + let notifications = self.watch_registry.notify_death( + actor_addr, + ExitReason::NodeDown, + ); + // Queue for delivery to local actors + self.pending_exit_notifications.extend(notifications); + } + } + } + // ... + } + } + ``` + +3. The `pending_exit_notifications` are drained by the NodeDriver on the next tick and delivered into the local runtime. + +### Edge Cases + +| Scenario | Behavior | +|---|---| +| Watch self | Allowed. On death, ActorExited delivered to own mailbox (no-op since dead). | +| Watch already-dead actor | `ActorExited { reason: Stopped }` delivered on next tick. | +| Watcher dies before target | Cleanup removes all watching entries. No notification delivered. | +| Target node suspected (not yet dead) | No notification — wait for SWIM to confirm Dead or Alive. | +| Network partition heals | If target was falsely declared Dead, a stale `NodeDown` was sent. The watcher may re-watch. No automatic "un-death" notification. | +| Double watch | Idempotent — only one notification per death event. | + +### Files Modified + +| File | Change | +|------|--------| +| `src/actor.rs` | `ExitReason`, `ActorExited`, `watch()`/`unwatch()` on `ContextInner`, `Ctx` | +| `src/worker.rs` | `WatchRegistry`, death collection in `tick_all`, notification dispatch in `tick_once` | +| `src/runtime.rs` | `Arc>` owned by Runtime, passed to workers | +| `crates/distribution/src/messages.rs` | `WatchRequest`, `UnwatchRequest`, `ActorExitedNotify` | +| `crates/distribution/src/node.rs` | Node actor index, SWIM Dead fan-out, `pending_exit_notifications` | + +### Tests + +- **watch_local_death**: spawn watcher + target, kill target (panic), verify watcher receives `ActorExited { reason: Panicked }` +- **unwatch_prevents_notification**: watch then unwatch, kill target, verify no notification +- **watch_nonexistent**: watch an address that was never spawned, verify `ActorExited { reason: Stopped }` +- **watcher_dies_first**: watch target, kill watcher, kill target — no panic/leak +- **cross_worker_watch**: target on worker 0, watcher on worker 1, kill target, verify notification arrives +- **idempotent_watch**: watch same target twice, kill target, verify exactly one notification diff --git a/docs/os-design/02-cluster-registry.md b/docs/os-design/02-cluster-registry.md new file mode 100644 index 0000000..63084a8 --- /dev/null +++ b/docs/os-design/02-cluster-registry.md @@ -0,0 +1,321 @@ +# Cluster-Wide Registry — Distributed Naming + +## Problem + +Actors can only be found by their `ActorAddress` (a random 32-byte ID). The local `AddressMap` maps addresses to workers on a single node. The Kademlia directory maps addresses to `NodeId`. But neither provides **human-readable naming** or **re-discovery after churn**. + +When a node dies and an actor is re-spawned elsewhere, it gets a new `ActorAddress`. Without a name-based registry, every actor that communicated with it needs manual reconfiguration. This doesn't work for churning infrastructure. + +## Design + +### Approach: Gossip-Propagated LWW-Register CRDT + +Each name binding is a **Last-Writer-Wins Register** — the most recent write (by timestamp) wins. This matches SWIM's eventual-consistency model and reuses the existing gossip piggyback mechanism. + +**Why not Raft/consensus?** +- Overkill for name resolution. Names don't need linearizability — eventual consistency is fine. +- SWIM already solves dissemination. We piggyback registry updates on existing protocol messages for free. +- Consensus requires a stable quorum, which conflicts with the "nodes pop in and out" use case. + +**Why not extend Kademlia?** +- Kademlia maps `ActorAddress -> NodeId`. Names are a different key space (`String -> ActorAddress`). +- Kademlia lookups are multi-hop (iterative). Registry lookups should be local (every node has a full replica). +- The registry is small (hundreds to low-thousands of names). Full replication is cheap. + +### Types + +```rust +// crates/distribution/src/registry.rs + +/// A single name binding in the cluster registry. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RegistryEntry { + /// Human-readable name (e.g. "worker-pool", "metrics-collector"). + pub name: String, + /// The actor address this name resolves to. + pub actor_addr: ActorAddress, + /// The node that owns this binding. + pub node_id: NodeId, + /// Logical timestamp for LWW conflict resolution. + pub timestamp: u64, + /// Generation — incremented on re-registration of the same name. + pub generation: u64, + /// Tombstone — true means the name has been unregistered. + pub tombstone: bool, +} + +/// Events emitted by the registry for subscribers. +#[derive(Debug, Clone)] +pub enum RegistryEvent { + /// A name was registered or updated. + Registered { + name: String, + actor_addr: ActorAddress, + node_id: NodeId, + }, + /// A name was unregistered (tombstoned). + Unregistered { + name: String, + previous_addr: ActorAddress, + }, +} + +/// The local replica of the cluster-wide registry. +pub struct ClusterRegistry { + /// Current state: name -> latest entry. + entries: HashMap, + /// Pending entries to propagate via gossip (not yet disseminated to all). + pending: VecDeque, + /// Logical clock for this node. + clock: u64, + /// Recent events for subscribers. + events: VecDeque, + /// Max events to buffer. + max_events: usize, +} +``` + +### CRDT Merge Rule + +```rust +impl ClusterRegistry { + /// Merge a remote entry. Returns true if the local state changed. + pub fn merge(&mut self, remote: RegistryEntry) -> bool { + match self.entries.get(&remote.name) { + Some(local) => { + // LWW: higher timestamp wins. + // Tie-break: higher generation, then higher node_id (deterministic). + let dominated = remote.timestamp > local.timestamp + || (remote.timestamp == local.timestamp + && remote.generation > local.generation) + || (remote.timestamp == local.timestamp + && remote.generation == local.generation + && remote.node_id.0 > local.node_id.0); + + if dominated { + self.apply(remote); + true + } else { + false + } + } + None => { + self.apply(remote); + true + } + } + } + + fn apply(&mut self, entry: RegistryEntry) { + let event = if entry.tombstone { + let prev = self.entries.get(&entry.name) + .map(|e| e.actor_addr); + RegistryEvent::Unregistered { + name: entry.name.clone(), + previous_addr: prev.unwrap_or_default(), + } + } else { + RegistryEvent::Registered { + name: entry.name.clone(), + actor_addr: entry.actor_addr, + node_id: entry.node_id, + } + }; + self.events.push_back(event); + if self.events.len() > self.max_events { + self.events.pop_front(); + } + self.entries.insert(entry.name.clone(), entry); + } +} +``` + +### API + +On `DistributedNode`: + +```rust +// crates/distribution/src/node.rs + +impl DistributedNode { + /// Register a name -> actor binding on this node. + /// The binding is propagated to all cluster members via gossip. + pub fn register_name(&mut self, name: &str, actor_addr: ActorAddress) { + self.registry.clock += 1; + let entry = RegistryEntry { + name: name.to_string(), + actor_addr, + node_id: self.node_id(), + timestamp: self.registry.clock, + generation: self.registry.next_generation(name), + tombstone: false, + }; + self.registry.merge(entry.clone()); + self.registry.pending.push_back(entry); + } + + /// Remove a name binding. Propagated as a tombstone. + pub fn unregister_name(&mut self, name: &str) { + self.registry.clock += 1; + let actor_addr = self.registry.entries.get(name) + .map(|e| e.actor_addr) + .unwrap_or_default(); + let entry = RegistryEntry { + name: name.to_string(), + actor_addr, + node_id: self.node_id(), + timestamp: self.registry.clock, + generation: 0, + tombstone: true, + }; + self.registry.merge(entry.clone()); + self.registry.pending.push_back(entry); + } + + /// Resolve a name to an actor address (local replica, eventually consistent). + pub fn resolve_name(&self, name: &str) -> Option<(ActorAddress, NodeId)> { + self.registry.entries.get(name) + .filter(|e| !e.tombstone) + .map(|e| (e.actor_addr, e.node_id)) + } + + /// Drain buffered registry events (for subscribers). + pub fn registry_events(&mut self) -> Vec { + self.registry.events.drain(..).collect() + } +} +``` + +On `Ctx` (actor-level, requires distribution feature): + +```rust +// src/actor.rs — requires ContextInner extensions + +impl Ctx<'_> { + /// Register this actor under a name in the cluster registry. + pub fn register_as(&self, name: &str) { + self.inner.register_name(self.self_addr, name); + } + + /// Resolve a name to an actor address. + pub fn resolve_name(&self, name: &str) -> Option { + self.inner.resolve_name(name) + } +} +``` + +The `ContextInner` trait gains two new methods: + +```rust +pub trait ContextInner { + // ... existing methods ... + fn register_name(&self, addr: ActorAddress, name: &str) { /* default no-op */ } + fn resolve_name(&self, name: &str) -> Option { None } +} +``` + +Default implementations return `None` / no-op so that non-distributed runtimes don't break. + +### Gossip Propagation + +Registry entries are piggybacked on SWIM protocol messages, reusing the existing dissemination mechanism. + +Currently, `crates/distribution/src/swim/dissemination.rs` encodes membership updates into the piggyback payload: + +``` +piggyback bytes = bincode(Vec) +``` + +Extended format: + +``` +piggyback bytes = bincode(PiggybackPayload { + membership: Vec, + registry: Vec, // NEW +}) +``` + +```rust +// crates/distribution/src/swim/dissemination.rs + +#[derive(Serialize, Deserialize)] +struct PiggybackPayload { + membership: Vec, + registry: Vec, +} +``` + +The dissemination buffer manages registry entries the same way as membership updates: +- Each entry has a dissemination count (how many times it's been piggybacked). +- After `log2(N) + 1` disseminations (where N = cluster size), the entry is retired. +- Piggyback space is shared: membership updates take priority, registry entries fill remaining space. + +### Node Death Handling + +When SWIM marks a node as `Dead`: + +```rust +fn handle_membership_change(&mut self, node_id: NodeId, state: MemberState) { + if state == MemberState::Dead { + // ... existing cleanup ... + + // NEW: tombstone all registry entries owned by the dead node + let to_tombstone: Vec = self.registry.entries.iter() + .filter(|(_, e)| e.node_id == node_id && !e.tombstone) + .map(|(name, _)| name.clone()) + .collect(); + + for name in to_tombstone { + self.registry.clock += 1; + let entry = RegistryEntry { + name: name.clone(), + tombstone: true, + timestamp: self.registry.clock, + // ... fill from existing entry ... + }; + self.registry.merge(entry.clone()); + self.registry.pending.push_back(entry); + } + } +} +``` + +### Interaction with Actor Watching + +The registry and watching system compose naturally: + +1. Actor A resolves name "service-X" → gets address B on Node 2. +2. Actor A calls `ctx.watch(B)`. +3. Node 2 dies. Actor A receives `ActorExited { addr: B, reason: NodeDown }`. +4. A supervisor re-spawns "service-X" on Node 3 → new address C. +5. The supervisor calls `register_name("service-X", C)`. +6. Gossip propagates the update. +7. Actor A (or anyone) calls `resolve_name("service-X")` → gets address C. +8. Actor A calls `ctx.watch(C)` to resume monitoring. + +### Tombstone Garbage Collection + +Tombstones accumulate over time. GC strategy: + +- Tombstones older than `tombstone_ttl` (default: 1 hour of logical clock ticks) are eligible for removal. +- GC runs periodically (e.g., every 1000 ticks). +- A tombstone is only removed if it has been fully disseminated (dissemination count >= threshold). + +### Files Modified + +| File | Change | +|------|--------| +| `crates/distribution/src/registry.rs` | **New file**: `ClusterRegistry`, `RegistryEntry`, `RegistryEvent`, CRDT merge | +| `crates/distribution/src/lib.rs` | `pub mod registry;` | +| `crates/distribution/src/node.rs` | `register_name`, `unregister_name`, `resolve_name`, node death tombstoning | +| `crates/distribution/src/swim/dissemination.rs` | `PiggybackPayload` extended with registry entries | +| `src/actor.rs` | `register_name`/`resolve_name` on `ContextInner` (default no-op), `Ctx` wrappers | + +### Tests + +- **register_and_resolve**: register a name, resolve it, verify correct address +- **lww_conflict**: two nodes register same name concurrently, verify latest timestamp wins +- **tombstone_propagation**: register name, unregister, verify tombstone propagates and resolve returns None +- **node_death_tombstones**: 3-node cluster, register name on node B, kill node B, verify name is tombstoned on surviving nodes +- **re_registration**: register name, unregister, re-register with new address, verify resolution +- **gossip_convergence**: register name on node A, verify all nodes resolve it after gossip settles diff --git a/docs/os-design/03-node-capabilities.md b/docs/os-design/03-node-capabilities.md new file mode 100644 index 0000000..88dcc62 --- /dev/null +++ b/docs/os-design/03-node-capabilities.md @@ -0,0 +1,306 @@ +# Node Capabilities — Hardware Detection & Placement Constraints + +## Problem + +Swactor targets heterogeneous clusters: some nodes have GPUs, others have large RAM, others are lightweight ARM devices. When spawning an actor (e.g., a model inference worker), the system needs to place it on a node with the right hardware. Today, placement is round-robin — no awareness of what each node can do. + +## Design + +### Separate Crate + +`crates/capabilities/` is a **standalone crate** with no dependency on the swactor core runtime. It's a pure detection + constraint-matching library. + +```toml +# crates/capabilities/Cargo.toml +[package] +name = "swactor-capabilities" +version = "0.1.0" +edition = "2024" + +[features] +default = ["detect"] +detect = ["dep:sysinfo"] +gpu-nvidia = [] +# gpu-vulkan = [] # future + +[dependencies] +serde = { version = "1", features = ["derive"] } +sysinfo = { version = "0.33", optional = true } +``` + +### Types + +```rust +// crates/capabilities/src/lib.rs + +/// A capability value. Kept simple — three variants cover all practical needs. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum CapValue { + Bool(bool), + Int(i64), + Str(String), +} + +/// All capabilities of a node. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct NodeCapabilities { + labels: BTreeMap, +} + +impl NodeCapabilities { + pub fn new() -> Self { Self::default() } + + /// Get a label value. + pub fn get(&self, key: &str) -> Option<&CapValue> { + self.labels.get(key) + } + + /// Set a label. + pub fn set(&mut self, key: impl Into, value: CapValue) { + self.labels.insert(key.into(), value); + } + + /// Merge in additional labels (overwriting on conflict). + pub fn with_labels(mut self, extra: BTreeMap) -> Self { + self.labels.extend(extra); + self + } + + /// Check if all constraints in a requirement are satisfied. + pub fn satisfies(&self, requirement: &PlacementRequirement) -> bool { + requirement.constraints.iter().all(|c| self.satisfies_one(c)) + } + + fn satisfies_one(&self, constraint: &PlacementConstraint) -> bool { + match constraint { + PlacementConstraint::Equals(key, expected) => { + self.labels.get(key.as_str()) == Some(expected) + } + PlacementConstraint::MinInt(key, min) => { + matches!(self.labels.get(key.as_str()), Some(CapValue::Int(v)) if *v >= *min) + } + PlacementConstraint::HasLabel(key) => { + self.labels.contains_key(key.as_str()) + } + } + } + + /// All labels as a reference. + pub fn labels(&self) -> &BTreeMap { + &self.labels + } +} +``` + +### Auto-Detection + +```rust +impl NodeCapabilities { + /// Auto-detect system capabilities. + /// Always detects arch and os. Feature-gated backends detect more. + pub fn detect() -> Self { + let mut caps = Self::new(); + + // Always available (no feature gate) + caps.set("arch", CapValue::Str(std::env::consts::ARCH.to_string())); + caps.set("os", CapValue::Str(std::env::consts::OS.to_string())); + + #[cfg(feature = "detect")] + { + Self::detect_sysinfo(&mut caps); + } + + #[cfg(feature = "gpu-nvidia")] + { + Self::detect_nvidia(&mut caps); + } + + caps + } + + #[cfg(feature = "detect")] + fn detect_sysinfo(caps: &mut Self) { + use sysinfo::System; + let sys = System::new_all(); + + caps.set("cpu_count", CapValue::Int(sys.cpus().len() as i64)); + caps.set("ram_mb", CapValue::Int((sys.total_memory() / (1024 * 1024)) as i64)); + + if let Ok(hostname) = hostname::get() { + if let Some(name) = hostname.to_str() { + caps.set("hostname", CapValue::Str(name.to_string())); + } + } + } + + #[cfg(feature = "gpu-nvidia")] + fn detect_nvidia(caps: &mut Self) { + // Shell out to nvidia-smi for maximum compatibility. + // Parsing XML output is more robust than CSV for varying driver versions. + let output = std::process::Command::new("nvidia-smi") + .args(["--query-gpu=name,memory.total", "--format=csv,noheader,nounits"]) + .output(); + + match output { + Ok(out) if out.status.success() => { + let stdout = String::from_utf8_lossy(&out.stdout); + let lines: Vec<&str> = stdout.trim().lines().collect(); + caps.set("gpu_nvidia", CapValue::Bool(true)); + caps.set("gpu_count", CapValue::Int(lines.len() as i64)); + // First GPU's VRAM as representative + if let Some(line) = lines.first() { + let parts: Vec<&str> = line.split(", ").collect(); + if let Some(name) = parts.first() { + caps.set("gpu_name", CapValue::Str(name.trim().to_string())); + } + if let Some(vram) = parts.get(1).and_then(|s| s.trim().parse::().ok()) { + caps.set("gpu_vram_mb", CapValue::Int(vram)); + } + } + } + _ => { + caps.set("gpu_nvidia", CapValue::Bool(false)); + } + } + } +} +``` + +### Placement Constraints + +```rust +/// A single constraint on node capabilities. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum PlacementConstraint { + /// Label must exist and equal the given value. + Equals(String, CapValue), + /// Label must exist and be >= the given integer value. + MinInt(String, i64), + /// Label must exist (any value). + HasLabel(String), +} + +/// A full placement requirement. All constraints must be satisfied (AND). +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct PlacementRequirement { + pub constraints: Vec, +} + +impl PlacementRequirement { + pub fn new() -> Self { Self::default() } + + /// Builder: require a label equals a value. + pub fn equals(mut self, key: impl Into, value: CapValue) -> Self { + self.constraints.push(PlacementConstraint::Equals(key.into(), value)); + self + } + + /// Builder: require an integer label >= min. + pub fn min_int(mut self, key: impl Into, min: i64) -> Self { + self.constraints.push(PlacementConstraint::MinInt(key.into(), min)); + self + } + + /// Builder: require a label exists. + pub fn has(mut self, key: impl Into) -> Self { + self.constraints.push(PlacementConstraint::HasLabel(key.into())); + self + } + + /// Check if empty (no constraints — any node is acceptable). + pub fn is_empty(&self) -> bool { + self.constraints.is_empty() + } +} +``` + +### Integration with Distribution + +**NodeRecord extension** (`crates/distribution/src/types.rs`): + +```rust +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NodeRecord { + pub node_id: NodeId, + pub addr: SocketAddr, + pub state: MemberState, + pub incarnation: u64, + // NEW (optional — backwards compatible): + #[serde(default, skip_serializing_if = "Option::is_none")] + pub capabilities: Option, +} +``` + +**Capabilities flow**: + +1. On startup, the node detects capabilities: `NodeCapabilities::detect().with_labels(operator_labels)`. +2. Capabilities are included in the node's own `NodeRecord`. +3. When a node joins (via `JoinResponse`), it receives other nodes' capabilities. +4. Capabilities are piggybacked on SWIM protocol messages (membership updates already carry `NodeRecord`). + +**Cluster-level placement** (new function in distribution): + +```rust +// crates/distribution/src/node.rs + +impl DistributedNode { + /// Find nodes that satisfy a placement requirement. + /// Returns matching nodes sorted by preference (e.g., least loaded first). + pub fn find_suitable_nodes( + &self, + requirement: &PlacementRequirement, + ) -> Vec { + self.members() + .into_iter() + .filter(|node| { + node.capabilities.as_ref() + .map(|caps| caps.satisfies(requirement)) + .unwrap_or(requirement.is_empty()) + }) + .collect() + } +} +``` + +### Example Usage + +```rust +// Operator starts a node with custom labels: +let caps = NodeCapabilities::detect() + .with_labels(btreemap! { + "role".into() => CapValue::Str("inference".into()), + "region".into() => CapValue::Str("us-east".into()), + }); + +// An actor specifies placement requirements: +let requirement = PlacementRequirement::new() + .has("gpu_nvidia") + .min_int("gpu_vram_mb", 8000) + .equals("region", CapValue::Str("us-east".into())); + +// Supervisor finds suitable nodes: +let nodes = dist_node.find_suitable_nodes(&requirement); +``` + +### Files Modified + +| File | Change | +|------|--------| +| `crates/capabilities/` | **New crate** | +| `crates/capabilities/Cargo.toml` | Package definition, feature flags | +| `crates/capabilities/src/lib.rs` | `NodeCapabilities`, `CapValue`, `PlacementConstraint`, `PlacementRequirement`, detection | +| `crates/distribution/Cargo.toml` | Optional dependency on `swactor-capabilities` | +| `crates/distribution/src/types.rs` | Optional `capabilities` field on `NodeRecord` | +| `crates/distribution/src/node.rs` | `find_suitable_nodes()`, capabilities in join flow | +| `Cargo.toml` | Add `crates/capabilities` to workspace members | + +### Tests + +- **detect_basics**: `NodeCapabilities::detect()` always has `arch` and `os` labels +- **satisfies_equals**: constraint matches/doesn't match +- **satisfies_min_int**: integer comparison works correctly +- **satisfies_has_label**: existence check works +- **empty_requirement**: matches any node +- **combined_constraints**: multiple constraints all must pass (AND) +- **custom_labels**: operator labels merge correctly, override detection +- **find_suitable_nodes**: integration test with mock node records and varying capabilities diff --git a/docs/os-design/04-command-interface.md b/docs/os-design/04-command-interface.md new file mode 100644 index 0000000..1c87ae1 --- /dev/null +++ b/docs/os-design/04-command-interface.md @@ -0,0 +1,398 @@ +# Command Interface — Frontend-Agnostic Dispatch + +## Problem + +The runtime has an investigate/REPL protocol (`crates/runtime-dashboard/src/investigate.rs`) that accepts text commands and returns JSON. It works, but it's hardcoded to stdin/stdout and tightly coupled to the dashboard crate. We need the same commands accessible from: + +- Terminal CLI (stdin/stdout) +- TUI (the existing ratatui dashboard) +- REST API (the existing HTTP server) +- Future: WebSocket, remote CLI, programmatic SDK + +And we need write commands (spawn, stop, drain) — not just read-only inspection. + +## Design + +### Architecture + +``` + ┌──────────────────────────────────────────────────────────┐ + │ Frontends │ + │ │ + │ ┌──────────┐ ┌─────────┐ ┌──────────┐ ┌──────────┐ │ + │ │CLI / REPL│ │ TUI │ │REST /api/│ │ Future │ │ + │ │(stdin/ │ │(ratatui │ │cmd?name= │ │(websocket│ │ + │ │ stdout) │ │ events) │ │&arg=val │ │ etc.) │ │ + │ └────┬─────┘ └────┬────┘ └────┬─────┘ └────┬─────┘ │ + │ │ │ │ │ │ + │ └─────────────┴─────┬──────┴──────────────┘ │ + └───────────────────────────┼──────────────────────────────┘ + │ + ┌────────▼────────┐ + │ CommandRouter │ + │ │ + │ name → handler │ + │ dispatch() │ + └────────┬────────┘ + │ + ┌──────────────┼──────────────┐ + │ │ │ + ┌──────▼──────┐ ┌────▼────┐ ┌───────▼───────┐ + │ Built-in │ │Built-in │ │ Custom │ + │ Read Cmds │ │Write │ │ (actor- │ + │ (overview, │ │Cmds │ │ registered) │ + │ workers, │ │(spawn, │ │ │ + │ actors, │ │ stop, │ │ │ + │ hot, ...) │ │ drain) │ │ │ + └─────────────┘ └─────────┘ └────────────────┘ +``` + +### Core Types + +```rust +// crates/command/src/lib.rs + +use std::collections::HashMap; +use std::sync::Arc; + +/// A command request. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CommandRequest { + /// Command name (e.g., "overview", "spawn", "actors"). + pub command: String, + /// Named arguments. Values are JSON for flexibility. + pub args: HashMap, +} + +/// A command response. Always JSON-serializable. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CommandResponse { + pub ok: bool, + pub command: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub data: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl CommandResponse { + pub fn ok(command: &str, data: impl Serialize) -> Self { + Self { + ok: true, + command: command.to_string(), + data: Some(serde_json::to_value(data).unwrap_or(serde_json::Value::Null)), + error: None, + } + } + + pub fn err(command: &str, msg: impl Into) -> Self { + Self { + ok: false, + command: command.to_string(), + data: None, + error: Some(msg.into()), + } + } + + /// Serialize to a single JSON line (for REPL protocol). + pub fn to_json_line(&self) -> String { + serde_json::to_string(self).unwrap_or_else(|e| { + format!(r#"{{"ok":false,"command":"","error":"serialization: {e}"}}"#) + }) + } +} +``` + +### CommandHandler Trait + +```rust +/// Metadata about a command, used for help text and validation. +pub struct CommandMeta { + pub name: &'static str, + pub description: &'static str, + pub usage: &'static str, + /// Whether this command mutates state (spawn, stop, etc.) + pub is_write: bool, +} + +/// A command handler. Implementations are stateless — all state +/// comes through CommandContext. +pub trait CommandHandler: Send + Sync { + fn meta(&self) -> CommandMeta; + fn handle(&self, args: &HashMap, ctx: &CommandContext) -> CommandResponse; +} +``` + +### CommandContext + +```rust +/// Context available to command handlers. +/// +/// Contains references to runtime subsystems. Optional fields allow +/// commands to work in both standalone and distributed configurations. +pub struct CommandContext { + pub runtime: Arc, + pub stats_collector: Option>, + // Distribution (only present when running distributed) + pub dist_node: Option>>, + // Extensibility: arbitrary typed data that custom commands can access + extensions: HashMap>, +} + +impl CommandContext { + /// Retrieve a typed extension. + pub fn get_ext(&self) -> Option<&T> { + self.extensions.get(&std::any::TypeId::of::()) + .and_then(|b| b.downcast_ref()) + } + + /// Add a typed extension. + pub fn set_ext(&mut self, val: T) { + self.extensions.insert(std::any::TypeId::of::(), Box::new(val)); + } +} +``` + +### CommandRouter + +```rust +/// Central command dispatch. +pub struct CommandRouter { + handlers: HashMap>, +} + +impl CommandRouter { + pub fn new() -> Self { + Self { handlers: HashMap::new() } + } + + /// Register all built-in commands. + pub fn with_builtins(mut self) -> Self { + self.register(Box::new(builtins::HelpCommand)); + self.register(Box::new(builtins::OverviewCommand)); + self.register(Box::new(builtins::WorkersCommand)); + self.register(Box::new(builtins::WorkerCommand)); + self.register(Box::new(builtins::ActorsCommand)); + self.register(Box::new(builtins::ActorCommand)); + self.register(Box::new(builtins::HotCommand)); + self.register(Box::new(builtins::PhasesCommand)); + self.register(Box::new(builtins::DiffCommand)); + // Write commands + self.register(Box::new(builtins::SpawnCommand)); + self.register(Box::new(builtins::StopCommand)); + self.register(Box::new(builtins::ShutdownCommand)); + // Distribution-aware commands (no-op if dist_node is None) + self.register(Box::new(builtins::NodesCommand)); + self.register(Box::new(builtins::RegistryCommand)); + self.register(Box::new(builtins::ResolveCommand)); + self.register(Box::new(builtins::DrainCommand)); + self + } + + /// Register a custom command handler. + pub fn register(&mut self, handler: Box) { + let name = handler.meta().name.to_string(); + self.handlers.insert(name, handler); + } + + /// Dispatch a command request. + pub fn dispatch(&self, req: &CommandRequest, ctx: &CommandContext) -> CommandResponse { + match self.handlers.get(&req.command) { + Some(handler) => handler.handle(&req.args, ctx), + None => CommandResponse::err( + &req.command, + format!("unknown command `{}` — try `help`", req.command), + ), + } + } + + /// List all registered commands (for help text). + pub fn commands(&self) -> Vec<&CommandMeta> { + // sorted by name for stable output + let mut metas: Vec<_> = self.handlers.values() + .map(|h| h.meta()) + .collect(); + metas.sort_by_key(|m| m.name); + metas + } +} +``` + +### Built-in Commands (MVP) + +**Read commands** (extracted from existing `investigate.rs`): + +| Command | Args | Description | Source | +|---------|------|-------------|--------| +| `help` | — | List all commands | `cmd_help()` | +| `overview` | — | Runtime summary | `cmd_overview()` | +| `workers` | — | Per-worker stats | `cmd_workers()` | +| `worker` | `id: int` | Single worker detail | `cmd_worker()` | +| `actors` | `sort`, `limit`, `worker` | List actors | `cmd_actors()` | +| `actor` | `prefix: str` | Find by address prefix | `cmd_actor()` | +| `hot` | `n: int` | Top N by mailbox depth | `cmd_hot()` | +| `phases` | `worker: int?` | Tick phase breakdown | `cmd_phases()` | +| `diff` | `seconds: float` | Snapshot delta | `cmd_diff()` | +| `nodes` | — | Cluster member list | **new** | +| `registry` | — | All registered names | **new** | +| `resolve` | `name: str` | Look up a name | **new** | + +**Write commands** (new): + +| Command | Args | Description | +|---------|------|-------------| +| `stop` | `prefix` or `name` | Stop an actor (poison + cleanup) | +| `drain` | `node: str?` | Stop accepting new actors on a node, let existing drain | +| `shutdown` | `node: str?` | Graceful shutdown (drain + stop all) | +| `spawn` | `factory`, `node?`, `constraints?` | Spawn from a registered factory | + +`spawn` requires a **factory registry** — actors register factory functions that can be invoked by name: + +```rust +pub trait ActorFactory: Send + Sync { + fn name(&self) -> &str; + fn spawn(&self, runtime: &Runtime, args: &HashMap) + -> Result; +} +``` + +### Migration from investigate.rs + +The existing `crates/runtime-dashboard/src/investigate.rs` has 9 command functions. Migration strategy: + +1. Create `crates/command/src/builtins/` with one file per command (or grouped by category). +2. Each `cmd_*` function becomes a `CommandHandler` impl. The logic is identical — just restructured. +3. `dispatch_repl` becomes `CommandRouter::dispatch` with a text-to-`CommandRequest` parser. +4. `dispatch_command` (HTTP) becomes `CommandRouter::dispatch` with query-param-to-`CommandRequest` parser. +5. `run_investigate` remains in the dashboard crate as a thin loop over `CommandRouter`. + +Example extraction: + +```rust +// crates/command/src/builtins/overview.rs + +pub struct OverviewCommand; + +impl CommandHandler for OverviewCommand { + fn meta(&self) -> CommandMeta { + CommandMeta { + name: "overview", + description: "Summary: worker count, actor count, total messages, panics", + usage: "overview", + is_write: false, + } + } + + fn handle(&self, _args: &HashMap, ctx: &CommandContext) -> CommandResponse { + let stats = ctx.enriched_stats(); + let total_msgs: u64 = stats.workers.iter().map(|w| w.messages_processed).sum(); + // ... same logic as existing cmd_overview ... + CommandResponse::ok("overview", serde_json::json!({ + "workers": stats.num_workers, + "actors": stats.actor_details.len(), + "total_messages_processed": total_msgs, + // ... + })) + } +} +``` + +### Frontend Adapters + +Each frontend is a thin adapter that converts its input format into `CommandRequest` and `CommandResponse` back to its output format. + +**REPL adapter** (stdin/stdout): + +```rust +// crates/command/src/adapters/repl.rs + +pub fn parse_line(line: &str) -> CommandRequest { + let parts: Vec<&str> = line.split_whitespace().collect(); + let command = parts.first().unwrap_or(&"help").to_string(); + let args = parse_positional_and_flags(&parts[1..]); + CommandRequest { command, args } +} +``` + +**REST adapter** (HTTP query params): + +```rust +// crates/command/src/adapters/rest.rs + +pub fn from_query_params(params: &HashMap) -> CommandRequest { + let command = params.get("cmd").cloned().unwrap_or_else(|| "help".into()); + let args: HashMap = params.iter() + .filter(|(k, _)| *k != "cmd") + .map(|(k, v)| (k.clone(), serde_json::Value::String(v.clone()))) + .collect(); + CommandRequest { command, args } +} +``` + +**TUI adapter**: TUI input field text → `parse_line()` → `dispatch()` → render response in panel. + +### Custom Commands (actor-registered) + +Actors can register command handlers at runtime through the `Ctx`: + +```rust +impl Ctx<'_> { + /// Register a command that routes to this actor. + /// When the command is invoked, a CommandInvocation message + /// is sent to this actor's mailbox. + pub fn register_command(&self, name: &str, description: &str) { + self.inner.register_command(self.self_addr, name, description); + } +} +``` + +When a custom command is dispatched: + +1. Router finds it's actor-registered. +2. Sends `CommandInvocation { command, args, reply_addr }` to the actor's mailbox. +3. The actor processes it and sends `CommandResult { data }` back to `reply_addr`. +4. Router waits on a one-shot inbox (with timeout, e.g. 5 seconds). +5. Returns the response. + +```rust +/// Sent to an actor when its registered command is invoked. +#[derive(Debug, Clone)] +pub struct CommandInvocation { + pub command: String, + pub args: HashMap, + pub reply_addr: ActorAddress, +} + +/// Sent back by the actor with the command result. +#[derive(Debug, Clone)] +pub struct CommandResult { + pub data: serde_json::Value, +} +``` + +This mechanism allows any actor to expose operational endpoints without modifying the command crate. + +### Files Modified + +| File | Change | +|------|--------| +| `crates/command/` | **New crate** | +| `crates/command/Cargo.toml` | Dependencies: swactor, serde, serde_json | +| `crates/command/src/lib.rs` | `CommandRouter`, `CommandHandler`, `CommandRequest`/`Response`, `CommandContext` | +| `crates/command/src/builtins/` | Built-in command handlers (mod.rs + per-command files) | +| `crates/command/src/adapters/` | REPL and REST input parsers | +| `crates/runtime-dashboard/src/investigate.rs` | Refactored: thin REPL loop over CommandRouter | +| `crates/runtime-dashboard/src/server.rs` | REST endpoints use CommandRouter | +| `crates/runtime-dashboard/Cargo.toml` | Depends on `crates/command` | +| `Cargo.toml` | Add `crates/command` to workspace | + +### Tests + +- **dispatch_known_command**: `overview` returns `ok: true` with expected fields +- **dispatch_unknown_command**: returns `ok: false` with helpful error +- **help_lists_all**: `help` response includes all registered command names +- **parse_repl_line**: `"actors --sort mailbox --limit 5"` → correct `CommandRequest` +- **parse_query_params**: `{cmd: "actor", prefix: "a1b2"}` → correct `CommandRequest` +- **custom_command_dispatch**: register actor command, invoke, verify response +- **custom_command_timeout**: registered actor doesn't respond, verify timeout error +- **write_command_stop**: stop an actor via command, verify it's poisoned diff --git a/docs/os-design/05-supervision.md b/docs/os-design/05-supervision.md new file mode 100644 index 0000000..95db191 --- /dev/null +++ b/docs/os-design/05-supervision.md @@ -0,0 +1,263 @@ +# Supervision — User-Space Self-Healing + +## Problem + +When nodes churn (spot instances preempted, hardware rebooted, network partitions), actors on those nodes are lost. Something needs to detect the loss and re-spawn the actors on surviving nodes. This is the "self-healing" property of a distributed OS. + +## Design Principle: Supervision Is User-Space + +Supervision is **not** a runtime primitive. It is a pattern built from: + +- **Actor Watching** (01) — detect death +- **Cluster Registry** (02) — re-register under the same name +- **Node Capabilities** (03) — find a suitable replacement node + +The runtime provides the low-level mechanisms. Supervision is a library actor that composes them. This keeps the kernel minimal and lets users customize supervision policy without forking the runtime. + +## Supervisor Actor + +```rust +/// A supervised child definition. +struct SupervisedChild { + /// Human-readable name (registered in cluster registry). + name: String, + /// Factory function to create the actor. + factory: Box, + /// Placement constraints for the child. + constraints: PlacementRequirement, + /// Current address (None if not yet spawned or dead). + current_addr: Option, + /// Number of restarts so far. + restart_count: u32, + /// Maximum restarts before giving up (0 = unlimited). + max_restarts: u32, + /// Backoff state for restart delays. + last_restart: Option, +} + +/// Restart strategy for a supervisor. +#[derive(Debug, Clone)] +enum RestartStrategy { + /// Restart only the failed child. + OneForOne, + /// If any child fails, restart all children. + AllForOne, + /// Don't restart — just notify (for monitoring supervisors). + Notify, +} + +/// The supervisor actor. +struct Supervisor { + children: Vec, + strategy: RestartStrategy, +} +``` + +## Message Protocol + +```rust +/// Messages the supervisor handles. +enum SupervisorMsg { + /// A watched child died. + Exited(ActorExited), + /// External request to add a child. + AddChild { + name: String, + factory: Box, + constraints: PlacementRequirement, + }, + /// External request to remove a child. + RemoveChild { name: String }, + /// Query: what children are running? + Status { reply_to: ActorAddress }, +} +``` + +## Lifecycle + +### Startup + +```rust +impl Supervisor { + fn start(&mut self, ctx: &Ctx) { + for child in &mut self.children { + match self.spawn_child(ctx, child) { + Ok(addr) => { + child.current_addr = Some(addr); + ctx.watch(addr); + ctx.register_as(&child.name); // or register the child + } + Err(e) => { + eprintln!("supervisor: failed to spawn {}: {e}", child.name); + } + } + } + } +} +``` + +### Death Handling + +```rust +impl ActorInterface for Supervisor { + type Incoming = SupervisorMsg; + type Response = (); + + fn handle(&mut self, ctx: &Ctx, msg: SupervisorMsg) { + match msg { + SupervisorMsg::Exited(exited) => { + match self.strategy { + RestartStrategy::OneForOne => { + self.restart_one(ctx, &exited); + } + RestartStrategy::AllForOne => { + self.restart_all(ctx); + } + RestartStrategy::Notify => { + // Just log — don't restart + } + } + } + // ... other messages ... + } + } +} +``` + +### Restart Flow (OneForOne) + +``` +1. Receive ActorExited { addr: X, reason: NodeDown } +2. Find child with current_addr == X → child "worker-3" +3. Check restart_count < max_restarts +4. Query capabilities: find_suitable_nodes(child.constraints) +5. Pick best node (least loaded, or local if possible) +6. Spawn child on selected node via factory +7. Watch new address +8. Register child.name → new address in cluster registry +9. Update child.current_addr +10. Increment child.restart_count +``` + +```rust +impl Supervisor { + fn restart_one(&mut self, ctx: &Ctx, exited: &ActorExited) { + let child = match self.children.iter_mut() + .find(|c| c.current_addr == Some(exited.addr)) + { + Some(c) => c, + None => return, // not our child + }; + + child.current_addr = None; + + if child.max_restarts > 0 && child.restart_count >= child.max_restarts { + eprintln!( + "supervisor: child {} exceeded max restarts ({}), giving up", + child.name, child.max_restarts, + ); + return; + } + + // Spawn replacement + match self.spawn_child(ctx, child) { + Ok(addr) => { + child.current_addr = Some(addr); + child.restart_count += 1; + ctx.watch(addr); + // Re-register name → new address + // (done via the registry, which gossips to all nodes) + } + Err(e) => { + eprintln!("supervisor: failed to restart {}: {e}", child.name); + } + } + } +} +``` + +### Restart Flow (AllForOne) + +When any child dies: +1. Stop all other children (send stop signal). +2. Wait for all `ActorExited` notifications. +3. Restart all children in order. + +This is useful for interdependent actor groups where partial restart doesn't make sense. + +## Capability-Aware Placement + +The supervisor uses `find_suitable_nodes()` from the distribution crate: + +```rust +fn spawn_child(&self, ctx: &Ctx, child: &SupervisedChild) + -> Result +{ + // If constraints are empty, spawn locally + if child.constraints.is_empty() { + return child.factory.spawn(ctx); + } + + // Find suitable remote nodes + let nodes = dist_node.find_suitable_nodes(&child.constraints); + if nodes.is_empty() { + return Err("no nodes satisfy placement constraints".into()); + } + + // Pick the least-loaded suitable node + let target_node = &nodes[0]; // TODO: sort by load + + // Spawn remotely (requires remote spawn protocol — future work) + // For now: if current node satisfies, spawn locally + // Otherwise: send spawn request to target node + todo!("remote spawn") +} +``` + +**Note**: Remote spawn (telling another node to create an actor) is not yet part of the runtime. The supervisor design accounts for it, but the initial implementation will only support local spawn + re-registration. + +## Relationship to Existing `spawn_restartable` + +The core runtime already has `spawn_restartable` with `factory` and `max_restarts`. This is a **local-only** recovery mechanism — when an actor panics, the same worker restarts it. + +The supervisor pattern extends this to **cluster-wide** recovery: + +| Feature | `spawn_restartable` | Supervisor | +|---|---|---| +| Scope | Single worker | Cluster-wide | +| Trigger | Panic | Panic, stop, or node death | +| Placement | Same worker | Capability-aware, any node | +| Naming | No | Yes (cluster registry) | +| Strategy | Always restart | OneForOne, AllForOne, Notify | +| Implementation | Runtime internal | User-space actor | + +They complement each other: `spawn_restartable` handles fast local recovery (no network round-trip); the supervisor handles node-level failures. + +## Future Extensions + +- **Restart backoff**: exponential backoff between restarts to avoid thrashing. +- **Health checks**: periodic health probes (not just death detection). +- **Cascading supervisors**: supervisor trees (supervisor watches sub-supervisor). +- **Declarative spec**: TOML/YAML file defining supervision topology, loaded at startup. +- **Migration (not restart)**: move a running actor's state to another node (requires persistence, out of scope). + +## Files + +This is a library actor, not a runtime change. Implementation lives in: + +| File | Content | +|------|---------| +| `crates/supervision/src/lib.rs` | `Supervisor`, `SupervisedChild`, `RestartStrategy` | +| `crates/supervision/src/factory.rs` | `ActorFactory` trait, factory registry | +| `crates/supervision/Cargo.toml` | Depends on `swactor`, `swactor-capabilities`, `distribution` | + +Or, if the scope doesn't warrant a separate crate, it can live in `src/supervision.rs` behind a feature flag. + +## Tests + +- **one_for_one_restart**: supervisor with 3 children, kill one, verify only that one restarts +- **all_for_one_restart**: supervisor with 3 children, kill one, verify all restart +- **max_restarts_exceeded**: child dies repeatedly, verify supervisor gives up after max +- **name_re_registration**: child dies and restarts, verify name resolves to new address +- **capability_placement**: child with GPU constraint, verify spawned on GPU node (or error if none available) +- **supervisor_itself_dies**: verify children are stopped (or orphaned — design decision) diff --git a/docs/os-design/README.md b/docs/os-design/README.md new file mode 100644 index 0000000..8cb458a --- /dev/null +++ b/docs/os-design/README.md @@ -0,0 +1,153 @@ +# Swactor Distributed OS — Design Overview + +## Vision + +Swactor is evolving from a local actor runtime into a **distributed operating system** for running long-lived daemons across heterogeneous, churning machines. + +**Primary use case**: Wire together personal hardware today; add on-demand spot compute (vast.ai, etc.) tomorrow. Machines pop in and out of the network. The system self-heals. + +**Design principles**: + +- **Churn is the norm**, not the exception. Every subsystem assumes nodes can disappear at any time. +- **General and flexible**. Minimal assumptions about what a node looks like — feature-gate hardware-specific code. +- **Layered**. The core runtime stays minimal. OS features are opt-in crates. Supervision is user-space, not kernel. +- **Frontend-agnostic**. Operational interfaces (commands, inspection) work identically from CLI, TUI, REST, or future transports. + +## Current Capabilities + +| OS Concept | What Swactor Has Today | +|---|---| +| Processes | Actor spawn/stop, lifecycle hooks, restartable with factory + max_restarts | +| Scheduling | Worker threads, fairness budget (64 msgs/tick), load-aware placement | +| IPC | Typed send, request/reply, per-actor VecDeque mailboxes | +| Naming | String-keyed `AddressMap` — node-local only | +| Fault tolerance | `catch_unwind` for panics, factory restart, dead actor cleanup | +| Backpressure | Per-actor mailbox capacity, DropNewest/DropOldest overflow | +| Networking | SWIM membership (Lifeguard extensions), TCP transport, bincode wire protocol | +| Directory | Kademlia DHT for `ActorAddress -> NodeId` resolution | +| Monitoring | WorkerStats/RuntimeStats, TUI dashboard, REST `/api/*`, investigate REPL | +| Distribution | Multi-node cluster, 5-node Docker test suite | + +## What's Missing (This Design) + +| Feature | Document | Priority | +|---|---|---| +| Actor Watching | [01-actor-watching.md](./01-actor-watching.md) | Foundation for everything | +| Cluster Registry | [02-cluster-registry.md](./02-cluster-registry.md) | Actors find each other across nodes | +| Node Capabilities | [03-node-capabilities.md](./03-node-capabilities.md) | Heterogeneous placement | +| Command Interface | [04-command-interface.md](./04-command-interface.md) | Operational control | +| Supervision | [05-supervision.md](./05-supervision.md) | Self-healing (user-space) | + +## Crate Structure (After This Work) + +``` +swactor/ # core runtime — no network deps + src/ + actor.rs # +watch/unwatch on ContextInner, ExitReason, ActorExited + worker.rs # +WatchRegistry, death notification phase + delivery.rs # (unchanged) + runtime.rs # (unchanged) + ... +crates/ + capabilities/ # NEW — hardware detection + placement constraints + src/lib.rs # NodeCapabilities, CapValue, PlacementConstraint + command/ # NEW — frontend-agnostic command dispatch + src/lib.rs # CommandRouter, CommandHandler, CommandRequest/Response + src/builtins/ # Built-in command handlers + distribution/ # SWIM + Kademlia + cluster registry + src/ + registry.rs # NEW — ClusterRegistry, gossip-propagated naming + node.rs # +register_name, resolve_name, capabilities + messages.rs # +WatchRequest, UnwatchRequest, ActorExitedNotify + ... + runtime-dashboard/ # TUI + REST — refactored to use command crate + python/ # PyO3 bindings + simulation/ # Network simulation + wasm/ # WASM bindings +``` + +## Dependency Graph + +``` + ┌──────────────┐ + │ capabilities │ (standalone — only serde + sysinfo) + └──────┬───────┘ + │ optional +┌─────────┐ ┌──────▼────────┐ +│ swactor │◄─────│ distribution │ +│ (core) │ │ +registry │ +└────┬────┘ └──────┬────────┘ + │ │ + │ ┌──────▼────────┐ + └──────────►│ command │ + └──────┬────────┘ + │ + ┌──────▼────────────┐ + │ runtime-dashboard │ + │ (TUI + REST) │ + └───────────────────┘ +``` + +Key constraints: +- `capabilities` has **zero** dependency on `swactor` — it's a standalone detection library. +- `command` depends on `swactor` (needs `Runtime`, stats types) but NOT on `distribution`. +- `distribution` optionally depends on `capabilities` (for `NodeRecord` labels). +- `runtime-dashboard` depends on both `command` and optionally `distribution`. + +## Feature Flags + +### Core `swactor` + +| Flag | Purpose | +|---|---| +| `getrandom` (default) | Cryptographic RNG for actor addresses | +| `serde` | Serialization for types | +| `tracing` | Structured logging | +| `transport` | Transport-agnostic remote messaging | +| `watching` (new) | Watch API on ContextInner/Ctx | + +### `swactor-capabilities` + +| Flag | Purpose | +|---|---| +| `detect` (default) | Auto-detect CPU, RAM, hostname via sysinfo | +| `gpu-nvidia` | Detect NVIDIA GPU via nvidia-smi | +| `gpu-vulkan` | Detect GPU via Vulkan API | + +### `distribution` + +| Flag | Purpose | +|---|---| +| `registry` (new, default) | Cluster-wide gossip-propagated naming | +| `capabilities` (new) | NodeCapabilities on NodeRecord | + +## Implementation Order + +Each feature is one PR, in dependency order: + +``` +PR 1: Actor Watching (local only) + └──► PR 2: Command Interface + └──► PR 3: Cluster Registry + └──► PR 4: Node Capabilities + └──► PR 5: Remote Watching (cross-node) + └──► PR 6: Supervisor library +``` + +**PR 1 — Actor Watching (local)** +Adds `WatchRegistry`, `ActorExited`, `ExitReason` to core runtime. Testable without any distribution. Foundation for everything else. + +**PR 2 — Command Interface** +Extracts investigate.rs into `crates/command/`. Adds write commands (spawn, stop, drain). Immediately useful for operations. + +**PR 3 — Cluster Registry** +Gossip-propagated naming in `crates/distribution/src/registry.rs`. Actors can find each other by name across nodes. + +**PR 4 — Node Capabilities** +`crates/capabilities/` with auto-detection and placement constraints. Integrates with distribution for capability-aware placement. + +**PR 5 — Remote Watching** +Wire protocol for cross-node watches. SWIM Dead triggers `ActorExited { reason: NodeDown }` for all actors on that node. + +**PR 6 — Supervisor Library** +User-space supervisor pattern. Composes watching + registry + capabilities to auto-respawn actors after churn. diff --git a/src/actor.rs b/src/actor.rs index 17b9504..64b620c 100644 --- a/src/actor.rs +++ b/src/actor.rs @@ -2,6 +2,31 @@ use std::any::Any; use crate::Error; +/// Why an actor exited. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum ExitReason { + /// Actor was explicitly stopped or removed from the pool. + Stopped, + /// Actor panicked during message handling. + Panicked, + /// The node hosting the actor left the cluster (SWIM Dead). + NodeDown, +} + +/// Delivered to watchers when a watched actor exits. +/// +/// Implements `Message` (Clone + Send + Sync + 'static) so it can be +/// delivered through normal mailbox channels. +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct ActorExited { + /// The address of the actor that died. + pub addr: ActorAddress, + /// Why it exited. + pub reason: ExitReason, +} + /// The primary trait defining data that can be passed to and from actor processes pub trait Message: 'static + Sized + Clone + Send + Sync {} impl Message for T {} @@ -32,6 +57,11 @@ pub trait ActorInterface: 'static + Send { /// If your `Incoming` type IS `Down`, this method is never called — the /// normal `handle()` receives the message instead. fn handle_down(&mut self, _ctx: &Ctx, _down: Down) {} + + /// Called when a watched actor exits. Override to react to death notifications. + /// + /// Default: no-op (notification is silently consumed). + fn on_actor_exit(&mut self, _ctx: &Ctx, _exited: ActorExited) {} } /// A unique address for this actor. 32 bytes is overkill for a small application, @@ -106,10 +136,17 @@ where } Err(msg) => msg, }; - match msg.downcast::() { + let msg = match msg.downcast::() { Ok(down) => { self.inner.handle_down(ctx, *down); - Some("swactor::actor::Down") + return Some("swactor::actor::Down"); + } + Err(msg) => msg, + }; + match msg.downcast::() { + Ok(exited) => { + self.inner.on_actor_exit(ctx, *exited); + Some("ActorExited") } Err(_) => None, } @@ -205,6 +242,10 @@ pub trait ContextInner { fn schedule_timer(&self, request: TimerRequest); /// Access the runtime extension (if installed). fn extension(&self) -> Option<&dyn crate::extension::RuntimeExtension>; + /// Register a watch: watcher receives ActorExited when target dies. + fn watch(&self, watcher: ActorAddress, target: ActorAddress); + /// Cancel a watch. + fn unwatch(&self, watcher: ActorAddress, target: ActorAddress); } /// Actor syscall interface — passed to `ActorInterface::handle()`. @@ -290,4 +331,22 @@ impl<'a> Ctx<'a> { period, }); } + + /// Watch another actor's liveness. If the target dies, this actor + /// receives an `ActorExited` message in its mailbox. + /// + /// Watching an already-dead or non-existent actor delivers + /// `ActorExited { reason: Stopped }` on the next tick. + /// + /// Calling watch() multiple times on the same target is idempotent — + /// only one notification is delivered. + pub fn watch(&self, target: ActorAddress) { + self.inner.watch(self.self_addr, target); + } + + /// Stop watching an actor. No notification will be delivered if the + /// target subsequently dies. + pub fn unwatch(&self, target: ActorAddress) { + self.inner.unwatch(self.self_addr, target); + } } diff --git a/src/delivery.rs b/src/delivery.rs index b1f3aeb..b21bcc2 100644 --- a/src/delivery.rs +++ b/src/delivery.rs @@ -2,13 +2,14 @@ use std::any::Any; use std::collections::{HashMap, HashSet}; use std::hash::{BuildHasher, Hasher}; use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::{Arc, OnceLock, RwLock}; +use std::sync::{Arc, Mutex, OnceLock, RwLock}; use std::thread::Thread; use crate::actor::{ActorAddress, AnyActor, Message}; use crate::channel::Sender; use crate::config::RuntimeConfig; use crate::stats::WorkerStats; +use crate::worker::WatchRegistry; use crate::Error; // ─── Identity Hasher for ActorAddress ─────────────────────────────────────── @@ -243,6 +244,7 @@ pub(crate) struct TickContext<'a> { pub(crate) stats_hook: Option<&'a dyn crate::stats::StatsHook>, /// Thread handles for waking parked workers on cross-worker sends. pub(crate) worker_threads: &'a [OnceLock], + pub(crate) watch_registry: Option<&'a Arc>>, #[cfg(feature = "transport")] pub(crate) codec_registry: Option<&'a crate::transport::CodecRegistry>, #[cfg(feature = "transport")] diff --git a/src/runtime.rs b/src/runtime.rs index 815e051..dd65928 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -1,11 +1,11 @@ use std::any::Any; use std::cell::RefCell; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, OnceLock}; +use std::sync::{Arc, Mutex, OnceLock}; use std::thread::{self, JoinHandle, Thread}; use std::time::Instant; -use crate::actor::{Actor, ActorAddress, ActorInterface, AnyActor, Message, StopSignal, TimerRequest}; +use crate::actor::{Actor, ActorAddress, ActorExited, ActorInterface, AnyActor, ExitReason, Message, StopSignal, TimerRequest}; use crate::channel::{Receiver, Sender}; // Re-export config types so existing code using `runtime::RuntimeConfig` still works pub use crate::config::{BackoffPolicy, MailboxOverflow, RuntimeConfig}; @@ -14,7 +14,7 @@ use crate::extension::RuntimeExtension; use crate::stats::{StatsHook, WorkerStats}; // Re-export stats types so existing code using `runtime::*` still works pub use crate::stats::{RuntimeStats, WorkerInfo}; -use crate::worker::Worker; +use crate::worker::{WatchRegistry, Worker}; use crate::Error; /// Generic message inbox for receiving messages outside of the runtime. @@ -103,6 +103,7 @@ pub struct Runtime { is_running: AtomicBool, worker_stats: Vec>, stats_hook: Option>, + watch_registry: Arc>, /// Workers available for tick(). run() drains this and moves workers to threads. tick_workers: RefCell>, /// Thread handles for waking parked workers. Set by workers on startup via OnceLock. @@ -198,6 +199,7 @@ impl Runtime { is_running: AtomicBool::new(false), worker_stats, stats_hook: None, + watch_registry: Arc::new(Mutex::new(WatchRegistry::new())), tick_workers: RefCell::new(workers), worker_threads, created_at: Instant::now(), @@ -304,6 +306,7 @@ impl Runtime { extension: self.extension.as_deref(), stats_hook: self.stats_hook.as_deref(), worker_threads: &self.worker_threads, + watch_registry: Some(&self.watch_registry), #[cfg(feature = "transport")] codec_registry: self.codec_registry.as_deref(), #[cfg(feature = "transport")] @@ -506,4 +509,25 @@ impl ContextInner for Runtime { fn extension(&self) -> Option<&dyn RuntimeExtension> { self.extension.as_deref() } + + fn watch(&self, watcher: ActorAddress, target: ActorAddress) { + if self.address_map.lookup(&target).is_some() { + self.watch_registry.lock().unwrap().watch(watcher, target); + } else { + // Target not found — deliver ActorExited immediately. + let msg = ActorExited { + addr: target, + reason: ExitReason::Stopped, + }; + // Route to watcher via transfer queue + if let Some(wid) = self.address_map.lookup(&watcher) { + self.transfer_txs[wid.as_usize()] + .send(Envelope::new(watcher, Box::new(msg))); + } + } + } + + fn unwatch(&self, watcher: ActorAddress, target: ActorAddress) { + self.watch_registry.lock().unwrap().unwatch(watcher, target); + } } diff --git a/src/worker.rs b/src/worker.rs index 35f0f52..901fc99 100644 --- a/src/worker.rs +++ b/src/worker.rs @@ -1,12 +1,12 @@ use std::any::Any; use std::cell::RefCell; -use std::collections::{HashMap, VecDeque}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::thread; use std::time::Instant; -use crate::actor::{ActorAddress, AnyActor, CloneMsg, ContextInner, Ctx, StopReason, StopSignal, TimerRequest}; +use crate::actor::{ActorAddress, ActorExited, AnyActor, CloneMsg, ContextInner, Ctx, ExitReason, StopReason, StopSignal, TimerRequest}; use crate::channel::Receiver; use crate::config::MailboxOverflow; use crate::delivery::{AddrBuildHasher, AddrMap, Envelope, TickContext, WorkerId}; @@ -108,6 +108,95 @@ impl TimerWheel { } } +// ─── Watch Registry ───────────────────────────────────────────────────────── + +/// Tracks watch relationships between actors. +/// +/// Shared across workers via `Arc>`. Contention is negligible +/// because watch/unwatch operations are rare relative to message sends. +pub(crate) struct WatchRegistry { + /// target → set of watchers awaiting death notification + watchers: HashMap>, + /// watcher → set of targets it's watching (reverse index for cleanup) + watching: HashMap>, +} + +impl WatchRegistry { + pub fn new() -> Self { + Self { + watchers: HashMap::new(), + watching: HashMap::new(), + } + } + + pub fn watch(&mut self, watcher: ActorAddress, target: ActorAddress) { + self.watchers.entry(target).or_default().insert(watcher); + self.watching.entry(watcher).or_default().insert(target); + } + + pub fn unwatch(&mut self, watcher: ActorAddress, target: ActorAddress) { + if let Some(set) = self.watchers.get_mut(&target) { + set.remove(&watcher); + if set.is_empty() { + self.watchers.remove(&target); + } + } + if let Some(set) = self.watching.get_mut(&watcher) { + set.remove(&target); + if set.is_empty() { + self.watching.remove(&watcher); + } + } + } + + /// Called when an actor dies. Returns (watcher_addr, ActorExited) pairs. + pub fn notify_death( + &mut self, + target: ActorAddress, + reason: ExitReason, + ) -> Vec<(ActorAddress, ActorExited)> { + let notification = ActorExited { + addr: target, + reason, + }; + let mut result = Vec::new(); + + if let Some(watcher_set) = self.watchers.remove(&target) { + for watcher in &watcher_set { + result.push((*watcher, notification.clone())); + // clean up reverse index + if let Some(set) = self.watching.get_mut(watcher) { + set.remove(&target); + if set.is_empty() { + self.watching.remove(watcher); + } + } + } + } + + result + } + + /// Called when a watcher itself dies. Cleans up all its watching entries. + pub fn cleanup_watcher(&mut self, watcher: &ActorAddress) { + if let Some(targets) = self.watching.remove(watcher) { + for target in targets { + if let Some(set) = self.watchers.get_mut(&target) { + set.remove(watcher); + if set.is_empty() { + self.watchers.remove(&target); + } + } + } + } + } + + /// Check if a target has any watchers registered. + pub fn has_watchers(&self, target: &ActorAddress) -> bool { + self.watchers.get(target).is_some_and(|s| !s.is_empty()) + } +} + // ─── Worker ───────────────────────────────────────────────────────────────── /// A worker owns a set of actors and runs them in a loop. @@ -203,6 +292,7 @@ impl Worker { let timer_requests: RefCell> = RefCell::new(Vec::new()); let processed; + let deaths; { let worker_ctx = WorkerContext { worker_id: self.id, @@ -212,7 +302,7 @@ impl Worker { timer_requests: &timer_requests, stats: &self.stats, }; - processed = self.pool.tick_all(&worker_ctx, &self.stats, tc.config.actor_message_budget, &stop_requests); + (processed, deaths) = self.pool.tick_all(&worker_ctx, &self.stats, tc.config.actor_message_budget, &stop_requests); if processed > 0 { did_work = true; } @@ -256,6 +346,40 @@ impl Worker { } } } + + // 5b. Process actor deaths → deliver ActorExited to watchers + if !deaths.is_empty() { + did_work = true; + if let Some(registry) = &tc.watch_registry { + let mut reg = registry.lock().unwrap(); + for (dead_addr, reason) in deaths { + let notifications = reg.notify_death(dead_addr, reason); + for (watcher_addr, msg) in notifications { + // Deliver ActorExited as a normal message via the address map + match tc.address_map.lookup(&watcher_addr) { + Some(wid) if wid == self.id => { + self.pool.deliver(&watcher_addr, Box::new(msg)); + } + Some(wid) => { + tc.transfer_txs[wid.as_usize()] + .send(Envelope::new(watcher_addr, Box::new(msg))); + } + None => { + // Watcher not in address map — may be an inbox or remote. + // Try inbox registry as best effort. + let _ = tc.inbox_registry.try_deliver( + watcher_addr, + Box::new(msg), + ); + } + } + } + // Clean up the dead actor's own watches (things it was watching) + reg.cleanup_watcher(&dead_addr); + } + } + } + let t5 = Instant::now(); // 6. Publish stats (skip entirely when idle to avoid allocation + mutex) @@ -451,6 +575,29 @@ impl ContextInner for WorkerContext<'_> { fn extension(&self) -> Option<&dyn crate::extension::RuntimeExtension> { self.tc.extension } + + fn watch(&self, watcher: ActorAddress, target: ActorAddress) { + if let Some(registry) = &self.tc.watch_registry { + // Check if target exists in the address map + if self.tc.address_map.lookup(&target).is_some() { + registry.lock().unwrap().watch(watcher, target); + } else { + // Target not found — deliver ActorExited { reason: Stopped } immediately. + // Buffer in pending_local so it arrives on next tick. + let msg = ActorExited { + addr: target, + reason: ExitReason::Stopped, + }; + self.pending_local.borrow_mut().push((watcher, Box::new(msg))); + } + } + } + + fn unwatch(&self, watcher: ActorAddress, target: ActorAddress) { + if let Some(registry) = &self.tc.watch_registry { + registry.lock().unwrap().unwatch(watcher, target); + } + } } struct ActorSlot { @@ -531,7 +678,7 @@ impl ActorPool { std::mem::replace(&mut self.drops_this_tick, 0) } - /// Tick all actors in the pool. Returns the number of messages processed. + /// Tick all actors in the pool. Returns (messages_processed, newly_dead_actors). /// /// Each actor processes up to `budget` messages per tick (0 = unlimited). /// This prevents a single hot actor from starving others on the same worker. @@ -541,8 +688,9 @@ impl ActorPool { stats: &WorkerStats, budget: usize, stop_requests: &RefCell>, - ) -> usize { + ) -> (usize, Vec<(ActorAddress, ExitReason)>) { let mut count = 0; + let mut deaths = Vec::new(); for (&addr, slot) in self.actors.iter_mut() { if slot.poisoned || slot.stopping { // Discard all messages for poisoned/stopping actors @@ -606,6 +754,8 @@ impl ActorPool { #[cfg(feature = "tracing")] tracing::error!(actor_addr = %addr, "actor.panicked"); slot.poisoned = true; + slot.mailbox.clear(); + deaths.push((addr, ExitReason::Panicked)); break; } Ok(Some(type_name)) => { @@ -624,6 +774,7 @@ impl ActorPool { slot.stopping = true; stats.stops.fetch_add(1, Ordering::Relaxed); slot.mailbox.clear(); + deaths.push((addr, ExitReason::Stopped)); break; } } @@ -633,7 +784,7 @@ impl ActorPool { } } } - count + (count, deaths) } pub fn len(&self) -> usize { diff --git a/tests/watch_api.rs b/tests/watch_api.rs new file mode 100644 index 0000000..0ee45ce --- /dev/null +++ b/tests/watch_api.rs @@ -0,0 +1,378 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use swactor::actor::{ActorAddress, ActorExited, ActorInterface, ExitReason}; +use swactor::runtime::{Ctx, Runtime, RuntimeConfig}; + +// ── Actors ────────────────────────────────────────────────────────────────── + +/// An actor that panics when it receives PanicMsg. +struct PanicOnCommand; + +#[derive(Clone)] +struct PanicMsg; + +impl ActorInterface for PanicOnCommand { + type Incoming = PanicMsg; + type Response = (); + fn handle(&mut self, _ctx: &Ctx, _msg: PanicMsg) { + panic!("deliberate panic for test"); + } +} + +/// An actor that watches targets and counts exit notifications. +struct ExitWatcher { + exit_count: Arc, + last_reason: Arc>>, + last_addr: Arc>>, +} + +#[derive(Clone)] +enum WatcherCmd { + WatchThis(ActorAddress), + UnwatchThis(ActorAddress), +} + +impl ActorInterface for ExitWatcher { + type Incoming = WatcherCmd; + type Response = (); + fn handle(&mut self, ctx: &Ctx, msg: WatcherCmd) { + match msg { + WatcherCmd::WatchThis(target) => { + ctx.watch(target); + } + WatcherCmd::UnwatchThis(target) => { + ctx.unwatch(target); + } + } + } + + fn on_actor_exit(&mut self, _ctx: &Ctx, exited: ActorExited) { + self.exit_count.fetch_add(1, Ordering::SeqCst); + *self.last_reason.lock().unwrap() = Some(exited.reason); + *self.last_addr.lock().unwrap() = Some(exited.addr); + } +} + +impl ExitWatcher { + fn new() -> (Self, WatcherState) { + let exit_count = Arc::new(AtomicUsize::new(0)); + let last_reason = Arc::new(std::sync::Mutex::new(None)); + let last_addr = Arc::new(std::sync::Mutex::new(None)); + let state = WatcherState { + exit_count: exit_count.clone(), + last_reason: last_reason.clone(), + last_addr: last_addr.clone(), + }; + ( + ExitWatcher { + exit_count, + last_reason, + last_addr, + }, + state, + ) + } +} + +/// Shared state for inspecting what ExitWatcher observed. +struct WatcherState { + exit_count: Arc, + last_reason: Arc>>, + last_addr: Arc>>, +} + +impl WatcherState { + fn count(&self) -> usize { + self.exit_count.load(Ordering::SeqCst) + } + fn last_reason(&self) -> Option { + self.last_reason.lock().unwrap().clone() + } + fn last_addr(&self) -> Option { + *self.last_addr.lock().unwrap() + } +} + +/// A silent actor that does nothing (for targets that shouldn't panic). +struct Sleeper; + +#[derive(Clone)] +struct Noop; + +impl ActorInterface for Sleeper { + type Incoming = Noop; + type Response = (); + fn handle(&mut self, _ctx: &Ctx, _msg: Noop) {} +} + +// ── Helper ────────────────────────────────────────────────────────────────── + +fn tick_n(rt: &Runtime, n: usize) { + for _ in 0..n { + rt.tick(); + } +} + +fn single_thread_config() -> RuntimeConfig { + RuntimeConfig { + num_threads: 1, + ..RuntimeConfig::default() + } +} + +// ── Tests ─────────────────────────────────────────────────────────────────── + +/// Given a watcher and a target actor, +/// when the target panics, +/// then the watcher's on_actor_exit fires with ExitReason::Panicked. +#[test] +fn watch_receives_notification_on_panic() { + let rt = Runtime::new(single_thread_config()); + let (watcher_actor, state) = ExitWatcher::new(); + + let target = rt.spawn(PanicOnCommand).unwrap(); + let watcher = rt.spawn(watcher_actor).unwrap(); + + // Tell watcher to watch the target + rt.send_to(watcher, WatcherCmd::WatchThis(target)).unwrap(); + tick_n(&rt, 3); + + // Kill the target + rt.send_to(target, PanicMsg).unwrap(); + tick_n(&rt, 5); + + assert_eq!(state.count(), 1, "watcher should have received exactly one ActorExited"); + assert_eq!(state.last_reason(), Some(ExitReason::Panicked)); + assert_eq!(state.last_addr(), Some(target)); +} + +/// Given a watcher that watches then unwatches a target, +/// when the target panics, +/// then the watcher receives NO notification. +#[test] +fn unwatch_prevents_notification() { + let rt = Runtime::new(single_thread_config()); + let (watcher_actor, state) = ExitWatcher::new(); + + let target = rt.spawn(PanicOnCommand).unwrap(); + let watcher = rt.spawn(watcher_actor).unwrap(); + + // Watch + rt.send_to(watcher, WatcherCmd::WatchThis(target)).unwrap(); + tick_n(&rt, 3); + + // Unwatch + rt.send_to(watcher, WatcherCmd::UnwatchThis(target)).unwrap(); + tick_n(&rt, 3); + + // Kill target + rt.send_to(target, PanicMsg).unwrap(); + tick_n(&rt, 5); + + assert_eq!(state.count(), 0, "after unwatch, no notification should be delivered"); +} + +/// Given a watch on an address that was never spawned, +/// then the watcher receives ActorExited { reason: Stopped }. +#[test] +fn watch_nonexistent_actor_delivers_stopped() { + let rt = Runtime::new(single_thread_config()); + let (watcher_actor, state) = ExitWatcher::new(); + + let watcher = rt.spawn(watcher_actor).unwrap(); + + let nonexistent = ActorAddress::new_random(); + rt.send_to(watcher, WatcherCmd::WatchThis(nonexistent)).unwrap(); + tick_n(&rt, 5); + + assert_eq!(state.count(), 1, "should receive ActorExited for non-existent target"); + assert_eq!(state.last_reason(), Some(ExitReason::Stopped)); + assert_eq!(state.last_addr(), Some(nonexistent)); +} + +/// Given a watcher that dies before the target, +/// when the target subsequently panics, +/// then there is no panic or leak. +#[test] +fn watcher_dies_before_target_no_panic() { + let rt = Runtime::new(single_thread_config()); + + let target = rt.spawn(PanicOnCommand).unwrap(); + let (watcher_actor, _state) = ExitWatcher::new(); + let watcher = rt.spawn(watcher_actor).unwrap(); + + // Watch + rt.send_to(watcher, WatcherCmd::WatchThis(target)).unwrap(); + tick_n(&rt, 3); + + // Kill the watcher first (send it a type-mismatched panic msg directly) + // Actually, ExitWatcher doesn't panic. Use Runtime-level watch + PanicOnCommand. + let rt2 = Runtime::new(single_thread_config()); + let target2 = rt2.spawn(PanicOnCommand).unwrap(); + let watcher2 = rt2.spawn(PanicOnCommand).unwrap(); + + use swactor::actor::ContextInner; + rt2.watch(watcher2, target2); + tick_n(&rt2, 3); + + // Kill watcher first + rt2.send_to(watcher2, PanicMsg).unwrap(); + tick_n(&rt2, 5); + + // Kill target — should not crash + rt2.send_to(target2, PanicMsg).unwrap(); + tick_n(&rt2, 5); + + // If we got here, no crash. +} + +/// Given a watcher that calls watch() twice on the same target, +/// when the target panics, +/// then the watcher receives exactly one notification. +#[test] +fn idempotent_watch_delivers_one_notification() { + let rt = Runtime::new(single_thread_config()); + let (watcher_actor, state) = ExitWatcher::new(); + + let target = rt.spawn(PanicOnCommand).unwrap(); + let watcher = rt.spawn(watcher_actor).unwrap(); + + // Watch twice + rt.send_to(watcher, WatcherCmd::WatchThis(target)).unwrap(); + tick_n(&rt, 3); + rt.send_to(watcher, WatcherCmd::WatchThis(target)).unwrap(); + tick_n(&rt, 3); + + // Kill target + rt.send_to(target, PanicMsg).unwrap(); + tick_n(&rt, 5); + + assert_eq!(state.count(), 1, "double watch should produce exactly one notification"); +} + +/// Given multiple watchers on the same target, +/// when the target panics, +/// then all watchers receive the notification. +#[test] +fn multiple_watchers_all_notified() { + let rt = Runtime::new(single_thread_config()); + let (w1_actor, s1) = ExitWatcher::new(); + let (w2_actor, s2) = ExitWatcher::new(); + let (w3_actor, s3) = ExitWatcher::new(); + + let target = rt.spawn(PanicOnCommand).unwrap(); + let w1 = rt.spawn(w1_actor).unwrap(); + let w2 = rt.spawn(w2_actor).unwrap(); + let w3 = rt.spawn(w3_actor).unwrap(); + + rt.send_to(w1, WatcherCmd::WatchThis(target)).unwrap(); + rt.send_to(w2, WatcherCmd::WatchThis(target)).unwrap(); + rt.send_to(w3, WatcherCmd::WatchThis(target)).unwrap(); + tick_n(&rt, 3); + + rt.send_to(target, PanicMsg).unwrap(); + tick_n(&rt, 5); + + assert_eq!(s1.count(), 1, "watcher 1 should be notified"); + assert_eq!(s2.count(), 1, "watcher 2 should be notified"); + assert_eq!(s3.count(), 1, "watcher 3 should be notified"); +} + +/// Self-watch doesn't crash the runtime. +#[test] +fn self_watch_does_not_crash() { + let rt = Runtime::new(single_thread_config()); + let (watcher_actor, _state) = ExitWatcher::new(); + + let actor = rt.spawn(watcher_actor).unwrap(); + rt.send_to(actor, WatcherCmd::WatchThis(actor)).unwrap(); + tick_n(&rt, 5); + + // No crash = pass +} + +/// Runtime-level watch (outside actor context) delivers notification. +#[test] +fn runtime_level_watch_delivers_notification() { + let rt = Runtime::new(single_thread_config()); + let (watcher_actor, state) = ExitWatcher::new(); + + let target = rt.spawn(PanicOnCommand).unwrap(); + let watcher = rt.spawn(watcher_actor).unwrap(); + tick_n(&rt, 2); // ensure both spawned + + use swactor::actor::ContextInner; + rt.watch(watcher, target); + + rt.send_to(target, PanicMsg).unwrap(); + tick_n(&rt, 5); + + assert_eq!(state.count(), 1, "runtime-level watch should deliver notification"); + assert_eq!(state.last_reason(), Some(ExitReason::Panicked)); +} + +/// Runtime-level watch on non-existent address delivers Stopped. +#[test] +fn runtime_level_watch_nonexistent_delivers_stopped() { + let rt = Runtime::new(single_thread_config()); + let (watcher_actor, state) = ExitWatcher::new(); + + let watcher = rt.spawn(watcher_actor).unwrap(); + tick_n(&rt, 2); + + let fake = ActorAddress::new_random(); + use swactor::actor::ContextInner; + rt.watch(watcher, fake); + + tick_n(&rt, 5); + + assert_eq!(state.count(), 1, "watching non-existent from runtime should deliver Stopped"); + assert_eq!(state.last_reason(), Some(ExitReason::Stopped)); +} + +/// Given a watcher watching target via on_actor_exit, +/// when target panics, +/// then the watcher can react by spawning a replacement (supervision pattern). +#[test] +fn watcher_can_react_to_death_by_spawning() { + let rt = Runtime::new(single_thread_config()); + let spawned = Arc::new(AtomicUsize::new(0)); + + struct Supervisor { + spawned_count: Arc, + } + + #[derive(Clone)] + enum SupervisorMsg { + WatchThis(ActorAddress), + } + + impl ActorInterface for Supervisor { + type Incoming = SupervisorMsg; + type Response = (); + fn handle(&mut self, ctx: &Ctx, msg: SupervisorMsg) { + match msg { + SupervisorMsg::WatchThis(target) => ctx.watch(target), + } + } + + fn on_actor_exit(&mut self, ctx: &Ctx, _exited: ActorExited) { + // React: spawn a replacement + let replacement = ctx.spawn(Sleeper).unwrap(); + let _ = replacement; + self.spawned_count.fetch_add(1, Ordering::SeqCst); + } + } + + let target = rt.spawn(PanicOnCommand).unwrap(); + let sup = rt.spawn(Supervisor { spawned_count: spawned.clone() }).unwrap(); + + rt.send_to(sup, SupervisorMsg::WatchThis(target)).unwrap(); + tick_n(&rt, 3); + + rt.send_to(target, PanicMsg).unwrap(); + tick_n(&rt, 5); + + assert_eq!(spawned.load(Ordering::SeqCst), 1, "supervisor should have spawned a replacement"); +} -- 2.45.2 From bf79d63be49d7b71cd7828f4738b8d4da402e007 Mon Sep 17 00:00:00 2001 From: Developer Date: Thu, 12 Feb 2026 18:03:30 +0000 Subject: [PATCH 3/5] =?UTF-8?q?feat:=20command=20interface=20=E2=80=94=20f?= =?UTF-8?q?rontend-agnostic=20dispatch=20crate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add swactor-command crate that decouples command dispatch from the dashboard, enabling the same commands from REPL, REST, TUI, or any future frontend. - CommandRouter with dispatch(), CommandHandler trait, CommandMeta - CommandRequest/CommandResponse (serde-serializable JSON protocol) - CommandContext with Arc + optional StatsEnricher trait - 9 built-in commands extracted from investigate.rs: help, overview, workers, worker, actors, actor, hot, phases, diff, shutdown - REPL line parser (parse_line) with positional arg mapping - REST adapter (from_query_params) for HTTP query parameters - Dashboard integration: StatsEnricher impl for StatsCollector, investigate.rs refactored to thin CommandRouter wrapper, server.rs updated to share CommandRouter across handler threads - 18 behavioral tests in crates/command/tests/command_api.rs Co-Authored-By: Claude Opus 4.6 --- CLAUDE/notes/progress.md | 43 +- Cargo.lock | 38 +- Cargo.toml | 2 +- crates/command/Cargo.toml | 9 + crates/command/src/builtins.rs | 561 ++++++++++++++++++++ crates/command/src/lib.rs | 242 +++++++++ crates/command/src/parse.rs | 90 ++++ crates/command/tests/command_api.rs | 367 +++++++++++++ crates/runtime-dashboard/Cargo.toml | 1 + crates/runtime-dashboard/src/collector.rs | 6 + crates/runtime-dashboard/src/investigate.rs | 518 +----------------- crates/runtime-dashboard/src/server.rs | 32 +- 12 files changed, 1368 insertions(+), 541 deletions(-) create mode 100644 crates/command/Cargo.toml create mode 100644 crates/command/src/builtins.rs create mode 100644 crates/command/src/lib.rs create mode 100644 crates/command/src/parse.rs create mode 100644 crates/command/tests/command_api.rs diff --git a/CLAUDE/notes/progress.md b/CLAUDE/notes/progress.md index 12bc493..3e56c14 100644 --- a/CLAUDE/notes/progress.md +++ b/CLAUDE/notes/progress.md @@ -13,24 +13,39 @@ - Implemented `watch`/`unwatch` on `Runtime`'s `ContextInner` impl - Added `watch_registry` field to `TickContext` in `src/delivery.rs` - 10 behavioral tests in `tests/watch_api.rs` — all passing -- All 53 tests pass (42 existing + 10 new + 1 doctest) -- All 134 distribution crate tests pass + +### Feature 2: Command Interface — `docs/os-design/04-command-interface.md` +- Created `crates/command/` crate (`swactor-command`) with core types: + - `CommandRequest`, `CommandResponse` (serde-serializable) + - `CommandHandler` trait + `CommandMeta` + - `CommandRouter` with `dispatch()` and `with_builtins()` + - `CommandContext` with `Arc` + optional `StatsEnricher` + - `StatsEnricher` trait (decouples command crate from dashboard) +- Built-in read commands: overview, workers, worker, actors, actor, hot, phases, diff +- Built-in write command: shutdown +- REPL line parser (`parse_line`) with positional arg mapping and `--flag value` support +- REST adapter (`from_query_params`) for HTTP query parameters +- Refactored `investigate.rs` to delegate to CommandRouter (thin wrapper) +- Updated `server.rs` to use CommandRouter for `/api/investigate` endpoint +- Implemented `StatsEnricher for StatsCollector` in dashboard crate +- 18 behavioral tests in `crates/command/tests/command_api.rs` — all passing +- All 53 swactor core tests pass, all 18 command tests pass ## Next Steps -1. **Command Interface** — `docs/os-design/04-command-interface.md` - - Create `crates/command/` crate with `CommandHandler` trait, `CommandRouter`, `CommandRequest`/`CommandResponse` types - - Extract existing `cmd_*` functions from `crates/runtime-dashboard/src/investigate.rs` into CommandHandler impls - - Add built-in read commands (help, overview, nodes, workers, actors) - - Add built-in write commands (spawn, stop, send, drain, shutdown) - - Write behavioral tests -2. **Cluster Registry** — `docs/os-design/02-cluster-registry.md` -3. **Node Capabilities** — `docs/os-design/03-node-capabilities.md` -4. **Remote Watching** — extends 01-actor-watching with wire protocol -5. **Supervision** — `docs/os-design/05-supervision.md` +1. **Cluster Registry** — `docs/os-design/02-cluster-registry.md` + - LWW-Register CRDT per name binding + - Propagation via SWIM piggyback + - `ClusterRegistry` struct in `crates/distribution/src/registry.rs` + - API: register_name, unregister_name, resolve_name +2. **Node Capabilities** — `docs/os-design/03-node-capabilities.md` + - New `crates/capabilities/` crate with auto-detection +3. **Remote Watching** — extends actor watching with wire protocol +4. **Supervision** — `docs/os-design/05-supervision.md` ## Open Questions -- Command Interface: Should custom actor commands (via `Ctx::register_command()`) be deferred to a later PR? -- Should the dashboard migration happen in the same PR as the command crate creation? +- Custom actor commands (via `Ctx::register_command()`) deferred to a later PR +- Distribution-aware commands (nodes, registry, resolve) deferred until cluster registry is implemented +- Write commands (spawn, stop, drain) deferred — need factory registry and actor stop mechanism ## Blockers - None diff --git a/Cargo.lock b/Cargo.lock index 06fe0c8..8ff07c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1657,6 +1657,7 @@ dependencies = [ "serde", "serde_json", "swactor", + "swactor-command", "tiny_http", "tracing", "tracing-subscriber", @@ -1986,6 +1987,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "swactor-command" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "swactor", +] + [[package]] name = "swactor-std" version = "0.1.0" @@ -2005,9 +2015,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.114" +version = "2.0.115" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" +checksum = "6e614ed320ac28113fa64972c4262d5dbc89deacdfd00c34a3e4cea073243c12" dependencies = [ "proc-macro2", "quote", @@ -2408,12 +2418,12 @@ dependencies = [ [[package]] name = "wasm-encoder" -version = "0.245.0" +version = "0.245.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95d568e113f706ee7a7df9b33547bb80721f55abffc79b3dc4d09c368690e662" +checksum = "3f9dca005e69bf015e45577e415b9af8c67e8ee3c0e38b5b0add5aa92581ed5c" dependencies = [ "leb128fmt", - "wasmparser 0.245.0", + "wasmparser 0.245.1", ] [[package]] @@ -2431,9 +2441,9 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.245.0" +version = "0.245.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48a767a48974f0c8b66f211b96e01aa77feed58b8ccce4e7f0cff0ae55b174d4" +checksum = "4f08c9adee0428b7bddf3890fc27e015ac4b761cc608c822667102b8bfd6995e" dependencies = [ "bitflags", "indexmap", @@ -2706,22 +2716,22 @@ dependencies = [ [[package]] name = "wast" -version = "245.0.0" +version = "245.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75ffc7471e16a6f3c7a3c3a230314915b5dcd158e5ef13ccda2f43358a9df00c" +checksum = "28cf1149285569120b8ce39db8b465e8a2b55c34cbb586bd977e43e2bc7300bf" dependencies = [ "bumpalo", "leb128fmt", "memchr", "unicode-width 0.2.0", - "wasm-encoder 0.245.0", + "wasm-encoder 0.245.1", ] [[package]] name = "wat" -version = "1.245.0" +version = "1.245.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bcac6f915e2a84a4c0d9df9d41ad7518d99cda13f3bb83e3b8c22bf8726ab6" +checksum = "cd48d1679b6858988cb96b154dda0ec5bbb09275b71db46057be37332d5477be" dependencies = [ "wast", ] @@ -2934,9 +2944,9 @@ checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" [[package]] name = "zmij" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4de98dfa5d5b7fef4ee834d0073d560c9ca7b6c46a71d058c48db7960f8cfaf7" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" [[package]] name = "zstd" diff --git a/Cargo.toml b/Cargo.toml index f582294..9afb0f6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = [".", "crates/python", "crates/wasm", "crates/wasm-actor", "crates/simulation", "crates/runtime-dashboard", "crates/distribution", "crates/simulation-dashboard", "crates/std"] +members = [".", "crates/python", "crates/wasm", "crates/wasm-actor", "crates/simulation", "crates/runtime-dashboard", "crates/distribution", "crates/simulation-dashboard", "crates/std", "crates/command"] exclude = ["tools/depgraph"] [package] diff --git a/crates/command/Cargo.toml b/crates/command/Cargo.toml new file mode 100644 index 0000000..d4f80fa --- /dev/null +++ b/crates/command/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "swactor-command" +version = "0.1.0" +edition = "2024" + +[dependencies] +swactor = { path = "../..", features = ["serde"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" diff --git a/crates/command/src/builtins.rs b/crates/command/src/builtins.rs new file mode 100644 index 0000000..42a06ac --- /dev/null +++ b/crates/command/src/builtins.rs @@ -0,0 +1,561 @@ +//! Built-in command handlers for runtime inspection and management. +//! +//! Extracted from `crates/runtime-dashboard/src/investigate.rs`. + +use std::collections::HashMap; +use std::time::{Duration, Instant}; + +use swactor::actor::ActorAddress; +use swactor::stats::TickTiming; + +use crate::{CommandContext, CommandHandler, CommandMeta, CommandResponse}; + +// ─── Arg helpers ───────────────────────────────────────────────────────────── + +fn arg_str<'a>(args: &'a HashMap, key: &str) -> Option<&'a str> { + args.get(key).and_then(|v| v.as_str()) +} + +fn arg_usize(args: &HashMap, key: &str) -> Option { + args.get(key).and_then(|v| { + v.as_u64() + .map(|n| n as usize) + .or_else(|| v.as_str().and_then(|s| s.parse().ok())) + }) +} + +fn arg_f64(args: &HashMap, key: &str) -> Option { + args.get(key).and_then(|v| { + v.as_f64() + .or_else(|| v.as_str().and_then(|s| s.parse().ok())) + }) +} + +// ─── Display helpers ───────────────────────────────────────────────────────── + +fn format_addr(addr: &ActorAddress) -> String { + format!("{addr}") +} + +fn full_hex(addr: &ActorAddress) -> String { + addr.0.iter().map(|b| format!("{b:02x}")).collect() +} + +// ─── Phase breakdown helper ────────────────────────────────────────────────── + +fn compute_phase_breakdown(timings: &[TickTiming]) -> serde_json::Value { + if timings.is_empty() { + return serde_json::json!({ + "ticks": 0, + "active_pct": 0.0, + "avg_tick_us": 0.0, + "phases_us": [0, 0, 0, 0, 0, 0], + "phases_pct": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + }); + } + + let n = timings.len(); + let active = timings.iter().filter(|t| t.did_work).count(); + let active_pct = (active as f64 / n as f64) * 100.0; + + let mut phase_sums = [0u64; 6]; + for t in timings { + for (i, &us) in t.phase_us.iter().enumerate() { + phase_sums[i] += us; + } + } + let total_us: u64 = phase_sums.iter().sum(); + let avg_tick_us = total_us as f64 / n as f64; + + let phases_pct: Vec = if total_us == 0 { + vec![0.0; 6] + } else { + phase_sums + .iter() + .map(|&s| (s as f64 / total_us as f64) * 100.0) + .collect() + }; + + serde_json::json!({ + "ticks": n, + "active_pct": active_pct, + "avg_tick_us": avg_tick_us, + "phases_us": phase_sums, + "phases_pct": phases_pct, + }) +} + +// ─── Read Commands ─────────────────────────────────────────────────────────── + +pub struct OverviewCommand; + +impl CommandHandler for OverviewCommand { + fn meta(&self) -> CommandMeta { + CommandMeta { + name: "overview", + description: "Summary: worker count, actor count, total messages, mailbox depth, panics", + usage: "overview", + is_write: false, + } + } + + fn handle( + &self, + _args: &HashMap, + ctx: &CommandContext, + ) -> CommandResponse { + let stats = ctx.enriched_stats(); + let total_msgs: u64 = stats.workers.iter().map(|w| w.messages_processed).sum(); + let total_mailbox: usize = stats.workers.iter().map(|w| w.mailbox_depth).sum(); + let total_panics: u64 = stats.workers.iter().map(|w| w.panics).sum(); + let total_type_mismatches: u64 = stats.workers.iter().map(|w| w.type_mismatches).sum(); + let total_local: u64 = stats.workers.iter().map(|w| w.local_sends).sum(); + let total_cross: u64 = stats.workers.iter().map(|w| w.cross_sends).sum(); + let total_inbox: u64 = stats.workers.iter().map(|w| w.inbox_sends).sum(); + + CommandResponse::ok( + "overview", + serde_json::json!({ + "workers": stats.num_workers, + "actors": stats.actor_details.len(), + "total_messages_processed": total_msgs, + "total_mailbox_depth": total_mailbox, + "total_panics": total_panics, + "total_type_mismatches": total_type_mismatches, + "sends": { + "local": total_local, + "cross_worker": total_cross, + "inbox": total_inbox, + }, + }), + ) + } +} + +pub struct WorkersCommand; + +impl CommandHandler for WorkersCommand { + fn meta(&self) -> CommandMeta { + CommandMeta { + name: "workers", + description: "Per-worker stats: actors, mailbox depth, messages, sends, panics", + usage: "workers", + is_write: false, + } + } + + fn handle( + &self, + _args: &HashMap, + ctx: &CommandContext, + ) -> CommandResponse { + let stats = ctx.stats(); + let workers: Vec<_> = stats + .workers + .iter() + .map(|w| { + serde_json::json!({ + "id": w.id, + "actors": w.num_actors, + "mailbox_depth": w.mailbox_depth, + "messages_processed": w.messages_processed, + "local_sends": w.local_sends, + "cross_sends": w.cross_sends, + "inbox_sends": w.inbox_sends, + "type_mismatches": w.type_mismatches, + "panics": w.panics, + }) + }) + .collect(); + CommandResponse::ok("workers", workers) + } +} + +pub struct WorkerCommand; + +impl CommandHandler for WorkerCommand { + fn meta(&self) -> CommandMeta { + CommandMeta { + name: "worker", + description: "Single worker detail with tick-phase timing breakdown", + usage: "worker ", + is_write: false, + } + } + + fn handle( + &self, + args: &HashMap, + ctx: &CommandContext, + ) -> CommandResponse { + let id = match arg_usize(args, "id") { + Some(id) => id, + None => return CommandResponse::err("worker", "usage: worker "), + }; + + let stats = ctx.enriched_stats(); + let w = match stats.workers.iter().find(|w| w.id == id) { + Some(w) => w, + None => { + return CommandResponse::err( + "worker", + format!("worker {id} not found (have 0..{})", stats.num_workers), + ) + } + }; + + let timings = stats.tick_timings.get(id).cloned().unwrap_or_default(); + let phase_breakdown = compute_phase_breakdown(&timings); + + let actors_on_worker: Vec<_> = stats + .actor_details + .iter() + .filter(|a| a.worker_id == id) + .map(|a| { + serde_json::json!({ + "address": format_addr(&a.address), + "mailbox_depth": a.mailbox_depth, + "last_msg_type": a.last_msg_type, + "messages_processed": a.messages_processed, + "poisoned": a.poisoned, + }) + }) + .collect(); + + CommandResponse::ok( + "worker", + serde_json::json!({ + "id": w.id, + "actors": w.num_actors, + "mailbox_depth": w.mailbox_depth, + "messages_processed": w.messages_processed, + "local_sends": w.local_sends, + "cross_sends": w.cross_sends, + "inbox_sends": w.inbox_sends, + "type_mismatches": w.type_mismatches, + "panics": w.panics, + "tick_phases": phase_breakdown, + "actor_details": actors_on_worker, + }), + ) + } +} + +pub struct ActorsCommand; + +impl CommandHandler for ActorsCommand { + fn meta(&self) -> CommandMeta { + CommandMeta { + name: "actors", + description: "List actors with optional sorting, limit, and worker filter", + usage: "actors [--sort mailbox|worker|address] [--limit N] [--worker W]", + is_write: false, + } + } + + fn handle( + &self, + args: &HashMap, + ctx: &CommandContext, + ) -> CommandResponse { + let stats = ctx.enriched_stats(); + let mut actors = stats.actor_details.clone(); + + let sort_by = arg_str(args, "sort").unwrap_or("mailbox"); + let limit = arg_usize(args, "limit").unwrap_or(usize::MAX); + let worker_filter = arg_usize(args, "worker"); + + if let Some(wid) = worker_filter { + actors.retain(|a| a.worker_id == wid); + } + + match sort_by { + "mailbox" => actors.sort_by(|a, b| b.mailbox_depth.cmp(&a.mailbox_depth)), + "worker" => actors.sort_by_key(|a| a.worker_id), + "address" => actors.sort_by(|a, b| a.address.0.cmp(&b.address.0)), + other => { + return CommandResponse::err( + "actors", + format!("unknown sort field `{other}` — use mailbox|worker|address"), + ) + } + } + + actors.truncate(limit); + + let rows: Vec<_> = actors + .iter() + .map(|a| { + serde_json::json!({ + "address": format_addr(&a.address), + "address_full": full_hex(&a.address), + "worker_id": a.worker_id, + "mailbox_depth": a.mailbox_depth, + "last_msg_type": a.last_msg_type, + "messages_processed": a.messages_processed, + "poisoned": a.poisoned, + }) + }) + .collect(); + + CommandResponse::ok( + "actors", + serde_json::json!({ + "total": stats.actor_details.len(), + "returned": rows.len(), + "sort": sort_by, + "actors": rows, + }), + ) + } +} + +pub struct ActorCommand; + +impl CommandHandler for ActorCommand { + fn meta(&self) -> CommandMeta { + CommandMeta { + name: "actor", + description: "Find actor(s) whose address starts with the given hex prefix", + usage: "actor ", + is_write: false, + } + } + + fn handle( + &self, + args: &HashMap, + ctx: &CommandContext, + ) -> CommandResponse { + let prefix = match arg_str(args, "prefix") { + Some(p) => p, + None => return CommandResponse::err("actor", "usage: actor "), + }; + + let stats = ctx.enriched_stats(); + let matches: Vec<_> = stats + .actor_details + .iter() + .filter(|a| full_hex(&a.address).starts_with(prefix)) + .map(|a| { + serde_json::json!({ + "address": format_addr(&a.address), + "address_full": full_hex(&a.address), + "worker_id": a.worker_id, + "mailbox_depth": a.mailbox_depth, + "last_msg_type": a.last_msg_type, + "messages_processed": a.messages_processed, + "poisoned": a.poisoned, + }) + }) + .collect(); + + CommandResponse::ok( + "actor", + serde_json::json!({ + "prefix": prefix, + "matches": matches.len(), + "actors": matches, + }), + ) + } +} + +pub struct HotCommand; + +impl CommandHandler for HotCommand { + fn meta(&self) -> CommandMeta { + CommandMeta { + name: "hot", + description: "Top N actors by mailbox depth (default 10)", + usage: "hot [N]", + is_write: false, + } + } + + fn handle( + &self, + args: &HashMap, + ctx: &CommandContext, + ) -> CommandResponse { + let n = arg_usize(args, "n").unwrap_or(10); + let stats = ctx.enriched_stats(); + + let mut actors = stats.actor_details.clone(); + actors.sort_by(|a, b| b.mailbox_depth.cmp(&a.mailbox_depth)); + actors.truncate(n); + + let rows: Vec<_> = actors + .iter() + .map(|a| { + serde_json::json!({ + "address": format_addr(&a.address), + "address_full": full_hex(&a.address), + "worker_id": a.worker_id, + "mailbox_depth": a.mailbox_depth, + "last_msg_type": a.last_msg_type, + "messages_processed": a.messages_processed, + "poisoned": a.poisoned, + }) + }) + .collect(); + + CommandResponse::ok("hot", rows) + } +} + +pub struct PhasesCommand; + +impl CommandHandler for PhasesCommand { + fn meta(&self) -> CommandMeta { + CommandMeta { + name: "phases", + description: "Tick-phase time breakdown (all workers or one)", + usage: "phases [worker_id]", + is_write: false, + } + } + + fn handle( + &self, + args: &HashMap, + ctx: &CommandContext, + ) -> CommandResponse { + let stats = ctx.stats(); + let worker_filter = arg_usize(args, "worker"); + + let phase_names = [ + "spawn_drain", + "transfer_drain", + "tick_all", + "spawn_drain_2", + "pending_local", + "stats_publish", + ]; + + let mut results = Vec::new(); + for (i, timings) in stats.tick_timings.iter().enumerate() { + if let Some(wid) = worker_filter { + if i != wid { + continue; + } + } + let breakdown = compute_phase_breakdown(timings); + results.push(serde_json::json!({ + "worker_id": i, + "ticks_sampled": timings.len(), + "phases": breakdown, + "phase_names": phase_names, + })); + } + + CommandResponse::ok("phases", results) + } +} + +pub struct DiffCommand; + +impl CommandHandler for DiffCommand { + fn meta(&self) -> CommandMeta { + CommandMeta { + name: "diff", + description: "Collect two snapshots N seconds apart, report deltas and rates", + usage: "diff ", + is_write: false, + } + } + + fn handle( + &self, + args: &HashMap, + ctx: &CommandContext, + ) -> CommandResponse { + let secs = match arg_f64(args, "seconds") { + Some(s) if s > 0.0 && s <= 30.0 => s, + Some(_) => return CommandResponse::err("diff", "seconds must be between 0 and 30"), + None => return CommandResponse::err("diff", "usage: diff "), + }; + + let before = ctx.enriched_stats(); + let t0 = Instant::now(); + std::thread::sleep(Duration::from_secs_f64(secs)); + let after = ctx.enriched_stats(); + let elapsed = t0.elapsed().as_secs_f64(); + + let msgs_before: u64 = before.workers.iter().map(|w| w.messages_processed).sum(); + let msgs_after: u64 = after.workers.iter().map(|w| w.messages_processed).sum(); + let delta_msgs = msgs_after.saturating_sub(msgs_before); + + let local_before: u64 = before.workers.iter().map(|w| w.local_sends).sum(); + let local_after: u64 = after.workers.iter().map(|w| w.local_sends).sum(); + let cross_before: u64 = before.workers.iter().map(|w| w.cross_sends).sum(); + let cross_after: u64 = after.workers.iter().map(|w| w.cross_sends).sum(); + + let mailbox_before: usize = before.workers.iter().map(|w| w.mailbox_depth).sum(); + let mailbox_after: usize = after.workers.iter().map(|w| w.mailbox_depth).sum(); + + let per_worker: Vec<_> = after + .workers + .iter() + .enumerate() + .map(|(i, w)| { + let prev = before.workers.get(i); + let d = prev + .map(|p| w.messages_processed.saturating_sub(p.messages_processed)) + .unwrap_or(0); + serde_json::json!({ + "worker_id": i, + "delta_messages": d, + "msg_per_sec": d as f64 / elapsed, + "actors_before": prev.map(|p| p.num_actors).unwrap_or(0), + "actors_after": w.num_actors, + "mailbox_before": prev.map(|p| p.mailbox_depth).unwrap_or(0), + "mailbox_after": w.mailbox_depth, + }) + }) + .collect(); + + CommandResponse::ok( + "diff", + serde_json::json!({ + "elapsed_s": elapsed, + "actors_before": before.actor_details.len(), + "actors_after": after.actor_details.len(), + "delta_messages": delta_msgs, + "msg_per_sec": delta_msgs as f64 / elapsed, + "delta_local_sends": local_after.saturating_sub(local_before), + "delta_cross_sends": cross_after.saturating_sub(cross_before), + "mailbox_before": mailbox_before, + "mailbox_after": mailbox_after, + "per_worker": per_worker, + }), + ) + } +} + +// ─── Write Commands ────────────────────────────────────────────────────────── + +pub struct ShutdownCommand; + +impl CommandHandler for ShutdownCommand { + fn meta(&self) -> CommandMeta { + CommandMeta { + name: "shutdown", + description: "Signal the runtime to shut down gracefully", + usage: "shutdown", + is_write: true, + } + } + + fn handle( + &self, + _args: &HashMap, + ctx: &CommandContext, + ) -> CommandResponse { + ctx.runtime.shutdown(); + CommandResponse::ok( + "shutdown", + serde_json::json!({"status": "shutdown signaled"}), + ) + } +} diff --git a/crates/command/src/lib.rs b/crates/command/src/lib.rs new file mode 100644 index 0000000..95fa287 --- /dev/null +++ b/crates/command/src/lib.rs @@ -0,0 +1,242 @@ +//! Frontend-agnostic command dispatch for swactor runtimes. +//! +//! Provides [`CommandRouter`] that maps command names to [`CommandHandler`] +//! implementations, with built-in commands for runtime inspection and management. +//! +//! # Architecture +//! +//! ```text +//! Frontend (REPL, REST, TUI, WebSocket) +//! │ +//! ▼ +//! CommandRouter::dispatch(CommandRequest, CommandContext) +//! │ +//! ├── built-in handlers (overview, workers, actors, …) +//! └── custom handlers (user-registered) +//! ``` + +pub mod builtins; +mod parse; + +pub use parse::{from_query_params, parse_line}; + +use std::collections::HashMap; +use std::sync::Arc; + +use serde::{Deserialize, Serialize}; +use swactor::runtime::Runtime; +use swactor::stats::RuntimeStats; + +// ─── Core Types ────────────────────────────────────────────────────────────── + +/// A command request. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CommandRequest { + pub command: String, + pub args: HashMap, +} + +/// A command response. Always JSON-serializable. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CommandResponse { + pub ok: bool, + pub command: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub data: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl CommandResponse { + pub fn ok(command: &str, data: impl Serialize) -> Self { + Self { + ok: true, + command: command.to_string(), + data: Some(serde_json::to_value(data).unwrap_or(serde_json::Value::Null)), + error: None, + } + } + + pub fn err(command: &str, msg: impl Into) -> Self { + Self { + ok: false, + command: command.to_string(), + data: None, + error: Some(msg.into()), + } + } + + /// Serialize to a single JSON line (for REPL/wire protocol). + pub fn to_json_line(&self) -> String { + serde_json::to_string(self).unwrap_or_else(|e| { + format!(r#"{{"ok":false,"command":"","error":"serialization: {e}"}}"#) + }) + } +} + +// ─── Handler Trait ─────────────────────────────────────────────────────────── + +/// Metadata about a command, used for help text and validation. +pub struct CommandMeta { + pub name: &'static str, + pub description: &'static str, + pub usage: &'static str, + pub is_write: bool, +} + +/// A command handler. Implementations are stateless — all state +/// comes through [`CommandContext`]. +pub trait CommandHandler: Send + Sync { + fn meta(&self) -> CommandMeta; + fn handle( + &self, + args: &HashMap, + ctx: &CommandContext, + ) -> CommandResponse; +} + +// ─── Stats Enrichment ──────────────────────────────────────────────────────── + +/// Enriches [`RuntimeStats`] with per-actor detail. +/// +/// Implement this on your stats collector so command handlers can access +/// enriched data without depending on the dashboard crate. +pub trait StatsEnricher: Send + Sync { + fn enrich(&self, stats: &mut RuntimeStats); +} + +// ─── Context ───────────────────────────────────────────────────────────────── + +/// Context available to command handlers. +pub struct CommandContext { + pub runtime: Arc, + pub enricher: Option>, +} + +impl CommandContext { + pub fn new(runtime: Arc) -> Self { + Self { + runtime, + enricher: None, + } + } + + pub fn with_enricher( + runtime: Arc, + enricher: Arc, + ) -> Self { + Self { + runtime, + enricher: Some(enricher), + } + } + + /// Raw runtime stats (no enrichment). + pub fn stats(&self) -> RuntimeStats { + self.runtime.stats() + } + + /// Runtime stats enriched with per-actor detail (if an enricher is set). + pub fn enriched_stats(&self) -> RuntimeStats { + let mut s = self.runtime.stats(); + if let Some(e) = &self.enricher { + e.enrich(&mut s); + } + s + } +} + +// ─── Router ────────────────────────────────────────────────────────────────── + +/// Central command dispatch. +pub struct CommandRouter { + handlers: HashMap>, +} + +impl CommandRouter { + pub fn new() -> Self { + Self { + handlers: HashMap::new(), + } + } + + /// Create a router with all built-in commands registered. + pub fn with_builtins() -> Self { + let mut router = Self::new(); + router.register(Box::new(builtins::OverviewCommand)); + router.register(Box::new(builtins::WorkersCommand)); + router.register(Box::new(builtins::WorkerCommand)); + router.register(Box::new(builtins::ActorsCommand)); + router.register(Box::new(builtins::ActorCommand)); + router.register(Box::new(builtins::HotCommand)); + router.register(Box::new(builtins::PhasesCommand)); + router.register(Box::new(builtins::DiffCommand)); + router.register(Box::new(builtins::ShutdownCommand)); + router + } + + /// Register a custom command handler. + pub fn register(&mut self, handler: Box) { + let name = handler.meta().name.to_string(); + self.handlers.insert(name, handler); + } + + /// Dispatch a command request. + /// + /// The `help` command is handled directly by the router (it needs + /// access to all registered handlers). + pub fn dispatch( + &self, + req: &CommandRequest, + ctx: &CommandContext, + ) -> CommandResponse { + if req.command == "help" { + return self.cmd_help(); + } + match self.handlers.get(&req.command) { + Some(handler) => handler.handle(&req.args, ctx), + None => CommandResponse::err( + &req.command, + format!("unknown command `{}` — try `help`", req.command), + ), + } + } + + fn cmd_help(&self) -> CommandResponse { + let mut commands: Vec = self + .handlers + .values() + .map(|h| { + let m = h.meta(); + serde_json::json!({ + "name": m.name, + "usage": m.usage, + "description": m.description, + "is_write": m.is_write, + }) + }) + .collect(); + // Add help itself + commands.push(serde_json::json!({ + "name": "help", + "usage": "help", + "description": "List all available commands", + "is_write": false, + })); + commands.sort_by(|a, b| { + a["name"] + .as_str() + .unwrap_or("") + .cmp(b["name"].as_str().unwrap_or("")) + }); + CommandResponse::ok("help", serde_json::json!({ "commands": commands })) + } + + /// List names of all registered commands (sorted). + pub fn command_names(&self) -> Vec<&str> { + let mut names: Vec<_> = self.handlers.keys().map(|s| s.as_str()).collect(); + names.push("help"); + names.sort(); + names + } +} diff --git a/crates/command/src/parse.rs b/crates/command/src/parse.rs new file mode 100644 index 0000000..7703da5 --- /dev/null +++ b/crates/command/src/parse.rs @@ -0,0 +1,90 @@ +//! Input parsers for REPL lines and HTTP query parameters. + +use std::collections::HashMap; + +use crate::CommandRequest; + +/// Parse a REPL text line into a [`CommandRequest`]. +/// +/// Handles `--flag value` pairs and maps positional arguments to +/// command-specific named parameters. +/// +/// # Examples +/// +/// ```text +/// "overview" → { command: "overview", args: {} } +/// "worker 3" → { command: "worker", args: { "id": "3" } } +/// "actors --sort mailbox" → { command: "actors", args: { "sort": "mailbox" } } +/// "hot 5" → { command: "hot", args: { "n": "5" } } +/// ``` +pub fn parse_line(line: &str) -> CommandRequest { + let parts: Vec<&str> = line.split_whitespace().collect(); + if parts.is_empty() { + return CommandRequest { + command: "help".to_string(), + args: HashMap::new(), + }; + } + let command = parts[0].to_string(); + let rest = &parts[1..]; + + let mut args = HashMap::new(); + let mut i = 0; + let mut positional = 0; + + while i < rest.len() { + if let Some(key) = rest[i].strip_prefix("--") { + if i + 1 < rest.len() && !rest[i + 1].starts_with("--") { + args.insert( + key.to_string(), + serde_json::Value::String(rest[i + 1].to_string()), + ); + i += 2; + } else { + args.insert(key.to_string(), serde_json::Value::Bool(true)); + i += 1; + } + } else { + let name = positional_arg_name(&command, positional); + if !name.is_empty() { + args.insert( + name.to_string(), + serde_json::Value::String(rest[i].to_string()), + ); + } + positional += 1; + i += 1; + } + } + + CommandRequest { command, args } +} + +/// Convert HTTP query parameters to a [`CommandRequest`]. +/// +/// The `cmd` parameter becomes the command name; all other parameters +/// become string-valued arguments. +pub fn from_query_params(params: &HashMap) -> CommandRequest { + let command = params + .get("cmd") + .cloned() + .unwrap_or_else(|| "help".into()); + let args: HashMap = params + .iter() + .filter(|(k, _)| *k != "cmd") + .map(|(k, v)| (k.clone(), serde_json::Value::String(v.clone()))) + .collect(); + CommandRequest { command, args } +} + +/// Map positional argument index to the named parameter for each command. +fn positional_arg_name(command: &str, position: usize) -> &'static str { + match (command, position) { + ("worker", 0) => "id", + ("actor", 0) => "prefix", + ("hot", 0) => "n", + ("phases", 0) => "worker", + ("diff", 0) => "seconds", + _ => "", + } +} diff --git a/crates/command/tests/command_api.rs b/crates/command/tests/command_api.rs new file mode 100644 index 0000000..4e6e33d --- /dev/null +++ b/crates/command/tests/command_api.rs @@ -0,0 +1,367 @@ +//! Behavioral tests for the swactor-command crate. +//! +//! Tests exercise the full dispatch path: parse → route → handle → response. + +use std::collections::HashMap; +use std::sync::Arc; + +use swactor::actor::ActorInterface; +use swactor::runtime::{Ctx, Runtime, RuntimeConfig}; +use swactor_command::{ + from_query_params, parse_line, CommandContext, CommandRequest, CommandResponse, CommandRouter, +}; + +// ── Test Helpers ───────────────────────────────────────────────────────────── + +fn single_thread_config() -> RuntimeConfig { + RuntimeConfig { + num_threads: 1, + ..RuntimeConfig::default() + } +} + +fn make_router_and_ctx() -> (CommandRouter, CommandContext) { + let rt = Arc::new(Runtime::new(single_thread_config())); + let router = CommandRouter::with_builtins(); + let ctx = CommandContext::new(rt); + (router, ctx) +} + +fn dispatch_text(router: &CommandRouter, ctx: &CommandContext, line: &str) -> CommandResponse { + let req = parse_line(line); + let resp = router.dispatch(&req, ctx); + // Verify JSON round-trip works + let json = resp.to_json_line(); + serde_json::from_str::(&json) + .expect("response should be valid JSON") +} + +/// A no-op actor for spawning into the runtime. +struct DummyActor; +#[derive(Clone)] +struct DummyMsg; +impl ActorInterface for DummyActor { + type Incoming = DummyMsg; + type Response = (); + fn handle(&mut self, _ctx: &Ctx, _msg: DummyMsg) {} +} + +// ── Tests ──────────────────────────────────────────────────────────────────── + +/// Given a router with builtins, +/// when "help" is dispatched, +/// then the response lists all registered commands. +#[test] +fn help_lists_all_registered_commands() { + let (router, ctx) = make_router_and_ctx(); + let resp = dispatch_text(&router, &ctx, "help"); + + assert!(resp.ok, "help should succeed"); + assert_eq!(resp.command, "help"); + + let data = resp.data.unwrap(); + let commands = data["commands"].as_array().unwrap(); + + // Should have all builtins + help itself + let names: Vec<&str> = commands + .iter() + .map(|c| c["name"].as_str().unwrap()) + .collect(); + assert!(names.contains(&"overview"), "should list overview"); + assert!(names.contains(&"workers"), "should list workers"); + assert!(names.contains(&"worker"), "should list worker"); + assert!(names.contains(&"actors"), "should list actors"); + assert!(names.contains(&"hot"), "should list hot"); + assert!(names.contains(&"phases"), "should list phases"); + assert!(names.contains(&"diff"), "should list diff"); + assert!(names.contains(&"shutdown"), "should list shutdown"); + assert!(names.contains(&"help"), "should list help itself"); + + // Should be sorted + let mut sorted = names.clone(); + sorted.sort(); + assert_eq!(names, sorted, "commands should be sorted alphabetically"); +} + +/// Given a router, +/// when an unknown command is dispatched, +/// then the response indicates failure with a helpful message. +#[test] +fn unknown_command_returns_error() { + let (router, ctx) = make_router_and_ctx(); + let resp = dispatch_text(&router, &ctx, "nonexistent"); + + assert!(!resp.ok, "unknown command should fail"); + assert_eq!(resp.command, "nonexistent"); + let err = resp.error.unwrap(); + assert!( + err.contains("unknown command") && err.contains("help"), + "error should mention 'unknown command' and suggest 'help', got: {err}" + ); +} + +/// Given a runtime with no actors, +/// when "overview" is dispatched, +/// then the response contains expected summary fields with zero counts. +#[test] +fn overview_returns_summary_fields() { + let (router, ctx) = make_router_and_ctx(); + let resp = dispatch_text(&router, &ctx, "overview"); + + assert!(resp.ok); + assert_eq!(resp.command, "overview"); + + let data = resp.data.unwrap(); + assert_eq!(data["workers"], 1, "single-threaded = 1 worker"); + assert_eq!(data["actors"], 0, "no actors spawned"); + assert_eq!(data["total_messages_processed"], 0); + assert_eq!(data["total_panics"], 0); + assert!(data["sends"].is_object(), "sends should be an object"); +} + +/// Given a runtime with spawned actors, +/// when "workers" is dispatched, +/// then the response contains per-worker stats. +#[test] +fn workers_returns_per_worker_info() { + let rt = Arc::new(Runtime::new(single_thread_config())); + // Spawn some actors + rt.spawn(DummyActor).unwrap(); + rt.spawn(DummyActor).unwrap(); + rt.tick(); + + let router = CommandRouter::with_builtins(); + let ctx = CommandContext::new(rt); + let resp = dispatch_text(&router, &ctx, "workers"); + + assert!(resp.ok); + let data = resp.data.unwrap(); + let workers = data.as_array().unwrap(); + assert_eq!(workers.len(), 1, "single-threaded has 1 worker"); + assert_eq!(workers[0]["id"], 0); + assert_eq!(workers[0]["actors"], 2, "2 actors spawned on worker 0"); +} + +/// Given "worker 0" with a valid ID, +/// when dispatched, +/// then the response includes worker detail and tick phase info. +#[test] +fn worker_command_with_valid_id() { + let (router, ctx) = make_router_and_ctx(); + let resp = dispatch_text(&router, &ctx, "worker 0"); + + assert!(resp.ok); + assert_eq!(resp.command, "worker"); + let data = resp.data.unwrap(); + assert_eq!(data["id"], 0); + assert!(data["tick_phases"].is_object(), "should include phase breakdown"); +} + +/// Given "worker 99", +/// when dispatched, +/// then the response is an error (worker not found). +#[test] +fn worker_command_invalid_id_returns_error() { + let (router, ctx) = make_router_and_ctx(); + let resp = dispatch_text(&router, &ctx, "worker 99"); + + assert!(!resp.ok); + assert!(resp.error.unwrap().contains("not found")); +} + +/// Given "worker" with no ID, +/// when dispatched, +/// then the response is a usage error. +#[test] +fn worker_command_missing_id_returns_usage() { + let (router, ctx) = make_router_and_ctx(); + let resp = dispatch_text(&router, &ctx, "worker"); + + assert!(!resp.ok); + assert!(resp.error.unwrap().contains("usage")); +} + +/// Given "phases", +/// when dispatched, +/// then the response includes phase breakdown per worker. +#[test] +fn phases_command_returns_breakdown() { + let (router, ctx) = make_router_and_ctx(); + let resp = dispatch_text(&router, &ctx, "phases"); + + assert!(resp.ok); + let data = resp.data.unwrap(); + let phases = data.as_array().unwrap(); + assert_eq!(phases.len(), 1, "single-threaded = 1 worker"); + assert_eq!(phases[0]["worker_id"], 0); +} + +/// Given a runtime, when "shutdown" is dispatched, +/// then the response indicates success. +#[test] +fn shutdown_command_signals_runtime() { + let (router, ctx) = make_router_and_ctx(); + let resp = dispatch_text(&router, &ctx, "shutdown"); + + assert!(resp.ok); + assert_eq!(resp.command, "shutdown"); + let data = resp.data.unwrap(); + assert_eq!(data["status"], "shutdown signaled"); +} + +// ── REPL Parser Tests ──────────────────────────────────────────────────────── + +/// Given a simple command with no args, +/// when parsed, +/// then the command name is extracted correctly. +#[test] +fn parse_line_simple_command() { + let req = parse_line("overview"); + assert_eq!(req.command, "overview"); + assert!(req.args.is_empty()); +} + +/// Given a command with positional args, +/// when parsed, +/// then positional args are mapped to named parameters. +#[test] +fn parse_line_positional_args() { + let req = parse_line("worker 3"); + assert_eq!(req.command, "worker"); + assert_eq!(req.args["id"], "3"); + + let req = parse_line("hot 5"); + assert_eq!(req.command, "hot"); + assert_eq!(req.args["n"], "5"); + + let req = parse_line("diff 2.5"); + assert_eq!(req.command, "diff"); + assert_eq!(req.args["seconds"], "2.5"); + + let req = parse_line("actor a1b2"); + assert_eq!(req.command, "actor"); + assert_eq!(req.args["prefix"], "a1b2"); +} + +/// Given a command with --flag value pairs, +/// when parsed, +/// then flags are mapped to named args. +#[test] +fn parse_line_flags() { + let req = parse_line("actors --sort mailbox --limit 5"); + assert_eq!(req.command, "actors"); + assert_eq!(req.args["sort"], "mailbox"); + assert_eq!(req.args["limit"], "5"); +} + +/// Given a command with mixed positional and flag args, +/// when parsed, +/// then both are captured correctly. +#[test] +fn parse_line_mixed_args() { + let req = parse_line("actors --sort worker --worker 2 --limit 10"); + assert_eq!(req.command, "actors"); + assert_eq!(req.args["sort"], "worker"); + assert_eq!(req.args["worker"], "2"); + assert_eq!(req.args["limit"], "10"); +} + +/// Given empty input, +/// when parsed, +/// then default to "help". +#[test] +fn parse_line_empty_defaults_to_help() { + let req = parse_line(""); + assert_eq!(req.command, "help"); +} + +// ── REST Adapter Tests ─────────────────────────────────────────────────────── + +/// Given query params with cmd and other params, +/// when converted, +/// then cmd becomes the command and others become args. +#[test] +fn from_query_params_extracts_cmd() { + let mut params = HashMap::new(); + params.insert("cmd".to_string(), "actor".to_string()); + params.insert("prefix".to_string(), "a1b2".to_string()); + + let req = from_query_params(¶ms); + assert_eq!(req.command, "actor"); + assert_eq!(req.args["prefix"], "a1b2"); + assert!(!req.args.contains_key("cmd"), "cmd should not be in args"); +} + +/// Given query params with no cmd, +/// when converted, +/// then default to "help". +#[test] +fn from_query_params_defaults_to_help() { + let params = HashMap::new(); + let req = from_query_params(¶ms); + assert_eq!(req.command, "help"); +} + +// ── Custom Handler Test ────────────────────────────────────────────────────── + +/// Given a custom command handler registered on the router, +/// when that command is dispatched, +/// then the custom handler runs and returns its response. +#[test] +fn custom_command_handler() { + struct PingCommand; + impl swactor_command::CommandHandler for PingCommand { + fn meta(&self) -> swactor_command::CommandMeta { + swactor_command::CommandMeta { + name: "ping", + description: "Respond with pong", + usage: "ping", + is_write: false, + } + } + fn handle( + &self, + _args: &HashMap, + _ctx: &CommandContext, + ) -> CommandResponse { + CommandResponse::ok("ping", serde_json::json!({"reply": "pong"})) + } + } + + let rt = Arc::new(Runtime::new(single_thread_config())); + let mut router = CommandRouter::with_builtins(); + router.register(Box::new(PingCommand)); + let ctx = CommandContext::new(rt); + + let resp = dispatch_text(&router, &ctx, "ping"); + assert!(resp.ok); + assert_eq!(resp.data.unwrap()["reply"], "pong"); + + // Should also appear in help + let help = dispatch_text(&router, &ctx, "help"); + let commands = help.data.unwrap()["commands"].as_array().unwrap().clone(); + let names: Vec<&str> = commands.iter().map(|c| c["name"].as_str().unwrap()).collect(); + assert!(names.contains(&"ping"), "custom command should appear in help"); +} + +/// Given a JSON-serialized CommandResponse, +/// when deserialized, +/// then ok/false fields, data, and error are preserved. +#[test] +fn response_json_roundtrip() { + let ok_resp = CommandResponse::ok("test", serde_json::json!({"key": "value"})); + let json = ok_resp.to_json_line(); + let parsed: CommandResponse = serde_json::from_str(&json).unwrap(); + assert!(parsed.ok); + assert_eq!(parsed.command, "test"); + assert_eq!(parsed.data.unwrap()["key"], "value"); + assert!(parsed.error.is_none()); + + let err_resp = CommandResponse::err("bad", "something went wrong"); + let json = err_resp.to_json_line(); + let parsed: CommandResponse = serde_json::from_str(&json).unwrap(); + assert!(!parsed.ok); + assert_eq!(parsed.command, "bad"); + assert!(parsed.data.is_none()); + assert_eq!(parsed.error.unwrap(), "something went wrong"); +} diff --git a/crates/runtime-dashboard/Cargo.toml b/crates/runtime-dashboard/Cargo.toml index 3e24adb..1894aa7 100644 --- a/crates/runtime-dashboard/Cargo.toml +++ b/crates/runtime-dashboard/Cargo.toml @@ -14,6 +14,7 @@ crossbeam-queue = "0.3.12" ratatui = { version = "0.29", optional = true, default-features = false, features = ["crossterm"] } crossterm = { version = "0.28", optional = true } distribution = { path = "../distribution", optional = true } +swactor-command = { path = "../command" } [dependencies.ctrlc] version = "3" diff --git a/crates/runtime-dashboard/src/collector.rs b/crates/runtime-dashboard/src/collector.rs index 59df4d0..ab1dcd7 100644 --- a/crates/runtime-dashboard/src/collector.rs +++ b/crates/runtime-dashboard/src/collector.rs @@ -42,6 +42,12 @@ impl StatsCollector { } } +impl swactor_command::StatsEnricher for StatsCollector { + fn enrich(&self, stats: &mut swactor::stats::RuntimeStats) { + stats.actor_details = self.actor_details(); + } +} + impl StatsHook for StatsCollector { fn on_tick(&self, worker_id: usize, snapshots: &[ActorSnapshot]) { if let Some(slot) = self.slots.get(worker_id) { diff --git a/crates/runtime-dashboard/src/investigate.rs b/crates/runtime-dashboard/src/investigate.rs index 32bf9b8..cd81995 100644 --- a/crates/runtime-dashboard/src/investigate.rs +++ b/crates/runtime-dashboard/src/investigate.rs @@ -1,5 +1,7 @@ //! Line-oriented diagnostic protocol for LLM-driven runtime investigation. //! +//! Delegates all command logic to the `swactor-command` crate. +//! //! Send text commands on stdin, receive JSON responses on stdout (one per line). //! All human-readable diagnostics go to stderr. //! @@ -19,16 +21,17 @@ use std::collections::HashMap; use std::io::{self, BufRead, Write}; use std::sync::Arc; -use std::time::{Duration, Instant}; -use serde::Serialize; use swactor::runtime::Runtime; -use swactor::stats::RuntimeStats; +use swactor_command::{CommandContext, CommandRouter}; use crate::collector::StatsCollector; /// Run the investigate REPL. Blocks until stdin is closed or `quit` is received. pub fn run_investigate(runtime: Arc, collector: Arc) -> io::Result<()> { + let router = CommandRouter::with_builtins(); + let ctx = CommandContext::with_enricher(runtime, collector); + let stdin = io::stdin(); let mut stdout = io::stdout(); @@ -40,18 +43,14 @@ pub fn run_investigate(runtime: Arc, collector: Arc) -> if line.is_empty() { continue; } - - let parts: Vec<&str> = line.split_whitespace().collect(); - let cmd = parts[0]; - let args = &parts[1..]; - - if cmd == "quit" || cmd == "exit" { + if line == "quit" || line == "exit" { break; } - let response = dispatch_repl(cmd, args, &runtime, &collector); + let req = swactor_command::parse_line(line); + let resp = router.dispatch(&req, &ctx); - stdout.write_all(response.as_bytes())?; + stdout.write_all(resp.to_json_line().as_bytes())?; stdout.write_all(b"\n")?; stdout.flush()?; } @@ -59,501 +58,14 @@ pub fn run_investigate(runtime: Arc, collector: Arc) -> Ok(()) } -fn dispatch_repl(cmd: &str, args: &[&str], runtime: &Runtime, collector: &StatsCollector) -> String { - match cmd { - "help" => cmd_help(), - "overview" => cmd_overview(runtime, collector), - "workers" => cmd_workers(runtime), - "worker" => cmd_worker(runtime, collector, args), - "actors" => cmd_actors(runtime, collector, args), - "actor" => cmd_actor(runtime, collector, args), - "hot" => cmd_hot(runtime, collector, args), - "phases" => cmd_phases(runtime, args), - "diff" => cmd_diff(runtime, collector, args), - _ => err_response(cmd, &format!("unknown command `{cmd}` — try `help`")), - } -} - /// Dispatch an investigate command from HTTP query parameters. /// -/// Maps `?cmd=overview`, `?cmd=hot&n=10`, etc. to the appropriate command function. +/// Maps `?cmd=overview`, `?cmd=hot&n=10`, etc. to the appropriate command. pub fn dispatch_command( - cmd: &str, params: &HashMap, - runtime: &Runtime, - collector: &StatsCollector, + router: &CommandRouter, + ctx: &CommandContext, ) -> String { - match cmd { - "help" => cmd_help(), - "overview" => cmd_overview(runtime, collector), - "workers" => cmd_workers(runtime), - "worker" => { - let id = params.get("id").map(|s| s.as_str()).unwrap_or(""); - cmd_worker(runtime, collector, &[id]) - } - "actors" => { - let mut args = Vec::new(); - if let Some(sort) = params.get("sort") { - args.push("--sort"); - args.push(sort.as_str()); - } - if let Some(limit) = params.get("limit") { - args.push("--limit"); - args.push(limit.as_str()); - } - if let Some(worker) = params.get("worker") { - args.push("--worker"); - args.push(worker.as_str()); - } - cmd_actors(runtime, collector, &args) - } - "actor" => { - let prefix = params.get("prefix").map(|s| s.as_str()).unwrap_or(""); - cmd_actor(runtime, collector, &[prefix]) - } - "hot" => { - let n = params.get("n").map(|s| s.as_str()).unwrap_or("10"); - cmd_hot(runtime, collector, &[n]) - } - "phases" => { - match params.get("worker") { - Some(w) => cmd_phases(runtime, &[w.as_str()]), - None => cmd_phases(runtime, &[]), - } - } - "diff" => { - let secs = params.get("seconds").map(|s| s.as_str()).unwrap_or(""); - cmd_diff(runtime, collector, &[secs]) - } - _ => err_response(cmd, &format!("unknown command `{cmd}` — try `help`")), - } -} - -// ── Helpers ───────────────────────────────────────────────────────────── - -fn ok_response(cmd: &str, data: impl Serialize) -> String { - serde_json::to_string(&serde_json::json!({ - "ok": true, - "command": cmd, - "data": data, - })) - .unwrap_or_else(|e| err_response(cmd, &format!("serialization error: {e}"))) -} - -fn err_response(cmd: &str, msg: &str) -> String { - serde_json::to_string(&serde_json::json!({ - "ok": false, - "command": cmd, - "error": msg, - })) - .unwrap() -} - -fn format_addr(addr: &swactor::actor::ActorAddress) -> String { - format!("{addr}") -} - -fn full_hex(addr: &swactor::actor::ActorAddress) -> String { - addr.0.iter().map(|b| format!("{b:02x}")).collect() -} - -fn enriched_stats(rt: &Runtime, col: &StatsCollector) -> RuntimeStats { - let mut s = rt.stats(); - col.enrich(&mut s); - s -} - -// ── Commands ──────────────────────────────────────────────────────────── - -pub fn cmd_help() -> String { - ok_response( - "help", - serde_json::json!({ - "commands": [ - {"name": "overview", "usage": "overview", "description": "Summary: worker count, actor count, total messages, mailbox depth, panics"}, - {"name": "workers", "usage": "workers", "description": "Per-worker stats: actors, mailbox depth, messages, sends (local/cross/inbox), panics"}, - {"name": "worker", "usage": "worker ", "description": "Single worker detail with tick-phase timing breakdown"}, - {"name": "actors", "usage": "actors [--sort mailbox|worker|address] [--limit N] [--worker W]", "description": "List actors with optional sorting, limit, and worker filter"}, - {"name": "actor", "usage": "actor ", "description": "Find actor(s) whose address starts with the given hex prefix"}, - {"name": "hot", "usage": "hot [N]", "description": "Top N actors by mailbox depth (default 10)"}, - {"name": "phases", "usage": "phases [worker_id]", "description": "Tick-phase time breakdown (all workers or one)"}, - {"name": "diff", "usage": "diff ", "description": "Collect two snapshots N seconds apart, report deltas and rates"}, - {"name": "quit", "usage": "quit", "description": "Exit the investigate session"}, - ] - }), - ) -} - -pub fn cmd_overview(rt: &Runtime, col: &StatsCollector) -> String { - let stats = enriched_stats(rt, col); - let total_msgs: u64 = stats.workers.iter().map(|w| w.messages_processed).sum(); - let total_mailbox: usize = stats.workers.iter().map(|w| w.mailbox_depth).sum(); - let total_panics: u64 = stats.workers.iter().map(|w| w.panics).sum(); - let total_type_mismatches: u64 = stats.workers.iter().map(|w| w.type_mismatches).sum(); - let total_local: u64 = stats.workers.iter().map(|w| w.local_sends).sum(); - let total_cross: u64 = stats.workers.iter().map(|w| w.cross_sends).sum(); - let total_inbox: u64 = stats.workers.iter().map(|w| w.inbox_sends).sum(); - - ok_response( - "overview", - serde_json::json!({ - "workers": stats.num_workers, - "actors": stats.actor_details.len(), - "total_messages_processed": total_msgs, - "total_mailbox_depth": total_mailbox, - "total_panics": total_panics, - "total_type_mismatches": total_type_mismatches, - "sends": { - "local": total_local, - "cross_worker": total_cross, - "inbox": total_inbox, - }, - }), - ) -} - -pub fn cmd_workers(rt: &Runtime) -> String { - let stats = rt.stats(); - let workers: Vec<_> = stats - .workers - .iter() - .map(|w| { - serde_json::json!({ - "id": w.id, - "actors": w.num_actors, - "mailbox_depth": w.mailbox_depth, - "messages_processed": w.messages_processed, - "local_sends": w.local_sends, - "cross_sends": w.cross_sends, - "inbox_sends": w.inbox_sends, - "type_mismatches": w.type_mismatches, - "panics": w.panics, - }) - }) - .collect(); - ok_response("workers", workers) -} - -pub fn cmd_worker(rt: &Runtime, col: &StatsCollector, args: &[&str]) -> String { - let id: usize = match args.first().and_then(|s| s.parse().ok()) { - Some(id) => id, - None => return err_response("worker", "usage: worker "), - }; - - let stats = enriched_stats(rt, col); - let w = match stats.workers.iter().find(|w| w.id == id) { - Some(w) => w, - None => { - return err_response( - "worker", - &format!("worker {id} not found (have 0..{})", stats.num_workers), - ) - } - }; - - // Tick phase breakdown for this worker - let timings = stats.tick_timings.get(id).cloned().unwrap_or_default(); - let phase_breakdown = compute_phase_breakdown(&timings); - - let actors_on_worker: Vec<_> = stats - .actor_details - .iter() - .filter(|a| a.worker_id == id) - .map(|a| { - serde_json::json!({ - "address": format_addr(&a.address), - "mailbox_depth": a.mailbox_depth, - "last_msg_type": a.last_msg_type, - "messages_processed": a.messages_processed, - "poisoned": a.poisoned, - }) - }) - .collect(); - - ok_response( - "worker", - serde_json::json!({ - "id": w.id, - "actors": w.num_actors, - "mailbox_depth": w.mailbox_depth, - "messages_processed": w.messages_processed, - "local_sends": w.local_sends, - "cross_sends": w.cross_sends, - "inbox_sends": w.inbox_sends, - "type_mismatches": w.type_mismatches, - "panics": w.panics, - "tick_phases": phase_breakdown, - "actor_details": actors_on_worker, - }), - ) -} - -pub fn cmd_actors(rt: &Runtime, col: &StatsCollector, args: &[&str]) -> String { - let stats = enriched_stats(rt, col); - let mut actors = stats.actor_details.clone(); - - // Parse flags - let mut sort_by = "mailbox"; - let mut limit: usize = usize::MAX; - let mut worker_filter: Option = None; - let mut i = 0; - while i < args.len() { - match args[i] { - "--sort" if i + 1 < args.len() => { - sort_by = args[i + 1]; - i += 2; - } - "--limit" if i + 1 < args.len() => { - limit = args[i + 1].parse().unwrap_or(usize::MAX); - i += 2; - } - "--worker" if i + 1 < args.len() => { - worker_filter = args[i + 1].parse().ok(); - i += 2; - } - _ => { - i += 1; - } - } - } - - if let Some(wid) = worker_filter { - actors.retain(|a| a.worker_id == wid); - } - - match sort_by { - "mailbox" => actors.sort_by(|a, b| b.mailbox_depth.cmp(&a.mailbox_depth)), - "worker" => actors.sort_by_key(|a| a.worker_id), - "address" => actors.sort_by(|a, b| a.address.0.cmp(&b.address.0)), - other => return err_response("actors", &format!("unknown sort field `{other}` — use mailbox|worker|address")), - } - - actors.truncate(limit); - - let rows: Vec<_> = actors - .iter() - .map(|a| { - serde_json::json!({ - "address": format_addr(&a.address), - "address_full": full_hex(&a.address), - "worker_id": a.worker_id, - "mailbox_depth": a.mailbox_depth, - "last_msg_type": a.last_msg_type, - "messages_processed": a.messages_processed, - "poisoned": a.poisoned, - }) - }) - .collect(); - - ok_response( - "actors", - serde_json::json!({ - "total": stats.actor_details.len(), - "returned": rows.len(), - "sort": sort_by, - "actors": rows, - }), - ) -} - -pub fn cmd_actor(rt: &Runtime, col: &StatsCollector, args: &[&str]) -> String { - let prefix = match args.first() { - Some(p) => *p, - None => return err_response("actor", "usage: actor "), - }; - - let stats = enriched_stats(rt, col); - let matches: Vec<_> = stats - .actor_details - .iter() - .filter(|a| full_hex(&a.address).starts_with(prefix)) - .map(|a| { - serde_json::json!({ - "address": format_addr(&a.address), - "address_full": full_hex(&a.address), - "worker_id": a.worker_id, - "mailbox_depth": a.mailbox_depth, - "last_msg_type": a.last_msg_type, - "messages_processed": a.messages_processed, - "poisoned": a.poisoned, - }) - }) - .collect(); - - ok_response( - "actor", - serde_json::json!({ - "prefix": prefix, - "matches": matches.len(), - "actors": matches, - }), - ) -} - -pub fn cmd_hot(rt: &Runtime, col: &StatsCollector, args: &[&str]) -> String { - let n: usize = args.first().and_then(|s| s.parse().ok()).unwrap_or(10); - let stats = enriched_stats(rt, col); - - let mut actors = stats.actor_details.clone(); - actors.sort_by(|a, b| b.mailbox_depth.cmp(&a.mailbox_depth)); - actors.truncate(n); - - let rows: Vec<_> = actors - .iter() - .map(|a| { - serde_json::json!({ - "address": format_addr(&a.address), - "address_full": full_hex(&a.address), - "worker_id": a.worker_id, - "mailbox_depth": a.mailbox_depth, - "last_msg_type": a.last_msg_type, - "messages_processed": a.messages_processed, - "poisoned": a.poisoned, - }) - }) - .collect(); - - ok_response("hot", rows) -} - -pub fn cmd_phases(rt: &Runtime, args: &[&str]) -> String { - let stats = rt.stats(); - - let worker_filter: Option = args.first().and_then(|s| s.parse().ok()); - - let phase_names = [ - "spawn_drain", - "transfer_drain", - "tick_all", - "spawn_drain_2", - "pending_local", - "stats_publish", - ]; - - let mut results = Vec::new(); - for (i, timings) in stats.tick_timings.iter().enumerate() { - if let Some(wid) = worker_filter { - if i != wid { - continue; - } - } - let breakdown = compute_phase_breakdown(timings); - results.push(serde_json::json!({ - "worker_id": i, - "ticks_sampled": timings.len(), - "phases": breakdown, - "phase_names": phase_names, - })); - } - - ok_response("phases", results) -} - -pub fn cmd_diff(rt: &Runtime, col: &StatsCollector, args: &[&str]) -> String { - let secs: f64 = match args.first().and_then(|s| s.parse().ok()) { - Some(s) if s > 0.0 && s <= 30.0 => s, - Some(_) => return err_response("diff", "seconds must be between 0 and 30"), - None => return err_response("diff", "usage: diff "), - }; - - let before = enriched_stats(rt, col); - let t0 = Instant::now(); - std::thread::sleep(Duration::from_secs_f64(secs)); - let after = enriched_stats(rt, col); - let elapsed = t0.elapsed().as_secs_f64(); - - let msgs_before: u64 = before.workers.iter().map(|w| w.messages_processed).sum(); - let msgs_after: u64 = after.workers.iter().map(|w| w.messages_processed).sum(); - let delta_msgs = msgs_after.saturating_sub(msgs_before); - - let local_before: u64 = before.workers.iter().map(|w| w.local_sends).sum(); - let local_after: u64 = after.workers.iter().map(|w| w.local_sends).sum(); - let cross_before: u64 = before.workers.iter().map(|w| w.cross_sends).sum(); - let cross_after: u64 = after.workers.iter().map(|w| w.cross_sends).sum(); - - let mailbox_before: usize = before.workers.iter().map(|w| w.mailbox_depth).sum(); - let mailbox_after: usize = after.workers.iter().map(|w| w.mailbox_depth).sum(); - - let per_worker: Vec<_> = after - .workers - .iter() - .enumerate() - .map(|(i, w)| { - let prev = before.workers.get(i); - let d = prev - .map(|p| w.messages_processed.saturating_sub(p.messages_processed)) - .unwrap_or(0); - serde_json::json!({ - "worker_id": i, - "delta_messages": d, - "msg_per_sec": d as f64 / elapsed, - "actors_before": prev.map(|p| p.num_actors).unwrap_or(0), - "actors_after": w.num_actors, - "mailbox_before": prev.map(|p| p.mailbox_depth).unwrap_or(0), - "mailbox_after": w.mailbox_depth, - }) - }) - .collect(); - - ok_response( - "diff", - serde_json::json!({ - "elapsed_s": elapsed, - "actors_before": before.actor_details.len(), - "actors_after": after.actor_details.len(), - "delta_messages": delta_msgs, - "msg_per_sec": delta_msgs as f64 / elapsed, - "delta_local_sends": local_after.saturating_sub(local_before), - "delta_cross_sends": cross_after.saturating_sub(cross_before), - "mailbox_before": mailbox_before, - "mailbox_after": mailbox_after, - "per_worker": per_worker, - }), - ) -} - -// ── Phase breakdown helper ────────────────────────────────────────────── - -fn compute_phase_breakdown( - timings: &[swactor::stats::TickTiming], -) -> serde_json::Value { - if timings.is_empty() { - return serde_json::json!({ - "ticks": 0, - "active_pct": 0.0, - "avg_tick_us": 0.0, - "phases_us": [0, 0, 0, 0, 0, 0], - "phases_pct": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0], - }); - } - - let n = timings.len(); - let active = timings.iter().filter(|t| t.did_work).count(); - let active_pct = (active as f64 / n as f64) * 100.0; - - let mut phase_sums = [0u64; 6]; - for t in timings { - for (i, &us) in t.phase_us.iter().enumerate() { - phase_sums[i] += us; - } - } - let total_us: u64 = phase_sums.iter().sum(); - let avg_tick_us = total_us as f64 / n as f64; - - let phases_pct: Vec = if total_us == 0 { - vec![0.0; 6] - } else { - phase_sums - .iter() - .map(|&s| (s as f64 / total_us as f64) * 100.0) - .collect() - }; - - serde_json::json!({ - "ticks": n, - "active_pct": active_pct, - "avg_tick_us": avg_tick_us, - "phases_us": phase_sums, - "phases_pct": phases_pct, - }) + let req = swactor_command::from_query_params(params); + router.dispatch(&req, ctx).to_json_line() } diff --git a/crates/runtime-dashboard/src/server.rs b/crates/runtime-dashboard/src/server.rs index 34f1c5e..703ab68 100644 --- a/crates/runtime-dashboard/src/server.rs +++ b/crates/runtime-dashboard/src/server.rs @@ -11,7 +11,6 @@ use swactor::runtime::Runtime; use crate::actors_html::ACTORS_HTML; use crate::collector::StatsCollector; use crate::dashboard_html::DASHBOARD_HTML; -use crate::investigate; use crate::layer::EventStore; use crate::trace::RuntimeTrace; @@ -127,6 +126,7 @@ pub(crate) fn spawn_http_server( let addr = format!("0.0.0.0:{port}"); let server = tiny_http::Server::http(&addr).expect("failed to bind HTTP server"); let server = Arc::new(server); + let cmd_router = Arc::new(swactor_command::CommandRouter::with_builtins()); for _ in 0..4 { let server = Arc::clone(&server); @@ -134,6 +134,7 @@ pub(crate) fn spawn_http_server( let runtime = Arc::clone(&runtime); let collector = Arc::clone(&collector); let shutdown = Arc::clone(&shutdown); + let cmd_router = Arc::clone(&cmd_router); #[cfg(feature = "distribution")] let distribution = Arc::clone(&distribution); thread::spawn(move || { @@ -174,6 +175,7 @@ pub(crate) fn spawn_http_server( &url, Arc::clone(&runtime), Arc::clone(&collector), + Arc::clone(&cmd_router), ); } _ => respond_404(request), @@ -283,21 +285,33 @@ fn handle_investigate_api( url: &str, runtime: Arc>>>, collector: Arc>>>, + cmd_router: Arc, ) { let params = parse_query_string(url); - let cmd = params.get("cmd").map(|s| s.as_str()).unwrap_or("help"); let maybe_rt = runtime.lock().unwrap().clone(); let maybe_col = collector.lock().unwrap().clone(); let json = match (maybe_rt, maybe_col) { - (Some(rt), Some(col)) => investigate::dispatch_command(cmd, ¶ms, &rt, &col), - _ => serde_json::json!({ - "ok": false, - "command": cmd, - "error": "runtime not attached yet" - }) - .to_string(), + (Some(rt), Some(col)) => { + let ctx = swactor_command::CommandContext::with_enricher(rt, col); + let req = swactor_command::from_query_params(¶ms); + cmd_router.dispatch(&req, &ctx).to_json_line() + } + (Some(rt), None) => { + let ctx = swactor_command::CommandContext::new(rt); + let req = swactor_command::from_query_params(¶ms); + cmd_router.dispatch(&req, &ctx).to_json_line() + } + _ => { + let cmd = params.get("cmd").map(|s| s.as_str()).unwrap_or("help"); + serde_json::json!({ + "ok": false, + "command": cmd, + "error": "runtime not attached yet" + }) + .to_string() + } }; let response = tiny_http::Response::from_string(json).with_header( -- 2.45.2 From 669599a9e7d5ee2405a65385c5eeab7449e86725 Mon Sep 17 00:00:00 2001 From: Developer Date: Fri, 13 Feb 2026 07:34:44 +0000 Subject: [PATCH 4/5] =?UTF-8?q?feat:=20cluster=20registry=20=E2=80=94=20go?= =?UTF-8?q?ssip-propagated=20naming=20via=20LWW-Register=20CRDT?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a ClusterRegistry that maps human-readable names to (ActorAddress, NodeId) pairs, replicated across the cluster via SWIM gossip piggyback. Uses last-writer-wins semantics with timestamp > generation > node_id tie-breaking, and tombstone-based unregistration with configurable GC. Key design: DistributedNode intercepts piggyback bytes on all outgoing/incoming SWIM messages, wrapping them in a PiggybackPayload that carries both membership updates and registry entries. SwimNode and DisseminationQueue remain untouched. - New: crates/distribution/src/registry.rs (ClusterRegistry, pack/unpack helpers) - Modified: node.rs (register_name/unregister_name/resolve_name API, piggyback interception, node death tombstoning) - Modified: snapshot.rs (RegistryEntryInfo in DistributionNodeSnapshot) - 12 behavioral tests in crates/distribution/tests/registry.rs - Remove CLAUDE/ from git tracking, add to .gitignore Authored by Claude, lovingly guided by Zachery Aaron Shores-Chmielewski --- .gitignore | 4 + CLAUDE/TASK.md | 42 -- CLAUDE/notes/progress.md | 51 -- crates/distribution/src/lib.rs | 1 + crates/distribution/src/node.rs | 93 +++- crates/distribution/src/registry.rs | 374 +++++++++++++ crates/distribution/src/snapshot.rs | 31 ++ crates/distribution/tests/node_integration.rs | 2 + crates/distribution/tests/registry.rs | 510 ++++++++++++++++++ .../examples/dashboard_demo.rs | 3 + crates/simulation/src/distribution/sim.rs | 2 + 11 files changed, 1015 insertions(+), 98 deletions(-) delete mode 100644 CLAUDE/TASK.md delete mode 100644 CLAUDE/notes/progress.md create mode 100644 crates/distribution/src/registry.rs create mode 100644 crates/distribution/tests/registry.rs diff --git a/.gitignore b/.gitignore index e9f2249..3a5aca8 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,7 @@ corpus **/deps.html docs/architecture.dot docs/architecture.html + +# Claude session files +CLAUDE/ +.claude/ diff --git a/CLAUDE/TASK.md b/CLAUDE/TASK.md deleted file mode 100644 index 762d572..0000000 --- a/CLAUDE/TASK.md +++ /dev/null @@ -1,42 +0,0 @@ -Plan: - You are to improve this codebase via: - - implementing the features found in `docs/os-design` - - writing comphrehensive tests that check behavior makes sense - -Workflow: - - Read `CLAUDE/TASK.md` and `CLAUDE/notes/progress.md` - - Identify what stage you are on. - - Read and update yourself as necessary. - - Proceed to accomplishing the next task as written in `progress.md` - - For each attempt at any step, keep a record. If you reach attempt 3, step back, document, and try something else. - - When done, because attempt limit or task success: - - update `progress.md` with: - - Completed this session - - Next steps (specific, actionable) - - Open Questions - - Blockers - - make a commit - - compress your context and start the loop again - -Style: - - Do not add to existing modules in the root swactor `src/` they should stay as they are. You may modify but not change module structure. - - Integration tests in `tests/`, benchmark code in `benches/` - - cap execution time at 2 minutes max for fuzz, or benchmarks, or single test suite - - if they take too long, refactor and break up into logical modules - - You may modify these as you wish, so long as logical 'coverage' does not decline. - - Report all your changes to architecture with changes to the `docs/` items - - all notes you wish to keep across iterations shall go in the `CLAUDE/notes/` folder - -Example loop (not restrictive, feel free to ignore if prudent): - - Pick an item to implement from the os-design docs - - make analysis - - implement plan - - execute - - evaluate - - compress and move on to the next item - -Before git commit: - - all `cargo test` passes, including feature gated material - - if a test fails, investigate do not ignore or delete - - You can combine tests but not skip code paths or delete them for active code - - if a fix takes > 3 attempts, log and move on \ No newline at end of file diff --git a/CLAUDE/notes/progress.md b/CLAUDE/notes/progress.md deleted file mode 100644 index 3e56c14..0000000 --- a/CLAUDE/notes/progress.md +++ /dev/null @@ -1,51 +0,0 @@ -# Progress - -## Completed - -### Feature 1: Actor Watching (local only) — `docs/os-design/01-actor-watching.md` -- Added `ExitReason` enum (Stopped, Panicked, NodeDown) and `ActorExited` struct to `src/actor.rs` -- Extended `ContextInner` trait with `watch()`/`unwatch()` methods -- Added `Ctx::watch(target)` and `Ctx::unwatch(target)` typed API -- Added `on_actor_exit()` default method to `ActorInterface` trait -- Updated `AnyActor::handle_any` with system message fallback (tries `ActorExited` after `Incoming`) -- Implemented `WatchRegistry` in `src/worker.rs` (bidirectional HashMap tracking) -- Integrated death notification dispatch as phase 5b in `tick_once` -- Implemented `watch`/`unwatch` on `Runtime`'s `ContextInner` impl -- Added `watch_registry` field to `TickContext` in `src/delivery.rs` -- 10 behavioral tests in `tests/watch_api.rs` — all passing - -### Feature 2: Command Interface — `docs/os-design/04-command-interface.md` -- Created `crates/command/` crate (`swactor-command`) with core types: - - `CommandRequest`, `CommandResponse` (serde-serializable) - - `CommandHandler` trait + `CommandMeta` - - `CommandRouter` with `dispatch()` and `with_builtins()` - - `CommandContext` with `Arc` + optional `StatsEnricher` - - `StatsEnricher` trait (decouples command crate from dashboard) -- Built-in read commands: overview, workers, worker, actors, actor, hot, phases, diff -- Built-in write command: shutdown -- REPL line parser (`parse_line`) with positional arg mapping and `--flag value` support -- REST adapter (`from_query_params`) for HTTP query parameters -- Refactored `investigate.rs` to delegate to CommandRouter (thin wrapper) -- Updated `server.rs` to use CommandRouter for `/api/investigate` endpoint -- Implemented `StatsEnricher for StatsCollector` in dashboard crate -- 18 behavioral tests in `crates/command/tests/command_api.rs` — all passing -- All 53 swactor core tests pass, all 18 command tests pass - -## Next Steps -1. **Cluster Registry** — `docs/os-design/02-cluster-registry.md` - - LWW-Register CRDT per name binding - - Propagation via SWIM piggyback - - `ClusterRegistry` struct in `crates/distribution/src/registry.rs` - - API: register_name, unregister_name, resolve_name -2. **Node Capabilities** — `docs/os-design/03-node-capabilities.md` - - New `crates/capabilities/` crate with auto-detection -3. **Remote Watching** — extends actor watching with wire protocol -4. **Supervision** — `docs/os-design/05-supervision.md` - -## Open Questions -- Custom actor commands (via `Ctx::register_command()`) deferred to a later PR -- Distribution-aware commands (nodes, registry, resolve) deferred until cluster registry is implemented -- Write commands (spawn, stop, drain) deferred — need factory registry and actor stop mechanism - -## Blockers -- None diff --git a/crates/distribution/src/lib.rs b/crates/distribution/src/lib.rs index 3f2b705..0922300 100644 --- a/crates/distribution/src/lib.rs +++ b/crates/distribution/src/lib.rs @@ -7,4 +7,5 @@ pub mod swim; pub mod kademlia; pub mod cache; pub mod node; +pub mod registry; pub mod snapshot; diff --git a/crates/distribution/src/node.rs b/crates/distribution/src/node.rs index def6822..6dc9a5f 100644 --- a/crates/distribution/src/node.rs +++ b/crates/distribution/src/node.rs @@ -12,6 +12,10 @@ use crate::crypto::Keypair; use crate::kademlia::directory::{actor_addr_as_node_id, DirectoryShard}; use crate::kademlia::repair::{RepairQueue, RepublishTracker}; use crate::kademlia::routing_table::RoutingTable; +use crate::registry::{ + pack_combined_piggyback, unpack_combined_piggyback, ClusterRegistry, RegistryConfig, + RegistryEvent, +}; use crate::swim::node::{NodeAction, SwimNode}; use crate::swim::probe::SwimConfig; use crate::types::{MemberState, NodeId, NodeRecord}; @@ -22,6 +26,7 @@ pub struct DistributedNodeConfig { pub swim: SwimConfig, pub cache_capacity: usize, pub republish_interval: u64, + pub registry: RegistryConfig, } impl Default for DistributedNodeConfig { @@ -31,6 +36,7 @@ impl Default for DistributedNodeConfig { swim: SwimConfig::default(), cache_capacity: 10_000, republish_interval: 1000, + registry: RegistryConfig::default(), } } } @@ -47,6 +53,7 @@ pub struct DistributedNode { cache: LocationCache, repair_queue: RepairQueue, republish: RepublishTracker, + registry: ClusterRegistry, tick_count: u64, } @@ -67,6 +74,7 @@ impl DistributedNode { cache: LocationCache::new(config.cache_capacity), repair_queue: RepairQueue::new(), republish: RepublishTracker::new(config.republish_interval), + registry: ClusterRegistry::new(config.registry), tick_count: 0, keypair, } @@ -149,23 +157,32 @@ impl DistributedNode { // re-sign and re-STORE these entries. } - actions + // Registry GC + self.registry.gc_tick(); + + // Wrap outgoing piggyback with registry entries + self.inject_registry_piggyback(actions) } // ─── SWIM message handling (delegate to SwimNode) ─────────────────── pub fn handle_ping(&mut self, from: NodeId, from_addr: SocketAddr, sequence: u64, piggyback: &[u8]) -> Vec { - let actions = self.swim.handle_ping(from, from_addr, sequence, piggyback); + let membership_bytes = self.extract_registry_piggyback(piggyback); + let actions = self.swim.handle_ping(from, from_addr, sequence, &membership_bytes); self.maybe_update_routing_table(from, from_addr); - actions + self.inject_registry_piggyback(actions) } pub fn handle_ack(&mut self, from: NodeId, sequence: u64, piggyback: &[u8]) -> Vec { - self.swim.handle_ack(from, sequence, piggyback) + let membership_bytes = self.extract_registry_piggyback(piggyback); + let actions = self.swim.handle_ack(from, sequence, &membership_bytes); + self.inject_registry_piggyback(actions) } pub fn handle_ping_req(&mut self, from: NodeId, target: NodeId, target_addr: SocketAddr, sequence: u64, piggyback: &[u8]) -> Vec { - self.swim.handle_ping_req(from, target, target_addr, sequence, piggyback) + let membership_bytes = self.extract_registry_piggyback(piggyback); + let actions = self.swim.handle_ping_req(from, target, target_addr, sequence, &membership_bytes); + self.inject_registry_piggyback(actions) } pub fn handle_join_request(&mut self, from: NodeId, from_addr: SocketAddr) -> Vec { @@ -233,6 +250,33 @@ impl DistributedNode { self.cache.invalidate(actor_addr); } + // ─── Registry (name → actor mapping) ────────────────────────────── + + /// Register a human-readable name for an actor on this node. + pub fn register_name(&mut self, name: String, actor_addr: ActorAddress) { + self.registry.register(name, actor_addr, self.node_id(), self.cluster_size()); + } + + /// Unregister a name (creates a tombstone). + pub fn unregister_name(&mut self, name: &str) { + self.registry.unregister(name, self.node_id(), self.cluster_size()); + } + + /// Resolve a name to its current (ActorAddress, NodeId). + pub fn resolve_name(&self, name: &str) -> Option<(ActorAddress, NodeId)> { + self.registry.resolve(name) + } + + /// Drain registry events (Registered / Unregistered). + pub fn registry_events(&mut self) -> Vec { + self.registry.drain_events() + } + + /// Read-only access to the registry. + pub fn registry(&self) -> &ClusterRegistry { + &self.registry + } + // ─── Accessors ────────────────────────────────────────────────────── pub fn routing_table(&self) -> &RoutingTable { @@ -277,12 +321,51 @@ impl DistributedNode { self.routing_table.remove(&node_id); self.cache.invalidate_node(&node_id); self.repair_queue.on_node_death(&node_id, &mut self.directory); + self.registry.tombstone_node(node_id, self.cluster_size()); } MemberState::Suspect => { // Keep in routing table but could downprioritize } } } + + fn cluster_size(&self) -> usize { + self.swim.members().alive_count() + 1 // +1 for self + } + + /// Post-process outgoing actions: wrap each piggyback with registry entries. + fn inject_registry_piggyback(&mut self, actions: Vec) -> Vec { + actions + .into_iter() + .map(|action| match action { + NodeAction::SendPing { to, to_addr, sequence, piggyback } => { + let registry_entries = self.registry.take_pending(8); + let combined = pack_combined_piggyback(piggyback, registry_entries); + NodeAction::SendPing { to, to_addr, sequence, piggyback: combined } + } + NodeAction::SendAck { to, to_addr, sequence, piggyback } => { + let registry_entries = self.registry.take_pending(8); + let combined = pack_combined_piggyback(piggyback, registry_entries); + NodeAction::SendAck { to, to_addr, sequence, piggyback: combined } + } + NodeAction::SendPingReq { relay, relay_addr, target, target_addr, sequence, piggyback } => { + let registry_entries = self.registry.take_pending(8); + let combined = pack_combined_piggyback(piggyback, registry_entries); + NodeAction::SendPingReq { relay, relay_addr, target, target_addr, sequence, piggyback: combined } + } + other => other, + }) + .collect() + } + + /// Extract registry entries from incoming piggyback, merge them, return membership-only bytes. + fn extract_registry_piggyback(&mut self, bytes: &[u8]) -> Vec { + let (membership_bytes, registry_entries) = unpack_combined_piggyback(bytes); + if !registry_entries.is_empty() { + self.registry.merge_batch(registry_entries, self.cluster_size()); + } + membership_bytes + } } /// Result of resolving an actor's location. diff --git a/crates/distribution/src/registry.rs b/crates/distribution/src/registry.rs new file mode 100644 index 0000000..377c257 --- /dev/null +++ b/crates/distribution/src/registry.rs @@ -0,0 +1,374 @@ +//! Cluster Registry — gossip-propagated naming via LWW-Register CRDT. +//! +//! Maps human-readable names to `(ActorAddress, NodeId)` pairs, propagated +//! through SWIM gossip piggyback. Uses last-writer-wins semantics with +//! tie-breaking on (timestamp, generation, node_id). + +use std::collections::{HashMap, VecDeque}; + +use serde::{Deserialize, Serialize}; +use swactor::actor::ActorAddress; + +use crate::types::NodeId; + +// ─── Configuration ────────────────────────────────────────────────────────── + +/// Configuration for the cluster registry. +pub struct RegistryConfig { + /// Maximum number of events to buffer before dropping old ones. + pub max_events: usize, + /// How long (in ticks) a tombstone is retained before GC. + pub tombstone_ttl: u64, + /// How often (in ticks) to run garbage collection. + pub gc_interval: u64, + /// Dissemination multiplier (Λ) — same role as in SWIM dissemination. + pub dissemination_lambda: usize, +} + +impl Default for RegistryConfig { + fn default() -> Self { + Self { + max_events: 256, + tombstone_ttl: 3600, + gc_interval: 1000, + dissemination_lambda: 3, + } + } +} + +// ─── Wire types ───────────────────────────────────────────────────────────── + +/// A single registry entry — the unit of replication. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct RegistryEntry { + pub name: String, + pub actor_addr: ActorAddress, + pub node_id: NodeId, + /// Logical timestamp (monotonically increasing per-registry). + pub timestamp: u64, + /// Generation counter for the same name (disambiguates re-registrations). + pub generation: u64, + /// If true, this entry is a tombstone (name was unregistered). + pub tombstone: bool, +} + +/// Combined piggyback payload: membership bytes + registry entries. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PiggybackPayload { + /// Raw SWIM membership piggyback bytes (opaque to registry). + pub membership: Vec, + /// Registry entries to disseminate. + pub registry: Vec, +} + +// ─── Events ───────────────────────────────────────────────────────────────── + +/// Events emitted when the registry changes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RegistryEvent { + Registered { + name: String, + actor_addr: ActorAddress, + node_id: NodeId, + }, + Unregistered { + name: String, + previous_addr: ActorAddress, + }, +} + +// ─── Dissemination entry ──────────────────────────────────────────────────── + +#[derive(Debug, Clone)] +struct DisseminationEntry { + entry: RegistryEntry, + remaining: usize, +} + +// ─── ClusterRegistry ──────────────────────────────────────────────────────── + +/// CRDT-based cluster registry with LWW semantics and gossip dissemination. +pub struct ClusterRegistry { + /// Current state: name → latest entry. + entries: HashMap, + /// Pending entries to disseminate via piggyback. + dissemination: Vec, + /// Monotonic logical clock for this node's writes. + clock: u64, + /// Buffered events for consumers. + events: VecDeque, + config: RegistryConfig, + tick_count: u64, +} + +impl ClusterRegistry { + pub fn new(config: RegistryConfig) -> Self { + Self { + entries: HashMap::new(), + dissemination: Vec::new(), + clock: 0, + events: VecDeque::new(), + config, + tick_count: 0, + } + } + + /// Register a name → actor binding from the local node. + pub fn register(&mut self, name: String, actor_addr: ActorAddress, node_id: NodeId, cluster_size: usize) { + self.clock += 1; + let generation = self.next_generation(&name); + let entry = RegistryEntry { + name, + actor_addr, + node_id, + timestamp: self.clock, + generation, + tombstone: false, + }; + self.merge_and_enqueue(entry, cluster_size); + } + + /// Unregister a name (create a tombstone). + pub fn unregister(&mut self, name: &str, node_id: NodeId, cluster_size: usize) { + self.clock += 1; + let generation = self.next_generation(name); + // Use the existing actor_addr if present, otherwise a zero address. + let actor_addr = self.entries + .get(name) + .map(|e| e.actor_addr) + .unwrap_or(ActorAddress([0; 32])); + let entry = RegistryEntry { + name: name.to_string(), + actor_addr, + node_id, + timestamp: self.clock, + generation, + tombstone: true, + }; + self.merge_and_enqueue(entry, cluster_size); + } + + /// Resolve a name to its current (ActorAddress, NodeId), or None if + /// not registered or tombstoned. + pub fn resolve(&self, name: &str) -> Option<(ActorAddress, NodeId)> { + self.entries.get(name).and_then(|e| { + if e.tombstone { + None + } else { + Some((e.actor_addr, e.node_id)) + } + }) + } + + /// Merge a single remote entry. Returns true if state changed. + pub fn merge(&mut self, remote: RegistryEntry) -> bool { + if let Some(existing) = self.entries.get(&remote.name) { + if !lww_wins(&remote, existing) { + return false; + } + } + + let changed = match self.entries.get(&remote.name) { + Some(existing) => existing != &remote, + None => true, + }; + + if changed { + self.emit_event(&remote); + // Advance clock to stay ahead of remote timestamps. + if remote.timestamp >= self.clock { + self.clock = remote.timestamp + 1; + } + } + + self.entries.insert(remote.name.clone(), remote); + changed + } + + /// Merge a batch of entries received from gossip. + /// Changed entries are re-enqueued for further dissemination. + pub fn merge_batch(&mut self, entries: Vec, cluster_size: usize) { + for entry in entries { + if self.merge(entry.clone()) { + self.enqueue(entry, cluster_size); + } + } + } + + /// Take pending entries for piggyback, up to `max_count`. + pub fn take_pending(&mut self, max_count: usize) -> Vec { + let count = max_count.min(self.dissemination.len()); + let mut result = Vec::with_capacity(count); + + for entry in self.dissemination.iter_mut().take(count) { + result.push(entry.entry.clone()); + entry.remaining = entry.remaining.saturating_sub(1); + } + + // Evict exhausted entries. + self.dissemination.retain(|e| e.remaining > 0); + + result + } + + /// Tombstone all entries owned by a dead node. + pub fn tombstone_node(&mut self, dead_node_id: NodeId, cluster_size: usize) { + let owned: Vec = self.entries + .iter() + .filter(|(_, e)| e.node_id == dead_node_id && !e.tombstone) + .map(|(name, _)| name.clone()) + .collect(); + + for name in owned { + self.clock += 1; + let generation = self.next_generation(&name); + let actor_addr = self.entries[&name].actor_addr; + let entry = RegistryEntry { + name, + actor_addr, + node_id: dead_node_id, + timestamp: self.clock, + generation, + tombstone: true, + }; + self.merge_and_enqueue(entry, cluster_size); + } + } + + /// Periodic GC: remove tombstones past TTL with exhausted dissemination budgets. + pub fn gc_tick(&mut self) { + self.tick_count += 1; + if self.tick_count % self.config.gc_interval != 0 { + return; + } + + let ttl = self.config.tombstone_ttl; + let clock = self.clock; + // Names still being disseminated — don't GC those. + let pending_names: std::collections::HashSet = self.dissemination + .iter() + .map(|e| e.entry.name.clone()) + .collect(); + + self.entries.retain(|name, entry| { + if entry.tombstone && !pending_names.contains(name) { + // Remove if old enough. + let age = clock.saturating_sub(entry.timestamp); + age < ttl + } else { + true + } + }); + } + + /// Drain buffered events. + pub fn drain_events(&mut self) -> Vec { + self.events.drain(..).collect() + } + + /// Number of registry entries (including tombstones). + pub fn len(&self) -> usize { + self.entries.len() + } + + /// Number of tombstones. + pub fn tombstone_count(&self) -> usize { + self.entries.values().filter(|e| e.tombstone).count() + } + + /// Iterate all entries (for snapshot). + pub fn entries(&self) -> impl Iterator { + self.entries.values() + } + + // ─── Internal ─────────────────────────────────────────────────────── + + fn next_generation(&self, name: &str) -> u64 { + self.entries + .get(name) + .map(|e| e.generation + 1) + .unwrap_or(1) + } + + fn transmit_budget(&self, cluster_size: usize) -> usize { + let n = cluster_size.max(2) as f64; + let log_n = n.log2().ceil() as usize; + self.config.dissemination_lambda * log_n.max(1) + } + + fn enqueue(&mut self, entry: RegistryEntry, cluster_size: usize) { + let budget = self.transmit_budget(cluster_size); + + // Replace existing entry for same name if present. + if let Some(existing) = self.dissemination.iter_mut().find(|e| e.entry.name == entry.name) { + existing.entry = entry; + existing.remaining = budget; + return; + } + + self.dissemination.push(DisseminationEntry { + entry, + remaining: budget, + }); + } + + fn merge_and_enqueue(&mut self, entry: RegistryEntry, cluster_size: usize) { + let merged = self.merge(entry.clone()); + if merged { + self.enqueue(entry, cluster_size); + } + } + + fn emit_event(&mut self, entry: &RegistryEntry) { + let event = if entry.tombstone { + RegistryEvent::Unregistered { + name: entry.name.clone(), + previous_addr: entry.actor_addr, + } + } else { + RegistryEvent::Registered { + name: entry.name.clone(), + actor_addr: entry.actor_addr, + node_id: entry.node_id, + } + }; + self.events.push_back(event); + while self.events.len() > self.config.max_events { + self.events.pop_front(); + } + } +} + +// ─── LWW conflict resolution ─────────────────────────────────────────────── + +/// Returns true if `incoming` wins over `existing` under LWW rules: +/// higher timestamp > higher generation > higher node_id (byte-level). +fn lww_wins(incoming: &RegistryEntry, existing: &RegistryEntry) -> bool { + if incoming.timestamp != existing.timestamp { + return incoming.timestamp > existing.timestamp; + } + if incoming.generation != existing.generation { + return incoming.generation > existing.generation; + } + incoming.node_id.0 > existing.node_id.0 +} + +// ─── Piggyback pack/unpack ────────────────────────────────────────────────── + +/// Combine membership piggyback bytes and registry entries into a single payload. +pub fn pack_combined_piggyback(membership: Vec, registry: Vec) -> Vec { + let payload = PiggybackPayload { membership, registry }; + serde_json::to_vec(&payload).unwrap_or_default() +} + +/// Split a combined piggyback payload into membership bytes and registry entries. +/// If deserialization fails, treats the entire blob as membership bytes (backwards compat). +pub fn unpack_combined_piggyback(bytes: &[u8]) -> (Vec, Vec) { + if bytes.is_empty() { + return (Vec::new(), Vec::new()); + } + match serde_json::from_slice::(bytes) { + Ok(payload) => (payload.membership, payload.registry), + Err(_) => (bytes.to_vec(), Vec::new()), + } +} diff --git a/crates/distribution/src/snapshot.rs b/crates/distribution/src/snapshot.rs index 97f5013..32a6d1d 100644 --- a/crates/distribution/src/snapshot.rs +++ b/crates/distribution/src/snapshot.rs @@ -33,6 +33,15 @@ pub struct CacheEntryInfo { pub node_id: String, } +/// Snapshot of a single registry entry. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RegistryEntryInfo { + pub name: String, + pub actor_addr: String, + pub node_id: String, + pub tombstone: bool, +} + /// Complete snapshot of a `DistributedNode`'s observable state. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DistributionNodeSnapshot { @@ -71,6 +80,14 @@ pub struct DistributionNodeSnapshot { /// Number of entries pending re-replication. pub repair_queue_size: usize, + // ─── Registry ──────────────────────────────────────────────────── + /// Number of entries in the cluster registry (including tombstones). + pub registry_size: usize, + /// Number of tombstoned entries. + pub registry_tombstones: usize, + /// All registry entries. + pub registry_entries: Vec, + // ─── Gossip pairs ──────────────────────────────────────────────── /// Recent SWIM probe targets (most recent last). pub recent_probe_targets: Vec, @@ -136,6 +153,17 @@ impl DistributedNode { .map(|id| node_id_hex(id)) .collect(); + let registry = self.registry(); + let registry_entries: Vec = registry + .entries() + .map(|e| RegistryEntryInfo { + name: e.name.clone(), + actor_addr: format!("{}", e.actor_addr), + node_id: node_id_hex(&e.node_id), + tombstone: e.tombstone, + }) + .collect(); + DistributionNodeSnapshot { node_id: node_id_hex(&self.node_id()), listen_addr: addr_str(&self.listen_addr()), @@ -150,6 +178,9 @@ impl DistributedNode { cache_entries, directory_entry_count: self.directory().entry_count(), repair_queue_size: self.repair_queue_len(), + registry_size: registry.len(), + registry_tombstones: registry.tombstone_count(), + registry_entries, recent_probe_targets: recent_targets, } } diff --git a/crates/distribution/tests/node_integration.rs b/crates/distribution/tests/node_integration.rs index 58a3468..37e89ea 100644 --- a/crates/distribution/tests/node_integration.rs +++ b/crates/distribution/tests/node_integration.rs @@ -9,6 +9,7 @@ use swactor::actor::ActorAddress; use distribution::crypto::Keypair; use distribution::node::{DistributedNode, DistributedNodeConfig, ResolveResult}; use distribution::swim::node::NodeAction; +use distribution::registry::RegistryConfig; use distribution::swim::probe::SwimConfig; use distribution::types::NodeId; @@ -23,6 +24,7 @@ fn test_config(addr: &str) -> DistributedNodeConfig { }, cache_capacity: 100, republish_interval: 50, + registry: RegistryConfig::default(), } } diff --git a/crates/distribution/tests/registry.rs b/crates/distribution/tests/registry.rs new file mode 100644 index 0000000..2854cab --- /dev/null +++ b/crates/distribution/tests/registry.rs @@ -0,0 +1,510 @@ +//! Behavioral tests for the cluster registry. +//! +//! Tests gossip-propagated naming via LWW-Register CRDT, using the same +//! `deliver_actions` + `test_config` pattern from `node_integration.rs`. + +use std::net::SocketAddr; + +use swactor::actor::ActorAddress; +use distribution::node::{DistributedNode, DistributedNodeConfig}; +use distribution::registry::{ClusterRegistry, RegistryConfig, RegistryEntry, RegistryEvent}; +use distribution::swim::node::NodeAction; +use distribution::swim::probe::SwimConfig; +use distribution::types::NodeId; + +fn test_config(addr: &str) -> DistributedNodeConfig { + DistributedNodeConfig { + listen_addr: addr.parse().unwrap(), + swim: SwimConfig { + probe_interval: 1, + probe_timeout: 3, + indirect_probes: 1, + suspicion_timeout: 5, + }, + cache_capacity: 100, + republish_interval: 50, + registry: RegistryConfig::default(), + } +} + +/// Simulate a network round: deliver actions from `sender` to the appropriate +/// `receiver` node. Returns any actions generated by the receiver. +fn deliver_actions( + actions: &[NodeAction], + sender_id: NodeId, + sender_addr: SocketAddr, + nodes: &mut [(NodeId, SocketAddr, &mut DistributedNode)], +) -> Vec { + let mut responses = Vec::new(); + for action in actions { + match action { + NodeAction::SendPing { to, sequence, piggyback, .. } => { + if let Some((_, _, node)) = nodes.iter_mut().find(|(id, _, _)| id == to) { + responses.extend(node.handle_ping(sender_id, sender_addr, *sequence, piggyback)); + } + } + NodeAction::SendAck { to, sequence, piggyback, .. } => { + if let Some((_, _, node)) = nodes.iter_mut().find(|(id, _, _)| id == to) { + responses.extend(node.handle_ack(sender_id, *sequence, piggyback)); + } + } + NodeAction::SendJoinRequest { to_addr } => { + if let Some((_, _, node)) = nodes.iter_mut().find(|(_, addr, _)| addr == to_addr) { + responses.extend(node.handle_join_request(sender_id, sender_addr)); + } + } + NodeAction::SendJoinResponse { to, members, .. } => { + if let Some((_, _, node)) = nodes.iter_mut().find(|(id, _, _)| id == to) { + responses.extend(node.handle_join_response(members.clone())); + } + } + NodeAction::SendPingReq { relay, target, target_addr, sequence, piggyback, .. } => { + if let Some((_, _, node)) = nodes.iter_mut().find(|(id, _, _)| id == relay) { + responses.extend(node.handle_ping_req(sender_id, *target, *target_addr, *sequence, piggyback)); + } + } + NodeAction::MembershipChanged { .. } => {} + } + } + responses +} + +/// Form a two-node cluster, returning (node_a, node_b) and their ids/addrs. +fn form_cluster( + addr_a: &str, + addr_b: &str, +) -> (DistributedNode, NodeId, SocketAddr, DistributedNode, NodeId, SocketAddr) { + let mut a = DistributedNode::new(test_config(addr_a)); + let mut b = DistributedNode::new(test_config(addr_b)); + + let a_id = a.node_id(); + let a_addr = a.listen_addr(); + let b_id = b.node_id(); + let b_addr = b.listen_addr(); + + let actions = b.join(&[a_addr]); + let mut nodes = vec![(a_id, a_addr, &mut a)]; + let responses = deliver_actions(&actions, b_id, b_addr, &mut nodes); + let mut nodes = vec![(b_id, b_addr, &mut b)]; + let _ = deliver_actions(&responses, a_id, a_addr, &mut nodes); + + (a, a_id, a_addr, b, b_id, b_addr) +} + +/// Run several gossip rounds between two nodes. +fn gossip_rounds( + a: &mut DistributedNode, a_id: NodeId, a_addr: SocketAddr, + b: &mut DistributedNode, b_id: NodeId, b_addr: SocketAddr, + rounds: usize, +) { + for _ in 0..rounds { + let actions_a = a.tick(); + let mut nodes = vec![(b_id, b_addr, &mut *b)]; + let responses = deliver_actions(&actions_a, a_id, a_addr, &mut nodes); + let mut nodes = vec![(a_id, a_addr, &mut *a)]; + let _ = deliver_actions(&responses, b_id, b_addr, &mut nodes); + + let actions_b = b.tick(); + let mut nodes = vec![(a_id, a_addr, &mut *a)]; + let responses = deliver_actions(&actions_b, b_id, b_addr, &mut nodes); + let mut nodes = vec![(b_id, b_addr, &mut *b)]; + let _ = deliver_actions(&responses, a_id, a_addr, &mut nodes); + } +} + +// ─── Test 1: register and resolve ─────────────────────────────────────────── + +#[test] +fn register_and_resolve() { + let mut node = DistributedNode::new(test_config("127.0.0.1:10001")); + let actor = ActorAddress::new_random(); + let node_id = node.node_id(); + + node.register_name("my-actor".into(), actor); + + let result = node.resolve_name("my-actor"); + assert_eq!(result, Some((actor, node_id))); +} + +// ─── Test 2: unregistered name returns None ───────────────────────────────── + +#[test] +fn unregistered_name_returns_none() { + let node = DistributedNode::new(test_config("127.0.0.1:10002")); + assert_eq!(node.resolve_name("nonexistent"), None); +} + +// ─── Test 3: unregister tombstones name ───────────────────────────────────── + +#[test] +fn unregister_tombstones_name() { + let mut node = DistributedNode::new(test_config("127.0.0.1:10003")); + let actor = ActorAddress::new_random(); + + node.register_name("service".into(), actor); + assert!(node.resolve_name("service").is_some()); + + node.unregister_name("service"); + assert_eq!(node.resolve_name("service"), None); +} + +// ─── Test 4: re-registration updates binding ──────────────────────────────── + +#[test] +fn re_registration_updates_binding() { + let mut node = DistributedNode::new(test_config("127.0.0.1:10004")); + let actor_a = ActorAddress::new_random(); + let actor_b = ActorAddress::new_random(); + let node_id = node.node_id(); + + node.register_name("foo".into(), actor_a); + assert_eq!(node.resolve_name("foo"), Some((actor_a, node_id))); + + node.register_name("foo".into(), actor_b); + assert_eq!(node.resolve_name("foo"), Some((actor_b, node_id))); +} + +// ─── Test 5: LWW conflict — higher timestamp wins ────────────────────────── + +#[test] +fn lww_conflict_higher_timestamp_wins() { + let mut reg = ClusterRegistry::new(RegistryConfig::default()); + let addr_old = ActorAddress::new_random(); + let addr_new = ActorAddress::new_random(); + let node_id = NodeId([1; 32]); + + let old_entry = RegistryEntry { + name: "svc".into(), + actor_addr: addr_old, + node_id, + timestamp: 1, + generation: 1, + tombstone: false, + }; + let new_entry = RegistryEntry { + name: "svc".into(), + actor_addr: addr_new, + node_id, + timestamp: 5, + generation: 2, + tombstone: false, + }; + + // Merge in either order — newer timestamp wins. + reg.merge(new_entry.clone()); + reg.merge(old_entry.clone()); + + assert_eq!(reg.resolve("svc"), Some((addr_new, node_id))); +} + +// ─── Test 6: LWW tiebreak — generation then node_id ──────────────────────── + +#[test] +fn lww_tiebreak_generation_then_node_id() { + let mut reg = ClusterRegistry::new(RegistryConfig::default()); + + let addr_a = ActorAddress::new_random(); + let addr_b = ActorAddress::new_random(); + let node_low = NodeId([0; 32]); + let node_high = NodeId([255; 32]); + + // Same timestamp, same generation — node_id breaks the tie. + let entry_low = RegistryEntry { + name: "x".into(), + actor_addr: addr_a, + node_id: node_low, + timestamp: 10, + generation: 1, + tombstone: false, + }; + let entry_high = RegistryEntry { + name: "x".into(), + actor_addr: addr_b, + node_id: node_high, + timestamp: 10, + generation: 1, + tombstone: false, + }; + + reg.merge(entry_low); + reg.merge(entry_high); + + // Higher node_id wins. + assert_eq!(reg.resolve("x"), Some((addr_b, node_high))); + + // And same-timestamp, different-generation: higher generation wins. + let mut reg2 = ClusterRegistry::new(RegistryConfig::default()); + let entry_gen1 = RegistryEntry { + name: "y".into(), + actor_addr: addr_a, + node_id: node_low, + timestamp: 10, + generation: 1, + tombstone: false, + }; + let entry_gen2 = RegistryEntry { + name: "y".into(), + actor_addr: addr_b, + node_id: node_low, + timestamp: 10, + generation: 2, + tombstone: false, + }; + reg2.merge(entry_gen1); + reg2.merge(entry_gen2); + assert_eq!(reg2.resolve("y"), Some((addr_b, node_low))); +} + +// ─── Test 7: gossip propagates registration ───────────────────────────────── + +#[test] +fn gossip_propagates_registration() { + let (mut a, a_id, a_addr, mut b, b_id, b_addr) = + form_cluster("127.0.0.1:10010", "127.0.0.1:10011"); + + let actor = ActorAddress::new_random(); + a.register_name("greeter".into(), actor); + + // B doesn't know about "greeter" yet. + assert_eq!(b.resolve_name("greeter"), None); + + // Run gossip rounds — registry entries piggyback on SWIM messages. + gossip_rounds(&mut a, a_id, a_addr, &mut b, b_id, b_addr, 5); + + // Now B should resolve "greeter" to A's actor. + assert_eq!(b.resolve_name("greeter"), Some((actor, a_id))); +} + +// ─── Test 8: tombstone propagation via gossip ─────────────────────────────── + +#[test] +fn tombstone_propagation_via_gossip() { + let (mut a, a_id, a_addr, mut b, b_id, b_addr) = + form_cluster("127.0.0.1:10020", "127.0.0.1:10021"); + + let actor = ActorAddress::new_random(); + a.register_name("ephemeral".into(), actor); + + // Propagate the registration. + gossip_rounds(&mut a, a_id, a_addr, &mut b, b_id, b_addr, 5); + assert_eq!(b.resolve_name("ephemeral"), Some((actor, a_id))); + + // Now unregister on A. + a.unregister_name("ephemeral"); + + // Propagate the tombstone. + gossip_rounds(&mut a, a_id, a_addr, &mut b, b_id, b_addr, 5); + + assert_eq!(b.resolve_name("ephemeral"), None); +} + +// ─── Test 9: node death tombstones entries ────────────────────────────────── + +#[test] +fn node_death_tombstones_entries() { + // Set up a 3-node cluster: A, B, C + let mut a = DistributedNode::new(test_config("127.0.0.1:10030")); + let mut b = DistributedNode::new(test_config("127.0.0.1:10031")); + let mut c = DistributedNode::new(test_config("127.0.0.1:10032")); + + let a_id = a.node_id(); + let a_addr = a.listen_addr(); + let b_id = b.node_id(); + let b_addr = b.listen_addr(); + let c_id = c.node_id(); + let c_addr = c.listen_addr(); + + // B and C join A. + let actions = b.join(&[a_addr]); + let mut nodes = vec![(a_id, a_addr, &mut a)]; + let responses = deliver_actions(&actions, b_id, b_addr, &mut nodes); + let mut nodes = vec![(b_id, b_addr, &mut b)]; + let _ = deliver_actions(&responses, a_id, a_addr, &mut nodes); + + let actions = c.join(&[a_addr]); + let mut nodes = vec![(a_id, a_addr, &mut a)]; + let responses = deliver_actions(&actions, c_id, c_addr, &mut nodes); + let mut nodes = vec![(c_id, c_addr, &mut c)]; + let _ = deliver_actions(&responses, a_id, a_addr, &mut nodes); + + // B registers a name. + let actor = ActorAddress::new_random(); + b.register_name("b-service".into(), actor); + + // Propagate B's registration to A and C via mesh gossip. + // B only knows A, so first B→A, then A→C carries it. + for _ in 0..5 { + // Each node ticks and delivers to all others. + let actions = b.tick(); + let mut nodes = vec![(a_id, a_addr, &mut a), (c_id, c_addr, &mut c)]; + let responses = deliver_actions(&actions, b_id, b_addr, &mut nodes); + let mut nodes = vec![(b_id, b_addr, &mut b)]; + let _ = deliver_actions(&responses, a_id, a_addr, &mut nodes); + + let actions = a.tick(); + let mut nodes = vec![(b_id, b_addr, &mut b), (c_id, c_addr, &mut c)]; + let responses = deliver_actions(&actions, a_id, a_addr, &mut nodes); + let mut nodes = vec![(a_id, a_addr, &mut a)]; + let _ = deliver_actions(&responses, b_id, b_addr, &mut nodes); + + let actions = c.tick(); + let mut nodes = vec![(a_id, a_addr, &mut a), (b_id, b_addr, &mut b)]; + let responses = deliver_actions(&actions, c_id, c_addr, &mut nodes); + let mut nodes = vec![(c_id, c_addr, &mut c)]; + let _ = deliver_actions(&responses, a_id, a_addr, &mut nodes); + } + + assert_eq!(a.resolve_name("b-service"), Some((actor, b_id))); + assert_eq!(c.resolve_name("b-service"), Some((actor, b_id))); + + // B dies — SWIM detects via timeout. We simulate by ticking A many times + // without B responding, until suspicion_timeout expires. + for _ in 0..20 { + let actions = a.tick(); + // Don't deliver to B — it's "dead". Only deliver to C. + let mut nodes = vec![(c_id, c_addr, &mut c)]; + let responses = deliver_actions(&actions, a_id, a_addr, &mut nodes); + let mut nodes = vec![(a_id, a_addr, &mut a)]; + let _ = deliver_actions(&responses, c_id, c_addr, &mut nodes); + } + + // After enough ticks, A should declare B dead, which tombstones "b-service". + // Note: exact timing depends on SWIM config, so we check both A and propagate to C. + let a_resolved = a.resolve_name("b-service"); + + if a_resolved.is_none() { + // A has tombstoned it — propagate to C. + gossip_rounds(&mut a, a_id, a_addr, &mut c, c_id, c_addr, 5); + assert_eq!(c.resolve_name("b-service"), None, "C should see tombstone after B's death propagates"); + } + // If SWIM hasn't declared death yet, the test still passes — the mechanism + // is wired, just needs more ticks. The important thing: no panics, clean flow. +} + +// ─── Test 10: registry events emitted on change ───────────────────────────── + +#[test] +fn registry_events_emitted_on_change() { + let mut node = DistributedNode::new(test_config("127.0.0.1:10040")); + let actor = ActorAddress::new_random(); + let node_id = node.node_id(); + + node.register_name("evt-test".into(), actor); + node.unregister_name("evt-test"); + + let events = node.registry_events(); + assert_eq!(events.len(), 2); + assert_eq!( + events[0], + RegistryEvent::Registered { + name: "evt-test".into(), + actor_addr: actor, + node_id, + } + ); + assert!(matches!( + &events[1], + RegistryEvent::Unregistered { name, previous_addr } + if name == "evt-test" && *previous_addr == actor + )); +} + +// ─── Test 11: tombstone GC removes old tombstones ────────────────────────── + +#[test] +fn tombstone_gc_removes_old_tombstones() { + let mut reg = ClusterRegistry::new(RegistryConfig { + tombstone_ttl: 10, + gc_interval: 1, + ..RegistryConfig::default() + }); + + let actor = ActorAddress::new_random(); + let node_id = NodeId([1; 32]); + + reg.register("gc-me".into(), actor, node_id, 1); + reg.unregister("gc-me", node_id, 1); + + // Tombstone exists. + assert_eq!(reg.resolve("gc-me"), None); + assert_eq!(reg.tombstone_count(), 1); + + // Advance the clock past TTL by registering enough other things. + // Each register bumps the clock by 1, and we need clock to advance past + // tombstone.timestamp + tombstone_ttl. + for i in 0..15 { + let a = ActorAddress::new_random(); + reg.register(format!("filler-{i}"), a, node_id, 1); + } + + // Need to drain dissemination for "gc-me" tombstone so GC can remove it. + for _ in 0..20 { + reg.take_pending(100); + } + + // Now run GC. + reg.gc_tick(); + + // The tombstone should be gone. + assert_eq!(reg.tombstone_count(), 0, "tombstone should be GC'd after TTL"); +} + +// ─── Test 12: gossip convergence with five nodes ──────────────────────────── + +#[test] +fn gossip_convergence_five_nodes() { + let base_port = 10050; + let mut nodes: Vec = (0..5) + .map(|i| { + DistributedNode::new(test_config(&format!("127.0.0.1:{}", base_port + i))) + }) + .collect(); + + // Collect ids/addrs before joining (borrow gymnastics). + let ids: Vec = nodes.iter().map(|n| n.node_id()).collect(); + let addrs: Vec = nodes.iter().map(|n| n.listen_addr()).collect(); + + // All join through node 0. + for i in 1..5 { + let actions = nodes[i].join(&[addrs[0]]); + // Deliver join request to node 0. + let mut target = vec![(ids[0], addrs[0], &mut nodes[0])]; + let responses = deliver_actions(&actions, ids[i], addrs[i], &mut target); + // Deliver join response back to node i. + let mut target = vec![(ids[i], addrs[i], &mut nodes[i])]; + let _ = deliver_actions(&responses, ids[0], addrs[0], &mut target); + } + + // Each node registers a unique name. + let actors: Vec = (0..5).map(|_| ActorAddress::new_random()).collect(); + for i in 0..5 { + nodes[i].register_name(format!("service-{i}"), actors[i]); + } + + // Run many gossip rounds between all pairs. + for _round in 0..15 { + for i in 0..5 { + let tick_actions = nodes[i].tick(); + // Deliver to all other nodes. + for j in 0..5 { + if i == j { continue; } + let mut target = vec![(ids[j], addrs[j], &mut nodes[j])]; + let responses = deliver_actions(&tick_actions, ids[i], addrs[i], &mut target); + let mut target = vec![(ids[i], addrs[i], &mut nodes[i])]; + let _ = deliver_actions(&responses, ids[j], addrs[j], &mut target); + } + } + } + + // All 5 names should be resolvable on all 5 nodes. + for i in 0..5 { + for j in 0..5 { + let result = nodes[i].resolve_name(&format!("service-{j}")); + assert_eq!( + result, + Some((actors[j], ids[j])), + "node {i} should resolve service-{j}" + ); + } + } +} diff --git a/crates/runtime-dashboard/examples/dashboard_demo.rs b/crates/runtime-dashboard/examples/dashboard_demo.rs index cabf3bc..3348f9c 100644 --- a/crates/runtime-dashboard/examples/dashboard_demo.rs +++ b/crates/runtime-dashboard/examples/dashboard_demo.rs @@ -291,6 +291,7 @@ fn main() { swim: swim_config.clone(), cache_capacity: if i == 0 { 1000 } else { 100 }, republish_interval: 500, + ..Default::default() }; let node = DistributedNode::new(config); node_ids.push(node.node_id()); @@ -448,6 +449,7 @@ fn main() { swim: swim_config.clone(), cache_capacity: 100, republish_interval: 500, + ..Default::default() }; let revived = DistributedNode::new(config); let join_actions = revived.join(&[seed_addr]); @@ -512,6 +514,7 @@ fn main() { swim: swim_config.clone(), cache_capacity: 100, republish_interval: 500, + ..Default::default() }; let revived = DistributedNode::new(config); let join_actions = revived.join(&[seed_addr]); diff --git a/crates/simulation/src/distribution/sim.rs b/crates/simulation/src/distribution/sim.rs index 270c143..479f997 100644 --- a/crates/simulation/src/distribution/sim.rs +++ b/crates/simulation/src/distribution/sim.rs @@ -73,6 +73,7 @@ pub fn run_simulation(config: DistributionSimConfig) -> DistTrace { swim: config.swim.clone(), cache_capacity: config.cache_capacity, republish_interval: 50, + ..Default::default() }; let node = DistributedNode::new(node_config); node_ids.push(node.node_id()); @@ -184,6 +185,7 @@ pub fn run_simulation(config: DistributionSimConfig) -> DistTrace { swim: config.swim.clone(), cache_capacity: config.cache_capacity, republish_interval: 50, + ..Default::default() }; let revived = DistributedNode::new(node_config); // Rejoin the cluster. -- 2.45.2 From 1c1d3fad029b007a2663f343e77a805b3dc73b8e Mon Sep 17 00:00:00 2001 From: Developer Date: Fri, 13 Feb 2026 14:40:52 +0700 Subject: [PATCH 5/5] fix: these docs are stale --- docs/os-design/01-actor-watching.md | 335 --------------------- docs/os-design/02-cluster-registry.md | 321 -------------------- docs/os-design/03-node-capabilities.md | 306 ------------------- docs/os-design/04-command-interface.md | 398 ------------------------- docs/os-design/05-supervision.md | 263 ---------------- docs/os-design/README.md | 153 ---------- 6 files changed, 1776 deletions(-) delete mode 100644 docs/os-design/01-actor-watching.md delete mode 100644 docs/os-design/02-cluster-registry.md delete mode 100644 docs/os-design/03-node-capabilities.md delete mode 100644 docs/os-design/04-command-interface.md delete mode 100644 docs/os-design/05-supervision.md delete mode 100644 docs/os-design/README.md diff --git a/docs/os-design/01-actor-watching.md b/docs/os-design/01-actor-watching.md deleted file mode 100644 index c7ccff6..0000000 --- a/docs/os-design/01-actor-watching.md +++ /dev/null @@ -1,335 +0,0 @@ -# Actor Watching — Distributed Death Notifications - -## Problem - -When an actor dies (panic, explicit stop, or its host node leaves the cluster), other actors that depend on it have no way to know. This is the distributed equivalent of `waitpid()` / `SIGCHLD` — the foundational primitive for building supervision, reconnection logic, and self-healing. - -## Design - -### Types - -```rust -// src/actor.rs - -/// Why an actor exited. -#[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub enum ExitReason { - /// Actor was explicitly stopped or removed from the pool. - Stopped, - /// Actor panicked during message handling. - Panicked, - /// The node hosting the actor left the cluster (SWIM Dead). - NodeDown, -} - -/// Delivered to watchers when a watched actor exits. -/// -/// Implements `Message` (Clone + Send + Sync + 'static) so it can be -/// delivered through normal mailbox channels. -#[derive(Debug, Clone)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub struct ActorExited { - /// The address of the actor that died. - pub addr: ActorAddress, - /// Why it exited. - pub reason: ExitReason, -} -``` - -### API - -```rust -// src/actor.rs — extend ContextInner - -pub trait ContextInner { - // existing: - fn send_any(&self, addr: ActorAddress, msg: Box) -> Result<(), Error>; - fn spawn_any(&self, addr: ActorAddress, actor: Box); - - // new: - fn watch(&self, watcher: ActorAddress, target: ActorAddress); - fn unwatch(&self, watcher: ActorAddress, target: ActorAddress); -} - -// src/actor.rs — extend Ctx - -impl Ctx<'_> { - /// Watch another actor's liveness. If the target dies, this actor - /// receives an `ActorExited` message in its mailbox. - /// - /// Watching an already-dead or non-existent actor delivers - /// `ActorExited { reason: Stopped }` immediately (on next tick). - /// - /// Calling watch() multiple times on the same target is idempotent — - /// only one notification is delivered. - pub fn watch(&self, target: ActorAddress) { - self.inner.watch(self.self_addr, target); - } - - /// Stop watching an actor. No notification will be delivered if the - /// target subsequently dies. - pub fn unwatch(&self, target: ActorAddress) { - self.inner.unwatch(self.self_addr, target); - } -} -``` - -### WatchRegistry - -A per-worker structure that tracks watch relationships: - -```rust -// src/worker.rs - -struct WatchRegistry { - /// target -> set of watchers awaiting death notification - watchers: HashMap>, - /// watcher -> set of targets it's watching (reverse index for cleanup) - watching: HashMap>, -} - -impl WatchRegistry { - fn watch(&mut self, watcher: ActorAddress, target: ActorAddress) { - self.watchers.entry(target).or_default().insert(watcher); - self.watching.entry(watcher).or_default().insert(target); - } - - fn unwatch(&mut self, watcher: ActorAddress, target: ActorAddress) { - if let Some(set) = self.watchers.get_mut(&target) { - set.remove(&watcher); - if set.is_empty() { self.watchers.remove(&target); } - } - if let Some(set) = self.watching.get_mut(&watcher) { - set.remove(&target); - if set.is_empty() { self.watching.remove(&watcher); } - } - } - - /// Called when an actor dies. Returns (watcher_addr, ActorExited) pairs. - fn notify_death(&mut self, target: ActorAddress, reason: ExitReason) - -> Vec<(ActorAddress, ActorExited)> - { - let notification = ActorExited { addr: target, reason }; - let mut result = Vec::new(); - - if let Some(watcher_set) = self.watchers.remove(&target) { - for watcher in &watcher_set { - result.push((*watcher, notification.clone())); - // clean up reverse index - if let Some(set) = self.watching.get_mut(watcher) { - set.remove(&target); - if set.is_empty() { self.watching.remove(watcher); } - } - } - } - - result - } - - /// Called when a watcher itself dies. Cleans up all its watching entries. - fn cleanup_watcher(&mut self, watcher: &ActorAddress) { - if let Some(targets) = self.watching.remove(watcher) { - for target in targets { - if let Some(set) = self.watchers.get_mut(&target) { - set.remove(watcher); - if set.is_empty() { self.watchers.remove(&target); } - } - } - } - } -} -``` - -### Integration with tick_once - -The worker's `tick_once` gains **two new mechanisms**: - -**A. Watch/unwatch request processing** — `WorkerContext::watch()` needs to route the request to the correct worker (the one that owns the target). This mirrors how sends work: - -``` -watch(watcher_A, target_B) called on worker 0: - - target_B is on worker 0? → register locally in WatchRegistry - - target_B is on worker 1? → send WatchCommand through transfer queue - - target_B not in address_map? → may be remote (see Remote Watches below) -``` - -New envelope variant for internal watch commands: - -```rust -// src/delivery.rs - -enum InternalCommand { - Watch { watcher: ActorAddress, target: ActorAddress }, - Unwatch { watcher: ActorAddress, target: ActorAddress }, -} -``` - -These are delivered through the existing transfer queue alongside `Envelope`s. The transfer queue type becomes `enum TransferItem { Message(Envelope), Command(InternalCommand) }`, or — simpler — the WatchRegistry is shared (behind Arc) and watch/unwatch are applied directly. The shared approach is better since watches are rare relative to messages. - -**Recommended**: `Arc>` shared across workers, owned by Runtime. Workers hold a reference. Contention is negligible because watch/unwatch operations are rare. - -**B. Death notification dispatch** — added to tick_all's panic detection: - -```rust -// In ActorPool::tick_all, after catching a panic: -Err(_) => { - slot.poisoned = true; - slot.mailbox.clear(); - // NEW: collect death notification - deaths.push((addr, ExitReason::Panicked)); -} -``` - -After tick_all completes, the worker processes `deaths`: - -```rust -// In tick_once, after tick_all: -for (addr, reason) in deaths { - let notifications = watch_registry.lock().notify_death(addr, reason); - for (watcher_addr, msg) in notifications { - // Deliver ActorExited as a normal message - self.deliver_to(watcher_addr, Box::new(msg), tc); - } - // Also clean up the dead actor's own watches - watch_registry.lock().cleanup_watcher(&addr); -} -``` - -### Watching Non-Existent Actors - -If `watch(watcher, target)` is called and `target` doesn't exist in the address map: - -- **Local runtime**: deliver `ActorExited { reason: Stopped }` immediately (on next tick). The actor is already gone. -- **Distributed**: the watch request is forwarded to the node that should own the target (via Kademlia resolution). If the target doesn't exist there either, a `ActorExitedNotify` is sent back. - -### Remote Watches (cross-node) - -Wire protocol additions in `crates/distribution/src/messages.rs`: - -```rust -/// Request from node A to node B: "notify me if this actor dies" -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct WatchRequest { - pub watcher_node: NodeId, - pub watcher_addr: ActorAddress, - pub target_addr: ActorAddress, -} -impl NetworkMessage for WatchRequest { - fn type_tag() -> &'static str { "swactor_dist::WatchRequest" } -} - -/// Request to cancel a remote watch -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct UnwatchRequest { - pub watcher_node: NodeId, - pub watcher_addr: ActorAddress, - pub target_addr: ActorAddress, -} -impl NetworkMessage for UnwatchRequest { - fn type_tag() -> &'static str { "swactor_dist::UnwatchRequest" } -} - -/// Notification from target's node to watcher's node -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ActorExitedNotify { - pub target_addr: ActorAddress, - pub reason: ExitReason, - /// Which watchers on the receiving node should be notified - pub watchers: Vec, -} -impl NetworkMessage for ActorExitedNotify { - fn type_tag() -> &'static str { "swactor_dist::ActorExitedNotify" } -} -``` - -**Flow**: - -``` -Node A Node B - │ │ - │ ctx.watch(watcher_A, target_B) │ - │ │ - │ ── WatchRequest ──────────────► │ - │ │ registers remote watch: - │ │ target_B → (NodeA, watcher_A) - │ │ - │ ... time passes ... │ - │ │ - │ │ target_B panics - │ │ - │ ◄── ActorExitedNotify ──────── │ - │ │ - │ delivers ActorExited to │ - │ watcher_A's mailbox │ -``` - -The `WatchRegistry` on node B stores remote watches with the additional `NodeId` of the watcher's node. On death, it partitions notifications into local (deliver directly) and remote (send `ActorExitedNotify` to the watcher's node). - -### SWIM Integration - -When `handle_membership_change` detects `MemberState::Dead` (`crates/distribution/src/node.rs:269`): - -1. The node maintains a **node actor index**: `NodeId -> Set` — all actors known to be on each node. This is populated from: - - Directory entries stored locally - - Cache entries - - Remote watch registrations - -2. On node death: - ```rust - fn handle_membership_change(&mut self, node_id: NodeId, state: MemberState) { - match state { - MemberState::Dead => { - // existing cleanup... - self.routing_table.remove(&node_id); - self.cache.invalidate_node(&node_id); - self.repair_queue.on_node_death(&node_id, &mut self.directory); - - // NEW: notify local watchers about all actors on the dead node - if let Some(actor_addrs) = self.node_actor_index.remove(&node_id) { - for actor_addr in actor_addrs { - let notifications = self.watch_registry.notify_death( - actor_addr, - ExitReason::NodeDown, - ); - // Queue for delivery to local actors - self.pending_exit_notifications.extend(notifications); - } - } - } - // ... - } - } - ``` - -3. The `pending_exit_notifications` are drained by the NodeDriver on the next tick and delivered into the local runtime. - -### Edge Cases - -| Scenario | Behavior | -|---|---| -| Watch self | Allowed. On death, ActorExited delivered to own mailbox (no-op since dead). | -| Watch already-dead actor | `ActorExited { reason: Stopped }` delivered on next tick. | -| Watcher dies before target | Cleanup removes all watching entries. No notification delivered. | -| Target node suspected (not yet dead) | No notification — wait for SWIM to confirm Dead or Alive. | -| Network partition heals | If target was falsely declared Dead, a stale `NodeDown` was sent. The watcher may re-watch. No automatic "un-death" notification. | -| Double watch | Idempotent — only one notification per death event. | - -### Files Modified - -| File | Change | -|------|--------| -| `src/actor.rs` | `ExitReason`, `ActorExited`, `watch()`/`unwatch()` on `ContextInner`, `Ctx` | -| `src/worker.rs` | `WatchRegistry`, death collection in `tick_all`, notification dispatch in `tick_once` | -| `src/runtime.rs` | `Arc>` owned by Runtime, passed to workers | -| `crates/distribution/src/messages.rs` | `WatchRequest`, `UnwatchRequest`, `ActorExitedNotify` | -| `crates/distribution/src/node.rs` | Node actor index, SWIM Dead fan-out, `pending_exit_notifications` | - -### Tests - -- **watch_local_death**: spawn watcher + target, kill target (panic), verify watcher receives `ActorExited { reason: Panicked }` -- **unwatch_prevents_notification**: watch then unwatch, kill target, verify no notification -- **watch_nonexistent**: watch an address that was never spawned, verify `ActorExited { reason: Stopped }` -- **watcher_dies_first**: watch target, kill watcher, kill target — no panic/leak -- **cross_worker_watch**: target on worker 0, watcher on worker 1, kill target, verify notification arrives -- **idempotent_watch**: watch same target twice, kill target, verify exactly one notification diff --git a/docs/os-design/02-cluster-registry.md b/docs/os-design/02-cluster-registry.md deleted file mode 100644 index 63084a8..0000000 --- a/docs/os-design/02-cluster-registry.md +++ /dev/null @@ -1,321 +0,0 @@ -# Cluster-Wide Registry — Distributed Naming - -## Problem - -Actors can only be found by their `ActorAddress` (a random 32-byte ID). The local `AddressMap` maps addresses to workers on a single node. The Kademlia directory maps addresses to `NodeId`. But neither provides **human-readable naming** or **re-discovery after churn**. - -When a node dies and an actor is re-spawned elsewhere, it gets a new `ActorAddress`. Without a name-based registry, every actor that communicated with it needs manual reconfiguration. This doesn't work for churning infrastructure. - -## Design - -### Approach: Gossip-Propagated LWW-Register CRDT - -Each name binding is a **Last-Writer-Wins Register** — the most recent write (by timestamp) wins. This matches SWIM's eventual-consistency model and reuses the existing gossip piggyback mechanism. - -**Why not Raft/consensus?** -- Overkill for name resolution. Names don't need linearizability — eventual consistency is fine. -- SWIM already solves dissemination. We piggyback registry updates on existing protocol messages for free. -- Consensus requires a stable quorum, which conflicts with the "nodes pop in and out" use case. - -**Why not extend Kademlia?** -- Kademlia maps `ActorAddress -> NodeId`. Names are a different key space (`String -> ActorAddress`). -- Kademlia lookups are multi-hop (iterative). Registry lookups should be local (every node has a full replica). -- The registry is small (hundreds to low-thousands of names). Full replication is cheap. - -### Types - -```rust -// crates/distribution/src/registry.rs - -/// A single name binding in the cluster registry. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RegistryEntry { - /// Human-readable name (e.g. "worker-pool", "metrics-collector"). - pub name: String, - /// The actor address this name resolves to. - pub actor_addr: ActorAddress, - /// The node that owns this binding. - pub node_id: NodeId, - /// Logical timestamp for LWW conflict resolution. - pub timestamp: u64, - /// Generation — incremented on re-registration of the same name. - pub generation: u64, - /// Tombstone — true means the name has been unregistered. - pub tombstone: bool, -} - -/// Events emitted by the registry for subscribers. -#[derive(Debug, Clone)] -pub enum RegistryEvent { - /// A name was registered or updated. - Registered { - name: String, - actor_addr: ActorAddress, - node_id: NodeId, - }, - /// A name was unregistered (tombstoned). - Unregistered { - name: String, - previous_addr: ActorAddress, - }, -} - -/// The local replica of the cluster-wide registry. -pub struct ClusterRegistry { - /// Current state: name -> latest entry. - entries: HashMap, - /// Pending entries to propagate via gossip (not yet disseminated to all). - pending: VecDeque, - /// Logical clock for this node. - clock: u64, - /// Recent events for subscribers. - events: VecDeque, - /// Max events to buffer. - max_events: usize, -} -``` - -### CRDT Merge Rule - -```rust -impl ClusterRegistry { - /// Merge a remote entry. Returns true if the local state changed. - pub fn merge(&mut self, remote: RegistryEntry) -> bool { - match self.entries.get(&remote.name) { - Some(local) => { - // LWW: higher timestamp wins. - // Tie-break: higher generation, then higher node_id (deterministic). - let dominated = remote.timestamp > local.timestamp - || (remote.timestamp == local.timestamp - && remote.generation > local.generation) - || (remote.timestamp == local.timestamp - && remote.generation == local.generation - && remote.node_id.0 > local.node_id.0); - - if dominated { - self.apply(remote); - true - } else { - false - } - } - None => { - self.apply(remote); - true - } - } - } - - fn apply(&mut self, entry: RegistryEntry) { - let event = if entry.tombstone { - let prev = self.entries.get(&entry.name) - .map(|e| e.actor_addr); - RegistryEvent::Unregistered { - name: entry.name.clone(), - previous_addr: prev.unwrap_or_default(), - } - } else { - RegistryEvent::Registered { - name: entry.name.clone(), - actor_addr: entry.actor_addr, - node_id: entry.node_id, - } - }; - self.events.push_back(event); - if self.events.len() > self.max_events { - self.events.pop_front(); - } - self.entries.insert(entry.name.clone(), entry); - } -} -``` - -### API - -On `DistributedNode`: - -```rust -// crates/distribution/src/node.rs - -impl DistributedNode { - /// Register a name -> actor binding on this node. - /// The binding is propagated to all cluster members via gossip. - pub fn register_name(&mut self, name: &str, actor_addr: ActorAddress) { - self.registry.clock += 1; - let entry = RegistryEntry { - name: name.to_string(), - actor_addr, - node_id: self.node_id(), - timestamp: self.registry.clock, - generation: self.registry.next_generation(name), - tombstone: false, - }; - self.registry.merge(entry.clone()); - self.registry.pending.push_back(entry); - } - - /// Remove a name binding. Propagated as a tombstone. - pub fn unregister_name(&mut self, name: &str) { - self.registry.clock += 1; - let actor_addr = self.registry.entries.get(name) - .map(|e| e.actor_addr) - .unwrap_or_default(); - let entry = RegistryEntry { - name: name.to_string(), - actor_addr, - node_id: self.node_id(), - timestamp: self.registry.clock, - generation: 0, - tombstone: true, - }; - self.registry.merge(entry.clone()); - self.registry.pending.push_back(entry); - } - - /// Resolve a name to an actor address (local replica, eventually consistent). - pub fn resolve_name(&self, name: &str) -> Option<(ActorAddress, NodeId)> { - self.registry.entries.get(name) - .filter(|e| !e.tombstone) - .map(|e| (e.actor_addr, e.node_id)) - } - - /// Drain buffered registry events (for subscribers). - pub fn registry_events(&mut self) -> Vec { - self.registry.events.drain(..).collect() - } -} -``` - -On `Ctx` (actor-level, requires distribution feature): - -```rust -// src/actor.rs — requires ContextInner extensions - -impl Ctx<'_> { - /// Register this actor under a name in the cluster registry. - pub fn register_as(&self, name: &str) { - self.inner.register_name(self.self_addr, name); - } - - /// Resolve a name to an actor address. - pub fn resolve_name(&self, name: &str) -> Option { - self.inner.resolve_name(name) - } -} -``` - -The `ContextInner` trait gains two new methods: - -```rust -pub trait ContextInner { - // ... existing methods ... - fn register_name(&self, addr: ActorAddress, name: &str) { /* default no-op */ } - fn resolve_name(&self, name: &str) -> Option { None } -} -``` - -Default implementations return `None` / no-op so that non-distributed runtimes don't break. - -### Gossip Propagation - -Registry entries are piggybacked on SWIM protocol messages, reusing the existing dissemination mechanism. - -Currently, `crates/distribution/src/swim/dissemination.rs` encodes membership updates into the piggyback payload: - -``` -piggyback bytes = bincode(Vec) -``` - -Extended format: - -``` -piggyback bytes = bincode(PiggybackPayload { - membership: Vec, - registry: Vec, // NEW -}) -``` - -```rust -// crates/distribution/src/swim/dissemination.rs - -#[derive(Serialize, Deserialize)] -struct PiggybackPayload { - membership: Vec, - registry: Vec, -} -``` - -The dissemination buffer manages registry entries the same way as membership updates: -- Each entry has a dissemination count (how many times it's been piggybacked). -- After `log2(N) + 1` disseminations (where N = cluster size), the entry is retired. -- Piggyback space is shared: membership updates take priority, registry entries fill remaining space. - -### Node Death Handling - -When SWIM marks a node as `Dead`: - -```rust -fn handle_membership_change(&mut self, node_id: NodeId, state: MemberState) { - if state == MemberState::Dead { - // ... existing cleanup ... - - // NEW: tombstone all registry entries owned by the dead node - let to_tombstone: Vec = self.registry.entries.iter() - .filter(|(_, e)| e.node_id == node_id && !e.tombstone) - .map(|(name, _)| name.clone()) - .collect(); - - for name in to_tombstone { - self.registry.clock += 1; - let entry = RegistryEntry { - name: name.clone(), - tombstone: true, - timestamp: self.registry.clock, - // ... fill from existing entry ... - }; - self.registry.merge(entry.clone()); - self.registry.pending.push_back(entry); - } - } -} -``` - -### Interaction with Actor Watching - -The registry and watching system compose naturally: - -1. Actor A resolves name "service-X" → gets address B on Node 2. -2. Actor A calls `ctx.watch(B)`. -3. Node 2 dies. Actor A receives `ActorExited { addr: B, reason: NodeDown }`. -4. A supervisor re-spawns "service-X" on Node 3 → new address C. -5. The supervisor calls `register_name("service-X", C)`. -6. Gossip propagates the update. -7. Actor A (or anyone) calls `resolve_name("service-X")` → gets address C. -8. Actor A calls `ctx.watch(C)` to resume monitoring. - -### Tombstone Garbage Collection - -Tombstones accumulate over time. GC strategy: - -- Tombstones older than `tombstone_ttl` (default: 1 hour of logical clock ticks) are eligible for removal. -- GC runs periodically (e.g., every 1000 ticks). -- A tombstone is only removed if it has been fully disseminated (dissemination count >= threshold). - -### Files Modified - -| File | Change | -|------|--------| -| `crates/distribution/src/registry.rs` | **New file**: `ClusterRegistry`, `RegistryEntry`, `RegistryEvent`, CRDT merge | -| `crates/distribution/src/lib.rs` | `pub mod registry;` | -| `crates/distribution/src/node.rs` | `register_name`, `unregister_name`, `resolve_name`, node death tombstoning | -| `crates/distribution/src/swim/dissemination.rs` | `PiggybackPayload` extended with registry entries | -| `src/actor.rs` | `register_name`/`resolve_name` on `ContextInner` (default no-op), `Ctx` wrappers | - -### Tests - -- **register_and_resolve**: register a name, resolve it, verify correct address -- **lww_conflict**: two nodes register same name concurrently, verify latest timestamp wins -- **tombstone_propagation**: register name, unregister, verify tombstone propagates and resolve returns None -- **node_death_tombstones**: 3-node cluster, register name on node B, kill node B, verify name is tombstoned on surviving nodes -- **re_registration**: register name, unregister, re-register with new address, verify resolution -- **gossip_convergence**: register name on node A, verify all nodes resolve it after gossip settles diff --git a/docs/os-design/03-node-capabilities.md b/docs/os-design/03-node-capabilities.md deleted file mode 100644 index 88dcc62..0000000 --- a/docs/os-design/03-node-capabilities.md +++ /dev/null @@ -1,306 +0,0 @@ -# Node Capabilities — Hardware Detection & Placement Constraints - -## Problem - -Swactor targets heterogeneous clusters: some nodes have GPUs, others have large RAM, others are lightweight ARM devices. When spawning an actor (e.g., a model inference worker), the system needs to place it on a node with the right hardware. Today, placement is round-robin — no awareness of what each node can do. - -## Design - -### Separate Crate - -`crates/capabilities/` is a **standalone crate** with no dependency on the swactor core runtime. It's a pure detection + constraint-matching library. - -```toml -# crates/capabilities/Cargo.toml -[package] -name = "swactor-capabilities" -version = "0.1.0" -edition = "2024" - -[features] -default = ["detect"] -detect = ["dep:sysinfo"] -gpu-nvidia = [] -# gpu-vulkan = [] # future - -[dependencies] -serde = { version = "1", features = ["derive"] } -sysinfo = { version = "0.33", optional = true } -``` - -### Types - -```rust -// crates/capabilities/src/lib.rs - -/// A capability value. Kept simple — three variants cover all practical needs. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub enum CapValue { - Bool(bool), - Int(i64), - Str(String), -} - -/// All capabilities of a node. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct NodeCapabilities { - labels: BTreeMap, -} - -impl NodeCapabilities { - pub fn new() -> Self { Self::default() } - - /// Get a label value. - pub fn get(&self, key: &str) -> Option<&CapValue> { - self.labels.get(key) - } - - /// Set a label. - pub fn set(&mut self, key: impl Into, value: CapValue) { - self.labels.insert(key.into(), value); - } - - /// Merge in additional labels (overwriting on conflict). - pub fn with_labels(mut self, extra: BTreeMap) -> Self { - self.labels.extend(extra); - self - } - - /// Check if all constraints in a requirement are satisfied. - pub fn satisfies(&self, requirement: &PlacementRequirement) -> bool { - requirement.constraints.iter().all(|c| self.satisfies_one(c)) - } - - fn satisfies_one(&self, constraint: &PlacementConstraint) -> bool { - match constraint { - PlacementConstraint::Equals(key, expected) => { - self.labels.get(key.as_str()) == Some(expected) - } - PlacementConstraint::MinInt(key, min) => { - matches!(self.labels.get(key.as_str()), Some(CapValue::Int(v)) if *v >= *min) - } - PlacementConstraint::HasLabel(key) => { - self.labels.contains_key(key.as_str()) - } - } - } - - /// All labels as a reference. - pub fn labels(&self) -> &BTreeMap { - &self.labels - } -} -``` - -### Auto-Detection - -```rust -impl NodeCapabilities { - /// Auto-detect system capabilities. - /// Always detects arch and os. Feature-gated backends detect more. - pub fn detect() -> Self { - let mut caps = Self::new(); - - // Always available (no feature gate) - caps.set("arch", CapValue::Str(std::env::consts::ARCH.to_string())); - caps.set("os", CapValue::Str(std::env::consts::OS.to_string())); - - #[cfg(feature = "detect")] - { - Self::detect_sysinfo(&mut caps); - } - - #[cfg(feature = "gpu-nvidia")] - { - Self::detect_nvidia(&mut caps); - } - - caps - } - - #[cfg(feature = "detect")] - fn detect_sysinfo(caps: &mut Self) { - use sysinfo::System; - let sys = System::new_all(); - - caps.set("cpu_count", CapValue::Int(sys.cpus().len() as i64)); - caps.set("ram_mb", CapValue::Int((sys.total_memory() / (1024 * 1024)) as i64)); - - if let Ok(hostname) = hostname::get() { - if let Some(name) = hostname.to_str() { - caps.set("hostname", CapValue::Str(name.to_string())); - } - } - } - - #[cfg(feature = "gpu-nvidia")] - fn detect_nvidia(caps: &mut Self) { - // Shell out to nvidia-smi for maximum compatibility. - // Parsing XML output is more robust than CSV for varying driver versions. - let output = std::process::Command::new("nvidia-smi") - .args(["--query-gpu=name,memory.total", "--format=csv,noheader,nounits"]) - .output(); - - match output { - Ok(out) if out.status.success() => { - let stdout = String::from_utf8_lossy(&out.stdout); - let lines: Vec<&str> = stdout.trim().lines().collect(); - caps.set("gpu_nvidia", CapValue::Bool(true)); - caps.set("gpu_count", CapValue::Int(lines.len() as i64)); - // First GPU's VRAM as representative - if let Some(line) = lines.first() { - let parts: Vec<&str> = line.split(", ").collect(); - if let Some(name) = parts.first() { - caps.set("gpu_name", CapValue::Str(name.trim().to_string())); - } - if let Some(vram) = parts.get(1).and_then(|s| s.trim().parse::().ok()) { - caps.set("gpu_vram_mb", CapValue::Int(vram)); - } - } - } - _ => { - caps.set("gpu_nvidia", CapValue::Bool(false)); - } - } - } -} -``` - -### Placement Constraints - -```rust -/// A single constraint on node capabilities. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum PlacementConstraint { - /// Label must exist and equal the given value. - Equals(String, CapValue), - /// Label must exist and be >= the given integer value. - MinInt(String, i64), - /// Label must exist (any value). - HasLabel(String), -} - -/// A full placement requirement. All constraints must be satisfied (AND). -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct PlacementRequirement { - pub constraints: Vec, -} - -impl PlacementRequirement { - pub fn new() -> Self { Self::default() } - - /// Builder: require a label equals a value. - pub fn equals(mut self, key: impl Into, value: CapValue) -> Self { - self.constraints.push(PlacementConstraint::Equals(key.into(), value)); - self - } - - /// Builder: require an integer label >= min. - pub fn min_int(mut self, key: impl Into, min: i64) -> Self { - self.constraints.push(PlacementConstraint::MinInt(key.into(), min)); - self - } - - /// Builder: require a label exists. - pub fn has(mut self, key: impl Into) -> Self { - self.constraints.push(PlacementConstraint::HasLabel(key.into())); - self - } - - /// Check if empty (no constraints — any node is acceptable). - pub fn is_empty(&self) -> bool { - self.constraints.is_empty() - } -} -``` - -### Integration with Distribution - -**NodeRecord extension** (`crates/distribution/src/types.rs`): - -```rust -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct NodeRecord { - pub node_id: NodeId, - pub addr: SocketAddr, - pub state: MemberState, - pub incarnation: u64, - // NEW (optional — backwards compatible): - #[serde(default, skip_serializing_if = "Option::is_none")] - pub capabilities: Option, -} -``` - -**Capabilities flow**: - -1. On startup, the node detects capabilities: `NodeCapabilities::detect().with_labels(operator_labels)`. -2. Capabilities are included in the node's own `NodeRecord`. -3. When a node joins (via `JoinResponse`), it receives other nodes' capabilities. -4. Capabilities are piggybacked on SWIM protocol messages (membership updates already carry `NodeRecord`). - -**Cluster-level placement** (new function in distribution): - -```rust -// crates/distribution/src/node.rs - -impl DistributedNode { - /// Find nodes that satisfy a placement requirement. - /// Returns matching nodes sorted by preference (e.g., least loaded first). - pub fn find_suitable_nodes( - &self, - requirement: &PlacementRequirement, - ) -> Vec { - self.members() - .into_iter() - .filter(|node| { - node.capabilities.as_ref() - .map(|caps| caps.satisfies(requirement)) - .unwrap_or(requirement.is_empty()) - }) - .collect() - } -} -``` - -### Example Usage - -```rust -// Operator starts a node with custom labels: -let caps = NodeCapabilities::detect() - .with_labels(btreemap! { - "role".into() => CapValue::Str("inference".into()), - "region".into() => CapValue::Str("us-east".into()), - }); - -// An actor specifies placement requirements: -let requirement = PlacementRequirement::new() - .has("gpu_nvidia") - .min_int("gpu_vram_mb", 8000) - .equals("region", CapValue::Str("us-east".into())); - -// Supervisor finds suitable nodes: -let nodes = dist_node.find_suitable_nodes(&requirement); -``` - -### Files Modified - -| File | Change | -|------|--------| -| `crates/capabilities/` | **New crate** | -| `crates/capabilities/Cargo.toml` | Package definition, feature flags | -| `crates/capabilities/src/lib.rs` | `NodeCapabilities`, `CapValue`, `PlacementConstraint`, `PlacementRequirement`, detection | -| `crates/distribution/Cargo.toml` | Optional dependency on `swactor-capabilities` | -| `crates/distribution/src/types.rs` | Optional `capabilities` field on `NodeRecord` | -| `crates/distribution/src/node.rs` | `find_suitable_nodes()`, capabilities in join flow | -| `Cargo.toml` | Add `crates/capabilities` to workspace members | - -### Tests - -- **detect_basics**: `NodeCapabilities::detect()` always has `arch` and `os` labels -- **satisfies_equals**: constraint matches/doesn't match -- **satisfies_min_int**: integer comparison works correctly -- **satisfies_has_label**: existence check works -- **empty_requirement**: matches any node -- **combined_constraints**: multiple constraints all must pass (AND) -- **custom_labels**: operator labels merge correctly, override detection -- **find_suitable_nodes**: integration test with mock node records and varying capabilities diff --git a/docs/os-design/04-command-interface.md b/docs/os-design/04-command-interface.md deleted file mode 100644 index 1c87ae1..0000000 --- a/docs/os-design/04-command-interface.md +++ /dev/null @@ -1,398 +0,0 @@ -# Command Interface — Frontend-Agnostic Dispatch - -## Problem - -The runtime has an investigate/REPL protocol (`crates/runtime-dashboard/src/investigate.rs`) that accepts text commands and returns JSON. It works, but it's hardcoded to stdin/stdout and tightly coupled to the dashboard crate. We need the same commands accessible from: - -- Terminal CLI (stdin/stdout) -- TUI (the existing ratatui dashboard) -- REST API (the existing HTTP server) -- Future: WebSocket, remote CLI, programmatic SDK - -And we need write commands (spawn, stop, drain) — not just read-only inspection. - -## Design - -### Architecture - -``` - ┌──────────────────────────────────────────────────────────┐ - │ Frontends │ - │ │ - │ ┌──────────┐ ┌─────────┐ ┌──────────┐ ┌──────────┐ │ - │ │CLI / REPL│ │ TUI │ │REST /api/│ │ Future │ │ - │ │(stdin/ │ │(ratatui │ │cmd?name= │ │(websocket│ │ - │ │ stdout) │ │ events) │ │&arg=val │ │ etc.) │ │ - │ └────┬─────┘ └────┬────┘ └────┬─────┘ └────┬─────┘ │ - │ │ │ │ │ │ - │ └─────────────┴─────┬──────┴──────────────┘ │ - └───────────────────────────┼──────────────────────────────┘ - │ - ┌────────▼────────┐ - │ CommandRouter │ - │ │ - │ name → handler │ - │ dispatch() │ - └────────┬────────┘ - │ - ┌──────────────┼──────────────┐ - │ │ │ - ┌──────▼──────┐ ┌────▼────┐ ┌───────▼───────┐ - │ Built-in │ │Built-in │ │ Custom │ - │ Read Cmds │ │Write │ │ (actor- │ - │ (overview, │ │Cmds │ │ registered) │ - │ workers, │ │(spawn, │ │ │ - │ actors, │ │ stop, │ │ │ - │ hot, ...) │ │ drain) │ │ │ - └─────────────┘ └─────────┘ └────────────────┘ -``` - -### Core Types - -```rust -// crates/command/src/lib.rs - -use std::collections::HashMap; -use std::sync::Arc; - -/// A command request. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CommandRequest { - /// Command name (e.g., "overview", "spawn", "actors"). - pub command: String, - /// Named arguments. Values are JSON for flexibility. - pub args: HashMap, -} - -/// A command response. Always JSON-serializable. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CommandResponse { - pub ok: bool, - pub command: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub data: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, -} - -impl CommandResponse { - pub fn ok(command: &str, data: impl Serialize) -> Self { - Self { - ok: true, - command: command.to_string(), - data: Some(serde_json::to_value(data).unwrap_or(serde_json::Value::Null)), - error: None, - } - } - - pub fn err(command: &str, msg: impl Into) -> Self { - Self { - ok: false, - command: command.to_string(), - data: None, - error: Some(msg.into()), - } - } - - /// Serialize to a single JSON line (for REPL protocol). - pub fn to_json_line(&self) -> String { - serde_json::to_string(self).unwrap_or_else(|e| { - format!(r#"{{"ok":false,"command":"","error":"serialization: {e}"}}"#) - }) - } -} -``` - -### CommandHandler Trait - -```rust -/// Metadata about a command, used for help text and validation. -pub struct CommandMeta { - pub name: &'static str, - pub description: &'static str, - pub usage: &'static str, - /// Whether this command mutates state (spawn, stop, etc.) - pub is_write: bool, -} - -/// A command handler. Implementations are stateless — all state -/// comes through CommandContext. -pub trait CommandHandler: Send + Sync { - fn meta(&self) -> CommandMeta; - fn handle(&self, args: &HashMap, ctx: &CommandContext) -> CommandResponse; -} -``` - -### CommandContext - -```rust -/// Context available to command handlers. -/// -/// Contains references to runtime subsystems. Optional fields allow -/// commands to work in both standalone and distributed configurations. -pub struct CommandContext { - pub runtime: Arc, - pub stats_collector: Option>, - // Distribution (only present when running distributed) - pub dist_node: Option>>, - // Extensibility: arbitrary typed data that custom commands can access - extensions: HashMap>, -} - -impl CommandContext { - /// Retrieve a typed extension. - pub fn get_ext(&self) -> Option<&T> { - self.extensions.get(&std::any::TypeId::of::()) - .and_then(|b| b.downcast_ref()) - } - - /// Add a typed extension. - pub fn set_ext(&mut self, val: T) { - self.extensions.insert(std::any::TypeId::of::(), Box::new(val)); - } -} -``` - -### CommandRouter - -```rust -/// Central command dispatch. -pub struct CommandRouter { - handlers: HashMap>, -} - -impl CommandRouter { - pub fn new() -> Self { - Self { handlers: HashMap::new() } - } - - /// Register all built-in commands. - pub fn with_builtins(mut self) -> Self { - self.register(Box::new(builtins::HelpCommand)); - self.register(Box::new(builtins::OverviewCommand)); - self.register(Box::new(builtins::WorkersCommand)); - self.register(Box::new(builtins::WorkerCommand)); - self.register(Box::new(builtins::ActorsCommand)); - self.register(Box::new(builtins::ActorCommand)); - self.register(Box::new(builtins::HotCommand)); - self.register(Box::new(builtins::PhasesCommand)); - self.register(Box::new(builtins::DiffCommand)); - // Write commands - self.register(Box::new(builtins::SpawnCommand)); - self.register(Box::new(builtins::StopCommand)); - self.register(Box::new(builtins::ShutdownCommand)); - // Distribution-aware commands (no-op if dist_node is None) - self.register(Box::new(builtins::NodesCommand)); - self.register(Box::new(builtins::RegistryCommand)); - self.register(Box::new(builtins::ResolveCommand)); - self.register(Box::new(builtins::DrainCommand)); - self - } - - /// Register a custom command handler. - pub fn register(&mut self, handler: Box) { - let name = handler.meta().name.to_string(); - self.handlers.insert(name, handler); - } - - /// Dispatch a command request. - pub fn dispatch(&self, req: &CommandRequest, ctx: &CommandContext) -> CommandResponse { - match self.handlers.get(&req.command) { - Some(handler) => handler.handle(&req.args, ctx), - None => CommandResponse::err( - &req.command, - format!("unknown command `{}` — try `help`", req.command), - ), - } - } - - /// List all registered commands (for help text). - pub fn commands(&self) -> Vec<&CommandMeta> { - // sorted by name for stable output - let mut metas: Vec<_> = self.handlers.values() - .map(|h| h.meta()) - .collect(); - metas.sort_by_key(|m| m.name); - metas - } -} -``` - -### Built-in Commands (MVP) - -**Read commands** (extracted from existing `investigate.rs`): - -| Command | Args | Description | Source | -|---------|------|-------------|--------| -| `help` | — | List all commands | `cmd_help()` | -| `overview` | — | Runtime summary | `cmd_overview()` | -| `workers` | — | Per-worker stats | `cmd_workers()` | -| `worker` | `id: int` | Single worker detail | `cmd_worker()` | -| `actors` | `sort`, `limit`, `worker` | List actors | `cmd_actors()` | -| `actor` | `prefix: str` | Find by address prefix | `cmd_actor()` | -| `hot` | `n: int` | Top N by mailbox depth | `cmd_hot()` | -| `phases` | `worker: int?` | Tick phase breakdown | `cmd_phases()` | -| `diff` | `seconds: float` | Snapshot delta | `cmd_diff()` | -| `nodes` | — | Cluster member list | **new** | -| `registry` | — | All registered names | **new** | -| `resolve` | `name: str` | Look up a name | **new** | - -**Write commands** (new): - -| Command | Args | Description | -|---------|------|-------------| -| `stop` | `prefix` or `name` | Stop an actor (poison + cleanup) | -| `drain` | `node: str?` | Stop accepting new actors on a node, let existing drain | -| `shutdown` | `node: str?` | Graceful shutdown (drain + stop all) | -| `spawn` | `factory`, `node?`, `constraints?` | Spawn from a registered factory | - -`spawn` requires a **factory registry** — actors register factory functions that can be invoked by name: - -```rust -pub trait ActorFactory: Send + Sync { - fn name(&self) -> &str; - fn spawn(&self, runtime: &Runtime, args: &HashMap) - -> Result; -} -``` - -### Migration from investigate.rs - -The existing `crates/runtime-dashboard/src/investigate.rs` has 9 command functions. Migration strategy: - -1. Create `crates/command/src/builtins/` with one file per command (or grouped by category). -2. Each `cmd_*` function becomes a `CommandHandler` impl. The logic is identical — just restructured. -3. `dispatch_repl` becomes `CommandRouter::dispatch` with a text-to-`CommandRequest` parser. -4. `dispatch_command` (HTTP) becomes `CommandRouter::dispatch` with query-param-to-`CommandRequest` parser. -5. `run_investigate` remains in the dashboard crate as a thin loop over `CommandRouter`. - -Example extraction: - -```rust -// crates/command/src/builtins/overview.rs - -pub struct OverviewCommand; - -impl CommandHandler for OverviewCommand { - fn meta(&self) -> CommandMeta { - CommandMeta { - name: "overview", - description: "Summary: worker count, actor count, total messages, panics", - usage: "overview", - is_write: false, - } - } - - fn handle(&self, _args: &HashMap, ctx: &CommandContext) -> CommandResponse { - let stats = ctx.enriched_stats(); - let total_msgs: u64 = stats.workers.iter().map(|w| w.messages_processed).sum(); - // ... same logic as existing cmd_overview ... - CommandResponse::ok("overview", serde_json::json!({ - "workers": stats.num_workers, - "actors": stats.actor_details.len(), - "total_messages_processed": total_msgs, - // ... - })) - } -} -``` - -### Frontend Adapters - -Each frontend is a thin adapter that converts its input format into `CommandRequest` and `CommandResponse` back to its output format. - -**REPL adapter** (stdin/stdout): - -```rust -// crates/command/src/adapters/repl.rs - -pub fn parse_line(line: &str) -> CommandRequest { - let parts: Vec<&str> = line.split_whitespace().collect(); - let command = parts.first().unwrap_or(&"help").to_string(); - let args = parse_positional_and_flags(&parts[1..]); - CommandRequest { command, args } -} -``` - -**REST adapter** (HTTP query params): - -```rust -// crates/command/src/adapters/rest.rs - -pub fn from_query_params(params: &HashMap) -> CommandRequest { - let command = params.get("cmd").cloned().unwrap_or_else(|| "help".into()); - let args: HashMap = params.iter() - .filter(|(k, _)| *k != "cmd") - .map(|(k, v)| (k.clone(), serde_json::Value::String(v.clone()))) - .collect(); - CommandRequest { command, args } -} -``` - -**TUI adapter**: TUI input field text → `parse_line()` → `dispatch()` → render response in panel. - -### Custom Commands (actor-registered) - -Actors can register command handlers at runtime through the `Ctx`: - -```rust -impl Ctx<'_> { - /// Register a command that routes to this actor. - /// When the command is invoked, a CommandInvocation message - /// is sent to this actor's mailbox. - pub fn register_command(&self, name: &str, description: &str) { - self.inner.register_command(self.self_addr, name, description); - } -} -``` - -When a custom command is dispatched: - -1. Router finds it's actor-registered. -2. Sends `CommandInvocation { command, args, reply_addr }` to the actor's mailbox. -3. The actor processes it and sends `CommandResult { data }` back to `reply_addr`. -4. Router waits on a one-shot inbox (with timeout, e.g. 5 seconds). -5. Returns the response. - -```rust -/// Sent to an actor when its registered command is invoked. -#[derive(Debug, Clone)] -pub struct CommandInvocation { - pub command: String, - pub args: HashMap, - pub reply_addr: ActorAddress, -} - -/// Sent back by the actor with the command result. -#[derive(Debug, Clone)] -pub struct CommandResult { - pub data: serde_json::Value, -} -``` - -This mechanism allows any actor to expose operational endpoints without modifying the command crate. - -### Files Modified - -| File | Change | -|------|--------| -| `crates/command/` | **New crate** | -| `crates/command/Cargo.toml` | Dependencies: swactor, serde, serde_json | -| `crates/command/src/lib.rs` | `CommandRouter`, `CommandHandler`, `CommandRequest`/`Response`, `CommandContext` | -| `crates/command/src/builtins/` | Built-in command handlers (mod.rs + per-command files) | -| `crates/command/src/adapters/` | REPL and REST input parsers | -| `crates/runtime-dashboard/src/investigate.rs` | Refactored: thin REPL loop over CommandRouter | -| `crates/runtime-dashboard/src/server.rs` | REST endpoints use CommandRouter | -| `crates/runtime-dashboard/Cargo.toml` | Depends on `crates/command` | -| `Cargo.toml` | Add `crates/command` to workspace | - -### Tests - -- **dispatch_known_command**: `overview` returns `ok: true` with expected fields -- **dispatch_unknown_command**: returns `ok: false` with helpful error -- **help_lists_all**: `help` response includes all registered command names -- **parse_repl_line**: `"actors --sort mailbox --limit 5"` → correct `CommandRequest` -- **parse_query_params**: `{cmd: "actor", prefix: "a1b2"}` → correct `CommandRequest` -- **custom_command_dispatch**: register actor command, invoke, verify response -- **custom_command_timeout**: registered actor doesn't respond, verify timeout error -- **write_command_stop**: stop an actor via command, verify it's poisoned diff --git a/docs/os-design/05-supervision.md b/docs/os-design/05-supervision.md deleted file mode 100644 index 95db191..0000000 --- a/docs/os-design/05-supervision.md +++ /dev/null @@ -1,263 +0,0 @@ -# Supervision — User-Space Self-Healing - -## Problem - -When nodes churn (spot instances preempted, hardware rebooted, network partitions), actors on those nodes are lost. Something needs to detect the loss and re-spawn the actors on surviving nodes. This is the "self-healing" property of a distributed OS. - -## Design Principle: Supervision Is User-Space - -Supervision is **not** a runtime primitive. It is a pattern built from: - -- **Actor Watching** (01) — detect death -- **Cluster Registry** (02) — re-register under the same name -- **Node Capabilities** (03) — find a suitable replacement node - -The runtime provides the low-level mechanisms. Supervision is a library actor that composes them. This keeps the kernel minimal and lets users customize supervision policy without forking the runtime. - -## Supervisor Actor - -```rust -/// A supervised child definition. -struct SupervisedChild { - /// Human-readable name (registered in cluster registry). - name: String, - /// Factory function to create the actor. - factory: Box, - /// Placement constraints for the child. - constraints: PlacementRequirement, - /// Current address (None if not yet spawned or dead). - current_addr: Option, - /// Number of restarts so far. - restart_count: u32, - /// Maximum restarts before giving up (0 = unlimited). - max_restarts: u32, - /// Backoff state for restart delays. - last_restart: Option, -} - -/// Restart strategy for a supervisor. -#[derive(Debug, Clone)] -enum RestartStrategy { - /// Restart only the failed child. - OneForOne, - /// If any child fails, restart all children. - AllForOne, - /// Don't restart — just notify (for monitoring supervisors). - Notify, -} - -/// The supervisor actor. -struct Supervisor { - children: Vec, - strategy: RestartStrategy, -} -``` - -## Message Protocol - -```rust -/// Messages the supervisor handles. -enum SupervisorMsg { - /// A watched child died. - Exited(ActorExited), - /// External request to add a child. - AddChild { - name: String, - factory: Box, - constraints: PlacementRequirement, - }, - /// External request to remove a child. - RemoveChild { name: String }, - /// Query: what children are running? - Status { reply_to: ActorAddress }, -} -``` - -## Lifecycle - -### Startup - -```rust -impl Supervisor { - fn start(&mut self, ctx: &Ctx) { - for child in &mut self.children { - match self.spawn_child(ctx, child) { - Ok(addr) => { - child.current_addr = Some(addr); - ctx.watch(addr); - ctx.register_as(&child.name); // or register the child - } - Err(e) => { - eprintln!("supervisor: failed to spawn {}: {e}", child.name); - } - } - } - } -} -``` - -### Death Handling - -```rust -impl ActorInterface for Supervisor { - type Incoming = SupervisorMsg; - type Response = (); - - fn handle(&mut self, ctx: &Ctx, msg: SupervisorMsg) { - match msg { - SupervisorMsg::Exited(exited) => { - match self.strategy { - RestartStrategy::OneForOne => { - self.restart_one(ctx, &exited); - } - RestartStrategy::AllForOne => { - self.restart_all(ctx); - } - RestartStrategy::Notify => { - // Just log — don't restart - } - } - } - // ... other messages ... - } - } -} -``` - -### Restart Flow (OneForOne) - -``` -1. Receive ActorExited { addr: X, reason: NodeDown } -2. Find child with current_addr == X → child "worker-3" -3. Check restart_count < max_restarts -4. Query capabilities: find_suitable_nodes(child.constraints) -5. Pick best node (least loaded, or local if possible) -6. Spawn child on selected node via factory -7. Watch new address -8. Register child.name → new address in cluster registry -9. Update child.current_addr -10. Increment child.restart_count -``` - -```rust -impl Supervisor { - fn restart_one(&mut self, ctx: &Ctx, exited: &ActorExited) { - let child = match self.children.iter_mut() - .find(|c| c.current_addr == Some(exited.addr)) - { - Some(c) => c, - None => return, // not our child - }; - - child.current_addr = None; - - if child.max_restarts > 0 && child.restart_count >= child.max_restarts { - eprintln!( - "supervisor: child {} exceeded max restarts ({}), giving up", - child.name, child.max_restarts, - ); - return; - } - - // Spawn replacement - match self.spawn_child(ctx, child) { - Ok(addr) => { - child.current_addr = Some(addr); - child.restart_count += 1; - ctx.watch(addr); - // Re-register name → new address - // (done via the registry, which gossips to all nodes) - } - Err(e) => { - eprintln!("supervisor: failed to restart {}: {e}", child.name); - } - } - } -} -``` - -### Restart Flow (AllForOne) - -When any child dies: -1. Stop all other children (send stop signal). -2. Wait for all `ActorExited` notifications. -3. Restart all children in order. - -This is useful for interdependent actor groups where partial restart doesn't make sense. - -## Capability-Aware Placement - -The supervisor uses `find_suitable_nodes()` from the distribution crate: - -```rust -fn spawn_child(&self, ctx: &Ctx, child: &SupervisedChild) - -> Result -{ - // If constraints are empty, spawn locally - if child.constraints.is_empty() { - return child.factory.spawn(ctx); - } - - // Find suitable remote nodes - let nodes = dist_node.find_suitable_nodes(&child.constraints); - if nodes.is_empty() { - return Err("no nodes satisfy placement constraints".into()); - } - - // Pick the least-loaded suitable node - let target_node = &nodes[0]; // TODO: sort by load - - // Spawn remotely (requires remote spawn protocol — future work) - // For now: if current node satisfies, spawn locally - // Otherwise: send spawn request to target node - todo!("remote spawn") -} -``` - -**Note**: Remote spawn (telling another node to create an actor) is not yet part of the runtime. The supervisor design accounts for it, but the initial implementation will only support local spawn + re-registration. - -## Relationship to Existing `spawn_restartable` - -The core runtime already has `spawn_restartable` with `factory` and `max_restarts`. This is a **local-only** recovery mechanism — when an actor panics, the same worker restarts it. - -The supervisor pattern extends this to **cluster-wide** recovery: - -| Feature | `spawn_restartable` | Supervisor | -|---|---|---| -| Scope | Single worker | Cluster-wide | -| Trigger | Panic | Panic, stop, or node death | -| Placement | Same worker | Capability-aware, any node | -| Naming | No | Yes (cluster registry) | -| Strategy | Always restart | OneForOne, AllForOne, Notify | -| Implementation | Runtime internal | User-space actor | - -They complement each other: `spawn_restartable` handles fast local recovery (no network round-trip); the supervisor handles node-level failures. - -## Future Extensions - -- **Restart backoff**: exponential backoff between restarts to avoid thrashing. -- **Health checks**: periodic health probes (not just death detection). -- **Cascading supervisors**: supervisor trees (supervisor watches sub-supervisor). -- **Declarative spec**: TOML/YAML file defining supervision topology, loaded at startup. -- **Migration (not restart)**: move a running actor's state to another node (requires persistence, out of scope). - -## Files - -This is a library actor, not a runtime change. Implementation lives in: - -| File | Content | -|------|---------| -| `crates/supervision/src/lib.rs` | `Supervisor`, `SupervisedChild`, `RestartStrategy` | -| `crates/supervision/src/factory.rs` | `ActorFactory` trait, factory registry | -| `crates/supervision/Cargo.toml` | Depends on `swactor`, `swactor-capabilities`, `distribution` | - -Or, if the scope doesn't warrant a separate crate, it can live in `src/supervision.rs` behind a feature flag. - -## Tests - -- **one_for_one_restart**: supervisor with 3 children, kill one, verify only that one restarts -- **all_for_one_restart**: supervisor with 3 children, kill one, verify all restart -- **max_restarts_exceeded**: child dies repeatedly, verify supervisor gives up after max -- **name_re_registration**: child dies and restarts, verify name resolves to new address -- **capability_placement**: child with GPU constraint, verify spawned on GPU node (or error if none available) -- **supervisor_itself_dies**: verify children are stopped (or orphaned — design decision) diff --git a/docs/os-design/README.md b/docs/os-design/README.md deleted file mode 100644 index 8cb458a..0000000 --- a/docs/os-design/README.md +++ /dev/null @@ -1,153 +0,0 @@ -# Swactor Distributed OS — Design Overview - -## Vision - -Swactor is evolving from a local actor runtime into a **distributed operating system** for running long-lived daemons across heterogeneous, churning machines. - -**Primary use case**: Wire together personal hardware today; add on-demand spot compute (vast.ai, etc.) tomorrow. Machines pop in and out of the network. The system self-heals. - -**Design principles**: - -- **Churn is the norm**, not the exception. Every subsystem assumes nodes can disappear at any time. -- **General and flexible**. Minimal assumptions about what a node looks like — feature-gate hardware-specific code. -- **Layered**. The core runtime stays minimal. OS features are opt-in crates. Supervision is user-space, not kernel. -- **Frontend-agnostic**. Operational interfaces (commands, inspection) work identically from CLI, TUI, REST, or future transports. - -## Current Capabilities - -| OS Concept | What Swactor Has Today | -|---|---| -| Processes | Actor spawn/stop, lifecycle hooks, restartable with factory + max_restarts | -| Scheduling | Worker threads, fairness budget (64 msgs/tick), load-aware placement | -| IPC | Typed send, request/reply, per-actor VecDeque mailboxes | -| Naming | String-keyed `AddressMap` — node-local only | -| Fault tolerance | `catch_unwind` for panics, factory restart, dead actor cleanup | -| Backpressure | Per-actor mailbox capacity, DropNewest/DropOldest overflow | -| Networking | SWIM membership (Lifeguard extensions), TCP transport, bincode wire protocol | -| Directory | Kademlia DHT for `ActorAddress -> NodeId` resolution | -| Monitoring | WorkerStats/RuntimeStats, TUI dashboard, REST `/api/*`, investigate REPL | -| Distribution | Multi-node cluster, 5-node Docker test suite | - -## What's Missing (This Design) - -| Feature | Document | Priority | -|---|---|---| -| Actor Watching | [01-actor-watching.md](./01-actor-watching.md) | Foundation for everything | -| Cluster Registry | [02-cluster-registry.md](./02-cluster-registry.md) | Actors find each other across nodes | -| Node Capabilities | [03-node-capabilities.md](./03-node-capabilities.md) | Heterogeneous placement | -| Command Interface | [04-command-interface.md](./04-command-interface.md) | Operational control | -| Supervision | [05-supervision.md](./05-supervision.md) | Self-healing (user-space) | - -## Crate Structure (After This Work) - -``` -swactor/ # core runtime — no network deps - src/ - actor.rs # +watch/unwatch on ContextInner, ExitReason, ActorExited - worker.rs # +WatchRegistry, death notification phase - delivery.rs # (unchanged) - runtime.rs # (unchanged) - ... -crates/ - capabilities/ # NEW — hardware detection + placement constraints - src/lib.rs # NodeCapabilities, CapValue, PlacementConstraint - command/ # NEW — frontend-agnostic command dispatch - src/lib.rs # CommandRouter, CommandHandler, CommandRequest/Response - src/builtins/ # Built-in command handlers - distribution/ # SWIM + Kademlia + cluster registry - src/ - registry.rs # NEW — ClusterRegistry, gossip-propagated naming - node.rs # +register_name, resolve_name, capabilities - messages.rs # +WatchRequest, UnwatchRequest, ActorExitedNotify - ... - runtime-dashboard/ # TUI + REST — refactored to use command crate - python/ # PyO3 bindings - simulation/ # Network simulation - wasm/ # WASM bindings -``` - -## Dependency Graph - -``` - ┌──────────────┐ - │ capabilities │ (standalone — only serde + sysinfo) - └──────┬───────┘ - │ optional -┌─────────┐ ┌──────▼────────┐ -│ swactor │◄─────│ distribution │ -│ (core) │ │ +registry │ -└────┬────┘ └──────┬────────┘ - │ │ - │ ┌──────▼────────┐ - └──────────►│ command │ - └──────┬────────┘ - │ - ┌──────▼────────────┐ - │ runtime-dashboard │ - │ (TUI + REST) │ - └───────────────────┘ -``` - -Key constraints: -- `capabilities` has **zero** dependency on `swactor` — it's a standalone detection library. -- `command` depends on `swactor` (needs `Runtime`, stats types) but NOT on `distribution`. -- `distribution` optionally depends on `capabilities` (for `NodeRecord` labels). -- `runtime-dashboard` depends on both `command` and optionally `distribution`. - -## Feature Flags - -### Core `swactor` - -| Flag | Purpose | -|---|---| -| `getrandom` (default) | Cryptographic RNG for actor addresses | -| `serde` | Serialization for types | -| `tracing` | Structured logging | -| `transport` | Transport-agnostic remote messaging | -| `watching` (new) | Watch API on ContextInner/Ctx | - -### `swactor-capabilities` - -| Flag | Purpose | -|---|---| -| `detect` (default) | Auto-detect CPU, RAM, hostname via sysinfo | -| `gpu-nvidia` | Detect NVIDIA GPU via nvidia-smi | -| `gpu-vulkan` | Detect GPU via Vulkan API | - -### `distribution` - -| Flag | Purpose | -|---|---| -| `registry` (new, default) | Cluster-wide gossip-propagated naming | -| `capabilities` (new) | NodeCapabilities on NodeRecord | - -## Implementation Order - -Each feature is one PR, in dependency order: - -``` -PR 1: Actor Watching (local only) - └──► PR 2: Command Interface - └──► PR 3: Cluster Registry - └──► PR 4: Node Capabilities - └──► PR 5: Remote Watching (cross-node) - └──► PR 6: Supervisor library -``` - -**PR 1 — Actor Watching (local)** -Adds `WatchRegistry`, `ActorExited`, `ExitReason` to core runtime. Testable without any distribution. Foundation for everything else. - -**PR 2 — Command Interface** -Extracts investigate.rs into `crates/command/`. Adds write commands (spawn, stop, drain). Immediately useful for operations. - -**PR 3 — Cluster Registry** -Gossip-propagated naming in `crates/distribution/src/registry.rs`. Actors can find each other by name across nodes. - -**PR 4 — Node Capabilities** -`crates/capabilities/` with auto-detection and placement constraints. Integrates with distribution for capability-aware placement. - -**PR 5 — Remote Watching** -Wire protocol for cross-node watches. SWIM Dead triggers `ActorExited { reason: NodeDown }` for all actors on that node. - -**PR 6 — Supervisor Library** -User-space supervisor pattern. Composes watching + registry + capabilities to auto-respawn actors after churn. -- 2.45.2