diff --git a/.deploy/deploy.example.toml b/.deploy/deploy.example.toml new file mode 100644 index 0000000..39404aa --- /dev/null +++ b/.deploy/deploy.example.toml @@ -0,0 +1,20 @@ +# Deploy configuration for `cargo xtask deploy` +# Copy to .deploy/deploy.toml (native) or .deploy/docker.toml (Docker) +# and fill in your machine details. + +[defaults] +dashboard_port = 9090 +relay_port = 3340 +# image = "swactor" # Required for --docker mode +# container = "swactor" # Required for --docker mode +# relay_hosts = ["1.2.3.4"] +# swactor_flags = ["--auth"] + +[[machines]] +name = "local-node" +local = true + +# [[machines]] +# name = "remote-node" +# ssh = "my-server" # SSH alias from ~/.ssh/config +# swactor_flags = ["--storage-path", "/var/lib/swactor/data"] diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..17eb168 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +* +!target/x86_64-unknown-linux-musl/release/swactor diff --git a/.gitignore b/.gitignore index ffbe531..81a3525 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,12 @@ docs/architecture.html crates/simulation/traces # xtask personal config -xtask/config.toml \ No newline at end of file +xtask/config.toml + +# Deploy config (machine-specific) +deploy.toml +.deploy/* +!.deploy/deploy.example.toml + +# Local dev node state +.dev-node/ \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 7da30f8..e00b73d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,6 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "getrandom 0.3.4", "once_cell", "version_check", "zerocopy", @@ -130,6 +131,15 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +[[package]] +name = "arc-swap" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f3647c145568cec02c42054e07bdf9a5a698e15b466fb2341bfc393cd24aa5" +dependencies = [ + "rustversion", +] + [[package]] name = "arrayref" version = "0.3.9" @@ -148,6 +158,45 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" +[[package]] +name = "asn1-rs" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56624a96882bb8c26d61312ae18cb45868e5a9992ea73c58e45c3101e56a1e60" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-compat" version = "0.2.5" @@ -238,6 +287,58 @@ dependencies = [ "fs_extra", ] +[[package]] +name = "axum" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "backon" version = "1.6.0" @@ -335,6 +436,15 @@ dependencies = [ "objc2", ] +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + [[package]] name = "bumpalo" version = "3.19.1" @@ -425,21 +535,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901" -[[package]] -name = "ci-relay" -version = "0.1.0" -dependencies = [ - "clap", - "hex", - "hmac", - "iroh", - "serde_json", - "sha2 0.10.9", - "swactor-ci", - "tiny_http", - "tokio", -] - [[package]] name = "ciborium" version = "0.2.2" @@ -1012,6 +1107,42 @@ dependencies = [ "syn", ] +[[package]] +name = "dashboard" +version = "0.1.0" +dependencies = [ + "axum", + "clap", + "crossbeam-queue", + "crossterm", + "ctrlc", + "distribution", + "iroh", + "ratatui", + "serde", + "serde_json", + "swactor", + "swactor-std", + "tokio", + "tokio-stream", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "data-encoding" version = "2.10.0" @@ -1048,6 +1179,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "deranged" version = "0.5.6" @@ -1187,8 +1332,10 @@ dependencies = [ name = "distribution" version = "0.1.0" dependencies = [ + "bs58", "ed25519-dalek 2.2.0", "iroh", + "iroh-relay", "rand_core 0.6.4", "serde", "serde_json", @@ -2289,12 +2436,17 @@ version = "0.38.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c946095f060e6e59b9ff30cc26c75cdb758e7fb0cde8312c89e2144654989fcb" dependencies = [ + "http-body-util", + "hyper", + "hyper-util", "iroh-metrics-derive", "itoa", "n0-error", "postcard", + "reqwest 0.12.28", "ryu", "serde", + "tokio", "tracing", ] @@ -2349,6 +2501,7 @@ dependencies = [ "rustc-hash", "rustls", "rustls-pki-types", + "rustls-platform-verifier", "slab", "sorted-index-buffer", "thiserror 2.0.18", @@ -2376,9 +2529,12 @@ version = "0.96.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd2b63e654b9dec799a73372cdc79b529ca6c7248c0c8de7da78a02e3a46f03c" dependencies = [ + "ahash", "blake3", "bytes", "cfg_aliases", + "clap", + "dashmap", "data-encoding", "derive_more", "getrandom 0.3.4", @@ -2399,17 +2555,28 @@ dependencies = [ "pkarr", "postcard", "rand", + "rcgen", + "reloadable-state", "reqwest 0.12.28", "rustls", + "rustls-cert-file-reader", + "rustls-cert-reloadable-resolver", "rustls-pki-types", "serde", "serde_bytes", + "serde_json", + "sha1", + "simdutf8", "strum 0.27.2", + "time", "tokio", "tokio-rustls", + "tokio-rustls-acme", "tokio-util", "tokio-websockets", + "toml 0.9.12+spec-1.1.0", "tracing", + "tracing-subscriber", "url", "vergen-gitcl", "webpki-roots 1.0.6", @@ -2593,20 +2760,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" -[[package]] -name = "local-runner" -version = "0.1.0" -dependencies = [ - "clap", - "ctrlc", - "iroh", - "runtime-dashboard", - "serde_json", - "swactor", - "swactor-ci", - "tokio", -] - [[package]] name = "lock_api" version = "0.4.14" @@ -2683,6 +2836,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "memchr" version = "2.8.0" @@ -2713,6 +2872,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -2952,6 +3117,16 @@ dependencies = [ "libc", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "ntimestamp" version = "1.0.0" @@ -2976,12 +3151,31 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -3096,6 +3290,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -3207,6 +3410,16 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64 0.22.1", + "serde_core", +] + [[package]] name = "pem-rfc7468" version = "1.0.0" @@ -3770,6 +3983,20 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rcgen" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10b99e0098aa4082912d4c649628623db6aba77335e4f4569ff5083a6448b32e" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "x509-parser", + "yasna", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -3833,6 +4060,23 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c" +[[package]] +name = "reloadable-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dc20ac1418988b60072d783c9f68e28a173fb63493c127952f6face3b40c6e0" + +[[package]] +name = "reloadable-state" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3853ef78d45b50f8b989896304a85239539d39b7f866a000e8846b9b72d74ce8" +dependencies = [ + "arc-swap", + "reloadable-core", + "tokio", +] + [[package]] name = "reqwest" version = "0.12.28" @@ -3936,26 +4180,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "runtime-dashboard" -version = "0.1.0" -dependencies = [ - "clap", - "crossbeam-queue", - "crossterm", - "ctrlc", - "distribution", - "iroh", - "ratatui", - "serde", - "serde_json", - "swactor", - "swactor-ci", - "tiny_http", - "tracing", - "tracing-subscriber", -] - [[package]] name = "rustc-demangle" version = "0.1.27" @@ -3977,6 +4201,15 @@ dependencies = [ "semver", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + [[package]] name = "rustix" version = "0.38.44" @@ -4019,6 +4252,40 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-cert-file-reader" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb47c2a50fdfdaf95b0ac8b12620fc327da1fd4adbb30d0c56d866b005873ff" +dependencies = [ + "rustls-cert-read", + "rustls-pki-types", + "thiserror 2.0.18", + "tokio", +] + +[[package]] +name = "rustls-cert-read" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd46e8c5ae4de3345c4786a83f99ec7aff287209b9e26fa883c473aeb28f19d5" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-cert-reloadable-resolver" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe1baa8a3a1f05eaa9fc55aed4342867f70e5c170ea3bfed1b38c51a4857c0c8" +dependencies = [ + "futures-util", + "reloadable-state", + "rustls", + "rustls-cert-read", + "thiserror 2.0.18", +] + [[package]] name = "rustls-native-certs" version = "0.8.3" @@ -4242,6 +4509,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_spanned" version = "0.6.9" @@ -4251,6 +4529,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8bbf91e5a4d6315eee45e704372590b30e260ee83af6639d64557f51b067776" +dependencies = [ + "serde_core", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -4276,6 +4563,17 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha1" +version = "0.11.0-rc.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c777f0a122a53fddb0beb6e706771197000b8eb5c9f42b5b850f450ef48c788" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.11.0-rc.10", +] + [[package]] name = "sha1_smol" version = "1.0.1" @@ -4407,7 +4705,7 @@ dependencies = [ "swactor", "swactor-ci", "tiny_http", - "toml", + "toml 0.8.23", ] [[package]] @@ -4607,35 +4905,32 @@ dependencies = [ name = "swactor-ci" version = "0.1.0" dependencies = [ + "clap", + "ctrlc", + "dashboard", "hex", "hmac", + "iroh", "serde", "serde_json", "serde_yaml", "sha2 0.10.9", "swactor", "tiny_http", + "tokio", "ureq", ] -[[package]] -name = "swactor-crypto-wasm" -version = "0.1.0" -dependencies = [ - "ed25519-dalek 2.2.0", -] - [[package]] name = "swactor-datastore" version = "0.1.0" dependencies = [ "blake3", "clap", - "ctrlc", + "dashboard", "distribution", "getrandom 0.2.17", "proptest", - "runtime-dashboard", "serde", "serde_json", "shared-types", @@ -4643,10 +4938,26 @@ dependencies = [ "swactor-std", "tempfile", "tiny_http", - "toml", "ureq", ] +[[package]] +name = "swactor-node" +version = "0.1.0" +dependencies = [ + "clap", + "ctrlc", + "dashboard", + "distribution", + "iroh", + "libc", + "serde", + "swactor", + "swactor-datastore", + "swactor-std", + "toml 0.8.23", +] + [[package]] name = "swactor-std" version = "0.1.0" @@ -4924,6 +5235,34 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls-acme" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31fcc374ec87d754358a5d0709ed1ab7671d51e0f70ddc3b17a11ac36604cfa" +dependencies = [ + "async-trait", + "base64 0.22.1", + "chrono", + "futures", + "log", + "num-bigint", + "pem", + "proc-macro2", + "rcgen", + "reqwest 0.12.28", + "ring", + "rustls", + "serde", + "serde_json", + "thiserror 2.0.18", + "time", + "tokio", + "tokio-rustls", + "webpki-roots 1.0.6", + "x509-parser", +] + [[package]] name = "tokio-stream" version = "0.1.18" @@ -4979,11 +5318,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ "serde", - "serde_spanned", + "serde_spanned 0.6.9", "toml_datetime 0.6.11", "toml_edit 0.22.27", ] +[[package]] +name = "toml" +version = "0.9.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned 1.0.4", + "toml_datetime 0.7.5+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow", +] + [[package]] name = "toml_datetime" version = "0.6.11" @@ -5010,7 +5364,7 @@ checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ "indexmap", "serde", - "serde_spanned", + "serde_spanned 0.6.9", "toml_datetime 0.6.11", "toml_write", "winnow", @@ -5043,6 +5397,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +[[package]] +name = "toml_writer" +version = "1.0.6+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab16f14aed21ee8bfd8ec22513f7287cd4a91aa92e44edfe2c17ddd004e92607" + [[package]] name = "tower" version = "0.5.3" @@ -5056,6 +5416,7 @@ dependencies = [ "tokio", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -5410,15 +5771,6 @@ dependencies = [ "wit-bindgen", ] -[[package]] -name = "wasm" -version = "0.1.0" -dependencies = [ - "swactor", - "swactor-std", - "wasm-bindgen", -] - [[package]] name = "wasm-bindgen" version = "0.2.108" @@ -5520,6 +5872,15 @@ dependencies = [ "wasmparser 0.244.0", ] +[[package]] +name = "wasm-runtime" +version = "0.1.0" +dependencies = [ + "swactor", + "swactor-std", + "wasm-bindgen", +] + [[package]] name = "wasm-streams" version = "0.4.2" @@ -5662,7 +6023,7 @@ dependencies = [ "serde", "serde_derive", "sha2 0.10.9", - "toml", + "toml 0.8.23", "windows-sys 0.59.0", "zstd", ] @@ -6550,6 +6911,24 @@ dependencies = [ "web-sys", ] +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "ring", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + [[package]] name = "xml-rs" version = "0.8.28" @@ -6572,7 +6951,17 @@ dependencies = [ "clap", "libc", "serde", - "toml", + "serde_json", + "toml 0.8.23", +] + +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 3fb1f43..071ee24 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,22 +2,20 @@ members = [ ".", "crates/python", - "crates/wasm", + "crates/wasm/runtime", "crates/bin-runner", "crates/simulation", - "crates/runtime-dashboard", + "crates/dashboard", "crates/distribution", "crates/std", "crates/datastore", "crates/shared-types", - "crates/crypto-wasm", + "crates/swactor-node", "tests/docker", "crates/ci", - "crates/local-runner", - "crates/ci-relay", "xtask", ] -exclude = ["tools/depgraph"] +exclude = ["tools/depgraph", "crates/wasm/crypto"] [package] name = "swactor" diff --git a/Dockerfile b/Dockerfile index 0e95fb8..1e65f4a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,3 @@ -FROM rust:1.93-slim AS builder -WORKDIR /build -COPY . . -RUN cargo build --release -p runtime-dashboard --features node - -FROM debian:bookworm-slim -COPY --from=builder /build/target/release/swactor-node /usr/local/bin/ -ENTRYPOINT ["swactor-node"] +FROM scratch +COPY target/x86_64-unknown-linux-musl/release/swactor /swactor +ENTRYPOINT ["/swactor"] diff --git a/.ci.yml b/ci/.ci.yml similarity index 100% rename from .ci.yml rename to ci/.ci.yml diff --git a/crates/ci-relay/Cargo.toml b/crates/ci-relay/Cargo.toml deleted file mode 100644 index 215dbcd..0000000 --- a/crates/ci-relay/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "ci-relay" -version = "0.1.0" -edition = "2024" - -[[bin]] -name = "ci-relay" -path = "src/main.rs" - -[dependencies] -swactor-ci = { path = "../ci", features = ["local"] } -iroh = "0.96" -tokio = { version = "1", features = ["rt-multi-thread"] } -tiny_http = "0.12" -serde_json = "1" -hmac = "0.12" -sha2 = "0.10" -hex = "0.4" -clap = { version = "4", features = ["derive"] } diff --git a/crates/ci/Cargo.toml b/crates/ci/Cargo.toml index 5a28b8e..4473d9c 100644 --- a/crates/ci/Cargo.toml +++ b/crates/ci/Cargo.toml @@ -9,13 +9,20 @@ serde = { version = "1", features = ["derive"] } serde_yaml = "0.9" serde_json = "1" -# Local runner library dependencies +# Shared optional deps (used by lib local feature and binaries) tiny_http = { version = "0.12", optional = true } ureq = { version = "2", features = ["json"], optional = true } hmac = { version = "0.12", optional = true } sha2 = { version = "0.10", optional = true } hex = { version = "0.4", optional = true } +# Binary-only optional deps +iroh = { version = "0.96", optional = true } +tokio = { version = "1", features = ["rt-multi-thread"], optional = true } +clap = { version = "4", features = ["derive"], optional = true } +ctrlc = { version = "3", optional = true } +dashboard = { path = "../dashboard", optional = true } + [features] default = [] local = [ @@ -25,5 +32,29 @@ local = [ "dep:sha2", "dep:hex", ] +relay = [ + "local", + "dep:iroh", + "dep:tokio", + "dep:clap", +] +runner = [ + "local", + "dep:iroh", + "dep:tokio", + "dep:clap", + "dep:ctrlc", + "dep:dashboard", +] [dev-dependencies] + +[[bin]] +name = "ci-relay" +path = "src/bin/ci_relay.rs" +required-features = ["relay"] + +[[bin]] +name = "local-runner" +path = "src/bin/local_runner.rs" +required-features = ["runner"] diff --git a/crates/ci-relay/src/main.rs b/crates/ci/src/bin/ci_relay.rs similarity index 100% rename from crates/ci-relay/src/main.rs rename to crates/ci/src/bin/ci_relay.rs diff --git a/crates/local-runner/src/main.rs b/crates/ci/src/bin/local_runner.rs similarity index 91% rename from crates/local-runner/src/main.rs rename to crates/ci/src/bin/local_runner.rs index 4519663..67adac4 100644 --- a/crates/local-runner/src/main.rs +++ b/crates/ci/src/bin/local_runner.rs @@ -14,7 +14,7 @@ use swactor::actor::ActorAddress; use swactor::config::RuntimeConfig; use swactor::runtime::Runtime; -use runtime_dashboard::ci_collector::{ +use dashboard::ci_collector::{ CiSnapshot, CiStatsProvider, JobSnapshot, PipelineSnapshot, ProvisionerStatus, }; @@ -92,22 +92,34 @@ impl CiStatsProvider for LocalCiSnapshotProvider { } fn pipeline_to_dashboard(p: &PipelineExecution) -> PipelineSnapshot { + // Convert between swactor_ci types and dashboard::ci_collector types + // (structurally identical but separate to avoid a dependency cycle). + let status: dashboard::ci_collector::PipelineStatus = + serde_json::from_value(serde_json::to_value(&p.status).unwrap()).unwrap(); + PipelineSnapshot { - pipeline_id: p.pipeline_id, + pipeline_id: dashboard::ci_collector::PipelineId(p.pipeline_id.0), pipeline_name: p.pipeline_name.clone(), repo_owner: p.repo_owner.clone(), repo_name: p.repo_name.clone(), commit_sha: p.commit_sha.clone(), branch: p.branch.clone(), - status: p.status.clone(), + status, jobs: p .jobs .values() - .map(|j| JobSnapshot { - job_id: j.job_id.clone(), - job_name: j.definition.name.clone(), - status: j.status.clone(), - output_line_count: j.output_lines.len(), + .map(|j| { + let job_status: dashboard::ci_collector::JobStatus = + serde_json::from_value(serde_json::to_value(&j.status).unwrap()).unwrap(); + JobSnapshot { + job_id: dashboard::ci_collector::JobId { + pipeline_id: dashboard::ci_collector::PipelineId(j.job_id.pipeline_id.0), + job_name: j.job_id.job_name.clone(), + }, + job_name: j.definition.name.clone(), + status: job_status, + output_line_count: j.output_lines.len(), + } }) .collect(), } @@ -128,7 +140,7 @@ fn main() { // Optionally start dashboard. let dash = args.dashboard_port.map(|port| { - let d = runtime_dashboard::start_dashboard(runtime_dashboard::DashboardConfig { + let d = dashboard::start_dashboard(dashboard::DashboardConfig { port, ..Default::default() }); @@ -138,7 +150,7 @@ fn main() { // Create 2-thread runtime. let num_threads = 2; - let collector = runtime_dashboard::collector::StatsCollector::new(num_threads); + let collector = dashboard::collector::StatsCollector::new(num_threads); let mut rt = Runtime::new(RuntimeConfig { num_threads, max_actors: 256, diff --git a/crates/runtime-dashboard/.gitignore b/crates/dashboard/.gitignore similarity index 100% rename from crates/runtime-dashboard/.gitignore rename to crates/dashboard/.gitignore diff --git a/crates/runtime-dashboard/AGENTS.md b/crates/dashboard/AGENTS.md similarity index 80% rename from crates/runtime-dashboard/AGENTS.md rename to crates/dashboard/AGENTS.md index 7ffd9d4..51ffad5 100644 --- a/crates/runtime-dashboard/AGENTS.md +++ b/crates/dashboard/AGENTS.md @@ -196,3 +196,50 @@ When diagnosing a runtime, a useful sequence: | Growing actors_after vs actors_before | `diff 5` | Unbounded actor spawning | | type_mismatches > 0 | `overview` | Messages routed to wrong actor type | | panics > 0 | `overview` | Actor handlers are panicking | + +### Distribution & Peer Management API + +#### GET /api/distribution +Returns the distribution node snapshot: node ID, members list, routing table, peer auth mode, etc. + +#### GET /api/peers +Returns the peer allow-list. No auth required. + +#### POST /api/peers/add (requires Bearer token) +Add a peer to the allow-list and trigger a SWIM join. +Body: `{"node_id": "", "label": "optional-label"}` +The `node_id` field accepts either 64-char hex or base58-encoded invite codes. + +#### POST /api/peers/remove (requires Bearer token) +Remove a peer from the allow-list. +Body: `{"node_id": ""}` + +### Agent Peering Playbook + +To programmatically join two nodes: + +1. Get each node's info: + ``` + curl http://NODE_A:9090/api/distribution + curl http://NODE_B:9090/api/distribution + ``` + Extract `invite_code` and `node_id` from each response. + +2. Add each node to the other's allow-list + trigger join: + ``` + curl -X POST http://NODE_A:9090/api/peers/add \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{"node_id": "", "label": "node-b"}' + + curl -X POST http://NODE_B:9090/api/peers/add \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{"node_id": "", "label": "node-a"}' + ``` + +3. Verify (poll until members appear, ~5-10s): + ``` + curl http://NODE_A:9090/api/distribution | jq '.members' + ``` + Should list node B as alive. diff --git a/crates/runtime-dashboard/Cargo.toml b/crates/dashboard/Cargo.toml similarity index 84% rename from crates/runtime-dashboard/Cargo.toml rename to crates/dashboard/Cargo.toml index 4556bd8..5d7fd03 100644 --- a/crates/runtime-dashboard/Cargo.toml +++ b/crates/dashboard/Cargo.toml @@ -1,15 +1,18 @@ [package] -name = "runtime-dashboard" +name = "dashboard" version = "0.1.0" edition = "2024" [dependencies] swactor = { path = "../..", features = ["serde", "tracing"] } +swactor-std = { path = "../std", default-features = false } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["registry"] } serde = { version = "1", features = ["derive"] } serde_json = "1" -tiny_http = "0.12" +axum = "0.8" +tokio = { version = "1", features = ["net", "rt-multi-thread", "sync", "time"] } +tokio-stream = "0.1" crossbeam-queue = "0.3.12" ratatui = { version = "0.29", optional = true, default-features = false, features = ["crossterm"] } crossterm = { version = "0.28", optional = true } @@ -17,7 +20,6 @@ distribution = { path = "../distribution", optional = true } clap = { version = "4", features = ["derive"], optional = true } ctrlc = "3" iroh = { version = "0.96", optional = true } -swactor-ci = { path = "../ci", optional = true } [features] default = ["distribution"] @@ -26,7 +28,6 @@ distribution = ["dep:distribution"] node = ["distribution", "dep:clap", "swactor/transport", "tcp"] tcp = ["distribution/tcp"] iroh = ["distribution/iroh", "dep:iroh"] -ci = ["dep:swactor-ci"] [[bin]] name = "swactor-tui" diff --git a/crates/runtime-dashboard/README.md b/crates/dashboard/README.md similarity index 100% rename from crates/runtime-dashboard/README.md rename to crates/dashboard/README.md diff --git a/crates/runtime-dashboard/examples/bench_dashboard.rs b/crates/dashboard/examples/bench_dashboard.rs similarity index 94% rename from crates/runtime-dashboard/examples/bench_dashboard.rs rename to crates/dashboard/examples/bench_dashboard.rs index aef4283..a2ed48f 100644 --- a/crates/runtime-dashboard/examples/bench_dashboard.rs +++ b/crates/dashboard/examples/bench_dashboard.rs @@ -5,8 +5,8 @@ use swactor::actor::{ActorAddress, ActorInterface, Ctx}; use swactor::config::{BackoffPolicy, RuntimeConfig}; use swactor::runtime::Runtime; -use runtime_dashboard::collector::StatsCollector; -use runtime_dashboard::{start_dashboard, DashboardConfig}; +use dashboard::collector::StatsCollector; +use dashboard::{start_dashboard, DashboardConfig}; // --------------------------------------------------------------------------- // Actors @@ -102,7 +102,7 @@ fn run_for(duration: Duration, mut tick: impl FnMut()) { // Scenarios // --------------------------------------------------------------------------- -fn scenario_single_actor(dash: &runtime_dashboard::DashboardHandle) { +fn scenario_single_actor(dash: &dashboard::DashboardHandle) { eprintln!(" [1/4] Single-actor bombardment (5s)"); let collector = StatsCollector::new(4); let mut rt = Runtime::new(bench_config(4, 64, 100_000)); @@ -122,7 +122,7 @@ fn scenario_single_actor(dash: &runtime_dashboard::DashboardHandle) { handle.join(); } -fn scenario_multi_actor(dash: &runtime_dashboard::DashboardHandle) { +fn scenario_multi_actor(dash: &dashboard::DashboardHandle) { eprintln!(" [2/4] Multi-actor fan-out (5s)"); let collector = StatsCollector::new(4); let mut rt = Runtime::new(bench_config(4, 128, 10_000)); @@ -146,7 +146,7 @@ fn scenario_multi_actor(dash: &runtime_dashboard::DashboardHandle) { handle.join(); } -fn scenario_ring(dash: &runtime_dashboard::DashboardHandle) { +fn scenario_ring(dash: &dashboard::DashboardHandle) { eprintln!(" [3/4] Ring topology (5s)"); let ring_size = 100; let collector = StatsCollector::new(4); @@ -182,7 +182,7 @@ fn scenario_ring(dash: &runtime_dashboard::DashboardHandle) { handle.join(); } -fn scenario_spawn_storm(dash: &runtime_dashboard::DashboardHandle) { +fn scenario_spawn_storm(dash: &dashboard::DashboardHandle) { eprintln!(" [4/4] Spawn storm (5s)"); let collector = StatsCollector::new(4); let mut rt = Runtime::new(bench_config(4, 50_000, 1_024)); diff --git a/crates/runtime-dashboard/examples/dashboard_demo.rs b/crates/dashboard/examples/dashboard_demo.rs similarity index 96% rename from crates/runtime-dashboard/examples/dashboard_demo.rs rename to crates/dashboard/examples/dashboard_demo.rs index f115b2f..0b3c9d8 100644 --- a/crates/runtime-dashboard/examples/dashboard_demo.rs +++ b/crates/dashboard/examples/dashboard_demo.rs @@ -13,9 +13,9 @@ use distribution::swim::node::NodeAction; use distribution::swim::probe::SwimConfig; use distribution::types::NodeId; -use runtime_dashboard::collector::StatsCollector; -use runtime_dashboard::distribution_collector::DistributionStatsProvider; -use runtime_dashboard::{start_dashboard, DashboardConfig}; +use dashboard::collector::StatsCollector; +use dashboard::distribution_collector::DistributionStatsProvider; +use dashboard::{start_dashboard, DashboardConfig}; // ── Demo actors ───────────────────────────────────────────────────────── @@ -190,6 +190,16 @@ fn deliver_actions_tagged( } } } + NodeAction::ForwardAck { to, target, sequence, piggyback } => { + if let Some(idx) = node_ids.iter().position(|id| id == to) { + if let Some(ref mut node) = nodes[idx] { + let resp = node.handle_indirect_ack(*target, *sequence, piggyback); + if !resp.is_empty() { + tagged_responses.push((idx, resp)); + } + } + } + } NodeAction::MembershipChanged { .. } => { // Notifications — no delivery needed } diff --git a/crates/runtime-dashboard/examples/record_and_replay_demo.rs b/crates/dashboard/examples/record_and_replay_demo.rs similarity index 97% rename from crates/runtime-dashboard/examples/record_and_replay_demo.rs rename to crates/dashboard/examples/record_and_replay_demo.rs index 71e0494..13d5292 100644 --- a/crates/runtime-dashboard/examples/record_and_replay_demo.rs +++ b/crates/dashboard/examples/record_and_replay_demo.rs @@ -7,8 +7,8 @@ use swactor::actor::{ActorAddress, ActorInterface, Ctx}; use swactor::config::RuntimeConfig; use swactor::runtime::Runtime; -use runtime_dashboard::collector::StatsCollector; -use runtime_dashboard::{serve_replay, start_dashboard, DashboardConfig, ReplayConfig}; +use dashboard::collector::StatsCollector; +use dashboard::{serve_replay, start_dashboard, DashboardConfig, ReplayConfig}; // ── Demo actors ───────────────────────────────────────────────────────── diff --git a/crates/runtime-dashboard/src/actor_detail_html.rs b/crates/dashboard/src/actor_detail_html.rs similarity index 95% rename from crates/runtime-dashboard/src/actor_detail_html.rs rename to crates/dashboard/src/actor_detail_html.rs index c5c2c27..4b4c41e 100644 --- a/crates/runtime-dashboard/src/actor_detail_html.rs +++ b/crates/dashboard/src/actor_detail_html.rs @@ -135,8 +135,11 @@ pub const ACTOR_DETAIL_HTML: &str = r##"
+
-
Address
—
+
Address
—
Worker
—
Status
—
@@ -237,7 +240,14 @@ pub const ACTOR_DETAIL_HTML: &str = r##" if (!actor) return; var hex = formatAddr(actor.address); - document.getElementById('addrBreadcrumb').textContent = shortAddr(hex); + if (actor.name) { + document.getElementById('addrBreadcrumb').textContent = actor.name; + document.getElementById('nameRow').style.display = ''; + document.getElementById('addrName').textContent = actor.name; + } else { + document.getElementById('addrBreadcrumb').textContent = shortAddr(hex); + document.getElementById('nameRow').style.display = 'none'; + } document.getElementById('addrFull').textContent = hex; document.getElementById('addrWorker').textContent = 'W' + actor.worker_id; diff --git a/crates/runtime-dashboard/src/actors_html.rs b/crates/dashboard/src/actors_html.rs similarity index 95% rename from crates/runtime-dashboard/src/actors_html.rs rename to crates/dashboard/src/actors_html.rs index 2c139fb..9595175 100644 --- a/crates/runtime-dashboard/src/actors_html.rs +++ b/crates/dashboard/src/actors_html.rs @@ -478,7 +478,8 @@ pub const ACTORS_HTML: &str = r##" if (filter) { var hex = addrToHex(a.address).toLowerCase(); var msgType = (a.last_msg_type || '').toLowerCase(); - if (hex.indexOf(filter) < 0 && ('w' + a.worker_id).indexOf(filter) < 0 && msgType.indexOf(filter) < 0) { + var actorName = (a.name || '').toLowerCase(); + if (hex.indexOf(filter) < 0 && ('w' + a.worker_id).indexOf(filter) < 0 && msgType.indexOf(filter) < 0 && actorName.indexOf(filter) < 0) { return false; } } @@ -542,8 +543,11 @@ pub const ACTORS_HTML: &str = r##" tr.className = 'clickable' + (fullHex === focusedAddrHex ? ' focused' : ''); if (a.poisoned) tr.style.opacity = '0.6'; tr.setAttribute('data-addr', fullHex); + var addrCell = a.name + ? '' + escapeHtml(a.name) + ' ' + escapeHtml(hex) + '' + (a.poisoned ? ' DEAD' : '') + '' + : '' + escapeHtml(hex) + (a.poisoned ? ' DEAD' : '') + ''; tr.innerHTML = - '' + escapeHtml(hex) + (a.poisoned ? ' DEAD' : '') + '' + + addrCell + 'W' + a.worker_id + '' + '' + a.mailbox_depth + '' + '' + (a.messages_processed || 0).toLocaleString() + '' + @@ -593,8 +597,14 @@ pub const ACTORS_HTML: &str = r##" return; } - document.getElementById('detailAddr').innerHTML = '' + escapeHtml(addrToHex(actor.address)) + ''; - document.getElementById('detailFullAddr').innerHTML = '' + escapeHtml(focusedAddrHex) + ''; + var detailAddrText = actor.name + ? '' + escapeHtml(actor.name) + '' + : '' + escapeHtml(addrToHex(actor.address)) + ''; + document.getElementById('detailAddr').innerHTML = detailAddrText; + var detailFullText = actor.name + ? '' + escapeHtml(actor.name) + ' ' + escapeHtml(focusedAddrHex) + '' + : '' + escapeHtml(focusedAddrHex) + ''; + document.getElementById('detailFullAddr').innerHTML = detailFullText; document.getElementById('detailWorker').textContent = 'W' + actor.worker_id; document.getElementById('detailMailbox').textContent = actor.mailbox_depth; document.getElementById('detailMsgCount').textContent = (actor.messages_processed || 0).toLocaleString(); diff --git a/crates/runtime-dashboard/src/bin/swactor-node.rs b/crates/dashboard/src/bin/swactor-node.rs similarity index 96% rename from crates/runtime-dashboard/src/bin/swactor-node.rs rename to crates/dashboard/src/bin/swactor-node.rs index 6737f5a..d2cd4f0 100644 --- a/crates/runtime-dashboard/src/bin/swactor-node.rs +++ b/crates/dashboard/src/bin/swactor-node.rs @@ -14,9 +14,9 @@ use distribution::registry::RegistryConfig; use distribution::snapshot::DistributionNodeSnapshot; use distribution::swim::probe::SwimConfig; -use runtime_dashboard::collector::StatsCollector; -use runtime_dashboard::distribution_collector::DistributionStatsProvider; -use runtime_dashboard::{start_dashboard, DashboardConfig}; +use dashboard::collector::StatsCollector; +use dashboard::distribution_collector::DistributionStatsProvider; +use dashboard::{start_dashboard, DashboardConfig}; // ── CLI ────────────────────────────────────────────────────────────────── @@ -95,6 +95,7 @@ fn main() { ..Default::default() }); dash.install_tracing(); + dash.start_http_standalone(); // Create actor runtime let num_threads = 2; @@ -154,7 +155,7 @@ fn run_tcp( args: Args, node_config: DistributedNodeConfig, handle: &swactor::runtime::RuntimeHandle, - dash: &runtime_dashboard::DashboardHandle, + dash: &dashboard::DashboardHandle, stop: &Arc, ) { use distribution::driver::NodeDriver; @@ -209,7 +210,7 @@ fn run_iroh( args: Args, node_config: DistributedNodeConfig, handle: &swactor::runtime::RuntimeHandle, - dash: &runtime_dashboard::DashboardHandle, + dash: &dashboard::DashboardHandle, stop: &Arc, ) { use distribution::iroh_driver::{IrohDriver, IrohDriverConfig}; @@ -219,6 +220,7 @@ fn run_iroh( secret_key: None, relay_mode: RelayMode::Default, node: node_config, + peer_auth: None, }; let mut driver = IrohDriver::new(iroh_config).expect("failed to create iroh driver"); diff --git a/crates/runtime-dashboard/src/bin/tui.rs b/crates/dashboard/src/bin/tui.rs similarity index 72% rename from crates/runtime-dashboard/src/bin/tui.rs rename to crates/dashboard/src/bin/tui.rs index 00060d9..1b58344 100644 --- a/crates/runtime-dashboard/src/bin/tui.rs +++ b/crates/dashboard/src/bin/tui.rs @@ -1,5 +1,5 @@ -use runtime_dashboard::tui::types::RuntimeEndpoint; -use runtime_dashboard::tui::{start_tui_remote, TuiConfig}; +use dashboard::tui::types::RuntimeEndpoint; +use dashboard::tui::{start_tui_remote, TuiConfig}; fn main() -> std::io::Result<()> { let url = std::env::args() diff --git a/crates/runtime-dashboard/src/ci_collector.rs b/crates/dashboard/src/ci_collector.rs similarity index 89% rename from crates/runtime-dashboard/src/ci_collector.rs rename to crates/dashboard/src/ci_collector.rs index 85b321e..8629fa2 100644 --- a/crates/runtime-dashboard/src/ci_collector.rs +++ b/crates/dashboard/src/ci_collector.rs @@ -2,7 +2,39 @@ use serde::{Deserialize, Serialize}; -use swactor_ci::{JobId, JobStatus, PipelineId, PipelineStatus}; +// CI types defined locally to avoid a cyclic dependency with swactor-ci. + +/// Unique identifier for a pipeline execution. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct PipelineId(pub u64); + +/// Unique identifier for a job within a pipeline. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct JobId { + pub pipeline_id: PipelineId, + pub job_name: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum JobStatus { + Pending, + WaitingForProvisioner, + Provisioning, + Running, + Passed, + Failed { reason: String }, + Skipped, + Interrupted, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum PipelineStatus { + Pending, + Running, + Passed, + Failed, + Error { reason: String }, +} // ─── Stats Provider ───────────────────────────────────────────────────────── diff --git a/crates/runtime-dashboard/src/collector.rs b/crates/dashboard/src/collector.rs similarity index 80% rename from crates/runtime-dashboard/src/collector.rs rename to crates/dashboard/src/collector.rs index 47837c3..ecd45b7 100644 --- a/crates/runtime-dashboard/src/collector.rs +++ b/crates/dashboard/src/collector.rs @@ -6,6 +6,7 @@ use std::sync::{Arc, RwLock}; +use swactor::runtime::Runtime; use swactor::stats::{ActorInfo, ActorSnapshot, RuntimeStats, StatsHook}; /// Collects per-actor snapshots pushed by worker threads. @@ -48,6 +49,21 @@ impl crate::command::StatsEnricher for StatsCollector { } } +/// Enrich actor names from the runtime's StdExtension name registry. +pub fn enrich_names(stats: &mut RuntimeStats, runtime: &Runtime) { + let ext = match runtime.extension() { + Some(ext) => ext, + None => return, + }; + let std_ext = match ext.as_any().downcast_ref::() { + Some(ext) => ext, + None => return, + }; + for actor in &mut stats.actor_details { + actor.name = std_ext.resolve_name(&actor.address); + } +} + impl StatsHook for StatsCollector { fn on_tick(&self, worker_id: usize, snapshots: &[ActorSnapshot]) { if let Some(slot) = self.slots.get(worker_id) { @@ -60,6 +76,7 @@ impl StatsHook for StatsCollector { last_msg_type: s.last_msg_type.map(|t| t.to_string()), messages_processed: s.messages_processed, poisoned: s.poisoned, + name: None, message_type_counts: s.message_type_counts.iter().map(|(k, v)| (k.to_string(), *v)).collect(), })); } diff --git a/crates/runtime-dashboard/src/command/builtins.rs b/crates/dashboard/src/command/builtins.rs similarity index 100% rename from crates/runtime-dashboard/src/command/builtins.rs rename to crates/dashboard/src/command/builtins.rs diff --git a/crates/runtime-dashboard/src/command/mod.rs b/crates/dashboard/src/command/mod.rs similarity index 100% rename from crates/runtime-dashboard/src/command/mod.rs rename to crates/dashboard/src/command/mod.rs diff --git a/crates/runtime-dashboard/src/command/parse.rs b/crates/dashboard/src/command/parse.rs similarity index 100% rename from crates/runtime-dashboard/src/command/parse.rs rename to crates/dashboard/src/command/parse.rs diff --git a/crates/runtime-dashboard/src/dashboard_html.rs b/crates/dashboard/src/dashboard_html.rs similarity index 95% rename from crates/runtime-dashboard/src/dashboard_html.rs rename to crates/dashboard/src/dashboard_html.rs index 0054723..0aa4f7f 100644 --- a/crates/runtime-dashboard/src/dashboard_html.rs +++ b/crates/dashboard/src/dashboard_html.rs @@ -356,6 +356,14 @@ pub const DASHBOARD_HTML: &str = r##" renderWorkerCards(data); + var nameMap = {}; + if (data.actor_details) { + data.actor_details.forEach(function(a) { + var h = formatAddr(a.address); + if (a.name) nameMap[h] = a.name; + }); + } + var tbody = document.getElementById('actorTableBody'); tbody.innerHTML = ''; if (data.actors) { @@ -372,7 +380,13 @@ pub const DASHBOARD_HTML: &str = r##" hex += '\u2026'; } var tr = document.createElement('tr'); - tr.innerHTML = '' + hex + 'W' + wid + ''; + var addrCell; + if (nameMap[hex]) { + addrCell = '' + escapeHtml(nameMap[hex]) + ' ' + hex + ''; + } else { + addrCell = '' + hex + ''; + } + tr.innerHTML = addrCell + 'W' + wid + ''; tbody.appendChild(tr); }); if (data.actors.length > 200) { @@ -493,7 +507,13 @@ pub const DASHBOARD_HTML: &str = r##" var msgShort = hasMsg ? shortTypeName(a.last_msg_type) : 'none'; var msgClass = hasMsg ? 'msg-type' : 'msg-type none'; var title = hasMsg ? ' title="' + escapeHtml(a.last_msg_type) + '"' : ''; - rows += '' + hex + '' + + var addrHtml; + if (a.name) { + addrHtml = '' + escapeHtml(a.name) + ' ' + hex + ''; + } else { + addrHtml = '' + hex + ''; + } + rows += '' + addrHtml + '' + a.mailbox_depth + '' + '' + escapeHtml(msgShort) + ''; }); diff --git a/crates/dashboard/src/datastore_collector.rs b/crates/dashboard/src/datastore_collector.rs new file mode 100644 index 0000000..b2e38b3 --- /dev/null +++ b/crates/dashboard/src/datastore_collector.rs @@ -0,0 +1,75 @@ +//! Datastore stats provider for the runtime dashboard. +//! +//! The trait returns a pre-serialized JSON string so that `runtime-dashboard` +//! has no compile-time dependency on `swactor-datastore` (which would create a +//! circular dependency since `swactor-datastore[node]` depends on us). +//! +//! The `swactor-datastore` crate implements this trait in its `node` feature. + +use std::sync::Arc; + +/// Scope filter for listing objects. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ListScope { + Local, + Swarm, +} + +/// Trait for providing datastore stats and CRUD operations to the dashboard. +/// +/// Implementations capture a point-in-time snapshot as serialized JSON. +/// The dashboard polls this every ~200ms via SSE. +/// +/// All command methods have default implementations returning `Err` so that +/// existing `DatastoreMetrics` impls continue to compile without changes. +pub trait DatastoreStatsProvider: Send + Sync { + /// Return a JSON-serialized datastore snapshot, or `None` if unavailable. + fn snapshot_json(&self) -> Option; + + /// List objects as a JSON string. `scope` selects local-only or swarm-wide. + fn list_objects(&self, _name_filter: Option<&str>, _scope: ListScope) -> Result { + Err("not supported".into()) + } + + /// Get a single object's metadata + manifest as JSON. + fn get_object(&self, _hash: &str) -> Result { + Err("not supported".into()) + } + + /// Get the raw binary data for an object. + fn get_data(&self, _hash: &str) -> Result, String> { + Err("not supported".into()) + } + + /// Store data, optionally with a name. Returns JSON with `content_hash`. + fn put_data(&self, _data: Vec, _name: Option) -> Result { + Err("not supported".into()) + } + + /// Delete an object by hash. Returns JSON confirmation. + fn delete_object(&self, _hash: &str) -> Result { + Err("not supported".into()) + } + + /// Get node status as JSON. + fn node_status(&self) -> Result { + Err("not supported".into()) + } + + /// Whether the datastore is currently running. + fn is_running(&self) -> bool { + false + } + + /// Shut down the datastore actors. + fn shutdown_datastore(&self) -> Result<(), String> { + Err("not supported".into()) + } +} + +/// Factory for creating a new datastore instance from the dashboard. +pub trait DatastoreFactory: Send + Sync { + /// Start a datastore with optional persistent storage path. + /// Returns a provider that can be installed into the dashboard. + fn start_datastore(&self, storage_path: Option) -> Result, String>; +} diff --git a/crates/dashboard/src/datastore_html.rs b/crates/dashboard/src/datastore_html.rs new file mode 100644 index 0000000..4522338 --- /dev/null +++ b/crates/dashboard/src/datastore_html.rs @@ -0,0 +1,571 @@ +pub const DATASTORE_HTML: &str = r##" + + + + +Swactor Runtime – Datastore + + + +
+
+

+ Swactor Runtime Dashboard + +

+ +
+
+ Waiting for data... + + +
+
+ +
+ + + + +
+

Datastore Stats

+
+
0
Objects
+
0
Total Size
+
0
Puts
+
0
Gets
+
0
Deletes
+
+
+ + +
+

Event Timeline

+
+
+ + +
+

Objects

+
+ + + +
HashNameOriginSizeActions
+
+
+ + +
+

Active Transfers

+
+
+
+ + + + +
+ + + + +"##; diff --git a/crates/runtime-dashboard/src/distribution_collector.rs b/crates/dashboard/src/distribution_collector.rs similarity index 100% rename from crates/runtime-dashboard/src/distribution_collector.rs rename to crates/dashboard/src/distribution_collector.rs diff --git a/crates/runtime-dashboard/src/distribution_html.rs b/crates/dashboard/src/distribution_html.rs similarity index 63% rename from crates/runtime-dashboard/src/distribution_html.rs rename to crates/dashboard/src/distribution_html.rs index ae9af8d..00ff106 100644 --- a/crates/runtime-dashboard/src/distribution_html.rs +++ b/crates/dashboard/src/distribution_html.rs @@ -94,6 +94,19 @@ pub const DISTRIBUTION_HTML: &str = r##" flex: 1; overflow-y: auto; font-size: 11px; } + .relay-badge { + display: inline-block; + background: #14b8a6; + color: #fff; + font-size: 9px; + font-weight: 700; + padding: 1px 5px; + border-radius: 3px; + margin-left: 6px; + vertical-align: middle; + letter-spacing: 0.5px; + } + .ego-hint { position: absolute; bottom: 12px; left: 12px; font-size: 10px; color: #555; pointer-events: none; @@ -105,6 +118,40 @@ pub const DISTRIBUTION_HTML: &str = r##" white-space: nowrap; } + .share-btn { + background: #6366f1; color: #fff; border: none; padding: 5px 14px; + font-size: 12px; font-family: inherit; border-radius: 4px; cursor: pointer; + } + .share-btn:hover { background: #5558e6; } + + .modal-overlay { + display: none; position: fixed; top: 0; left: 0; width: 100%; height: 100%; + background: rgba(0,0,0,0.6); z-index: 1000; align-items: center; justify-content: center; + } + .modal-overlay.active { display: flex; } + .modal { + background: #1c1f2e; border: 1px solid #2a2d3e; border-radius: 8px; + padding: 24px; width: 360px; max-width: 90vw; + } + .modal h2 { font-size: 14px; color: #fff; margin-bottom: 16px; } + .modal-close { + float: right; background: none; border: none; color: #888; font-size: 18px; + cursor: pointer; line-height: 1; + } + .modal-close:hover { color: #fff; } + .invite-input { + width: 100%; background: #0f1117; border: 1px solid #2a2d3e; color: #e0e0e0; + padding: 8px 10px; font-size: 12px; font-family: monospace; border-radius: 4px; + margin-bottom: 10px; + } + .copy-btn { + background: #6366f1; color: #fff; border: none; padding: 5px 14px; + font-size: 12px; font-family: inherit; border-radius: 4px; cursor: pointer; + margin-bottom: 16px; + } + .copy-btn:hover { background: #5558e6; } + .qr-canvas { display: block; margin: 0 auto; image-rendering: pixelated; } + ::-webkit-scrollbar { width: 6px; } ::-webkit-scrollbar-track { background: #0f1117; } ::-webkit-scrollbar-thumb { background: #2a2d3e; border-radius: 3px; } @@ -126,6 +173,17 @@ pub const DISTRIBUTION_HTML: &str = r##"
Waiting for data... + +
+ + + @@ -193,6 +251,17 @@ pub const DISTRIBUTION_HTML: &str = r##"

Routing Buckets

+
+

Trusted Peers

+
+
+
+ + + +
+
+
@@ -544,15 +613,27 @@ pub const DISTRIBUTION_HTML: &str = r##" ctx.stroke(); } + // Teal ring around relay nodes + var isRelay = (i === 0 && data && data.relay_url) + || (i > 0 && data && data.members[i-1] && data.members[i-1].relay_url); + if (isRelay) { + ctx.beginPath(); + ctx.arc(posX[i], posY[i], nr + 3/vs, 0, Math.PI * 2); + ctx.lineWidth = 2/vs; + ctx.strokeStyle = '#14b8a6'; + ctx.globalAlpha = opacity * 0.9; + ctx.stroke(); + } + ctx.globalAlpha = 1; - // Label (short ID) + // Label (name for self, short ID for others) if (vs > 0.5) { ctx.fillStyle = '#ccc'; ctx.globalAlpha = opacity; ctx.font = Math.round(9 / vs) + 'px monospace'; ctx.textAlign = 'center'; - var label = nodeIds[i] ? nodeIds[i].substring(0, 8) : ''; + var label = (i === 0 && data && data.node_name) ? data.node_name : (nodeIds[i] ? nodeIds[i].substring(0, 8) : ''); ctx.fillText(label, posX[i], posY[i] - nr - 3/vs); ctx.globalAlpha = 1; } @@ -614,8 +695,8 @@ pub const DISTRIBUTION_HTML: &str = r##" function updateUI(d) { // Self label - document.getElementById('selfLabel').textContent = 'Node: ' + d.node_id.substring(0, 16) + '\u2026'; - document.getElementById('nodeLabel').textContent = d.listen_addr; + document.getElementById('selfLabel').textContent = 'Node: ' + (d.node_name || d.node_id.substring(0, 16) + '\u2026'); + document.getElementById('nodeLabel').textContent = d.node_name || d.listen_addr; // Stats cards document.getElementById('statMembers').textContent = d.members.length; @@ -641,9 +722,14 @@ pub const DISTRIBUTION_HTML: &str = r##" var m = d.members[i]; var cls = 'state-' + m.state; var tr = document.createElement('tr'); + var idCell = m.node_id.substring(0, 16) + '\u2026'; + if (m.label) idCell = m.label + ' (' + m.node_id.substring(0, 8) + ')'; + var relayBadge = m.relay_url + ? ' RELAY' + : ''; tr.innerHTML = '' + m.state + '' + - '' + m.node_id.substring(0, 16) + '\u2026' + + '' + idCell + relayBadge + '' + '' + m.addr + '' + '' + m.incarnation + ''; body.appendChild(tr); @@ -760,8 +846,309 @@ pub const DISTRIBUTION_HTML: &str = r##" dot.className = 'status-dot'; }; + // ── Peer management ────────────────────────────────────────── + var dashToken = ''; + + function setToken(t) { dashToken = t; } + + function fetchPeers() { + fetch('/api/peers') + .then(function(r) { return r.json(); }) + .then(function(d) { renderPeers(d); }) + .catch(function() {}); + } + + function renderPeers(d) { + var el = document.getElementById('peerAuthMode'); + el.textContent = '(' + d.mode + ')'; + var list = document.getElementById('peersList'); + list.innerHTML = ''; + if (d.mode === 'open') { + list.innerHTML = '
Open mode — all peers accepted
'; + return; + } + // Cross-reference with members for online status + var memberMap = {}; + if (data && data.members) { + for (var i = 0; i < data.members.length; i++) { + memberMap[data.members[i].node_id] = data.members[i].state; + } + } + for (var i = 0; i < d.peers.length; i++) { + var p = d.peers[i]; + var state = memberMap[p.node_id] || 'offline'; + var color = state === 'alive' ? '#4caf50' : state === 'suspect' ? '#ff9800' : '#555'; + var div = document.createElement('div'); + div.style.cssText = 'display:flex;align-items:center;gap:6px;padding:2px 0;font-size:10px;'; + div.innerHTML = + '' + + '' + p.node_id.substring(0, 16) + '\u2026' + + '' + (p.label || '') + '' + + '' + state + '' + + ''; + list.appendChild(div); + } + } + + window.addPeer = function() { + var nid = document.getElementById('peerNodeId').value.trim(); + var label = document.getElementById('peerLabel').value.trim(); + if (!nid) return; + fetch('/api/peers/add', { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'Authorization': 'Bearer ' + dashToken }, + body: JSON.stringify({ node_id: nid, label: label }) + }).then(function() { + document.getElementById('peerNodeId').value = ''; + document.getElementById('peerLabel').value = ''; + fetchPeers(); + }); + }; + + window.removePeer = function(nid) { + fetch('/api/peers/remove', { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'Authorization': 'Bearer ' + dashToken }, + body: JSON.stringify({ node_id: nid }) + }).then(function() { fetchPeers(); }); + }; + + // Poll peers every 5 seconds + fetchPeers(); + setInterval(fetchPeers, 5000); + resizeCanvas(); drawGraph(); + + // ── Share modal ───────────────────────────────────────────── + var currentInvite = ''; + + window.openShareModal = function() { + if (!data || !data.invite_code) return; + currentInvite = data.invite_code; + document.getElementById('inviteInput').value = currentInvite; + document.getElementById('shareModal').classList.add('active'); + document.getElementById('copyBtn').textContent = 'Copy'; + renderQR(currentInvite); + }; + + window.closeShareModal = function() { + document.getElementById('shareModal').classList.remove('active'); + }; + + document.getElementById('shareModal').addEventListener('click', function(e) { + if (e.target === this) closeShareModal(); + }); + + window.copyInvite = function() { + navigator.clipboard.writeText(currentInvite).then(function() { + document.getElementById('copyBtn').textContent = 'Copied!'; + setTimeout(function() { document.getElementById('copyBtn').textContent = 'Copy'; }, 2000); + }); + }; + + // ── Minimal QR code renderer (Mode 2 alphanumeric, version auto) ── + function renderQR(text) { + var cv = document.getElementById('qrCanvas'); + var c = cv.getContext('2d'); + // Use a simple approach: encode text into a QR-like visual via a data URL + // For proper QR, we use a compact encoder below. + try { + var modules = qrEncode(text); + var size = modules.length; + var scale = Math.floor(200 / size); + cv.width = size * scale; + cv.height = size * scale; + c.fillStyle = '#fff'; + c.fillRect(0, 0, cv.width, cv.height); + c.fillStyle = '#000'; + for (var y = 0; y < size; y++) { + for (var x = 0; x < size; x++) { + if (modules[y][x]) c.fillRect(x * scale, y * scale, scale, scale); + } + } + } catch(e) { + // Fallback: just show the text + cv.width = 200; cv.height = 40; + c.fillStyle = '#1c1f2e'; c.fillRect(0, 0, 200, 40); + c.fillStyle = '#888'; c.font = '11px monospace'; c.textAlign = 'center'; + c.fillText('QR unavailable', 100, 24); + } + } + + // ── Compact QR encoder (byte mode, ECC L) ───────────────────────── + // Supports up to ~150 bytes (versions 1-10). Minimal but functional. + function qrEncode(text) { + var data = []; + for (var i = 0; i < text.length; i++) data.push(text.charCodeAt(i)); + var ver = 1; + // Byte mode capacities at ECC L: v1=17, v2=32, v3=53, v4=78, v5=106, v6=134, v7=154... + var caps = [0,17,32,53,78,106,134,154,192,230,271]; + while (ver < caps.length - 1 && data.length > caps[ver]) ver++; + if (data.length > caps[ver]) throw 'too long'; + var size = ver * 4 + 17; + var modules = [], isFunc = []; + for (var y = 0; y < size; y++) { modules.push([]); isFunc.push([]); for (var x = 0; x < size; x++) { modules[y].push(false); isFunc[y].push(false); } } + + // Finder patterns + function setFinder(row, col) { + for (var dy = -1; dy <= 7; dy++) for (var dx = -1; dx <= 7; dx++) { + var r = row+dy, c = col+dx; + if (r < 0 || r >= size || c < 0 || c >= size) continue; + var v = (dy >= 0 && dy <= 6 && dx >= 0 && dx <= 6) && + (dy === 0 || dy === 6 || dx === 0 || dx === 6 || (dy >= 2 && dy <= 4 && dx >= 2 && dx <= 4)); + modules[r][c] = v; isFunc[r][c] = true; + } + } + setFinder(0, 0); setFinder(0, size - 7); setFinder(size - 7, 0); + + // Timing patterns + for (var i = 8; i < size - 8; i++) { + modules[6][i] = i % 2 === 0; isFunc[6][i] = true; + modules[i][6] = i % 2 === 0; isFunc[i][6] = true; + } + + // Alignment patterns (simplified for small versions) + var alignPos = [[], [], [6,18], [6,22], [6,26], [6,30], [6,34], [6,22,38], [6,24,42], [6,26,46], [6,28,50]]; + if (ver >= 2) { + var ap = alignPos[ver]; + for (var ai = 0; ai < ap.length; ai++) for (var aj = 0; aj < ap.length; aj++) { + var ay = ap[ai], ax = ap[aj]; + if (isFunc[ay][ax]) continue; + for (var dy = -2; dy <= 2; dy++) for (var dx = -2; dx <= 2; dx++) { + modules[ay+dy][ax+dx] = Math.abs(dy) === 2 || Math.abs(dx) === 2 || (dy === 0 && dx === 0); + isFunc[ay+dy][ax+dx] = true; + } + } + } + + // Format info area reservation + for (var i = 0; i < 9; i++) { + if (i < size) { isFunc[8][i] = true; isFunc[i][8] = true; } + } + for (var i = 0; i < 8; i++) { + isFunc[8][size-1-i] = true; isFunc[size-1-i][8] = true; + } + modules[size-8][8] = true; isFunc[size-8][8] = true; // dark module + + // Version info (v >= 7 only, skip for simplicity) + + // Encode data + var totalBits = qrDataBits(ver); + var bits = []; + function pushBits(val, len) { for (var i = len - 1; i >= 0; i--) bits.push((val >> i) & 1); } + pushBits(4, 4); // byte mode indicator + pushBits(data.length, ver <= 9 ? 8 : 16); + for (var i = 0; i < data.length; i++) pushBits(data[i], 8); + pushBits(0, Math.min(4, totalBits - bits.length)); // terminator + while (bits.length % 8 !== 0) bits.push(0); + var pad = [0xEC, 0x11], pi = 0; + while (bits.length < totalBits) { pushBits(pad[pi], 8); pi = 1 - pi; } + + // ECC (Reed-Solomon with GF(256)) + var codewords = []; + for (var i = 0; i < bits.length; i += 8) { + var b = 0; for (var j = 0; j < 8; j++) b = (b << 1) | (bits[i+j] || 0); + codewords.push(b); + } + var eccInfo = qrEccInfo(ver); + var eccLen = eccInfo.eccPer; + var blocks = eccInfo.blocks; + var dataPerBlock = Math.floor(codewords.length / blocks); + var extra = codewords.length % blocks; + var allData = [], allEcc = []; + var idx = 0; + for (var b = 0; b < blocks; b++) { + var len = dataPerBlock + (b >= blocks - extra ? 1 : 0); + var block = codewords.slice(idx, idx + len); + idx += len; + allData.push(block); + allEcc.push(rsEncode(block, eccLen)); + } + + // Interleave + var result = []; + var maxData = allData[blocks-1].length; + for (var i = 0; i < maxData; i++) for (var b = 0; b < blocks; b++) { if (i < allData[b].length) result.push(allData[b][i]); } + for (var i = 0; i < eccLen; i++) for (var b = 0; b < blocks; b++) result.push(allEcc[b][i]); + + // Place data bits + var resultBits = []; + for (var i = 0; i < result.length; i++) for (var j = 7; j >= 0; j--) resultBits.push((result[i] >> j) & 1); + var bi = 0; + for (var right = size - 1; right >= 1; right -= 2) { + if (right === 6) right = 5; + for (var vert = 0; vert < size; vert++) { + for (var j = 0; j < 2; j++) { + var x = right - j; + var upward = ((right + 1) & 2) === 0; + var y = upward ? size - 1 - vert : vert; + if (!isFunc[y][x] && bi < resultBits.length) { + modules[y][x] = resultBits[bi++] === 1; + } + } + } + } + + // Apply mask 0 (checkerboard) and write format bits + for (var y = 0; y < size; y++) for (var x = 0; x < size; x++) { + if (!isFunc[y][x]) { if ((y + x) % 2 === 0) modules[y][x] = !modules[y][x]; } + } + // Format info for mask 0, ECC L = 0b111011111000100 + var fmtBits = [1,1,1,0,1,1,1,1,1,0,0,0,1,0,0]; + for (var i = 0; i < 6; i++) modules[8][i] = fmtBits[i] === 1; + modules[8][7] = fmtBits[6] === 1; modules[8][8] = fmtBits[7] === 1; modules[7][8] = fmtBits[8] === 1; + for (var i = 9; i < 15; i++) modules[14-i][8] = fmtBits[i] === 1; + for (var i = 0; i < 8; i++) modules[8][size-8+i] = fmtBits[i] === 1; + for (var i = 8; i < 15; i++) modules[size-15+i][8] = fmtBits[i] === 1; + + return modules; + } + + function qrDataBits(ver) { + // Total data codewords * 8 for ECC level L + var total = [0,19,34,55,80,108,136,156,194,232,274]; + return total[ver] * 8; + } + + function qrEccInfo(ver) { + // ECC codewords per block and number of blocks for ECC L + var info = [ + null, + {eccPer:7,blocks:1},{eccPer:10,blocks:1},{eccPer:15,blocks:1},{eccPer:20,blocks:1}, + {eccPer:26,blocks:1},{eccPer:18,blocks:2},{eccPer:20,blocks:2},{eccPer:24,blocks:2}, + {eccPer:30,blocks:2},{eccPer:18,blocks:2} + ]; + return info[ver]; + } + + // GF(256) Reed-Solomon + var gfExp = new Uint8Array(512), gfLog = new Uint8Array(256); + (function() { + var x = 1; + for (var i = 0; i < 255; i++) { gfExp[i] = x; gfLog[x] = i; x = (x << 1) ^ (x >= 128 ? 0x11d : 0); } + for (var i = 255; i < 512; i++) gfExp[i] = gfExp[i - 255]; + })(); + + function gfMul(a, b) { return (a === 0 || b === 0) ? 0 : gfExp[gfLog[a] + gfLog[b]]; } + + function rsEncode(data, eccLen) { + var gen = [1]; + for (var i = 0; i < eccLen; i++) { + var ng = new Array(gen.length + 1).fill(0); + for (var j = 0; j < gen.length; j++) { + ng[j] ^= gen[j]; + ng[j+1] ^= gfMul(gen[j], gfExp[i]); + } + gen = ng; + } + var msg = data.concat(new Array(eccLen).fill(0)); + for (var i = 0; i < data.length; i++) { + var coef = msg[i]; + if (coef !== 0) for (var j = 0; j < gen.length; j++) msg[i+j] ^= gfMul(gen[j], coef); + } + return msg.slice(data.length); + } })(); diff --git a/crates/runtime-dashboard/src/history.rs b/crates/dashboard/src/history.rs similarity index 99% rename from crates/runtime-dashboard/src/history.rs rename to crates/dashboard/src/history.rs index 51be11d..8a0b629 100644 --- a/crates/runtime-dashboard/src/history.rs +++ b/crates/dashboard/src/history.rs @@ -253,6 +253,7 @@ mod tests { fn make_actor(id: u8, worker: usize, depth: usize, msgs: u64) -> ActorInfo { ActorInfo { address: ActorAddress([id; 32]), + name: None, worker_id: worker, mailbox_depth: depth, last_msg_type: None, diff --git a/crates/runtime-dashboard/src/investigate.rs b/crates/dashboard/src/investigate.rs similarity index 100% rename from crates/runtime-dashboard/src/investigate.rs rename to crates/dashboard/src/investigate.rs diff --git a/crates/runtime-dashboard/src/layer.rs b/crates/dashboard/src/layer.rs similarity index 100% rename from crates/runtime-dashboard/src/layer.rs rename to crates/dashboard/src/layer.rs diff --git a/crates/runtime-dashboard/src/lib.rs b/crates/dashboard/src/lib.rs similarity index 65% rename from crates/runtime-dashboard/src/lib.rs rename to crates/dashboard/src/lib.rs index bc0b5a2..9c3ce92 100644 --- a/crates/runtime-dashboard/src/lib.rs +++ b/crates/dashboard/src/lib.rs @@ -23,7 +23,6 @@ pub mod distribution_collector; mod datastore_html; pub mod datastore_collector; -#[cfg(feature = "ci")] pub mod ci_collector; use std::io; @@ -43,6 +42,9 @@ use crate::history::{DashboardHistory, HistoryConfig}; use crate::layer::{now_ms, DashboardLayer, EventStore}; use crate::trace::{RuntimeTrace, TimestampedStats}; +/// Peer info sent through the join channel: (public_key, optional_relay_url). +pub type JoinPeerInfo = ([u8; 32], Option); + /// Configuration for the runtime dashboard. #[derive(Debug, Clone)] pub struct DashboardConfig { @@ -92,14 +94,19 @@ pub struct DashboardHandle { runtime: Arc>>>, collector: Arc>>>, shutdown: Arc, + shutdown_notify: Arc, stats_timeline: Arc>, history: Arc, recording: bool, + port: u16, #[cfg(feature = "distribution")] distribution: Arc>>>, datastore: Arc>>>, - #[cfg(feature = "ci")] + datastore_factory: Arc>>>, ci: Arc>>>, + peer_auth: Arc>>>>, + join_sender: Arc>>>, + standalone_rt: Mutex>, } impl DashboardHandle { @@ -137,12 +144,31 @@ impl DashboardHandle { *self.datastore.lock().unwrap() = Some(provider); } + /// Attach a datastore factory, enabling start/stop from the dashboard. + pub fn set_datastore_factory(&self, factory: Arc) { + *self.datastore_factory.lock().unwrap() = Some(factory); + } + + /// Get the shared datastore provider mutex (for external wiring). + pub fn datastore_provider(&self) -> &Arc>>> { + &self.datastore + } + /// Attach a CI stats provider, enabling the `/api/ci/*` endpoints. - #[cfg(feature = "ci")] pub fn set_ci(&self, provider: Arc) { *self.ci.lock().unwrap() = Some(provider); } + /// Attach a peer allow-list for the peer management API. + pub fn set_peer_auth(&self, auth: Arc>) { + *self.peer_auth.lock().unwrap() = Some(auth); + } + + /// Set a sender that triggers `driver.join()` when a peer is added via the dashboard. + pub fn set_join_sender(&self, tx: std::sync::mpsc::Sender) { + *self.join_sender.lock().unwrap() = Some(tx); + } + /// Access the time-series history store (for TUI sparklines, etc.). pub fn history(&self) -> &Arc { &self.history @@ -151,6 +177,49 @@ impl DashboardHandle { /// Signal the dashboard to shut down (SSE clients receive "done"). pub fn shutdown(&self) { self.shutdown.store(true, Ordering::Release); + self.shutdown_notify.notify_waiters(); + } + + /// Start the HTTP server on the provided tokio handle. + /// Use this when a tokio runtime already exists (e.g. IrohDriver's runtime). + pub fn start_http(&self, handle: tokio::runtime::Handle) { + let state = self.build_app_state(); + let port = self.port; + handle.spawn(async move { + server::run_server(state, port).await; + }); + } + + /// Start the HTTP server on a standalone tokio runtime (1 worker thread). + /// Use this when no external tokio runtime is available (e.g. TCP transport). + pub fn start_http_standalone(&self) { + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .build() + .expect("failed to create tokio runtime for dashboard HTTP"); + let handle = rt.handle().clone(); + *self.standalone_rt.lock().unwrap() = Some(rt); + self.start_http(handle); + } + + fn build_app_state(&self) -> server::AppState { + server::AppState { + store: Arc::clone(&self.store), + runtime: Arc::clone(&self.runtime), + collector: Arc::clone(&self.collector), + shutdown: Arc::clone(&self.shutdown), + shutdown_notify: Arc::clone(&self.shutdown_notify), + history: Arc::clone(&self.history), + cmd_router: Arc::new(crate::command::CommandRouter::with_builtins()), + #[cfg(feature = "distribution")] + distribution: Arc::clone(&self.distribution), + datastore: Arc::clone(&self.datastore), + datastore_factory: Arc::clone(&self.datastore_factory), + ci: Arc::clone(&self.ci), + peer_auth: Arc::clone(&self.peer_auth), + join_sender: Arc::clone(&self.join_sender), + } } /// Save the recorded trace to a JSON file. @@ -178,10 +247,12 @@ impl DashboardHandle { } } -/// Start a dashboard server and return a handle. +/// Start a dashboard and return a handle. /// -/// The dashboard starts serving immediately. Call `install_tracing()` to set up -/// the global subscriber, and `set_runtime()` to enable stats polling. +/// The dashboard state is created immediately but the HTTP server is NOT started. +/// Call `start_http()` or `start_http_standalone()` to begin serving. +/// Call `install_tracing()` to set up the global subscriber, and `set_runtime()` +/// to enable stats polling. pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle { let store = Arc::new(EventStore::new( config.event_capacity, @@ -191,6 +262,7 @@ pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle { let runtime: Arc>>> = Arc::new(Mutex::new(None)); let collector: Arc>>> = Arc::new(Mutex::new(None)); let shutdown = Arc::new(AtomicBool::new(false)); + let shutdown_notify = Arc::new(tokio::sync::Notify::new()); let stats_timeline = Arc::new(ArrayQueue::new(config.record_stats_capacity.max(1))); let history = Arc::new(DashboardHistory::new(HistoryConfig::default())); @@ -201,23 +273,17 @@ pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle { let datastore: Arc>>> = Arc::new(Mutex::new(None)); - #[cfg(feature = "ci")] - let ci: Arc>>> = + let datastore_factory: Arc>>> = Arc::new(Mutex::new(None)); - server::spawn_http_server( - Arc::clone(&store), - Arc::clone(&runtime), - Arc::clone(&collector), - Arc::clone(&shutdown), - Arc::clone(&history), - config.port, - #[cfg(feature = "distribution")] - Arc::clone(&distribution), - Arc::clone(&datastore), - #[cfg(feature = "ci")] - Arc::clone(&ci), - ); + let ci: Arc>>> = + Arc::new(Mutex::new(None)); + + let peer_auth: Arc>>>> = + Arc::new(Mutex::new(None)); + + let join_sender: Arc>>> = + Arc::new(Mutex::new(None)); // Start stats recorder thread when recording is enabled if config.record { @@ -247,21 +313,26 @@ pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle { }); } - eprintln!("Runtime dashboard at http://localhost:{}", config.port); + let port = config.port; DashboardHandle { store, runtime, collector, shutdown, + shutdown_notify, stats_timeline, history, recording: config.record, + port, #[cfg(feature = "distribution")] distribution, datastore, - #[cfg(feature = "ci")] - ci, + datastore_factory, + ci, + peer_auth, + join_sender, + standalone_rt: Mutex::new(None), } } @@ -274,6 +345,7 @@ pub fn run_with_dashboard( ) -> (RuntimeHandle, DashboardHandle) { let dash = start_dashboard(dash_config); dash.install_tracing(); + dash.start_http_standalone(); let num_workers = if rt_config.num_threads < 2 { 1 } else { rt_config.num_threads }; let collector = StatsCollector::new(num_workers); @@ -292,7 +364,21 @@ pub fn serve_replay(path: &str, config: ReplayConfig) -> io::Result<()> { let trace: RuntimeTrace = serde_json::from_str(&data) .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; - server::spawn_replay_server(Arc::new(trace), config.port, config.speed); + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .build() + .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + + let state = server::ReplayState { + trace: Arc::new(trace), + speed: config.speed, + }; + let port = config.port; + + rt.spawn(async move { + server::run_replay_server(state, port).await; + }); eprintln!("Replay dashboard at http://localhost:{}", config.port); eprintln!("Press Ctrl+C to stop"); diff --git a/crates/dashboard/src/server.rs b/crates/dashboard/src/server.rs new file mode 100644 index 0000000..b1a83c3 --- /dev/null +++ b/crates/dashboard/src/server.rs @@ -0,0 +1,960 @@ +use std::collections::HashMap; +use std::convert::Infallible; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use axum::body::Bytes; +use axum::extract::{Path, Query, State}; +use axum::http::{header, StatusCode}; +use axum::response::sse::{Event, KeepAlive, Sse}; +use axum::response::{IntoResponse, Response}; +use axum::routing::{get, post}; +use axum::Router; +use tokio_stream::wrappers::ReceiverStream; +use tokio_stream::StreamExt; + +use swactor::runtime::Runtime; + +use crate::actor_detail_html::ACTOR_DETAIL_HTML; +use crate::actors_html::ACTORS_HTML; +use crate::collector::StatsCollector; +use crate::command::CommandRouter; +use crate::dashboard_html::DASHBOARD_HTML; +use crate::datastore_collector::{DatastoreFactory, DatastoreStatsProvider, ListScope}; +use crate::datastore_html::DATASTORE_HTML; +use crate::history::DashboardHistory; +use crate::layer::EventStore; +use crate::topology; +use crate::topology_html::TOPOLOGY_HTML; +use crate::warnings::{WarningConfig, WarningDetector}; + +#[cfg(feature = "distribution")] +use crate::distribution_collector::DistributionStatsProvider; +#[cfg(feature = "distribution")] +use crate::distribution_html::DISTRIBUTION_HTML; + +use crate::ci_collector::CiStatsProvider; + +use crate::trace::RuntimeTrace; + +/// Format a server-sent event. +fn format_sse(event: &str, data: &str) -> Event { + Event::default().event(event).data(data) +} + +// ── Shared application state ──────────────────────────────────────────── + +#[derive(Clone)] +pub(crate) struct AppState { + pub store: Arc, + pub runtime: Arc>>>, + pub collector: Arc>>>, + pub shutdown: Arc, + pub shutdown_notify: Arc, + pub history: Arc, + pub cmd_router: Arc, + #[cfg(feature = "distribution")] + pub distribution: Arc>>>, + pub datastore: Arc>>>, + pub datastore_factory: Arc>>>, + pub ci: Arc>>>, + pub peer_auth: Arc>>>>, + pub join_sender: Arc>>>, +} + +// ── Router builders ───────────────────────────────────────────────────── + +pub(crate) fn build_live_router(state: AppState) -> Router { + let router = Router::new() + .route("/", get(page_dashboard)) + .route("/actors", get(page_actors)) + .route("/topology", get(page_topology)) + .route("/datastore", get(page_datastore)) + .route("/events", get(handle_live_sse)) + .route("/api/stats", get(handle_stats_api)) + .route("/api/history", get(handle_history_api)) + .route("/api/topology", get(handle_topology_api)) + .route("/api/investigate", get(handle_investigate_api)) + .route("/api/datastore", get(handle_datastore_api)) + .route("/api/logs", get(handle_logs_api)) + .route("/api/datastore/list", get(handle_ds_list)) + .route("/api/datastore/get", get(handle_ds_get)) + .route("/api/datastore/data", get(handle_ds_data)) + .route("/api/datastore/status", get(handle_ds_status)) + .route("/api/datastore/put", post(handle_ds_put)) + .route("/api/datastore/delete", post(handle_ds_delete)) + .route("/api/datastore/start", post(handle_ds_start)) + .route("/api/datastore/shutdown", post(handle_ds_shutdown)) + .route("/api/peers", get(handle_peers_list)) + .route("/api/peers/add", post(handle_peers_add)) + .route("/api/peers/sync", post(handle_peers_sync)) + .route("/api/peers/remove", post(handle_peers_remove)) + .route("/actor/{hex}", get(handle_actor_detail)); + + #[cfg(feature = "distribution")] + let router = router + .route("/distribution", get(page_distribution)) + .route("/api/distribution", get(handle_distribution_api)); + + let router = router.route("/api/ci/{*rest}", get(handle_ci_api)); + + router.with_state(state) +} + +pub(crate) fn build_replay_router(state: ReplayState) -> Router { + Router::new() + .route("/", get(replay_page_dashboard)) + .route("/actors", get(replay_page_actors)) + .route("/events", get(handle_replay_sse)) + .with_state(state) +} + +// ── Server startup ────────────────────────────────────────────────────── + +pub(crate) async fn run_server(state: AppState, port: u16) { + let listener = tokio::net::TcpListener::bind(format!("0.0.0.0:{port}")) + .await + .expect("failed to bind HTTP server"); + let shutdown = state.shutdown_notify.clone(); + axum::serve(listener, build_live_router(state)) + .with_graceful_shutdown(async move { shutdown.notified().await }) + .await + .expect("HTTP server error"); +} + +pub(crate) async fn run_replay_server(state: ReplayState, port: u16) { + let listener = tokio::net::TcpListener::bind(format!("0.0.0.0:{port}")) + .await + .expect("failed to bind HTTP server"); + axum::serve(listener, build_replay_router(state)) + .await + .expect("HTTP server error"); +} + +// ── HTML page handlers ────────────────────────────────────────────────── + +fn html_response(template: &str, mode: &str) -> Response { + let html = template.replace("__DASHBOARD_MODE__", mode); + ([(header::CONTENT_TYPE, "text/html; charset=utf-8")], html).into_response() +} + +async fn page_dashboard() -> Response { + html_response(DASHBOARD_HTML, "live") +} + +async fn page_actors() -> Response { + html_response(ACTORS_HTML, "live") +} + +async fn page_topology() -> Response { + html_response(TOPOLOGY_HTML, "live") +} + +async fn page_datastore() -> Response { + html_response(DATASTORE_HTML, "live") +} + +#[cfg(feature = "distribution")] +async fn page_distribution() -> Response { + html_response(DISTRIBUTION_HTML, "live") +} + +async fn handle_actor_detail(Path(hex_addr): Path) -> Response { + let html = ACTOR_DETAIL_HTML + .replace("__DASHBOARD_MODE__", "live") + .replace("__ACTOR_ADDR__", &hex_addr); + ([(header::CONTENT_TYPE, "text/html; charset=utf-8")], html).into_response() +} + +// ── Live SSE handler ──────────────────────────────────────────────────── + +async fn handle_live_sse( + State(state): State, +) -> Sse>> { + let (tx, rx) = tokio::sync::mpsc::channel::(32); + + tokio::spawn(async move { + let mut cursor: u64 = 0; + let mut warning_detector = WarningDetector::new(WarningConfig::default()); + let mut tick_count: u64 = 0; + + // Send initial history snapshot so sparklines render immediately + if state.history.sample_count() > 0 { + let json = state.history.worker_history_json(); + if tx.send(format_sse("history", &json)).await.is_err() { + return; + } + } + + loop { + // Send stats if runtime is available + { + let maybe_rt = state.runtime.lock().unwrap().clone(); + if let Some(rt) = maybe_rt { + let mut stats = rt.stats(); + if let Some(col) = state.collector.lock().unwrap().as_ref() { + col.enrich(&mut stats); + } + crate::collector::enrich_names(&mut stats, &rt); + state.history.record(&stats); + + // Run warning detection + let warnings = warning_detector.check(&stats); + if !warnings.is_empty() { + if let Ok(wjson) = serde_json::to_string(&warnings) { + if tx.send(format_sse("warnings", &wjson)).await.is_err() { + return; + } + } + } + + let json = serde_json::to_string(&stats).unwrap(); + if tx.send(format_sse("stats", &json)).await.is_err() { + return; + } + + // Send topology every 5th tick (~1/sec) + tick_count += 1; + if tick_count % 5 == 0 { + let topo = topology::worker_topology(&stats); + if let Ok(tjson) = serde_json::to_string(&topo) { + if tx.send(format_sse("topology", &tjson)).await.is_err() { + return; + } + } + } + } + } + + // Send distribution snapshot if provider is attached + #[cfg(feature = "distribution")] + { + let maybe_dist = state.distribution.lock().unwrap().clone(); + if let Some(provider) = maybe_dist { + if let Some(snapshot) = provider.snapshot() { + if let Ok(json) = serde_json::to_string(&snapshot) { + if tx.send(format_sse("distribution", &json)).await.is_err() { + return; + } + } + } + } + } + + // Send datastore snapshot if provider is attached + { + let maybe_ds = state.datastore.lock().unwrap().clone(); + match maybe_ds { + Some(provider) => { + let is_running = provider.is_running(); + let snap_json = provider.snapshot_json().unwrap_or_else(|| "null".into()); + let envelope = format!( + r#"{{"is_running":{},"snapshot":{}}}"#, + is_running, snap_json + ); + if tx.send(format_sse("datastore", &envelope)).await.is_err() { + return; + } + } + None => { + let envelope = r#"{"is_running":false,"snapshot":null}"#; + if tx.send(format_sse("datastore", envelope)).await.is_err() { + return; + } + } + } + } + + // Send CI snapshot if provider is attached + { + let maybe_ci = state.ci.lock().unwrap().clone(); + if let Some(provider) = maybe_ci { + let snapshot = provider.snapshot(); + if let Ok(json) = serde_json::to_string(&snapshot) { + if tx.send(format_sse("ci", &json)).await.is_err() { + return; + } + } + } + } + + // Send new activity events + let (batch, new_cursor) = state.store.read_from(cursor); + if !batch.is_empty() { + let json = serde_json::to_string(&batch).unwrap(); + if tx.send(format_sse("activity", &json)).await.is_err() { + return; + } + cursor = new_cursor; + } + + if state.shutdown.load(Ordering::Relaxed) { + let _ = tx.send(format_sse("done", "{}")).await; + return; + } + + tokio::time::sleep(Duration::from_millis(200)).await; + } + }); + + Sse::new(ReceiverStream::new(rx).map(Ok)) + .keep_alive(KeepAlive::default()) +} + +// ── JSON API handlers ─────────────────────────────────────────────────── + +fn json_response(json: String) -> Response { + ([(header::CONTENT_TYPE, "application/json")], json).into_response() +} + +fn json_error(status: StatusCode, msg: &str) -> Response { + let json = serde_json::json!({ "error": msg }).to_string(); + (status, [(header::CONTENT_TYPE, "application/json")], json).into_response() +} + +async fn handle_stats_api(State(state): State) -> Response { + let maybe_rt = state.runtime.lock().unwrap().clone(); + let json = match maybe_rt { + Some(rt) => { + let mut stats = rt.stats(); + if let Some(col) = state.collector.lock().unwrap().as_ref() { + col.enrich(&mut stats); + } + crate::collector::enrich_names(&mut stats, &rt); + serde_json::to_string(&stats).unwrap() + } + None => "{}".to_string(), + }; + json_response(json) +} + +async fn handle_investigate_api( + State(state): State, + Query(params): Query>, +) -> Response { + let maybe_rt = state.runtime.lock().unwrap().clone(); + let maybe_col = state.collector.lock().unwrap().clone(); + + let json = match (maybe_rt, maybe_col) { + (Some(rt), Some(col)) => { + let ctx = crate::command::CommandContext::with_enricher(rt, col); + let req = crate::command::from_query_params(¶ms); + state.cmd_router.dispatch(&req, &ctx).to_json_line() + } + (Some(rt), None) => { + let ctx = crate::command::CommandContext::new(rt); + let req = crate::command::from_query_params(¶ms); + state.cmd_router.dispatch(&req, &ctx).to_json_line() + } + _ => { + let cmd = params.get("cmd").map(|s| s.as_str()).unwrap_or("help"); + serde_json::json!({ + "ok": false, + "command": cmd, + "error": "runtime not attached yet" + }) + .to_string() + } + }; + + json_response(json) +} + +#[cfg(feature = "distribution")] +async fn handle_distribution_api(State(state): State) -> Response { + let json = match state.distribution.lock().unwrap().as_ref() { + Some(provider) => match provider.snapshot() { + Some(snapshot) => serde_json::to_string(&snapshot).unwrap_or_else(|_| "{}".into()), + None => "{}".to_string(), + }, + None => serde_json::json!({ + "error": "distribution provider not attached" + }) + .to_string(), + }; + json_response(json) +} + +async fn handle_datastore_api(State(state): State) -> Response { + let json = match state.datastore.lock().unwrap().as_ref() { + Some(provider) => provider.snapshot_json().unwrap_or_else(|| "{}".into()), + None => serde_json::json!({ + "error": "datastore provider not attached" + }) + .to_string(), + }; + json_response(json) +} + +async fn handle_ci_api( + State(state): State, + axum::extract::OriginalUri(uri): axum::extract::OriginalUri, +) -> Response { + use crate::ci_collector; + + let path = uri.path(); + let route = ci_collector::parse_route(path); + let json = match state.ci.lock().unwrap().as_ref() { + Some(provider) => { + let snapshot = provider.snapshot(); + ci_collector::handle_route(&route, &snapshot) + .unwrap_or_else(|| r#"{"error":"not found"}"#.to_string()) + } + None => serde_json::json!({ + "error": "CI provider not attached" + }) + .to_string(), + }; + json_response(json) +} + +async fn handle_topology_api(State(state): State) -> Response { + let maybe_rt = state.runtime.lock().unwrap().clone(); + let json = match maybe_rt { + Some(rt) => { + let mut stats = rt.stats(); + if let Some(col) = state.collector.lock().unwrap().as_ref() { + col.enrich(&mut stats); + } + let topo = topology::worker_topology(&stats); + serde_json::to_string(&topo).unwrap_or_else(|_| "{}".into()) + } + None => "{}".to_string(), + }; + json_response(json) +} + +async fn handle_logs_api( + State(state): State, + Query(params): Query>, +) -> Response { + let actor = params.get("actor").cloned().unwrap_or_default(); + let limit: usize = params + .get("limit") + .and_then(|s| s.parse().ok()) + .unwrap_or(200); + let level = params.get("level").cloned(); + + let mut events = state.store.read_for_actor(&actor, limit); + + // Filter by level if specified + if let Some(ref lvl) = level { + let lvl_upper = lvl.to_uppercase(); + events.retain(|e| e.level == lvl_upper); + } + + let json = serde_json::to_string(&events).unwrap_or_else(|_| "[]".into()); + json_response(json) +} + +async fn handle_history_api(State(state): State) -> Response { + let json = state.history.worker_history_json(); + json_response(json) +} + +// ── Datastore CRUD API handlers ───────────────────────────────────────── + +fn get_ds_provider( + datastore: &Arc>>>, +) -> Option> { + datastore.lock().unwrap().clone() +} + +async fn handle_ds_list( + State(state): State, + Query(params): Query>, +) -> Response { + let provider = match get_ds_provider(&state.datastore) { + Some(p) => p, + None => return json_error(StatusCode::SERVICE_UNAVAILABLE, "datastore not running"), + }; + let scope = match params.get("scope").map(|s| s.as_str()) { + Some("local") => ListScope::Local, + _ => ListScope::Swarm, + }; + let name_filter = params.get("name").map(|s| s.as_str()); + match provider.list_objects(name_filter, scope) { + Ok(json) => json_response(json), + Err(e) => json_error(StatusCode::INTERNAL_SERVER_ERROR, &e), + } +} + +async fn handle_ds_get( + State(state): State, + Query(params): Query>, +) -> Response { + let provider = match get_ds_provider(&state.datastore) { + Some(p) => p, + None => return json_error(StatusCode::SERVICE_UNAVAILABLE, "datastore not running"), + }; + let hash = match params.get("hash") { + Some(h) => h.as_str(), + None => return json_error(StatusCode::BAD_REQUEST, "missing ?hash= parameter"), + }; + match provider.get_object(hash) { + Ok(json) => json_response(json), + Err(e) if e.contains("not found") => json_error(StatusCode::NOT_FOUND, &e), + Err(e) => json_error(StatusCode::INTERNAL_SERVER_ERROR, &e), + } +} + +async fn handle_ds_data( + State(state): State, + Query(params): Query>, +) -> Response { + let provider = match get_ds_provider(&state.datastore) { + Some(p) => p, + None => return json_error(StatusCode::SERVICE_UNAVAILABLE, "datastore not running"), + }; + let hash = match params.get("hash") { + Some(h) => h.as_str(), + None => return json_error(StatusCode::BAD_REQUEST, "missing ?hash= parameter"), + }; + match provider.get_data(hash) { + Ok(data) => { + ([(header::CONTENT_TYPE, "application/octet-stream")], data).into_response() + } + Err(e) if e.contains("not found") => json_error(StatusCode::NOT_FOUND, &e), + Err(e) => json_error(StatusCode::INTERNAL_SERVER_ERROR, &e), + } +} + +async fn handle_ds_status(State(state): State) -> Response { + let provider = match get_ds_provider(&state.datastore) { + Some(p) => p, + None => return json_error(StatusCode::SERVICE_UNAVAILABLE, "datastore not running"), + }; + match provider.node_status() { + Ok(json) => json_response(json), + Err(e) => json_error(StatusCode::INTERNAL_SERVER_ERROR, &e), + } +} + +async fn handle_ds_put( + State(state): State, + Query(params): Query>, + body: Bytes, +) -> Response { + let provider = match get_ds_provider(&state.datastore) { + Some(p) => p, + None => return json_error(StatusCode::SERVICE_UNAVAILABLE, "datastore not running"), + }; + let name = params.get("name").cloned(); + match provider.put_data(body.to_vec(), name) { + Ok(json) => json_response(json), + Err(e) => json_error(StatusCode::INTERNAL_SERVER_ERROR, &e), + } +} + +async fn handle_ds_delete( + State(state): State, + Query(params): Query>, +) -> Response { + let provider = match get_ds_provider(&state.datastore) { + Some(p) => p, + None => return json_error(StatusCode::SERVICE_UNAVAILABLE, "datastore not running"), + }; + let hash = match params.get("hash") { + Some(h) => h.as_str(), + None => return json_error(StatusCode::BAD_REQUEST, "missing ?hash= parameter"), + }; + match provider.delete_object(hash) { + Ok(json) => json_response(json), + Err(e) if e.contains("not found") => json_error(StatusCode::NOT_FOUND, &e), + Err(e) => json_error(StatusCode::INTERNAL_SERVER_ERROR, &e), + } +} + +async fn handle_ds_start( + State(state): State, + Query(params): Query>, +) -> Response { + // Check if already running + { + let ds = state.datastore.lock().unwrap(); + if ds.is_some() { + return json_error(StatusCode::CONFLICT, "datastore already running"); + } + } + + let fac = match state.datastore_factory.lock().unwrap().clone() { + Some(f) => f, + None => return json_error(StatusCode::NOT_IMPLEMENTED, "no datastore factory configured"), + }; + + let storage_path = params.get("storage_path").cloned(); + + match fac.start_datastore(storage_path) { + Ok(provider) => { + *state.datastore.lock().unwrap() = Some(provider); + json_response(r#"{"ok":true}"#.to_string()) + } + Err(e) => json_error(StatusCode::INTERNAL_SERVER_ERROR, &e), + } +} + +async fn handle_ds_shutdown(State(state): State) -> Response { + let provider = match get_ds_provider(&state.datastore) { + Some(p) => p, + None => return json_error(StatusCode::SERVICE_UNAVAILABLE, "datastore not running"), + }; + + match provider.shutdown_datastore() { + Ok(()) => { + *state.datastore.lock().unwrap() = None; + json_response(r#"{"ok":true}"#.to_string()) + } + Err(e) => json_error(StatusCode::INTERNAL_SERVER_ERROR, &e), + } +} + +// ── Peer Management API ───────────────────────────────────────────────── + +async fn handle_peers_list(State(state): State) -> Response { + let maybe_auth = state.peer_auth.lock().unwrap().clone(); + match maybe_auth { + Some(auth) => { + let list = auth.lock().unwrap(); + let is_open = list.is_open(); + let peers: Vec = list + .list_peers() + .iter() + .map(|p| { + serde_json::json!({ + "node_id": p.node_id, + "label": p.label, + }) + }) + .collect(); + let json = serde_json::json!({ + "mode": if is_open { "open" } else { "allow-list" }, + "peers": peers, + }) + .to_string(); + json_response(json) + } + None => { + let json = serde_json::json!({ + "mode": "open", + "peers": [], + }) + .to_string(); + json_response(json) + } + } +} + +async fn handle_peers_add(State(state): State, body: String) -> Response { + let maybe_auth = state.peer_auth.lock().unwrap().clone(); + let auth = match maybe_auth { + Some(a) => a, + None => return json_error(StatusCode::BAD_REQUEST, "peer auth not configured"), + }; + + let parsed: serde_json::Value = match serde_json::from_str(&body) { + Ok(v) => v, + Err(e) => return json_error(StatusCode::BAD_REQUEST, &format!("invalid JSON: {e}")), + }; + + let node_id_str = match parsed.get("node_id").and_then(|v| v.as_str()) { + Some(s) => s, + None => return json_error(StatusCode::BAD_REQUEST, "missing node_id field"), + }; + let label = parsed + .get("label") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + + // Accept hex (64 chars) or base58 (~44 chars) + let bytes: [u8; 32] = if let Some(b) = distribution::identity::hex_decode(node_id_str) { + match b.try_into() { + Ok(arr) => arr, + Err(_) => { + return json_error( + StatusCode::BAD_REQUEST, + "invalid node_id (hex decoded to wrong length)", + ); + } + } + } else if let Some(arr) = distribution::identity::base58_decode(node_id_str) { + arr + } else { + return json_error( + StatusCode::BAD_REQUEST, + "invalid node_id (expected 64-char hex or base58)", + ); + }; + + let relay_url = parsed + .get("relay_url") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + + let node_id = distribution::types::NodeId(bytes); + let mut list = auth.lock().unwrap(); + list.add_peer(node_id, label); + if let Err(e) = list.save() { + eprintln!("warning: failed to persist peers.json: {e}"); + } + drop(list); + + // Trigger a SWIM join for the newly added peer + if let Some(tx) = state.join_sender.lock().unwrap().as_ref() { + let _ = tx.send((bytes, relay_url)); + } + + json_response(r#"{"ok":true}"#.to_string()) +} + +async fn handle_peers_sync(State(state): State, body: String) -> Response { + let maybe_auth = state.peer_auth.lock().unwrap().clone(); + let auth = match maybe_auth { + Some(a) => a, + None => return json_error(StatusCode::BAD_REQUEST, "peer auth not configured"), + }; + + let parsed: serde_json::Value = match serde_json::from_str(&body) { + Ok(v) => v, + Err(e) => return json_error(StatusCode::BAD_REQUEST, &format!("invalid JSON: {e}")), + }; + + let peers = match parsed.get("peers").and_then(|v| v.as_array()) { + Some(arr) => arr, + None => return json_error(StatusCode::BAD_REQUEST, "missing peers array"), + }; + + // Parse all peers first, bail on any error + let mut parsed_peers: Vec<(distribution::types::NodeId, String)> = Vec::new(); + for peer in peers { + let node_id_str = match peer.get("node_id").and_then(|v| v.as_str()) { + Some(s) => s, + None => return json_error(StatusCode::BAD_REQUEST, "peer missing node_id"), + }; + let label = peer + .get("label") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + + let bytes: [u8; 32] = if let Some(b) = distribution::identity::hex_decode(node_id_str) { + match b.try_into() { + Ok(arr) => arr, + Err(_) => { + return json_error( + StatusCode::BAD_REQUEST, + &format!("invalid node_id hex length for {node_id_str}"), + ); + } + } + } else if let Some(arr) = distribution::identity::base58_decode(node_id_str) { + arr + } else { + return json_error( + StatusCode::BAD_REQUEST, + &format!("invalid node_id: {node_id_str}"), + ); + }; + + parsed_peers.push((distribution::types::NodeId(bytes), label)); + } + + // Add all peers in a single lock acquisition + { + let mut list = auth.lock().unwrap(); + for (node_id, label) in &parsed_peers { + list.add_peer(*node_id, label.clone()); + } + if let Err(e) = list.save() { + eprintln!("warning: failed to persist peers.json: {e}"); + } + } + + // Trigger a SWIM join to the seed peer if specified + let join_seed = parsed.get("join_seed").and_then(|v| v.as_str()); + if let Some(seed_str) = join_seed { + let seed_bytes: Option<[u8; 32]> = + if let Some(b) = distribution::identity::hex_decode(seed_str) { + b.try_into().ok() + } else { + distribution::identity::base58_decode(seed_str) + }; + + if let Some(bytes) = seed_bytes { + // Find the relay_url for the seed from the peers array + let relay_url = peers.iter().find_map(|p| { + let nid = p.get("node_id").and_then(|v| v.as_str())?; + // Match by checking if this peer's node_id resolves to the same bytes + let peer_bytes: [u8; 32] = + if let Some(b) = distribution::identity::hex_decode(nid) { + b.try_into().ok()? + } else { + distribution::identity::base58_decode(nid)? + }; + if peer_bytes == bytes { + p.get("relay_url").and_then(|v| v.as_str()).map(|s| s.to_string()) + } else { + None + } + }); + + if let Some(tx) = state.join_sender.lock().unwrap().as_ref() { + let _ = tx.send((bytes, relay_url)); + } + } + } + + let added = parsed_peers.len(); + json_response(format!(r#"{{"ok":true,"added":{added}}}"#)) +} + +async fn handle_peers_remove(State(state): State, body: String) -> Response { + let maybe_auth = state.peer_auth.lock().unwrap().clone(); + let auth = match maybe_auth { + Some(a) => a, + None => return json_error(StatusCode::BAD_REQUEST, "peer auth not configured"), + }; + + let parsed: serde_json::Value = match serde_json::from_str(&body) { + Ok(v) => v, + Err(e) => return json_error(StatusCode::BAD_REQUEST, &format!("invalid JSON: {e}")), + }; + + let node_id_hex = match parsed.get("node_id").and_then(|v| v.as_str()) { + Some(s) => s, + None => return json_error(StatusCode::BAD_REQUEST, "missing node_id field"), + }; + + let bytes = match distribution::identity::hex_decode(node_id_hex) { + Some(b) if b.len() == 32 => b, + _ => { + return json_error( + StatusCode::BAD_REQUEST, + "invalid node_id hex (must be 64 hex chars)", + ); + } + }; + + let node_id = distribution::types::NodeId(bytes.try_into().unwrap()); + let mut list = auth.lock().unwrap(); + list.remove_peer(&node_id); + if let Err(e) = list.save() { + eprintln!("warning: failed to persist peers.json: {e}"); + } + + json_response(r#"{"ok":true}"#.to_string()) +} + +// ── Replay server ─────────────────────────────────────────────────────── + +#[derive(Clone)] +pub(crate) struct ReplayState { + pub trace: Arc, + pub speed: f64, +} + +async fn replay_page_dashboard() -> Response { + html_response(DASHBOARD_HTML, "replay") +} + +async fn replay_page_actors() -> Response { + html_response(ACTORS_HTML, "replay") +} + +async fn handle_replay_sse( + State(state): State, +) -> Sse>> { + let (tx, rx) = tokio::sync::mpsc::channel::(32); + + tokio::spawn(async move { + let trace = &state.trace; + let speed = state.speed; + + // Send replay metadata + let meta = serde_json::json!({ + "total_events": trace.events.len(), + "total_stats": trace.stats_timeline.len(), + "speed": speed, + }); + if tx.send(format_sse("replay_meta", &meta.to_string())).await.is_err() { + return; + } + + // Find the earliest timestamp across events and stats + let base_time = trace + .events + .first() + .map(|e| e.timestamp_ms) + .into_iter() + .chain(trace.stats_timeline.first().map(|s| s.timestamp_ms)) + .min() + .unwrap_or(0); + + let playback_start = tokio::time::Instant::now(); + let mut event_idx = 0; + let mut stats_idx = 0; + + loop { + let elapsed_ms = (playback_start.elapsed().as_millis() as f64 * speed) as u64; + let virtual_time = base_time + elapsed_ms; + + // Batch events up to virtual_time + let mut batch = Vec::new(); + while event_idx < trace.events.len() + && trace.events[event_idx].timestamp_ms <= virtual_time + { + batch.push(trace.events[event_idx].clone()); + event_idx += 1; + } + if !batch.is_empty() { + let json = serde_json::to_string(&batch).unwrap(); + if tx.send(format_sse("activity", &json)).await.is_err() { + return; + } + } + + // Send stats snapshots up to virtual_time + while stats_idx < trace.stats_timeline.len() + && trace.stats_timeline[stats_idx].timestamp_ms <= virtual_time + { + let json = + serde_json::to_string(&trace.stats_timeline[stats_idx].stats).unwrap(); + if tx.send(format_sse("stats", &json)).await.is_err() { + return; + } + stats_idx += 1; + } + + // Send progress + let total = trace.events.len() + trace.stats_timeline.len(); + let done_count = event_idx + stats_idx; + let progress = if total > 0 { + done_count as f64 / total as f64 + } else { + 1.0 + }; + let progress_json = serde_json::json!({ "progress": progress }); + if tx + .send(format_sse("replay_progress", &progress_json.to_string())) + .await + .is_err() + { + return; + } + + // Check if replay is complete + if event_idx >= trace.events.len() + && stats_idx >= trace.stats_timeline.len() + { + let _ = tx.send(format_sse("done", "{}")).await; + return; + } + + tokio::time::sleep(Duration::from_millis(50)).await; + } + }); + + Sse::new(ReceiverStream::new(rx).map(Ok)) + .keep_alive(KeepAlive::default()) +} diff --git a/crates/runtime-dashboard/src/topology.rs b/crates/dashboard/src/topology.rs similarity index 100% rename from crates/runtime-dashboard/src/topology.rs rename to crates/dashboard/src/topology.rs diff --git a/crates/runtime-dashboard/src/topology_html.rs b/crates/dashboard/src/topology_html.rs similarity index 100% rename from crates/runtime-dashboard/src/topology_html.rs rename to crates/dashboard/src/topology_html.rs diff --git a/crates/runtime-dashboard/src/trace.rs b/crates/dashboard/src/trace.rs similarity index 100% rename from crates/runtime-dashboard/src/trace.rs rename to crates/dashboard/src/trace.rs diff --git a/crates/runtime-dashboard/src/tui/app.rs b/crates/dashboard/src/tui/app.rs similarity index 99% rename from crates/runtime-dashboard/src/tui/app.rs rename to crates/dashboard/src/tui/app.rs index 7406d89..844345a 100644 --- a/crates/runtime-dashboard/src/tui/app.rs +++ b/crates/dashboard/src/tui/app.rs @@ -34,6 +34,7 @@ pub struct ActorRow { pub last_msg_type: Option, pub messages_processed: u64, pub poisoned: bool, + pub name: Option, /// Per-actor mailbox sparkline (from local ring buffer). pub sparkline_mailbox: Vec, /// Per-actor message rate sparkline. @@ -191,7 +192,8 @@ impl App { let addr = format!("{}", r.address).to_lowercase(); let msg = r.last_msg_type.as_deref().unwrap_or("").to_lowercase(); let worker = format!("w{}", r.worker_id); - addr.contains(&q) || msg.contains(&q) || worker.contains(&q) + let name = r.name.as_deref().unwrap_or("").to_lowercase(); + addr.contains(&q) || msg.contains(&q) || worker.contains(&q) || name.contains(&q) }).collect() } } @@ -335,6 +337,7 @@ impl App { last_msg_type: a.last_msg_type.clone(), messages_processed: a.messages_processed, poisoned: a.poisoned, + name: a.name.clone(), sparkline_rates: rates_buf.iter().copied().collect(), sparkline_mailbox: mbox_buf.iter().copied().collect(), message_type_counts: a.message_type_counts.clone(), diff --git a/crates/runtime-dashboard/src/tui/event.rs b/crates/dashboard/src/tui/event.rs similarity index 100% rename from crates/runtime-dashboard/src/tui/event.rs rename to crates/dashboard/src/tui/event.rs diff --git a/crates/runtime-dashboard/src/tui/mod.rs b/crates/dashboard/src/tui/mod.rs similarity index 98% rename from crates/runtime-dashboard/src/tui/mod.rs rename to crates/dashboard/src/tui/mod.rs index ceb22a1..b3297d3 100644 --- a/crates/runtime-dashboard/src/tui/mod.rs +++ b/crates/dashboard/src/tui/mod.rs @@ -107,6 +107,7 @@ fn run_loop( // Initial stats poll let mut stats = runtime.stats(); collector.enrich(&mut stats); + crate::collector::enrich_names(&mut stats, &runtime); app.update(stats); loop { @@ -116,6 +117,7 @@ fn run_loop( Ok(AppEvent::Tick) => { let mut stats = runtime.stats(); collector.enrich(&mut stats); + crate::collector::enrich_names(&mut stats, &runtime); app.update(stats); #[cfg(feature = "distribution")] if let Some(ref provider) = distribution { diff --git a/crates/runtime-dashboard/src/tui/sse_client.rs b/crates/dashboard/src/tui/sse_client.rs similarity index 100% rename from crates/runtime-dashboard/src/tui/sse_client.rs rename to crates/dashboard/src/tui/sse_client.rs diff --git a/crates/runtime-dashboard/src/tui/types.rs b/crates/dashboard/src/tui/types.rs similarity index 100% rename from crates/runtime-dashboard/src/tui/types.rs rename to crates/dashboard/src/tui/types.rs diff --git a/crates/runtime-dashboard/src/tui/ui.rs b/crates/dashboard/src/tui/ui.rs similarity index 92% rename from crates/runtime-dashboard/src/tui/ui.rs rename to crates/dashboard/src/tui/ui.rs index db26d13..adfbc2a 100644 --- a/crates/runtime-dashboard/src/tui/ui.rs +++ b/crates/dashboard/src/tui/ui.rs @@ -306,21 +306,30 @@ fn draw_actor_table(f: &mut Frame, app: &App, table_state: &mut TableState, area } fn actor_row_cells(a: &super::app::ActorRow) -> Row<'static> { - let addr = format!("{}", a.address); - let addr_short = if a.poisoned { - let s = if addr.len() > 16 { - format!("!{}...", &addr[..14]) + let addr_short = if let Some(ref name) = a.name { + if a.poisoned { + format!("!{}", name) } else { - format!("!{}", addr) - }; - s - } else if addr.len() > 18 { - format!("{}...", &addr[..16]) + name.clone() + } } else { - addr + let addr = format!("{}", a.address); + if a.poisoned { + if addr.len() > 16 { + format!("!{}...", &addr[..14]) + } else { + format!("!{}", addr) + } + } else if addr.len() > 18 { + format!("{}...", &addr[..16]) + } else { + addr + } }; let addr_style = if a.poisoned { Style::default().fg(Color::Red) + } else if a.name.is_some() { + Style::default().fg(Color::Cyan) } else { Style::default() }; @@ -465,21 +474,30 @@ fn draw_focused_actor_table(f: &mut Frame, app: &App, table_state: &mut TableSta let rows: Vec = focused_rows .iter() .map(|a| { - let addr = format!("{}", a.address); - let addr_short = if a.poisoned { - let s = if addr.len() > 16 { - format!("!{}...", &addr[..14]) + let addr_short = if let Some(ref name) = a.name { + if a.poisoned { + format!("!{}", name) } else { - format!("!{}", addr) - }; - s - } else if addr.len() > 18 { - format!("{}...", &addr[..16]) + name.clone() + } } else { - addr + let addr = format!("{}", a.address); + if a.poisoned { + if addr.len() > 16 { + format!("!{}...", &addr[..14]) + } else { + format!("!{}", addr) + } + } else if addr.len() > 18 { + format!("{}...", &addr[..16]) + } else { + addr + } }; let addr_style = if a.poisoned { Style::default().fg(Color::Red) + } else if a.name.is_some() { + Style::default().fg(Color::Cyan) } else { Style::default() }; @@ -563,32 +581,40 @@ fn draw_actor_detail(f: &mut Frame, app: &App) { .map(|s| short_type_name(Some(s))) .unwrap_or_else(|| "\u{2014}".to_string()); - let info_lines = vec![ - Line::from(vec![ + let mut info_lines = Vec::new(); + if let Some(ref name) = actor.name { + info_lines.push(Line::from(vec![ + Span::styled(" Name: ", Style::default().fg(Color::DarkGray)), + Span::styled(name.clone(), Style::default().fg(Color::Cyan).add_modifier(Modifier::BOLD)), + Span::styled(" Address: ", Style::default().fg(Color::DarkGray)), + Span::styled(addr_str, Style::default().fg(Color::White)), + ])); + } else { + info_lines.push(Line::from(vec![ Span::styled(" Address: ", Style::default().fg(Color::DarkGray)), Span::styled(addr_str, Style::default().fg(Color::White).add_modifier(Modifier::BOLD)), - ]), - Line::from(vec![ - Span::styled(" Worker: ", Style::default().fg(Color::DarkGray)), - Span::styled(format!("W{}", actor.worker_id), Style::default().fg(Color::Cyan)), - Span::styled(" Status: ", Style::default().fg(Color::DarkGray)), - Span::styled(status, Style::default().fg(status_color).add_modifier(Modifier::BOLD)), - ]), - Line::from(vec![ - Span::styled(" Messages: ", Style::default().fg(Color::DarkGray)), - Span::styled( - format_num(actor.messages_processed), - Style::default().fg(Color::Cyan).add_modifier(Modifier::BOLD), - ), - Span::styled(" Mailbox: ", Style::default().fg(Color::DarkGray)), - Span::styled( - format!("{}", actor.mailbox_depth), - Style::default().fg(Color::Blue).add_modifier(Modifier::BOLD), - ), - Span::styled(" Last Msg: ", Style::default().fg(Color::DarkGray)), - Span::styled(msg_type, Style::default().fg(Color::Green)), - ]), - ]; + ])); + } + info_lines.push(Line::from(vec![ + Span::styled(" Worker: ", Style::default().fg(Color::DarkGray)), + Span::styled(format!("W{}", actor.worker_id), Style::default().fg(Color::Cyan)), + Span::styled(" Status: ", Style::default().fg(Color::DarkGray)), + Span::styled(status, Style::default().fg(status_color).add_modifier(Modifier::BOLD)), + ])); + info_lines.push(Line::from(vec![ + Span::styled(" Messages: ", Style::default().fg(Color::DarkGray)), + Span::styled( + format_num(actor.messages_processed), + Style::default().fg(Color::Cyan).add_modifier(Modifier::BOLD), + ), + Span::styled(" Mailbox: ", Style::default().fg(Color::DarkGray)), + Span::styled( + format!("{}", actor.mailbox_depth), + Style::default().fg(Color::Blue).add_modifier(Modifier::BOLD), + ), + Span::styled(" Last Msg: ", Style::default().fg(Color::DarkGray)), + Span::styled(msg_type, Style::default().fg(Color::Green)), + ])); let info_block = Block::default() .borders(Borders::ALL) @@ -782,7 +808,7 @@ fn draw_dist_summary(f: &mut Frame, app: &App, area: Rect) { match &app.distribution { Some(d) => ( &d.node_id[..d.node_id.len().min(16)], - d.listen_addr.as_str(), + d.listen_addr.as_deref().unwrap_or("—"), d.alive_count, d.suspect_count, d.dead_count, @@ -863,7 +889,7 @@ fn draw_dist_members(f: &mut Frame, app: &App, table_state: &mut TableState, are Row::new(vec![ Cell::from(m.state.clone()).style(state_style), Cell::from(id_short), - Cell::from(m.addr.clone()), + Cell::from(m.addr.clone().unwrap_or_default()), Cell::from(format!("{}", m.incarnation)), ]) }) diff --git a/crates/runtime-dashboard/src/warnings.rs b/crates/dashboard/src/warnings.rs similarity index 99% rename from crates/runtime-dashboard/src/warnings.rs rename to crates/dashboard/src/warnings.rs index d1756d1..8fece1e 100644 --- a/crates/runtime-dashboard/src/warnings.rs +++ b/crates/dashboard/src/warnings.rs @@ -227,6 +227,7 @@ mod tests { fn make_actor(id: u8, depth: usize, msgs: u64, poisoned: bool) -> ActorInfo { ActorInfo { address: ActorAddress([id; 32]), + name: None, worker_id: 0, mailbox_depth: depth, last_msg_type: None, diff --git a/crates/runtime-dashboard/tests/command_api.rs b/crates/dashboard/tests/command_api.rs similarity index 98% rename from crates/runtime-dashboard/tests/command_api.rs rename to crates/dashboard/tests/command_api.rs index 3f68142..20412cf 100644 --- a/crates/runtime-dashboard/tests/command_api.rs +++ b/crates/dashboard/tests/command_api.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use swactor::actor::ActorInterface; use swactor::runtime::{Ctx, Runtime, RuntimeConfig}; -use runtime_dashboard::command::{ +use dashboard::command::{ from_query_params, parse_line, CommandContext, CommandResponse, CommandRouter, }; @@ -310,9 +310,9 @@ fn from_query_params_defaults_to_help() { #[test] fn custom_command_handler() { struct PingCommand; - impl runtime_dashboard::command::CommandHandler for PingCommand { - fn meta(&self) -> runtime_dashboard::command::CommandMeta { - runtime_dashboard::command::CommandMeta { + impl dashboard::command::CommandHandler for PingCommand { + fn meta(&self) -> dashboard::command::CommandMeta { + dashboard::command::CommandMeta { name: "ping", description: "Respond with pong", usage: "ping", diff --git a/crates/runtime-dashboard/tests/dashboard_core.rs b/crates/dashboard/tests/dashboard_core.rs similarity index 98% rename from crates/runtime-dashboard/tests/dashboard_core.rs rename to crates/dashboard/tests/dashboard_core.rs index 8bfe6c4..1beff8f 100644 --- a/crates/runtime-dashboard/tests/dashboard_core.rs +++ b/crates/dashboard/tests/dashboard_core.rs @@ -1,8 +1,8 @@ use std::sync::Arc; -use runtime_dashboard::collector::StatsCollector; -use runtime_dashboard::layer::{DashboardEvent, EventStore}; -use runtime_dashboard::trace::RuntimeTrace; +use dashboard::collector::StatsCollector; +use dashboard::layer::{DashboardEvent, EventStore}; +use dashboard::trace::RuntimeTrace; use swactor::actor::ActorAddress; use swactor::stats::{ActorSnapshot, StatsHook}; diff --git a/crates/datastore/Cargo.toml b/crates/datastore/Cargo.toml index d610dad..fcd9323 100644 --- a/crates/datastore/Cargo.toml +++ b/crates/datastore/Cargo.toml @@ -10,33 +10,23 @@ shared-types = { path = "../shared-types" } serde = { version = "1", features = ["derive"] } serde_json = "1" blake3 = "1" -tiny_http = { version = "0.12", optional = true } +tiny_http = "0.12" clap = { version = "4", features = ["derive"], optional = true } ureq = { version = "2", features = ["json"], optional = true } getrandom = { version = "0.2", optional = true } -ctrlc = { version = "3", optional = true } -runtime-dashboard = { path = "../runtime-dashboard", optional = true } -toml = { version = "0.8", optional = true } +dashboard = { path = "../dashboard" } +swactor-std = { path = "../std" } [dev-dependencies] serde_json = "1" proptest = "1" tempfile = "3" swactor = { path = "../.." } -swactor-std = { path = "../std" } ureq = { version = "2", features = ["json"] } -tiny_http = "0.12" -runtime-dashboard = { path = "../runtime-dashboard" } [features] -node = ["dep:tiny_http", "dep:clap", "dep:ctrlc", "dep:runtime-dashboard", "dep:toml"] cli = ["dep:clap", "dep:ureq", "dep:getrandom"] -[[bin]] -name = "swactor-store-node" -path = "src/bin/store_node.rs" -required-features = ["node"] - [[bin]] name = "swactor-store" path = "src/bin/store_cli.rs" diff --git a/crates/datastore/src/bin/store_cli.rs b/crates/datastore/src/bin/store_cli.rs index 001ab2f..37ded32 100644 --- a/crates/datastore/src/bin/store_cli.rs +++ b/crates/datastore/src/bin/store_cli.rs @@ -316,6 +316,75 @@ fn resolve_authorized_key(base: &str, name_input: &str, kp: &Keypair) -> String } } +// ── Key file helpers ──────────────────────────────────────────────────────── + +fn hex_decode(hex: &str) -> Option> { + if hex.len() % 2 != 0 { + return None; + } + let mut bytes = Vec::with_capacity(hex.len() / 2); + for chunk in hex.as_bytes().chunks(2) { + let hi = hex_digit(chunk[0])?; + let lo = hex_digit(chunk[1])?; + bytes.push((hi << 4) | lo); + } + Some(bytes) +} + +fn hex_digit(b: u8) -> Option { + match b { + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(b - b'a' + 10), + b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + } +} + +fn load_keypair(path: &std::path::Path) -> Keypair { + let data = fs::read_to_string(path).unwrap_or_else(|e| { + eprintln!("Error reading key file {}: {e}", path.display()); + std::process::exit(1); + }); + let json: serde_json::Value = serde_json::from_str(&data).unwrap_or_else(|e| { + eprintln!("Error parsing key file: {e}"); + std::process::exit(1); + }); + let secret_hex = json + .get("secret_key") + .and_then(|v| v.as_str()) + .unwrap_or_else(|| { + eprintln!("Key file missing secret_key field"); + std::process::exit(1); + }); + let secret_bytes = hex_decode(secret_hex).unwrap_or_else(|| { + eprintln!("Invalid secret_key hex in key file"); + std::process::exit(1); + }); + let secret: [u8; 32] = secret_bytes.try_into().unwrap_or_else(|_| { + eprintln!("secret_key must be exactly 32 bytes"); + std::process::exit(1); + }); + Keypair::from_bytes(&secret) +} + +// ── Auth signing ──────────────────────────────────────────────────────────── + +fn sign_action(keypair: &Keypair, action: DatastoreAction) -> String { + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + let mut nonce = [0u8; 16]; + getrandom::getrandom(&mut nonce).expect("failed to generate random nonce"); + let payload = SignedRequestPayload { + action, + timestamp, + nonce, + }; + let signed = sign_request(keypair, payload); + serde_json::to_string(&signed).expect("SignedRequest is always serializable") +} + fn main() { let args = Args::parse(); let base = args.url.trim_end_matches('/'); diff --git a/crates/datastore/src/bin/store_node.rs b/crates/datastore/src/bin/store_node.rs deleted file mode 100644 index 02e80f5..0000000 --- a/crates/datastore/src/bin/store_node.rs +++ /dev/null @@ -1,447 +0,0 @@ -//! swactor-store-node — standalone datastore node with HTTP API. -//! -//! Starts the actor runtime, spawns datastore actors, and serves -//! a REST API for external tools (the `swactor-store` CLI). - -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; -use std::thread; -use std::time::Duration; - -use clap::Parser; -use serde::Deserialize; - -use swactor::config::RuntimeConfig; -use swactor::runtime::Runtime; - -use swactor_datastore::actors::{BlobStoreActor, DatastoreNode, GatewayActor, MetadataActor}; -use swactor_datastore::api::start_api_server; -use swactor_datastore::auth::{AccessControlList, AuthzEngine}; -use swactor_datastore::messages::{GatewayMsg, MetadataMsg}; -use swactor_datastore::metrics::DatastoreMetrics; -use swactor_datastore::storage::{FilesystemBackend, InMemoryBackend}; -use swactor_datastore::DatastoreConfig; - -use distribution::crypto::Keypair; -use distribution::types::NodeId; - -#[derive(Parser)] -#[command(name = "swactor-store-node", about = "Swactor distributed datastore node")] -struct Args { - /// Path to a TOML config file - #[arg(long)] - config: Option, - - /// HTTP API port - #[arg(long)] - port: Option, - - /// Storage directory (omit for in-memory) - #[arg(long)] - storage_path: Option, - - /// Dashboard HTTP port (omit to disable dashboard) - #[arg(long)] - dashboard_port: Option, - - /// Chunk size in bytes - #[arg(long)] - chunk_size: Option, - - /// GC interval in ticks (each tick is ~100ms) - #[arg(long)] - gc_interval: Option, - - /// Dissemination interval in ticks - #[arg(long)] - disseminate_interval: Option, - - /// Enable auth (generates owner keypair if needed) - #[arg(long)] - auth: bool, - - /// Directory for owner.key.json + acl.json (default: "auth") - #[arg(long, default_value = "auth")] - auth_dir: String, -} - -#[derive(Deserialize, Default)] -struct NodeConfig { - port: Option, - storage_path: Option, - dashboard_port: Option, - chunk_size: Option, - gc_interval: Option, - disseminate_interval: Option, -} - -/// Resolved configuration with CLI > config file > defaults applied. -struct ResolvedConfig { - port: u16, - storage_path: Option, - dashboard_port: Option, - chunk_size: u32, - gc_interval: u64, - disseminate_interval: u64, -} - -// ── Key file helpers ──────────────────────────────────────────────────────── - -fn hex_encode(bytes: &[u8]) -> String { - bytes.iter().map(|b| format!("{b:02x}")).collect() -} - -fn hex_decode(hex: &str) -> Option> { - if hex.len() % 2 != 0 { - return None; - } - let mut bytes = Vec::with_capacity(hex.len() / 2); - for chunk in hex.as_bytes().chunks(2) { - let hi = hex_digit(chunk[0])?; - let lo = hex_digit(chunk[1])?; - bytes.push((hi << 4) | lo); - } - Some(bytes) -} - -fn hex_digit(b: u8) -> Option { - match b { - b'0'..=b'9' => Some(b - b'0'), - b'a'..=b'f' => Some(b - b'a' + 10), - b'A'..=b'F' => Some(b - b'A' + 10), - _ => None, - } -} - -fn load_or_generate_keypair(path: &std::path::Path) -> Keypair { - if path.exists() { - let data = std::fs::read_to_string(path).expect("failed to read key file"); - let json: serde_json::Value = serde_json::from_str(&data).expect("invalid key file JSON"); - let secret_hex = json - .get("secret_key") - .and_then(|v| v.as_str()) - .expect("key file missing secret_key"); - let secret_bytes = hex_decode(secret_hex).expect("invalid secret_key hex"); - let secret: [u8; 32] = secret_bytes - .try_into() - .expect("secret_key must be 32 bytes"); - Keypair::from_bytes(&secret) - } else { - let keypair = Keypair::generate(); - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_secs(); - let json = serde_json::json!({ - "version": 1, - "secret_key": hex_encode(&keypair.secret_bytes()), - "public_key": hex_encode(&keypair.node_id().0), - "created_at": format_timestamp(now), - }); - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent).expect("failed to create key file directory"); - } - std::fs::write(path, serde_json::to_string_pretty(&json).unwrap()) - .expect("failed to write key file"); - keypair - } -} - -fn format_timestamp(secs: u64) -> String { - // Simple ISO-8601 UTC timestamp - let s = secs % 60; - let m = (secs / 60) % 60; - let h = (secs / 3600) % 24; - let days = secs / 86400; - // Days since epoch to Y-M-D (simplified) - let (y, mo, d) = days_to_ymd(days); - format!("{y:04}-{mo:02}-{d:02}T{h:02}:{m:02}:{s:02}Z") -} - -fn days_to_ymd(mut days: u64) -> (u64, u64, u64) { - // Algorithm from http://howardhinnant.github.io/date_algorithms.html - days += 719468; - let era = days / 146097; - let doe = days - era * 146097; - let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; - let y = yoe + era * 400; - let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); - let mp = (5 * doy + 2) / 153; - let d = doy - (153 * mp + 2) / 5 + 1; - let m = if mp < 10 { mp + 3 } else { mp - 9 }; - let y = if m <= 2 { y + 1 } else { y }; - (y, m, d) -} - -fn resolve_config(args: &Args) -> ResolvedConfig { - let file_cfg = match &args.config { - Some(path) => { - let contents = std::fs::read_to_string(path) - .unwrap_or_else(|e| panic!("failed to read config file {}: {e}", path.display())); - toml::from_str::(&contents) - .unwrap_or_else(|e| panic!("failed to parse config file {}: {e}", path.display())) - } - None => NodeConfig::default(), - }; - - ResolvedConfig { - port: args.port.or(file_cfg.port).unwrap_or(9091), - storage_path: args.storage_path.clone().or(file_cfg.storage_path), - dashboard_port: args.dashboard_port.or(file_cfg.dashboard_port), - chunk_size: args.chunk_size.or(file_cfg.chunk_size).unwrap_or(1_048_576), - gc_interval: args.gc_interval.or(file_cfg.gc_interval).unwrap_or(1000), - disseminate_interval: args.disseminate_interval.or(file_cfg.disseminate_interval).unwrap_or(50), - } -} - -fn main() { - let args = Args::parse(); - let cfg = resolve_config(&args); - let stop = Arc::new(AtomicBool::new(false)); - - // Signal handler — double Ctrl-C forces immediate exit - { - let stop = Arc::clone(&stop); - ctrlc::set_handler(move || { - if stop.load(Ordering::Relaxed) { - eprintln!("\nForced exit."); - std::process::exit(1); - } - stop.store(true, Ordering::Relaxed); - }) - .expect("failed to set signal handler"); - } - - // Optionally start dashboard - let dash = cfg.dashboard_port.map(|port| { - let d = runtime_dashboard::start_dashboard(runtime_dashboard::DashboardConfig { - port, - ..Default::default() - }); - d.install_tracing(); - d - }); - - // Create runtime - let num_threads = 2; - let collector = runtime_dashboard::collector::StatsCollector::new(num_threads); - let mut rt = Runtime::new(RuntimeConfig { - num_threads, - max_actors: 1024, - channel_buffer_size: 2000, - ..Default::default() - }); - rt.set_stats_hook(collector.clone()); - - // Generate or load node identity - let (node_id, owner_keypair) = if args.auth { - let auth_dir = std::path::PathBuf::from(&args.auth_dir); - std::fs::create_dir_all(&auth_dir).expect("failed to create auth directory"); - let key_path = auth_dir.join("owner.key.json"); - let keypair = load_or_generate_keypair(&key_path); - let nid = keypair.node_id(); - eprintln!( - "Auth enabled — owner key: {}", - hex_encode(&nid.0) - ); - eprintln!("Key file: {}", key_path.display()); - (nid, Some((keypair, auth_dir))) - } else { - let node_id = { - let mut bytes = [0u8; 32]; - for (i, b) in std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - .to_le_bytes() - .iter() - .enumerate() - { - bytes[i % 32] ^= *b; - } - // Mix in process id for uniqueness - let pid = std::process::id(); - for (i, b) in pid.to_le_bytes().iter().enumerate() { - bytes[i + 16] ^= *b; - } - NodeId(bytes) - }; - (node_id, None) - }; - - // Datastore config - let config = DatastoreConfig { - chunk_size: cfg.chunk_size, - storage_path: cfg - .storage_path - .as_ref() - .map(|s| s.into()) - .unwrap_or_else(|| "datastore".into()), - gc_interval: cfg.gc_interval, - ..Default::default() - }; - - // Create storage backend - let backend: Box = match &cfg.storage_path { - Some(path) => { - let p = std::path::PathBuf::from(path); - std::fs::create_dir_all(&p).expect("failed to create storage directory"); - Box::new(FilesystemBackend::new(p)) - } - None => Box::new(InMemoryBackend::new()), - }; - - // Spawn actors before starting runtime threads - let blob_store_addr = rt - .spawn(BlobStoreActor::new(backend)) - .expect("failed to spawn BlobStoreActor"); - - let mut metadata = MetadataActor::new(node_id, &config); - metadata.set_blob_store(blob_store_addr); - let metadata_addr = rt - .spawn(metadata) - .expect("failed to spawn MetadataActor"); - - let datastore_node = DatastoreNode::new(node_id, blob_store_addr, metadata_addr, config); - let datastore_addr = rt - .spawn(datastore_node) - .expect("failed to spawn DatastoreNode"); - - // Spawn GatewayActor if auth is enabled - let gateway_addr = if let Some((_, ref auth_dir)) = owner_keypair { - let acl_path = auth_dir.join("acl.json"); - let acl = AccessControlList::load_or_create(&acl_path, node_id) - .expect("failed to load/create ACL"); - let engine = AuthzEngine::new(acl); - let gateway = GatewayActor::new(engine, datastore_addr, Some(acl_path)); - let addr = rt - .spawn(gateway) - .expect("failed to spawn GatewayActor"); - Some(addr) - } else { - None - }; - - // Start runtime - let handle = rt.run().expect("failed to start runtime"); - - // Load persisted entries from storage - { - let inbox = handle - .runtime - .new_inbox::() - .expect("failed to create inbox"); - let _ = handle.runtime.send_to( - blob_store_addr, - swactor_datastore::BlobStoreMsg::LoadAll { - reply_to: *inbox.addr(), - }, - ); - // Poll for response (up to 5 seconds) - let start = std::time::Instant::now(); - let mut loaded = false; - while start.elapsed() < Duration::from_secs(5) { - if let Some(resp) = inbox.try_recv() { - match resp { - swactor_datastore::DatastoreResponse::LoadedAll { entries } => { - let n = entries.len(); - let _ = handle.runtime.send_to( - metadata_addr, - swactor_datastore::MetadataMsg::BulkLoad { entries }, - ); - if n > 0 { - eprintln!("Loaded {n} entries from storage"); - } - loaded = true; - } - swactor_datastore::DatastoreResponse::Error { reason } => { - eprintln!("Warning: failed to load entries: {reason}"); - loaded = true; - } - _ => {} - } - break; - } - thread::sleep(Duration::from_millis(1)); - } - if !loaded { - eprintln!("Warning: timed out loading entries from storage"); - } - } - - // Create datastore metrics - let node_hex: String = node_id.0.iter().map(|b| format!("{b:02x}")).collect(); - let metrics = Arc::new(DatastoreMetrics::new()); - metrics.set_node_id(node_hex.clone()); - - if let Some(ref d) = dash { - d.set_runtime(handle.runtime.clone(), collector); - d.set_datastore(Arc::clone(&metrics) as Arc); - } - - // Start HTTP API - let (api_shutdown, _peers) = start_api_server( - handle.runtime.clone(), - datastore_addr, - metadata_addr, - blob_store_addr, - gateway_addr, - cfg.port, - Arc::clone(&metrics), - ); - - eprintln!("──────────────────────────────────────"); - eprintln!(" swactor-store node {}", &node_hex[..16]); - eprintln!(" API: http://0.0.0.0:{}", cfg.port); - if let Some(port) = cfg.dashboard_port { - eprintln!(" Dashboard: http://0.0.0.0:{port}"); - } - if cfg.storage_path.is_some() { - eprintln!(" Storage: {} (filesystem)", cfg.storage_path.as_ref().unwrap()); - } else { - eprintln!(" Storage: in-memory"); - } - if owner_keypair.is_some() { - eprintln!(" Auth: enabled (owner {})", &node_hex[..16]); - } else { - eprintln!(" Auth: disabled"); - } - eprintln!("──────────────────────────────────────"); - - // Main loop - let mut round: u64 = 0; - while !stop.load(Ordering::Relaxed) { - round += 1; - - if round % cfg.gc_interval == 0 { - let _ = handle - .runtime - .send_to(metadata_addr, MetadataMsg::GcTick); - - if let Some(gw) = gateway_addr { - let _ = handle.runtime.send_to(gw, GatewayMsg::NonceGcTick); - } - } - - if round % cfg.disseminate_interval == 0 { - let _ = handle - .runtime - .send_to(metadata_addr, MetadataMsg::DisseminateTick); - } - - thread::sleep(Duration::from_millis(100)); - } - - eprintln!("\nShutting down..."); - api_shutdown.store(true, Ordering::Relaxed); - handle.shutdown(); - if let Some(d) = dash { - d.shutdown(); - } - // Brief pause for threads to flush I/O, then exit. - // No join — cargo run already died from SIGINT so there's - // no parent waiting on us; just exit cleanly. - thread::sleep(Duration::from_millis(50)); - eprintln!("Shutdown complete."); - std::process::exit(0); -} diff --git a/crates/datastore/src/bridge.rs b/crates/datastore/src/bridge.rs new file mode 100644 index 0000000..5e77b38 --- /dev/null +++ b/crates/datastore/src/bridge.rs @@ -0,0 +1,519 @@ +//! Bridge between the runtime dashboard's `DatastoreStatsProvider` trait and +//! the datastore actor system. Allows the dashboard to perform CRUD operations +//! and lifecycle management without depending on `swactor-datastore` types. + +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::sync::Arc; +use std::thread; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use swactor::actor::ActorAddress; +use swactor::runtime::{Inbox, Runtime}; +use swactor_std::RuntimeNaming; + +use distribution::types::NodeId; +use dashboard::datastore_collector::{ + DatastoreFactory, DatastoreStatsProvider, ListScope, +}; + +use crate::actors::{BlobStoreActor, DatastoreNode, GatewayActor, MetadataActor}; +use crate::auth::{AccessControlList, AuthzEngine}; +use crate::chunking::reassemble_blob; +use crate::messages::{DatastoreNodeMsg, DatastoreResponse, GatewayMsg, MetadataMsg}; +use crate::metrics::DatastoreMetrics; +use crate::storage::{FilesystemBackend, InMemoryBackend}; +use crate::types::{ContentHash, DatastoreConfig}; + +const POLL_TIMEOUT: Duration = Duration::from_secs(5); +const POLL_INTERVAL: Duration = Duration::from_millis(1); + +fn poll_response(inbox: &Inbox, timeout: Duration) -> Option { + let start = Instant::now(); + loop { + if let Some(resp) = inbox.try_recv() { + return Some(resp); + } + if start.elapsed() > timeout { + return None; + } + thread::sleep(POLL_INTERVAL); + } +} + +fn entry_to_json(entry: &crate::types::ObjectEntry) -> serde_json::Value { + let node_hex: String = entry.node_id.0.iter().map(|b| format!("{b:02x}")).collect(); + serde_json::json!({ + "content_hash": entry.content_hash.to_hex(), + "name": entry.name, + "node_id": node_hex, + "tags": entry.tags, + "size_bytes": entry.size_bytes, + "created_at": entry.created_at, + }) +} + +fn manifest_to_json(manifest: &crate::types::ObjectManifest) -> serde_json::Value { + let chunks: Vec = manifest + .chunks + .iter() + .map(|c| { + serde_json::json!({ + "hash": c.hash.to_hex(), + "offset": c.offset, + "size": c.size, + }) + }) + .collect(); + serde_json::json!({ + "content_hash": manifest.content_hash.to_hex(), + "chunks": chunks, + "total_size": manifest.total_size, + "chunk_size": manifest.chunk_size, + "content_type": manifest.content_type, + }) +} + +fn entries_to_json(entries: &[crate::types::ObjectEntry]) -> Vec { + entries.iter().map(entry_to_json).collect() +} + +/// Bridges the dashboard trait to the datastore actor system. +pub struct DatastoreBridge { + metrics: Arc, + runtime: Arc, + datastore_addr: ActorAddress, + metadata_addr: ActorAddress, + #[allow(dead_code)] + blob_store_addr: ActorAddress, +} + +impl DatastoreBridge { + pub fn new( + metrics: Arc, + runtime: Arc, + datastore_addr: ActorAddress, + metadata_addr: ActorAddress, + blob_store_addr: ActorAddress, + ) -> Self { + Self { + metrics, + runtime, + datastore_addr, + metadata_addr, + blob_store_addr, + } + } +} + +impl DatastoreStatsProvider for DatastoreBridge { + fn snapshot_json(&self) -> Option { + let snap = self.metrics.snapshot(); + serde_json::to_string(&snap).ok() + } + + fn is_running(&self) -> bool { + true + } + + fn list_objects(&self, name_filter: Option<&str>, scope: ListScope) -> Result { + let inbox = self.runtime.new_inbox::() + .map_err(|e| format!("failed to create inbox: {e}"))?; + + match scope { + ListScope::Local => { + let _ = self.runtime.send_to( + self.datastore_addr, + DatastoreNodeMsg::List { + name_filter: name_filter.map(|s| s.to_string()), + all: false, + reply_to: *inbox.addr(), + }, + ); + } + ListScope::Swarm => { + let _ = self.runtime.send_to( + self.metadata_addr, + MetadataMsg::ListLocal { + name_filter: name_filter.map(|s| s.to_string()), + reply_to: *inbox.addr(), + }, + ); + } + } + + match poll_response(&inbox, POLL_TIMEOUT) { + Some(DatastoreResponse::ListOk { entries }) => { + let json = serde_json::json!({ "entries": entries_to_json(&entries) }).to_string(); + Ok(json) + } + Some(DatastoreResponse::Error { reason }) => Err(reason), + _ => Err("timeout".into()), + } + } + + fn get_object(&self, hash: &str) -> Result { + let content_hash = ContentHash::from_hex(hash) + .ok_or_else(|| "invalid content hash hex".to_string())?; + + let inbox = self.runtime.new_inbox::() + .map_err(|e| format!("failed to create inbox: {e}"))?; + + let _ = self.runtime.send_to( + self.datastore_addr, + DatastoreNodeMsg::Get { + content_hash, + reply_to: *inbox.addr(), + }, + ); + + match poll_response(&inbox, POLL_TIMEOUT) { + Some(DatastoreResponse::GetOk { entry, manifest }) => { + self.metrics.record_get(&content_hash.to_hex()); + let json = serde_json::json!({ + "entry": entry_to_json(&entry), + "manifest": manifest_to_json(&manifest), + }) + .to_string(); + Ok(json) + } + Some(DatastoreResponse::NotFound) => Err("not found".into()), + Some(DatastoreResponse::Error { reason }) => Err(reason), + _ => Err("timeout".into()), + } + } + + fn get_data(&self, hash: &str) -> Result, String> { + let content_hash = ContentHash::from_hex(hash) + .ok_or_else(|| "invalid content hash hex".to_string())?; + + // Get manifest + let inbox = self.runtime.new_inbox::() + .map_err(|e| format!("failed to create inbox: {e}"))?; + + let _ = self.runtime.send_to( + self.datastore_addr, + DatastoreNodeMsg::Get { + content_hash, + reply_to: *inbox.addr(), + }, + ); + + self.metrics.record_get(&content_hash.to_hex()); + + let manifest = match poll_response(&inbox, POLL_TIMEOUT) { + Some(DatastoreResponse::GetOk { manifest, .. }) => manifest, + Some(DatastoreResponse::NotFound) => return Err("not found".into()), + Some(DatastoreResponse::Error { reason }) => return Err(reason), + _ => return Err("timeout".into()), + }; + + // Read chunks + let mut chunk_data = Vec::new(); + for chunk_ref in &manifest.chunks { + let chunk_inbox = self.runtime.new_inbox::() + .map_err(|e| format!("failed to create inbox: {e}"))?; + + let _ = self.runtime.send_to( + self.datastore_addr, + DatastoreNodeMsg::ReadChunk { + hash: chunk_ref.hash, + reply_to: *chunk_inbox.addr(), + }, + ); + + match poll_response(&chunk_inbox, POLL_TIMEOUT) { + Some(DatastoreResponse::ChunkOk { hash, data }) => { + chunk_data.push((hash, data)); + } + _ => return Err("failed to read chunk".into()), + } + } + + reassemble_blob(&manifest, &chunk_data) + .map_err(|e| format!("reassembly failed: {e:?}")) + } + + fn put_data(&self, data: Vec, name: Option) -> Result { + let body_len = data.len(); + + let inbox = self.runtime.new_inbox::() + .map_err(|e| format!("failed to create inbox: {e}"))?; + + let _ = self.runtime.send_to( + self.datastore_addr, + DatastoreNodeMsg::Put { + data, + name: name.clone(), + tags: BTreeMap::new(), + reply_to: *inbox.addr(), + }, + ); + + match poll_response(&inbox, POLL_TIMEOUT) { + Some(DatastoreResponse::PutOk { content_hash }) => { + let hex = content_hash.to_hex(); + self.metrics.record_put(&hex, name.as_deref(), body_len as u64); + let json = serde_json::json!({ "content_hash": hex }).to_string(); + Ok(json) + } + Some(DatastoreResponse::Error { reason }) => Err(reason), + _ => Err("timeout waiting for put response".into()), + } + } + + fn delete_object(&self, hash: &str) -> Result { + let content_hash = ContentHash::from_hex(hash) + .ok_or_else(|| "invalid content hash hex".to_string())?; + + let inbox = self.runtime.new_inbox::() + .map_err(|e| format!("failed to create inbox: {e}"))?; + + let _ = self.runtime.send_to( + self.datastore_addr, + DatastoreNodeMsg::Delete { + content_hash, + reply_to: *inbox.addr(), + }, + ); + + match poll_response(&inbox, POLL_TIMEOUT) { + Some(DatastoreResponse::DeleteOk { content_hash }) => { + let hex = content_hash.to_hex(); + self.metrics.record_delete(&hex, 0); + let json = serde_json::json!({ "content_hash": hex }).to_string(); + Ok(json) + } + Some(DatastoreResponse::NotFound) => Err("not found".into()), + Some(DatastoreResponse::Error { reason }) => Err(reason), + _ => Err("timeout".into()), + } + } + + fn node_status(&self) -> Result { + let inbox = self.runtime.new_inbox::() + .map_err(|e| format!("failed to create inbox: {e}"))?; + + let _ = self.runtime.send_to( + self.datastore_addr, + DatastoreNodeMsg::Status { + reply_to: *inbox.addr(), + }, + ); + + match poll_response(&inbox, POLL_TIMEOUT) { + Some(DatastoreResponse::NodeStatus { node_id }) => { + let hex: String = node_id.0.iter().map(|b| format!("{b:02x}")).collect(); + let json = serde_json::json!({ "node_id": hex }).to_string(); + Ok(json) + } + _ => Err("timeout".into()), + } + } + + fn shutdown_datastore(&self) -> Result<(), String> { + // We can't actually stop the actors from here without a runtime handle, + // but we can signal shutdown. The caller (server handler) clears the + // provider reference which effectively disables the datastore. + Ok(()) + } +} + +/// Factory that can spawn a new set of datastore actors on a shared runtime. +pub struct DatastoreNodeFactory { + runtime: Arc, + default_chunk_size: u32, +} + +impl DatastoreNodeFactory { + pub fn new(runtime: Arc, default_chunk_size: u32) -> Self { + Self { + runtime, + default_chunk_size, + } + } +} + +impl DatastoreFactory for DatastoreNodeFactory { + fn start_datastore( + &self, + storage_path: Option, + ) -> Result, String> { + // Generate a unique node ID + let node_id = generate_node_id(); + let node_hex: String = node_id.0.iter().map(|b| format!("{b:02x}")).collect(); + + let group = DatastoreGroup::spawn( + Arc::clone(&self.runtime), + DatastoreGroupConfig { + node_id, + node_id_hex: node_hex, + chunk_size: self.default_chunk_size, + storage_path, + auth: None, + gc_interval: 1000, + disseminate_interval: 50, + }, + )?; + + Ok(group.bridge().clone()) + } +} + +// ─── DatastoreGroup ───────────────────────────────────────────────────────── + +/// Configuration for spawning a complete datastore actor group. +pub struct DatastoreGroupConfig { + pub node_id: NodeId, + pub node_id_hex: String, + pub chunk_size: u32, + pub storage_path: Option, + pub auth: Option, + pub gc_interval: u64, + pub disseminate_interval: u64, +} + +/// Auth configuration for the datastore gateway. +pub struct DatastoreAuthConfig { + pub auth_dir: PathBuf, +} + +/// Owns the full lifecycle of a datastore actor group: BlobStore, Metadata, +/// DatastoreNode, and optional GatewayActor. +pub struct DatastoreGroup { + metadata_addr: ActorAddress, + gateway_addr: Option, + bridge: Arc, + gc_interval: u64, + disseminate_interval: u64, + runtime: Arc, +} + +impl DatastoreGroup { + /// Spawn all datastore actors, wire them together, and register names. + pub fn spawn(runtime: Arc, config: DatastoreGroupConfig) -> Result { + let ds_config = DatastoreConfig { + chunk_size: config.chunk_size, + storage_path: config + .storage_path + .as_ref() + .map(|s| s.into()) + .unwrap_or_else(|| "datastore".into()), + ..Default::default() + }; + + let backend: Box = match &config.storage_path { + Some(path) => { + let p = PathBuf::from(path); + std::fs::create_dir_all(&p) + .map_err(|e| format!("failed to create storage directory: {e}"))?; + Box::new(FilesystemBackend::new(p)) + } + None => Box::new(InMemoryBackend::new()), + }; + + let blob_store_addr = runtime + .spawn(BlobStoreActor::new(backend)) + .map_err(|e| format!("failed to spawn BlobStoreActor: {e}"))?; + let _ = runtime.register_name("BlobStore", blob_store_addr); + + let mut metadata = MetadataActor::new(config.node_id, &ds_config); + metadata.set_blob_store(blob_store_addr); + let metadata_addr = runtime + .spawn(metadata) + .map_err(|e| format!("failed to spawn MetadataActor: {e}"))?; + let _ = runtime.register_name("Metadata", metadata_addr); + + let datastore_node = + DatastoreNode::new(config.node_id, blob_store_addr, metadata_addr, ds_config); + let datastore_addr = runtime + .spawn(datastore_node) + .map_err(|e| format!("failed to spawn DatastoreNode: {e}"))?; + let _ = runtime.register_name("Datastore", datastore_addr); + + // Spawn GatewayActor if auth is configured + let gateway_addr = if let Some(auth_cfg) = &config.auth { + std::fs::create_dir_all(&auth_cfg.auth_dir) + .map_err(|e| format!("failed to create auth directory: {e}"))?; + let acl_path = auth_cfg.auth_dir.join("acl.json"); + let acl = AccessControlList::load_or_create(&acl_path, config.node_id) + .map_err(|e| format!("failed to load/create ACL: {e}"))?; + let engine = AuthzEngine::new(acl); + let gateway = GatewayActor::new(engine, datastore_addr, Some(acl_path)); + let addr = runtime + .spawn(gateway) + .map_err(|e| format!("failed to spawn GatewayActor: {e}"))?; + let _ = runtime.register_name("Gateway", addr); + eprintln!("Auth: enabled (owner {})", &config.node_id_hex[..16]); + Some(addr) + } else { + eprintln!("Auth: disabled"); + None + }; + + let metrics = Arc::new(DatastoreMetrics::new()); + metrics.set_node_id(config.node_id_hex.clone()); + + let bridge: Arc = Arc::new(DatastoreBridge::new( + metrics, + Arc::clone(&runtime), + datastore_addr, + metadata_addr, + blob_store_addr, + )); + + if config.storage_path.is_some() { + eprintln!( + "Datastore: persistent ({})", + config.storage_path.as_ref().unwrap() + ); + } else { + eprintln!("Datastore: in-memory"); + } + + Ok(Self { + metadata_addr, + gateway_addr, + bridge, + gc_interval: config.gc_interval, + disseminate_interval: config.disseminate_interval, + runtime, + }) + } + + /// Send periodic ticks to the datastore actors based on the current round. + pub fn tick(&self, round: u64) { + if round % self.gc_interval == 0 { + let _ = self.runtime.send_to(self.metadata_addr, MetadataMsg::GcTick); + if let Some(gw) = self.gateway_addr { + let _ = self.runtime.send_to(gw, GatewayMsg::NonceGcTick); + } + } + if round % self.disseminate_interval == 0 { + let _ = self + .runtime + .send_to(self.metadata_addr, MetadataMsg::DisseminateTick); + } + } + + /// Access the bridge (as a trait object for the dashboard). + pub fn bridge(&self) -> &Arc { + &self.bridge + } +} + +fn generate_node_id() -> NodeId { + let mut bytes = [0u8; 32]; + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + for (i, b) in nanos.to_le_bytes().iter().enumerate() { + bytes[i % 32] ^= *b; + } + let pid = std::process::id(); + for (i, b) in pid.to_le_bytes().iter().enumerate() { + bytes[i + 16] ^= *b; + } + NodeId(bytes) +} diff --git a/crates/datastore/src/lib.rs b/crates/datastore/src/lib.rs index 1768caa..1578c07 100644 --- a/crates/datastore/src/lib.rs +++ b/crates/datastore/src/lib.rs @@ -6,13 +6,13 @@ pub mod actors; pub mod auth; pub mod cli; pub mod metrics; -#[cfg(feature = "node")] pub mod api; -#[cfg(feature = "node")] pub mod ui_html; +pub mod bridge; pub use types::{ChunkRef, ContentHash, DatastoreConfig, ObjectEntry, ObjectManifest}; pub use messages::{BlobStoreMsg, DatastoreNodeMsg, DatastoreResponse, MetadataMsg, TransferMsg}; pub use chunking::{chunk_blob, reassemble_blob, verify_integrity, ChunkingError}; pub use storage::{StorageBackend, FilesystemBackend, InMemoryBackend}; pub use actors::{BlobStoreActor, DatastoreNode, MetadataActor, TransferActor}; +pub use bridge::{DatastoreGroup, DatastoreGroupConfig, DatastoreAuthConfig}; diff --git a/crates/datastore/src/messages.rs b/crates/datastore/src/messages.rs index fd5f8ae..92d0d4b 100644 --- a/crates/datastore/src/messages.rs +++ b/crates/datastore/src/messages.rs @@ -460,3 +460,4 @@ pub enum GatewayMsg { /// Periodic nonce garbage collection tick. NonceGcTick, } + diff --git a/crates/datastore/src/metrics.rs b/crates/datastore/src/metrics.rs index 5e6f2b0..9db347e 100644 --- a/crates/datastore/src/metrics.rs +++ b/crates/datastore/src/metrics.rs @@ -186,10 +186,9 @@ fn now_ms() -> u64 { .as_millis() as u64 } -// ── Dashboard integration (only when runtime-dashboard is available) ──────── +// ── Dashboard integration ──────────────────────────────────────────────────── -#[cfg(feature = "node")] -impl runtime_dashboard::datastore_collector::DatastoreStatsProvider for DatastoreMetrics { +impl dashboard::datastore_collector::DatastoreStatsProvider for DatastoreMetrics { fn snapshot_json(&self) -> Option { let snap = self.snapshot(); serde_json::to_string(&snap).ok() diff --git a/crates/datastore/tests/api_integration_test.rs b/crates/datastore/tests/api_integration_test.rs index 2786728..280a7c4 100644 --- a/crates/datastore/tests/api_integration_test.rs +++ b/crates/datastore/tests/api_integration_test.rs @@ -1,5 +1,3 @@ -#![cfg(feature = "node")] - //! Integration test: spins up a real datastore node with HTTP API and exercises //! the full CRUD lifecycle over HTTP. @@ -34,7 +32,7 @@ fn http_crud_lifecycle() { let base = format!("http://127.0.0.1:{port}"); // Set up runtime with worker threads (needed for HTTP server) - let collector = runtime_dashboard::collector::StatsCollector::new(2); + let collector = dashboard::collector::StatsCollector::new(2); let mut rt = Runtime::new(RuntimeConfig { num_threads: 2, max_actors: 1024, diff --git a/crates/datastore/tests/dashboard_integration_test.rs b/crates/datastore/tests/dashboard_integration_test.rs index 8fa8def..74c7adf 100644 --- a/crates/datastore/tests/dashboard_integration_test.rs +++ b/crates/datastore/tests/dashboard_integration_test.rs @@ -1,5 +1,3 @@ -#![cfg(feature = "node")] - //! End-to-end integration test: spins up a datastore node with an HTTP API //! **and** a runtime dashboard, performs CRUD over HTTP, then verifies: //! @@ -46,12 +44,12 @@ fn dashboard_reflects_datastore_operations() { // ── Infrastructure: runtime + actors + dashboard + API ────────────── - let dash = runtime_dashboard::start_dashboard(runtime_dashboard::DashboardConfig { + let dash = dashboard::start_dashboard(dashboard::DashboardConfig { port: dash_port, ..Default::default() }); - let collector = runtime_dashboard::collector::StatsCollector::new(2); + let collector = dashboard::collector::StatsCollector::new(2); let mut rt = Runtime::new(RuntimeConfig { num_threads: 2, max_actors: 1024, @@ -86,7 +84,7 @@ fn dashboard_reflects_datastore_operations() { dash.set_runtime(handle.runtime.clone(), collector); dash.set_datastore( Arc::clone(&metrics) - as Arc, + as Arc, ); let (api_shutdown, _peers) = start_api_server( diff --git a/crates/datastore/tests/http_auth_integration.rs b/crates/datastore/tests/http_auth_integration.rs index 879de9a..aa0aba6 100644 --- a/crates/datastore/tests/http_auth_integration.rs +++ b/crates/datastore/tests/http_auth_integration.rs @@ -4,8 +4,6 @@ //! and uses ureq to prove that authorized requests succeed while unauthorized //! ones get 403 and missing-auth requests get 401. -#![cfg(feature = "node")] - use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::atomic::Ordering; use std::time::{Duration, SystemTime, UNIX_EPOCH}; diff --git a/crates/distribution/Cargo.toml b/crates/distribution/Cargo.toml index cba55a8..16fc876 100644 --- a/crates/distribution/Cargo.toml +++ b/crates/distribution/Cargo.toml @@ -7,6 +7,7 @@ edition = "2024" default = ["tcp"] tcp = [] iroh = ["dep:iroh", "dep:tokio"] +relay = ["iroh", "dep:iroh-relay"] [dependencies] swactor = { path = "../..", features = ["serde", "transport"] } @@ -15,8 +16,12 @@ ed25519-dalek = { version = "2", features = ["rand_core"] } rand_core = { version = "0.6", features = ["getrandom"] } serde = { version = "1", features = ["derive"] } serde_json = "1" +bs58 = "0.5" iroh = { version = "0.96", optional = true } +iroh-relay = { version = "0.96", features = ["server"], optional = true } tokio = { version = "1", features = ["rt-multi-thread"], optional = true } [dev-dependencies] serde_json = "1" +iroh-relay = { version = "0.96", features = ["server", "test-utils"] } +tokio = { version = "1", features = ["rt-multi-thread"] } diff --git a/crates/distribution/src/driver.rs b/crates/distribution/src/driver.rs index 3be39ef..5500b2d 100644 --- a/crates/distribution/src/driver.rs +++ b/crates/distribution/src/driver.rs @@ -95,6 +95,25 @@ impl NodeDriver { }) } + /// Create a driver with a specific keypair for persistent identity. + pub fn with_keypair( + listen_addr: SocketAddr, + keypair: crate::crypto::Keypair, + config: DistributedNodeConfig, + ) -> Result { + let acceptor = TcpAcceptor::bind(listen_addr)?; + let actual_addr = acceptor.local_addr(); + let node = DistributedNode::with_keypair(keypair, config); + Ok(Self { + node, + transport: TcpTransport::pool(), + acceptor, + streams: Vec::new(), + address_book: PeerAddressBook::new(), + listen_addr: actual_addr, + }) + } + /// The node's identity. pub fn node_id(&self) -> NodeId { self.node.node_id() @@ -192,6 +211,42 @@ impl NodeDriver { } } + /// Process incoming TCP messages with peer auth filtering. + /// + /// Same as `recv()`, but checks the sender's NodeId against the + /// peer allow-list before dispatching. Unauthorized messages are dropped. + pub fn recv_with_auth( + &mut self, + peer_auth: &std::sync::Arc>, + ) { + let envelopes = self.acceptor.try_recv(&mut self.streams); + for (envelope, _peer_addr, hints_bytes) in envelopes { + // Extract sender NodeId from hints + let mut sender_node_id = None; + if !hints_bytes.is_empty() { + if let Ok(hints) = serde_json::from_slice::>(&hints_bytes) { + if let Some(first) = hints.first() { + sender_node_id = Some(first.node_id); + } + self.learn_hints(&hints); + } + } + + // Check peer auth if we know the sender + if let Some(node_id) = sender_node_id { + let allowed = peer_auth.lock().unwrap().is_allowed(&node_id); + if !allowed { + let hex: String = node_id.0[..4].iter().map(|b| format!("{b:02x}")).collect(); + eprintln!("driver: rejected message from unauthorized peer {hex}"); + continue; + } + } + + let response_actions = self.dispatch_incoming(envelope); + self.send_actions(&response_actions); + } + } + // ─── Outgoing: NodeAction → TCP ───────────────────────────────────── fn send_actions(&mut self, actions: &[NodeAction]) { @@ -282,6 +337,21 @@ impl NodeDriver { self.send_wire_with_hints::(&msg, dest, &hints) } + NodeAction::ForwardAck { + to, + target, + sequence, + piggyback, + } => { + let dest = self.resolve_addr(to)?; + let msg = IndirectAck { + target: *target, + sequence: *sequence, + piggyback: piggyback.clone(), + }; + self.send_wire_with_hints::(&msg, dest, &[sender_hint]) + } + NodeAction::MembershipChanged { .. } => { // Internal notification — no network I/O. Ok(()) @@ -386,6 +456,14 @@ impl NodeDriver { } }, + "swactor_dist::IndirectAck" => match decode::(&envelope.payload) { + Ok(msg) => self.node.handle_indirect_ack(msg.target, msg.sequence, &msg.piggyback), + Err(e) => { + eprintln!("driver: decode IndirectAck: {e}"); + Vec::new() + } + }, + other => { eprintln!("driver: unknown message type: {other}"); Vec::new() diff --git a/crates/distribution/src/identity.rs b/crates/distribution/src/identity.rs new file mode 100644 index 0000000..5867dc0 --- /dev/null +++ b/crates/distribution/src/identity.rs @@ -0,0 +1,118 @@ +//! Shared keypair persistence — load or generate an ed25519 keypair on disk. +//! +//! Reused by `swactor-node`, `store_node`, and xtask tooling. + +use std::path::Path; + +use crate::crypto::Keypair; + +/// Hex-encode a byte slice. +pub fn hex_encode(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// Hex-decode a string into bytes. Returns `None` on invalid input. +pub fn hex_decode(hex: &str) -> Option> { + if hex.len() % 2 != 0 { + return None; + } + let mut bytes = Vec::with_capacity(hex.len() / 2); + for chunk in hex.as_bytes().chunks(2) { + let hi = hex_digit(chunk[0])?; + let lo = hex_digit(chunk[1])?; + bytes.push((hi << 4) | lo); + } + Some(bytes) +} + +fn hex_digit(b: u8) -> Option { + match b { + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(b - b'a' + 10), + b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + } +} + +/// Base58-encode a byte slice (Bitcoin alphabet). +pub fn base58_encode(bytes: &[u8]) -> String { + bs58::encode(bytes).into_string() +} + +/// Base58-decode a string into a 32-byte array. Returns `None` on invalid input. +pub fn base58_decode(s: &str) -> Option<[u8; 32]> { + let bytes = bs58::decode(s).into_vec().ok()?; + bytes.try_into().ok() +} + +/// Load a keypair from `path`, or generate a new one and persist it. +/// +/// The file format is JSON: +/// ```json +/// { +/// "version": 1, +/// "secret_key": "", +/// "public_key": "", +/// "created_at": "2025-01-01T00:00:00Z" +/// } +/// ``` +pub fn load_or_generate_keypair(path: &Path) -> Keypair { + if path.exists() { + let data = std::fs::read_to_string(path).expect("failed to read key file"); + let json: serde_json::Value = + serde_json::from_str(&data).expect("invalid key file JSON"); + let secret_hex = json + .get("secret_key") + .and_then(|v| v.as_str()) + .expect("key file missing secret_key"); + let secret_bytes = hex_decode(secret_hex).expect("invalid secret_key hex"); + let secret: [u8; 32] = secret_bytes + .try_into() + .expect("secret_key must be 32 bytes"); + Keypair::from_bytes(&secret) + } else { + let keypair = Keypair::generate(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + let json = serde_json::json!({ + "version": 1, + "secret_key": hex_encode(&keypair.secret_bytes()), + "public_key": hex_encode(&keypair.node_id().0), + "created_at": format_timestamp(now), + }); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).expect("failed to create key file directory"); + } + std::fs::write(path, serde_json::to_string_pretty(&json).unwrap()) + .expect("failed to write key file"); + keypair + } +} + +/// Simple ISO-8601 UTC timestamp from epoch seconds. +pub fn format_timestamp(secs: u64) -> String { + let s = secs % 60; + let m = (secs / 60) % 60; + let h = (secs / 3600) % 24; + let days = secs / 86400; + let (y, mo, d) = days_to_ymd(days); + format!("{y:04}-{mo:02}-{d:02}T{h:02}:{m:02}:{s:02}Z") +} + +/// Convert days since epoch to (year, month, day). +/// Algorithm from . +pub fn days_to_ymd(mut days: u64) -> (u64, u64, u64) { + days += 719468; + let era = days / 146097; + let doe = days - era * 146097; + let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; + let y = yoe + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = if m <= 2 { y + 1 } else { y }; + (y, m, d) +} diff --git a/crates/distribution/src/iroh_driver.rs b/crates/distribution/src/iroh_driver.rs index a6cc41e..cab5ee7 100644 --- a/crates/distribution/src/iroh_driver.rs +++ b/crates/distribution/src/iroh_driver.rs @@ -8,16 +8,17 @@ //! (`tick()`, `recv()`, `join()`) to match the existing main loop pattern. use std::collections::HashMap; +use std::sync::{Arc, Mutex}; use std::time::Duration; use iroh::endpoint::Connection; -use iroh::{Endpoint, PublicKey, RelayMode, SecretKey}; +use iroh::{Endpoint, EndpointAddr, PublicKey, RelayMode, SecretKey}; use tokio::runtime::Runtime as TokioRuntime; - use crate::crypto::Keypair; use crate::messages::*; use crate::node::{DistributedNode, DistributedNodeConfig}; +use crate::peer_auth::PeerAllowList; use crate::snapshot::DistributionNodeSnapshot; use crate::swim::node::NodeAction; use crate::types::NodeId; @@ -37,6 +38,26 @@ pub struct IrohDriverConfig { pub relay_mode: RelayMode, /// Protocol-layer configuration. pub node: DistributedNodeConfig, + /// Optional peer allow-list. If provided, only allowed peers can connect. + pub peer_auth: Option>>, + /// If set, start an embedded relay server on this address. + /// Requires the `relay` feature. On success, the driver uses the embedded + /// relay for `RelayMode::Custom`; on failure, falls back to `relay_mode`. + #[cfg(feature = "relay")] + pub embedded_relay_bind: Option, + /// Public IP to advertise in the relay URL instead of the bind address. + /// When `Some`, the relay URL uses this IP; when `None`, falls back to the + /// bind address (which may be `0.0.0.0`). + #[cfg(feature = "relay")] + pub relay_public_ip: Option, +} + +// ─── Pending join result ──────────────────────────────────────────────────── + +/// Result of a background join attempt, collected during `recv()`. +struct JoinResult { + node_id: NodeId, + conn: Connection, } // ─── Driver ───────────────────────────────────────────────────────────────── @@ -50,6 +71,18 @@ pub struct IrohDriver { endpoint: Endpoint, rt: TokioRuntime, connections: HashMap, + peer_auth: Option>>, + /// Collects connections from background join tasks. + pending_joins: Arc>>, + /// Connections accepted by the background accept loop. + accepted_conns: Arc>>, + /// Relay URLs learned from join seeds, used for reconnection. + peer_relay_urls: HashMap, + /// Embedded relay server (if started). + #[cfg(feature = "relay")] + relay_server: Option, + /// URL of the embedded relay server (if started). + relay_url: Option, } impl IrohDriver { @@ -57,15 +90,40 @@ impl IrohDriver { /// /// Builds a tokio runtime, creates an iroh `Endpoint`, and initializes /// the protocol-layer `DistributedNode`. + /// + /// If `embedded_relay_bind` is set (requires `relay` feature), the driver + /// starts an embedded relay server on the tokio runtime before creating + /// the endpoint. On success the endpoint uses the embedded relay; on + /// failure it falls back to `config.relay_mode`. pub fn new(config: IrohDriverConfig) -> Result> { let rt = tokio::runtime::Builder::new_multi_thread() .enable_all() .build()?; + // Try to start embedded relay if configured + #[cfg(feature = "relay")] + let (relay_server, relay_url, effective_relay_mode) = match config.embedded_relay_bind { + Some(bind_addr) => { + match rt.block_on(start_embedded_relay(bind_addr, config.relay_public_ip)) { + Ok((server, url)) => { + let url_str = url.to_string(); + eprintln!("Relay: embedded relay started at {url}"); + (Some(server), Some(url_str), RelayMode::Custom(url.into())) + } + Err(e) => { + eprintln!("Relay: failed to start embedded relay: {e}, falling back"); + (None, None, config.relay_mode) + } + } + } + None => (None, None, config.relay_mode), + }; + #[cfg(not(feature = "relay"))] + let (relay_url, effective_relay_mode) = (None::, config.relay_mode); + let endpoint = rt.block_on(async { - let mut builder = Endpoint::builder() - .alpns(vec![ALPN.to_vec()]) - .relay_mode(config.relay_mode); + let mut builder = Endpoint::empty_builder(effective_relay_mode) + .alpns(vec![ALPN.to_vec()]); if let Some(key) = config.secret_key { builder = builder.secret_key(key); @@ -80,19 +138,100 @@ impl IrohDriver { let keypair = Keypair::from_bytes(&iroh_secret); let node = DistributedNode::with_keypair(keypair, config.node); + // Spawn background accept loop so incoming connections are never missed + let accepted_conns: Arc>> = + Arc::new(Mutex::new(Vec::new())); + { + let ep = endpoint.clone(); + let peer_auth = config.peer_auth.clone(); + let buf = Arc::clone(&accepted_conns); + rt.spawn(async move { + loop { + match ep.accept().await { + Some(incoming) => match incoming.await { + Ok(conn) => { + let remote_id = conn.remote_id(); + let node_id = NodeId(*remote_id.as_bytes()); + // Peer auth check + let allowed = match &peer_auth { + None => true, + Some(auth) => auth.lock().unwrap().is_allowed(&node_id), + }; + if !allowed { + eprintln!( + "iroh driver: rejected connection from unauthorized peer {}", + crate::identity::hex_encode(&node_id.0[..4]) + ); + conn.close(0u32.into(), b"unauthorized"); + continue; + } + eprintln!( + "iroh driver: accepted connection from {}", + crate::identity::hex_encode(&node_id.0[..4]) + ); + buf.lock().unwrap().push((node_id, conn)); + } + Err(e) => { + eprintln!("iroh driver: incoming connection error: {e}"); + } + }, + None => break, // endpoint closed + } + } + }); + } + Ok(Self { node, endpoint, rt, connections: HashMap::new(), + peer_auth: config.peer_auth, + pending_joins: Arc::new(Mutex::new(Vec::new())), + accepted_conns, + peer_relay_urls: HashMap::new(), + #[cfg(feature = "relay")] + relay_server, + relay_url, }) } + /// Get a handle to the tokio runtime owned by this driver. + pub fn tokio_handle(&self) -> tokio::runtime::Handle { + self.rt.handle().clone() + } + /// The node's identity. pub fn node_id(&self) -> NodeId { self.node.node_id() } + /// The endpoint's full address (public key + direct socket addresses). + /// + /// Constructs the address from the endpoint's public key and bound + /// sockets. Unspecified addresses (`0.0.0.0` / `[::]`) are mapped to + /// their loopback equivalents so peers on the same host can connect. + pub fn endpoint_addr(&self) -> EndpointAddr { + use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; + + let key = PublicKey::from_bytes(&self.node.node_id().0) + .expect("node_id is a valid public key"); + let mut addr = EndpointAddr::new(key); + for sock in self.endpoint.bound_sockets() { + let resolved = match sock.ip() { + IpAddr::V4(ip) if ip.is_unspecified() => { + SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), sock.port()) + } + IpAddr::V6(ip) if ip.is_unspecified() => { + SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), sock.port()) + } + _ => sock, + }; + addr = addr.with_ip_addr(resolved); + } + addr + } + /// Access the underlying node (read-only). pub fn node(&self) -> &DistributedNode { &self.node @@ -116,38 +255,98 @@ impl IrohDriver { /// Join a cluster by connecting to seed nodes via iroh. /// - /// Each seed is identified by its iroh `PublicKey` (= our `NodeId`). - pub fn join(&mut self, seeds: &[PublicKey]) { - for seed_key in seeds { - let seed_node_id = NodeId(*seed_key.as_bytes()); - if let Err(e) = self.send_join_request(*seed_key, seed_node_id) { - eprintln!("iroh driver: join error to {seed_key}: {e}"); + /// Each seed is identified by its `EndpointAddr` (public key + optional + /// direct addresses). Connect+send is spawned as a background task so + /// that the peer can accept the connection during its `recv()` cycle. + /// Results are collected in the next `recv()` call. + pub fn join(&mut self, seeds: &[EndpointAddr]) { + for seed_addr in seeds { + // Store relay URL for future reconnection + let seed_node_id = NodeId(*seed_addr.id.as_bytes()); + if let Some(relay) = seed_addr.relay_urls().next() { + self.peer_relay_urls.insert(seed_node_id, relay.clone()); } + self.spawn_join_request(seed_addr.clone()); } } - fn send_join_request( - &mut self, - seed_key: PublicKey, - seed_node_id: NodeId, - ) -> Result<(), Box> { + fn spawn_join_request(&self, seed_addr: EndpointAddr) { let msg = JoinRequest { from: self.node.node_id(), }; - let payload = serde_json::to_vec(&msg)?; + let payload = serde_json::to_vec(&msg).expect("serialize JoinRequest"); let tag = ::type_tag(); - let endpoint = self.endpoint.clone(); - let conn = self.rt.block_on(async { - let conn = endpoint.connect(seed_key, ALPN).await?; - let mut send = conn.open_uni().await?; - write_message(&mut send, tag.as_bytes(), &payload).await?; - send.finish()?; - Ok::<_, Box>(conn) - })?; + let seed_node_id = NodeId(*seed_addr.id.as_bytes()); + let pending = Arc::clone(&self.pending_joins); - self.connections.insert(seed_node_id, conn); - Ok(()) + self.rt.spawn(async move { + let mut delay = Duration::from_secs(2); + let max_delay = Duration::from_secs(30); + let max_attempts = 5; + + for attempt in 1..=max_attempts { + if attempt > 1 { + tokio::time::sleep(delay).await; + delay = (delay * 2).min(max_delay); + } + + eprintln!("iroh driver: join attempt {attempt}/{max_attempts} connecting to {}...", seed_addr.id); + let connect_result = tokio::time::timeout( + Duration::from_secs(10), + endpoint.connect(seed_addr.clone(), ALPN), + ).await; + + match connect_result { + Ok(Ok(conn)) => { + eprintln!("iroh driver: join attempt {attempt}/{max_attempts} connected to {}, sending...", seed_addr.id); + let send_result: Result<(), String> = async { + let mut send = conn.open_uni().await.map_err(|e| e.to_string())?; + let tag_len = (tag.len() as u32).to_be_bytes(); + send.write_all(&tag_len).await.map_err(|e| e.to_string())?; + send.write_all(tag.as_bytes()).await.map_err(|e| e.to_string())?; + send.write_all(&payload).await.map_err(|e| e.to_string())?; + send.finish().map_err(|e| e.to_string())?; + Ok(()) + } + .await; + + match send_result { + Ok(()) => { + eprintln!("iroh driver: join attempt {attempt}/{max_attempts} sent to {}", seed_addr.id); + pending.lock().unwrap().push(JoinResult { + node_id: seed_node_id, + conn, + }); + return; + } + Err(e) => { + eprintln!( + "iroh driver: join attempt {attempt}/{max_attempts} send error to {}: {e}", + seed_addr.id + ); + continue; + } + } + } + Ok(Err(e)) => { + eprintln!( + "iroh driver: join attempt {attempt}/{max_attempts} connect error to {}: {e}", + seed_addr.id + ); + continue; + } + Err(_) => { + eprintln!( + "iroh driver: join attempt {attempt}/{max_attempts} connect timeout to {}", + seed_addr.id + ); + continue; + } + } + } + eprintln!("iroh driver: join failed after {max_attempts} attempts to {}", seed_addr.id); + }); } /// Advance the node by one tick. @@ -158,12 +357,23 @@ impl IrohDriver { /// Process incoming iroh connections and messages (non-blocking). pub fn recv(&mut self) { + // Collect completed background join connections + { + let mut pending = self.pending_joins.lock().unwrap(); + if !pending.is_empty() { + eprintln!("iroh driver: collecting {} pending join connection(s)", pending.len()); + } + for result in pending.drain(..) { + self.connections.insert(result.node_id, result.conn); + } + } + let (incoming, new_conns) = self.rt.block_on(async { self.receive_pending().await }); - // Cache connections accepted from remote peers + // Cache connections accepted from remote peers (replace stale ones) for (node_id, conn) in new_conns { - self.connections.entry(node_id).or_insert(conn); + self.connections.insert(node_id, conn); } for (tag, payload, from_key) in incoming { let from = NodeId(*from_key.as_bytes()); @@ -232,6 +442,11 @@ impl IrohDriver { self.send_message(to, &msg) } + NodeAction::ForwardAck { to, target, sequence, piggyback } => { + let msg = IndirectAck { target: *target, sequence: *sequence, piggyback: piggyback.clone() }; + self.send_message(to, &msg) + } + NodeAction::MembershipChanged { .. } => Ok(()), } } @@ -274,6 +489,15 @@ impl IrohDriver { node_id: NodeId, key: PublicKey, ) -> Result> { + // Defense in depth: check peer auth before connecting + if !self.is_peer_allowed(&node_id) { + return Err(format!( + "peer {} not in allow-list", + crate::identity::hex_encode(&node_id.0[..4]) + ) + .into()); + } + // Check for cached connection that's still open if let Some(conn) = self.connections.get(&node_id) { if conn.close_reason().is_none() { @@ -284,9 +508,16 @@ impl IrohDriver { } let endpoint = self.endpoint.clone(); - let conn = self.rt.block_on(async { - endpoint.connect(key, ALPN).await - })?; + let conn = if let Some(relay) = self.peer_relay_urls.get(&node_id) { + let addr = EndpointAddr::new(key).with_relay_url(relay.clone()); + self.rt.block_on(async { + endpoint.connect(addr, ALPN).await + })? + } else { + self.rt.block_on(async { + endpoint.connect(key, ALPN).await + })? + }; self.connections.insert(node_id, conn.clone()); Ok(conn) @@ -294,26 +525,26 @@ impl IrohDriver { // ─── Incoming: iroh → handler ──────────────────────────────────── + fn is_peer_allowed(&self, node_id: &NodeId) -> bool { + match &self.peer_auth { + None => true, + Some(auth) => auth.lock().unwrap().is_allowed(node_id), + } + } + async fn receive_pending(&self) -> (Vec<(String, Vec, PublicKey)>, Vec<(NodeId, Connection)>) { let mut messages = Vec::new(); - let mut new_connections = Vec::new(); - // Poll for incoming connections with a short timeout - loop { - let accept_fut = self.endpoint.accept(); - let result = tokio::time::timeout(Duration::from_millis(1), accept_fut).await; + // Drain connections accepted by the background accept loop + let new_connections: Vec<(NodeId, Connection)> = { + let mut buf = self.accepted_conns.lock().unwrap(); + buf.drain(..).collect() + }; - match result { - Ok(Some(incoming)) => { - if let Ok(conn) = incoming.await { - let remote_id = conn.remote_id(); - self.read_streams(&conn, remote_id, &mut messages).await; - let node_id = NodeId(*remote_id.as_bytes()); - new_connections.push((node_id, conn)); - } - } - _ => break, - } + // Read streams from newly accepted connections + for (node_id, conn) in &new_connections { + let remote_id = PublicKey::from_bytes(&node_id.0).unwrap(); + self.read_streams(conn, remote_id, &mut messages).await; } // Also read from existing cached connections @@ -328,6 +559,10 @@ impl IrohDriver { self.read_streams(&conn, remote_id, &mut messages).await; } + if !messages.is_empty() { + eprintln!("iroh driver: received {} message(s)", messages.len()); + } + (messages, new_connections) } @@ -409,6 +644,14 @@ impl IrohDriver { } } + "swactor_dist::IndirectAck" => match serde_json::from_slice::(payload) { + Ok(msg) => self.node.handle_indirect_ack(msg.target, msg.sequence, &msg.piggyback), + Err(e) => { + eprintln!("iroh driver: decode IndirectAck: {e}"); + Vec::new() + } + }, + other => { eprintln!("iroh driver: unknown message type: {other}"); Vec::new() @@ -416,14 +659,65 @@ impl IrohDriver { } } - /// Shut down the iroh endpoint. - pub fn shutdown(&self) { + /// URL of the embedded relay server, if one was started. + pub fn relay_url(&self) -> Option<&str> { + self.relay_url.as_deref() + } + + /// The endpoint's home relay URL (from RelayMode::Custom), if connected. + pub fn home_relay_url(&self) -> Option { + self.endpoint.addr().relay_urls().next().cloned() + } + + /// Shut down the driver: stop the embedded relay (if any), then close the + /// iroh endpoint. + pub fn shutdown(&mut self) { + // Shut down embedded relay first (must stop before endpoint closes) + #[cfg(feature = "relay")] + if let Some(server) = self.relay_server.take() { + self.rt.block_on(async { + let _ = server.shutdown().await; + }); + } self.rt.block_on(async { self.endpoint.close().await; }); } } +// ─── Embedded Relay ───────────────────────────────────────────────────────── + +#[cfg(feature = "relay")] +async fn start_embedded_relay( + bind_addr: std::net::SocketAddr, + public_ip: Option, +) -> Result<(iroh_relay::server::Server, iroh::RelayUrl), Box> { + let server = iroh_relay::server::Server::spawn( + iroh_relay::server::ServerConfig::<(), ()> { + relay: Some(iroh_relay::server::RelayConfig { + http_bind_addr: bind_addr, + tls: None, + limits: Default::default(), + key_cache_capacity: Some(256), + access: iroh_relay::server::AccessConfig::Everyone, + }), + quic: None, + metrics_addr: None, + }, + ) + .await?; + + let url: iroh::RelayUrl = match server.http_addr() { + Some(addr) => { + let host = public_ip.unwrap_or_else(|| addr.ip()); + format!("http://{}:{}/", host, addr.port()).parse()? + } + None => return Err("relay server has no HTTP address".into()), + }; + + Ok((server, url)) +} + // ─── Wire Framing Over QUIC Streams ───────────────────────────────────────── /// Write a tagged message to a QUIC send stream. diff --git a/crates/distribution/src/lib.rs b/crates/distribution/src/lib.rs index 503bbf0..eb62498 100644 --- a/crates/distribution/src/lib.rs +++ b/crates/distribution/src/lib.rs @@ -1,5 +1,7 @@ pub mod types; pub mod crypto; +pub mod identity; +pub mod peer_auth; pub mod messages; pub mod codec; #[cfg(feature = "tcp")] @@ -9,6 +11,7 @@ pub mod kademlia; pub mod cache; pub mod node; pub mod registry; +pub mod node_metadata; pub mod snapshot; #[cfg(feature = "tcp")] pub mod driver; diff --git a/crates/distribution/src/messages.rs b/crates/distribution/src/messages.rs index 8cba98e..5ef8d14 100644 --- a/crates/distribution/src/messages.rs +++ b/crates/distribution/src/messages.rs @@ -61,6 +61,24 @@ impl NetworkMessage for PingReq { } } +/// SWIM indirect ack — "the target you asked me to ping is alive" +/// +/// Sent by a relay node back to the original prober after the relay +/// receives an ack from the indirect-ping target. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IndirectAck { + pub target: NodeId, + pub sequence: u64, + #[serde(default)] + pub piggyback: Vec, +} + +impl NetworkMessage for IndirectAck { + fn type_tag() -> &'static str { + "swactor_dist::IndirectAck" + } +} + /// SWIM join request — "I want to join the cluster" #[derive(Debug, Clone, Serialize, Deserialize)] pub struct JoinRequest { diff --git a/crates/distribution/src/node.rs b/crates/distribution/src/node.rs index d7e4690..566447c 100644 --- a/crates/distribution/src/node.rs +++ b/crates/distribution/src/node.rs @@ -10,6 +10,7 @@ use crate::crypto::Keypair; use crate::kademlia::directory::{actor_addr_as_node_id, DirectoryShard}; use crate::kademlia::repair::{RepairQueue, RepublishTracker}; use crate::kademlia::routing_table::RoutingTable; +use crate::node_metadata::{NodeMetadataDisseminator, NodeMetadataEntry}; use crate::registry::{ pack_combined_piggyback, unpack_combined_piggyback, ClusterRegistry, RegistryConfig, RegistryEntry, RegistryEvent, @@ -25,6 +26,8 @@ pub struct DistributedNodeConfig { pub cache_capacity: usize, pub republish_interval: u64, pub registry: RegistryConfig, + /// Dissemination multiplier for node metadata (default: 3). + pub metadata_lambda: usize, } impl Default for DistributedNodeConfig { @@ -34,6 +37,7 @@ impl Default for DistributedNodeConfig { cache_capacity: 10_000, republish_interval: 1000, registry: RegistryConfig::default(), + metadata_lambda: 3, } } } @@ -51,6 +55,7 @@ pub struct DistributedNode { repair_queue: RepairQueue, republish: RepublishTracker, registry: ClusterRegistry, + metadata: NodeMetadataDisseminator, tick_count: u64, } @@ -72,6 +77,7 @@ impl DistributedNode { repair_queue: RepairQueue::new(), republish: RepublishTracker::new(config.republish_interval), registry: ClusterRegistry::new(config.registry), + metadata: NodeMetadataDisseminator::new(config.metadata_lambda), tick_count: 0, keypair, } @@ -138,35 +144,51 @@ impl DistributedNode { // Registry GC self.registry.gc_tick(); - // Wrap outgoing piggyback with registry entries - self.inject_registry_piggyback(actions) + // Wrap outgoing piggyback with registry + metadata entries + self.inject_piggyback(actions) } // ─── SWIM message handling (delegate to SwimNode) ─────────────────── pub fn handle_ping(&mut self, from: NodeId, sequence: u64, piggyback: &[u8]) -> Vec { - let (membership_bytes, registry_entries) = unpack_combined_piggyback(piggyback); + let (membership_bytes, registry_entries, metadata_entries) = + unpack_combined_piggyback(piggyback); let actions = self.swim.handle_ping(from, sequence, &membership_bytes); self.process_membership_changes(&actions); self.merge_registry_entries(registry_entries); + self.merge_metadata_entries(metadata_entries); self.maybe_update_routing_table(from); - self.inject_registry_piggyback(actions) + self.inject_piggyback(actions) } pub fn handle_ack(&mut self, from: NodeId, sequence: u64, piggyback: &[u8]) -> Vec { - let (membership_bytes, registry_entries) = unpack_combined_piggyback(piggyback); + let (membership_bytes, registry_entries, metadata_entries) = + unpack_combined_piggyback(piggyback); let actions = self.swim.handle_ack(from, sequence, &membership_bytes); self.process_membership_changes(&actions); self.merge_registry_entries(registry_entries); - self.inject_registry_piggyback(actions) + self.merge_metadata_entries(metadata_entries); + self.inject_piggyback(actions) } pub fn handle_ping_req(&mut self, from: NodeId, target: NodeId, sequence: u64, piggyback: &[u8]) -> Vec { - let (membership_bytes, registry_entries) = unpack_combined_piggyback(piggyback); + let (membership_bytes, registry_entries, metadata_entries) = + unpack_combined_piggyback(piggyback); let actions = self.swim.handle_ping_req(from, target, sequence, &membership_bytes); self.process_membership_changes(&actions); self.merge_registry_entries(registry_entries); - self.inject_registry_piggyback(actions) + self.merge_metadata_entries(metadata_entries); + self.inject_piggyback(actions) + } + + pub fn handle_indirect_ack(&mut self, target: NodeId, sequence: u64, piggyback: &[u8]) -> Vec { + let (membership_bytes, registry_entries, metadata_entries) = + unpack_combined_piggyback(piggyback); + let actions = self.swim.handle_indirect_ack(target, sequence, &membership_bytes); + self.process_membership_changes(&actions); + self.merge_registry_entries(registry_entries); + self.merge_metadata_entries(metadata_entries); + self.inject_piggyback(actions) } pub fn handle_join_request(&mut self, from: NodeId) -> Vec { @@ -261,6 +283,24 @@ impl DistributedNode { &self.registry } + // ─── Node metadata (relay URL) ───────────────────────────────────── + + /// Set this node's relay URL and begin gossiping it to the cluster. + pub fn set_relay_url(&mut self, url: Option) { + self.metadata + .set_local(self.node_id(), url, self.cluster_size()); + } + + /// Look up a node's relay URL. + pub fn relay_url(&self, node_id: &NodeId) -> Option<&str> { + self.metadata.relay_url(node_id) + } + + /// Read-only access to the metadata disseminator. + pub fn metadata(&self) -> &NodeMetadataDisseminator { + &self.metadata + } + // ─── Accessors ────────────────────────────────────────────────────── pub fn routing_table(&self) -> &RoutingTable { @@ -306,15 +346,18 @@ impl DistributedNode { match state { MemberState::Alive => { self.routing_table.insert(node_id); - // Re-disseminate registry entries so the recovering node - // catches up on state accumulated during the partition. - self.registry.re_disseminate_all(self.cluster_size()); + // Re-disseminate registry + metadata entries so the recovering + // node catches up on state accumulated during the partition. + let size = self.cluster_size(); + self.registry.re_disseminate_all(size); + self.metadata.re_disseminate_all(size); } MemberState::Dead => { self.routing_table.remove(&node_id); self.cache.invalidate_node(&node_id); self.repair_queue.on_node_death(&node_id, &mut self.directory); self.registry.tombstone_node(node_id, self.cluster_size()); + self.metadata.remove_node(&node_id); } MemberState::Suspect => { // Keep in routing table but could downprioritize @@ -326,26 +369,35 @@ impl DistributedNode { self.swim.members().alive_count() + 1 // +1 for self } - /// Post-process outgoing actions: wrap each piggyback with registry entries. - fn inject_registry_piggyback(&mut self, actions: Vec) -> Vec { + /// Post-process outgoing actions: wrap each piggyback with registry + metadata entries. + fn inject_piggyback(&mut self, actions: Vec) -> Vec { actions .into_iter() .map(|action| match action { NodeAction::SendPing { to, sequence, piggyback } => { let registry_entries = self.registry.take_pending(8); - let combined = pack_combined_piggyback(piggyback, registry_entries); + let metadata_entries = self.metadata.take_pending(4); + let combined = pack_combined_piggyback(piggyback, registry_entries, metadata_entries); NodeAction::SendPing { to, sequence, piggyback: combined } } NodeAction::SendAck { to, sequence, piggyback } => { let registry_entries = self.registry.take_pending(8); - let combined = pack_combined_piggyback(piggyback, registry_entries); + let metadata_entries = self.metadata.take_pending(4); + let combined = pack_combined_piggyback(piggyback, registry_entries, metadata_entries); NodeAction::SendAck { to, sequence, piggyback: combined } } NodeAction::SendPingReq { relay, target, sequence, piggyback } => { let registry_entries = self.registry.take_pending(8); - let combined = pack_combined_piggyback(piggyback, registry_entries); + let metadata_entries = self.metadata.take_pending(4); + let combined = pack_combined_piggyback(piggyback, registry_entries, metadata_entries); NodeAction::SendPingReq { relay, target, sequence, piggyback: combined } } + NodeAction::ForwardAck { to, target, sequence, piggyback } => { + let registry_entries = self.registry.take_pending(8); + let metadata_entries = self.metadata.take_pending(4); + let combined = pack_combined_piggyback(piggyback, registry_entries, metadata_entries); + NodeAction::ForwardAck { to, target, sequence, piggyback: combined } + } other => other, }) .collect() @@ -357,6 +409,13 @@ impl DistributedNode { self.registry.merge_batch(entries, self.cluster_size()); } } + + /// Merge metadata entries received from a piggyback payload. + fn merge_metadata_entries(&mut self, entries: Vec) { + if !entries.is_empty() { + self.metadata.apply_incoming(entries, self.cluster_size()); + } + } } /// Result of resolving an actor's location. diff --git a/crates/distribution/src/node_metadata.rs b/crates/distribution/src/node_metadata.rs new file mode 100644 index 0000000..e4b4dbb --- /dev/null +++ b/crates/distribution/src/node_metadata.rs @@ -0,0 +1,129 @@ +//! Node metadata dissemination — gossip-propagated per-node metadata. +//! +//! Each node may have metadata (currently: relay URL) that should be visible +//! cluster-wide. Uses higher-generation-wins semantics and SWIM-style +//! dissemination budgets (Λ * log₂(n)). + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +use crate::types::NodeId; + +/// A single metadata entry for one node. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct NodeMetadataEntry { + pub node_id: NodeId, + pub relay_url: Option, + pub generation: u64, +} + +/// Dissemination state for one entry. +#[derive(Debug, Clone)] +struct PendingEntry { + entry: NodeMetadataEntry, + remaining: usize, +} + +/// Per-node metadata store with gossip dissemination. +pub struct NodeMetadataDisseminator { + store: HashMap, + pending: Vec, + lambda: usize, + local_generation: u64, +} + +impl NodeMetadataDisseminator { + pub fn new(lambda: usize) -> Self { + Self { + store: HashMap::new(), + pending: Vec::new(), + lambda, + local_generation: 0, + } + } + + /// Set this node's relay URL and enqueue for dissemination. + pub fn set_local(&mut self, node_id: NodeId, relay_url: Option, cluster_size: usize) { + self.local_generation += 1; + let entry = NodeMetadataEntry { + node_id, + relay_url, + generation: self.local_generation, + }; + self.store.insert(node_id, entry.clone()); + self.enqueue(entry, cluster_size); + } + + /// Merge incoming entries from gossip. Re-enqueues changed entries for forwarding. + pub fn apply_incoming(&mut self, entries: Vec, cluster_size: usize) { + for entry in entries { + let dominated = match self.store.get(&entry.node_id) { + Some(existing) => entry.generation <= existing.generation, + None => false, + }; + if dominated { + continue; + } + self.store.insert(entry.node_id, entry.clone()); + self.enqueue(entry, cluster_size); + } + } + + /// Take pending entries for piggyback, up to `max_count`. + pub fn take_pending(&mut self, max_count: usize) -> Vec { + let count = max_count.min(self.pending.len()); + let mut result = Vec::with_capacity(count); + + for entry in self.pending.iter_mut().take(count) { + result.push(entry.entry.clone()); + entry.remaining = entry.remaining.saturating_sub(1); + } + + self.pending.retain(|e| e.remaining > 0); + result + } + + /// Look up a node's relay URL. + pub fn relay_url(&self, node_id: &NodeId) -> Option<&str> { + self.store + .get(node_id) + .and_then(|e| e.relay_url.as_deref()) + } + + /// Remove metadata for a dead node. + pub fn remove_node(&mut self, node_id: &NodeId) { + self.store.remove(node_id); + self.pending.retain(|e| &e.entry.node_id != node_id); + } + + /// Re-enqueue all entries for dissemination (anti-entropy on membership recovery). + pub fn re_disseminate_all(&mut self, cluster_size: usize) { + for entry in self.store.values().cloned().collect::>() { + self.enqueue(entry, cluster_size); + } + } + + fn transmit_budget(&self, cluster_size: usize) -> usize { + let n = cluster_size.max(2) as f64; + let log_n = n.log2().ceil() as usize; + self.lambda * log_n.max(1) + } + + fn enqueue(&mut self, entry: NodeMetadataEntry, cluster_size: usize) { + let budget = self.transmit_budget(cluster_size); + + // Replace existing pending entry for same node if present. + if let Some(existing) = self.pending.iter_mut().find(|e| e.entry.node_id == entry.node_id) + { + existing.entry = entry; + existing.remaining = budget; + return; + } + + self.pending.push(PendingEntry { + entry, + remaining: budget, + }); + } +} diff --git a/crates/distribution/src/peer_auth.rs b/crates/distribution/src/peer_auth.rs new file mode 100644 index 0000000..2bcfb2a --- /dev/null +++ b/crates/distribution/src/peer_auth.rs @@ -0,0 +1,140 @@ +//! Runtime-mutable peer allow-list for network-level authentication. +//! +//! When enabled, only peers whose `NodeId` appears in the allow-list +//! can join the cluster or exchange messages with this node. + +use std::collections::HashMap; +use std::io; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +use crate::identity::{hex_decode, hex_encode}; +use crate::types::NodeId; + +/// A single trusted peer entry. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PeerEntry { + pub node_id: String, + pub label: String, +} + +/// File format for the peers.json allow-list. +#[derive(Debug, Serialize, Deserialize)] +struct PeersFile { + version: u32, + peers: Vec, +} + +/// Runtime-mutable peer allow-list. +/// +/// - `None` inner map = open mode (all peers accepted) +/// - `Some(map)` = only listed peers accepted +pub struct PeerAllowList { + peers: Option>, + path: Option, +} + +impl PeerAllowList { + /// Create an open allow-list (no restrictions). + pub fn open() -> Self { + Self { + peers: None, + path: None, + } + } + + /// Load an allow-list from a JSON file. + /// + /// If the file doesn't exist, creates an empty allow-list (restrictive mode + /// with zero peers). The file will be created on the first `save()`. + pub fn from_file(path: &Path) -> io::Result { + let peers = if path.exists() { + let data = std::fs::read_to_string(path)?; + let file: PeersFile = serde_json::from_str(&data) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + let mut map = HashMap::new(); + for entry in file.peers { + if let Some(bytes) = hex_decode(&entry.node_id) { + if let Ok(arr) = <[u8; 32]>::try_from(bytes.as_slice()) { + map.insert(NodeId(arr), entry); + } + } + } + map + } else { + HashMap::new() + }; + + Ok(Self { + peers: Some(peers), + path: Some(path.to_path_buf()), + }) + } + + /// Check whether a peer is allowed. + pub fn is_allowed(&self, node_id: &NodeId) -> bool { + match &self.peers { + None => true, + Some(map) => map.contains_key(node_id), + } + } + + /// Whether the allow-list is in open mode. + pub fn is_open(&self) -> bool { + self.peers.is_none() + } + + /// Add a peer to the allow-list. + pub fn add_peer(&mut self, node_id: NodeId, label: String) { + let map = self.peers.get_or_insert_with(HashMap::new); + map.insert( + node_id, + PeerEntry { + node_id: hex_encode(&node_id.0), + label, + }, + ); + } + + /// Remove a peer from the allow-list. + pub fn remove_peer(&mut self, node_id: &NodeId) { + if let Some(ref mut map) = self.peers { + map.remove(node_id); + } + } + + /// List all trusted peers. + pub fn list_peers(&self) -> Vec<&PeerEntry> { + match &self.peers { + None => Vec::new(), + Some(map) => map.values().collect(), + } + } + + /// Persist the allow-list to disk. + pub fn save(&self) -> io::Result<()> { + let path = match &self.path { + Some(p) => p, + None => return Ok(()), + }; + + let entries: Vec = match &self.peers { + Some(map) => map.values().cloned().collect(), + None => Vec::new(), + }; + + let file = PeersFile { + version: 1, + peers: entries, + }; + + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + + let json = serde_json::to_string_pretty(&file) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + std::fs::write(path, json) + } +} diff --git a/crates/distribution/src/registry.rs b/crates/distribution/src/registry.rs index e132709..711321d 100644 --- a/crates/distribution/src/registry.rs +++ b/crates/distribution/src/registry.rs @@ -9,6 +9,7 @@ use std::collections::{HashMap, VecDeque}; use serde::{Deserialize, Serialize}; use swactor::actor::ActorAddress; +use crate::node_metadata::NodeMetadataEntry; use crate::types::NodeId; // ─── Configuration ────────────────────────────────────────────────────────── @@ -53,13 +54,16 @@ pub struct RegistryEntry { pub tombstone: bool, } -/// Combined piggyback payload: membership bytes + registry entries. +/// Combined piggyback payload: membership bytes + registry entries + node metadata. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PiggybackPayload { /// Raw SWIM membership piggyback bytes (opaque to registry). pub membership: Vec, /// Registry entries to disseminate. pub registry: Vec, + /// Node metadata entries (relay URLs, etc.) to disseminate. + #[serde(default)] + pub node_metadata: Vec, } // ─── Events ───────────────────────────────────────────────────────────────── @@ -367,20 +371,30 @@ fn lww_wins(incoming: &RegistryEntry, existing: &RegistryEntry) -> bool { // ─── Piggyback pack/unpack ────────────────────────────────────────────────── -/// Combine membership piggyback bytes and registry entries into a single payload. -pub fn pack_combined_piggyback(membership: Vec, registry: Vec) -> Vec { - let payload = PiggybackPayload { membership, registry }; +/// Combine membership piggyback bytes, registry entries, and node metadata into a single payload. +pub fn pack_combined_piggyback( + membership: Vec, + registry: Vec, + node_metadata: Vec, +) -> Vec { + let payload = PiggybackPayload { + membership, + registry, + node_metadata, + }; serde_json::to_vec(&payload).unwrap_or_default() } -/// Split a combined piggyback payload into membership bytes and registry entries. +/// Split a combined piggyback payload into membership bytes, registry entries, and node metadata. /// If deserialization fails, treats the entire blob as membership bytes (backwards compat). -pub fn unpack_combined_piggyback(bytes: &[u8]) -> (Vec, Vec) { +pub fn unpack_combined_piggyback( + bytes: &[u8], +) -> (Vec, Vec, Vec) { if bytes.is_empty() { - return (Vec::new(), Vec::new()); + return (Vec::new(), Vec::new(), Vec::new()); } match serde_json::from_slice::(bytes) { - Ok(payload) => (payload.membership, payload.registry), - Err(_) => (bytes.to_vec(), Vec::new()), + Ok(payload) => (payload.membership, payload.registry, payload.node_metadata), + Err(_) => (bytes.to_vec(), Vec::new(), Vec::new()), } } diff --git a/crates/distribution/src/snapshot.rs b/crates/distribution/src/snapshot.rs index a0a568e..257a90d 100644 --- a/crates/distribution/src/snapshot.rs +++ b/crates/distribution/src/snapshot.rs @@ -15,6 +15,12 @@ pub struct MemberInfo { pub addr: Option, pub state: String, pub incarnation: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub is_authorized: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub label: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub relay_url: Option, } /// Snapshot of a node in the Kademlia routing table. @@ -89,6 +95,26 @@ pub struct DistributionNodeSnapshot { // ─── Gossip pairs ──────────────────────────────────────────────── /// Recent SWIM probe targets (most recent last). pub recent_probe_targets: Vec, + + // ─── Peer auth ────────────────────────────────────────────────── + /// "open" or "allow-list". + #[serde(default)] + pub peer_auth_mode: String, + /// Number of authorized peers (None if open mode). + #[serde(skip_serializing_if = "Option::is_none")] + pub authorized_peer_count: Option, + + /// Human-readable node name (e.g. "swift-falcon"). + #[serde(skip_serializing_if = "Option::is_none", default)] + pub node_name: Option, + + /// Base58-encoded invite code for this node. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub invite_code: Option, + + /// This node's relay URL, if running an embedded relay server. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub relay_url: Option, } fn node_id_hex(id: &NodeId) -> String { @@ -117,6 +143,9 @@ impl DistributedNode { addr: None, state: state_str(m.state), incarnation: m.incarnation, + is_authorized: None, + label: None, + relay_url: self.metadata().relay_url(&m.node_id).map(String::from), }) .collect(); @@ -179,6 +208,11 @@ impl DistributedNode { registry_tombstones: registry.tombstone_count(), registry_entries, recent_probe_targets: recent_targets, + peer_auth_mode: "open".into(), + authorized_peer_count: None, + node_name: None, + invite_code: None, + relay_url: self.metadata().relay_url(&self.node_id()).map(String::from), } } } diff --git a/crates/distribution/src/swim/node.rs b/crates/distribution/src/swim/node.rs index 6b5b25e..577dff2 100644 --- a/crates/distribution/src/swim/node.rs +++ b/crates/distribution/src/swim/node.rs @@ -3,6 +3,7 @@ //! This is the top-level SWIM state machine that a `DistributedNode` will drive. //! It produces `SwimAction`s that the caller translates into real network I/O. +use crate::identity::hex_encode; use crate::messages::MembershipUpdate; use crate::types::{MemberState, NodeId, NodeRecord}; @@ -26,6 +27,8 @@ pub enum NodeAction { }, /// Send a SWIM ack. SendAck { to: NodeId, sequence: u64, piggyback: Vec }, + /// Forward an indirect ack back to the original prober. + ForwardAck { to: NodeId, target: NodeId, sequence: u64, piggyback: Vec }, /// Send a join response with the current member list. SendJoinResponse { to: NodeId, members: Vec }, /// Notification: a node state changed (for wiring into Kademlia). @@ -40,6 +43,9 @@ pub struct SwimNode { dissemination: DisseminationQueue, /// Maximum piggybacked updates per message. max_piggyback: usize, + /// PingReqs we forwarded: (requester, target, sequence). + /// When we receive an ack matching (target, sequence), forward it to requester. + pending_relays: Vec<(NodeId, NodeId, u64)>, } impl SwimNode { @@ -49,6 +55,7 @@ impl SwimNode { probe: SwimProbe::new(config), dissemination: DisseminationQueue::new(3), // Λ = 3 max_piggyback: 8, + pending_relays: Vec::new(), } } @@ -96,19 +103,35 @@ impl SwimNode { &mut self.members, ); actions.extend(self.translate_probe_actions(probe_actions)); + + // Check if this ack completes a pending relay (indirect ping path) + if let Some(pos) = self.pending_relays.iter().position(|(_, t, s)| *t == from && *s == sequence) { + let (requester, target, seq) = self.pending_relays.remove(pos); + let pb = self.dissemination.pack_piggyback(self.max_piggyback); + actions.push(NodeAction::ForwardAck { + to: requester, target, sequence: seq, piggyback: pb, + }); + } + actions } /// Handle a received indirect ping request. pub fn handle_ping_req( &mut self, - _from: NodeId, + from: NodeId, target: NodeId, sequence: u64, piggyback: &[u8], ) -> Vec { let mut actions = self.apply_piggyback(piggyback); + // Record the pending relay so we can forward the ack back + if self.pending_relays.len() >= 16 { + self.pending_relays.remove(0); + } + self.pending_relays.push((from, target, sequence)); + // Forward a ping to the target on behalf of the requester let pb = self.dissemination.pack_piggyback(self.max_piggyback); actions.push(NodeAction::SendPing { @@ -119,6 +142,17 @@ impl SwimNode { actions } + /// Handle a received indirect ack (forwarded by a relay node). + pub fn handle_indirect_ack(&mut self, target: NodeId, sequence: u64, piggyback: &[u8]) -> Vec { + let mut actions = self.apply_piggyback(piggyback); + let probe_actions = self.probe.step( + SwimEvent::IndirectAckReceived { target, sequence }, + &mut self.members, + ); + actions.extend(self.translate_probe_actions(probe_actions)); + actions + } + /// Handle a join request from a new node. pub fn handle_join_request(&mut self, from: NodeId) -> Vec { // Add the new node to our member list @@ -223,6 +257,9 @@ impl SwimNode { update.incarnation, ); if changed { + if update.state == MemberState::Alive { + eprintln!("SWIM: alive {}", &hex_encode(&update.node_id.0)[..8]); + } // Re-disseminate the update self.dissemination.enqueue( membership_update(update.node_id, update.state, update.incarnation), @@ -272,6 +309,7 @@ impl SwimNode { }); } SwimAction::Suspect(node_id) => { + eprintln!("SWIM: suspect {}", &hex_encode(&node_id.0)[..8]); if self.members.suspect(node_id) { if let Some(entry) = self.members.get(&node_id) { self.dissemination.enqueue( @@ -287,6 +325,7 @@ impl SwimNode { } } SwimAction::DeclareDead(node_id) => { + eprintln!("SWIM: dead {}", &hex_encode(&node_id.0)[..8]); if let Some(entry) = self.members.get(&node_id) { let inc = entry.incarnation; self.dissemination.enqueue( diff --git a/crates/distribution/tests/common/iroh.rs b/crates/distribution/tests/common/iroh.rs new file mode 100644 index 0000000..f4e18f5 --- /dev/null +++ b/crates/distribution/tests/common/iroh.rs @@ -0,0 +1,230 @@ +//! Test helpers for iroh-based integration tests. +//! +//! Provides `IrohTestCluster` — an N-node harness that owns real iroh +//! endpoints with `RelayMode::Disabled`, connected via direct addresses. +#![allow(dead_code)] + +use std::ops::{Index, IndexMut}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use distribution::iroh_driver::{IrohDriver, IrohDriverConfig}; +use distribution::peer_auth::PeerAllowList; +use iroh::{PublicKey, RelayMode}; + +use super::test_config; + +// ─── Single-driver helpers ────────────────────────────────────────────── + +pub fn make_driver() -> IrohDriver { + IrohDriver::new(IrohDriverConfig { + secret_key: None, + relay_mode: RelayMode::Disabled, + node: test_config(), + peer_auth: None, + #[cfg(feature = "relay")] + embedded_relay_bind: None, + #[cfg(feature = "relay")] + relay_public_ip: None, + }) + .expect("failed to create iroh driver") +} + +pub fn make_driver_with_auth(auth: Arc>) -> IrohDriver { + IrohDriver::new(IrohDriverConfig { + secret_key: None, + relay_mode: RelayMode::Disabled, + node: test_config(), + peer_auth: Some(auth), + #[cfg(feature = "relay")] + embedded_relay_bind: None, + #[cfg(feature = "relay")] + relay_public_ip: None, + }) + .expect("failed to create iroh driver") +} + +pub fn make_driver_with_relay(relay_url: iroh::RelayUrl) -> IrohDriver { + IrohDriver::new(IrohDriverConfig { + secret_key: None, + relay_mode: RelayMode::Custom(relay_url.into()), + node: test_config(), + peer_auth: None, + #[cfg(feature = "relay")] + embedded_relay_bind: None, + #[cfg(feature = "relay")] + relay_public_ip: None, + }) + .expect("failed to create iroh driver with relay") +} + +/// Pump one driver: recv + tick. +pub fn pump_one(driver: &mut IrohDriver) { + driver.recv(); + driver.tick(); +} + +/// Pump a slice of drivers: recv + tick on each. +pub fn pump_all(drivers: &mut [IrohDriver]) { + for d in drivers.iter_mut() { + d.recv(); + d.tick(); + } +} + +/// Pump two drivers until a condition is met or timeout expires. +pub fn pump_until_pair( + a: &mut IrohDriver, + b: &mut IrohDriver, + timeout: Duration, + check_fn: fn(&IrohDriver, &IrohDriver) -> bool, +) -> bool { + let start = Instant::now(); + while start.elapsed() < timeout { + pump_one(a); + pump_one(b); + if check_fn(a, b) { + return true; + } + std::thread::sleep(Duration::from_millis(10)); + } + false +} + +/// Pump N drivers until a condition is met or timeout expires. +pub fn pump_until( + drivers: &mut [IrohDriver], + timeout: Duration, + check_fn: F, +) -> bool +where + F: Fn(&[IrohDriver]) -> bool, +{ + let start = Instant::now(); + while start.elapsed() < timeout { + pump_all(drivers); + if check_fn(drivers) { + return true; + } + std::thread::sleep(Duration::from_millis(10)); + } + false +} + +/// Check whether `driver` sees `peer_key` as alive. +pub fn sees_alive(driver: &IrohDriver, peer_key: &PublicKey) -> bool { + let snap = driver.snapshot(); + let peer_hex: String = peer_key + .as_bytes() + .iter() + .map(|b| format!("{:02x}", b)) + .collect(); + snap.members + .iter() + .any(|m| m.node_id == peer_hex && m.state == "alive") +} + +// ─── Local relay ──────────────────────────────────────────────────────── + +/// Guard that keeps the relay server alive while it exists. +pub struct RelayGuard { + _server: iroh_relay::server::Server, + _rt: tokio::runtime::Runtime, +} + +/// Spawn a local HTTP relay server for tests. Returns the relay URL and a +/// guard that shuts the server down on drop. +pub fn spawn_test_relay() -> (iroh::RelayUrl, RelayGuard) { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let server = rt.block_on(async { + iroh_relay::server::Server::spawn(iroh_relay::server::ServerConfig::<(), ()> { + relay: Some(iroh_relay::server::RelayConfig { + http_bind_addr: (std::net::Ipv4Addr::LOCALHOST, 0).into(), + tls: None, + limits: Default::default(), + key_cache_capacity: Some(256), + access: iroh_relay::server::AccessConfig::Everyone, + }), + quic: None, + metrics_addr: None, + }) + .await + }) + .unwrap(); + let url = server.http_url().expect("relay has no HTTP URL"); + ( + url, + RelayGuard { + _server: server, + _rt: rt, + }, + ) +} + +// ─── N-node cluster ───────────────────────────────────────────────────── + +/// An N-node iroh test cluster with real QUIC endpoints. +pub struct IrohTestCluster { + drivers: Vec, +} + +impl IrohTestCluster { + /// Create N disconnected drivers (no joins). + pub fn disconnected(n: usize) -> Self { + let drivers = (0..n).map(|_| make_driver()).collect(); + Self { drivers } + } + + /// Create N drivers connected in a star topology through node 0. + /// Nodes 1..N join node 0 using its full `EndpointAddr`. + pub fn star(n: usize) -> Self { + assert!(n >= 2, "star cluster requires at least 2 nodes"); + let mut drivers: Vec = (0..n).map(|_| make_driver()).collect(); + + let addr_0 = drivers[0].endpoint_addr(); + for i in 1..n { + drivers[i].join(&[addr_0.clone()]); + } + + Self { drivers } + } + + /// Pump all drivers until a condition is met or timeout expires. + pub fn pump_until(&mut self, timeout: Duration, check_fn: F) -> bool + where + F: Fn(&[IrohDriver]) -> bool, + { + let start = Instant::now(); + while start.elapsed() < timeout { + pump_all(&mut self.drivers); + if check_fn(&self.drivers) { + return true; + } + std::thread::sleep(Duration::from_millis(10)); + } + false + } + + /// Shut down all drivers. + pub fn shutdown(&mut self) { + for d in &mut self.drivers { + d.shutdown(); + } + } +} + +impl Index for IrohTestCluster { + type Output = IrohDriver; + fn index(&self, idx: usize) -> &Self::Output { + &self.drivers[idx] + } +} + +impl IndexMut for IrohTestCluster { + fn index_mut(&mut self, idx: usize) -> &mut Self::Output { + &mut self.drivers[idx] + } +} diff --git a/crates/distribution/tests/common/mod.rs b/crates/distribution/tests/common/mod.rs index 51c58ae..7e03906 100644 --- a/crates/distribution/tests/common/mod.rs +++ b/crates/distribution/tests/common/mod.rs @@ -25,6 +25,7 @@ pub fn test_config() -> DistributedNodeConfig { cache_capacity: 100, republish_interval: 50, registry: RegistryConfig::default(), + metadata_lambda: 3, } } @@ -99,6 +100,16 @@ fn deliver_actions_tagged( } } } + NodeAction::ForwardAck { to, target, sequence, piggyback } => { + if let Some(idx) = ids.iter().position(|id| id == to) { + if !excluded.contains(&idx) { + let resp = nodes[idx].handle_indirect_ack(*target, *sequence, piggyback); + if !resp.is_empty() { + tagged.push((idx, resp)); + } + } + } + } NodeAction::MembershipChanged { .. } => {} } } @@ -206,3 +217,6 @@ impl IndexMut for TestCluster { &mut self.nodes[idx] } } + +#[cfg(feature = "iroh")] +pub mod iroh; diff --git a/crates/distribution/tests/iroh_driver.rs b/crates/distribution/tests/iroh_driver.rs index 82cf7c5..dc3c24d 100644 --- a/crates/distribution/tests/iroh_driver.rs +++ b/crates/distribution/tests/iroh_driver.rs @@ -4,45 +4,27 @@ //! - Create endpoints with matching identities //! - Form clusters via join //! - Detect membership changes through SWIM +//! - Reject unauthorized peers //! //! Requires the `iroh` feature. #![cfg(feature = "iroh")] -use distribution::iroh_driver::{IrohDriver, IrohDriverConfig}; -use distribution::node::DistributedNodeConfig; -use distribution::registry::RegistryConfig; -use distribution::swim::probe::SwimConfig; -use iroh::RelayMode; +mod common; -fn test_config() -> DistributedNodeConfig { - DistributedNodeConfig { - swim: SwimConfig { - probe_interval: 1, - probe_timeout: 3, - indirect_probes: 1, - suspicion_timeout: 5, - dead_reprobe_interval: 0, - }, - cache_capacity: 100, - republish_interval: 50, - registry: RegistryConfig::default(), - } -} +use std::sync::{Arc, Mutex}; +use std::time::Duration; -fn make_driver() -> IrohDriver { - IrohDriver::new(IrohDriverConfig { - secret_key: None, - relay_mode: RelayMode::Disabled, - node: test_config(), - }) - .expect("failed to create iroh driver") -} +use common::iroh::*; +use distribution::peer_auth::PeerAllowList; +use iroh::PublicKey; + +// ─── Identity tests ───────────────────────────────────────────────────── #[test] fn iroh_driver_creates_with_unique_identity() { - let d1 = make_driver(); - let d2 = make_driver(); + let mut d1 = make_driver(); + let mut d2 = make_driver(); assert_ne!(d1.node_id(), d2.node_id()); d1.shutdown(); d2.shutdown(); @@ -50,7 +32,7 @@ fn iroh_driver_creates_with_unique_identity() { #[test] fn iroh_driver_snapshot_contains_node_id() { - let driver = make_driver(); + let mut driver = make_driver(); let snap = driver.snapshot(); assert!(!snap.node_id.is_empty()); assert_eq!(snap.members.len(), 0); @@ -59,11 +41,147 @@ fn iroh_driver_snapshot_contains_node_id() { #[test] fn iroh_driver_identity_matches_iroh_endpoint() { - let driver = make_driver(); + let mut driver = make_driver(); let node_id = driver.node_id(); - // The snapshot's node_id hex should match the NodeId bytes let snap = driver.snapshot(); let expected_hex: String = node_id.0.iter().map(|b| format!("{:02x}", b)).collect(); assert_eq!(snap.node_id, expected_hex); driver.shutdown(); } + +// ─── Join integration tests ───────────────────────────────────────────── + +#[test] +fn two_nodes_form_cluster_via_join() { + let mut node_a = make_driver(); + let mut node_b = make_driver(); + + let b_addr = node_b.endpoint_addr(); + node_a.join(&[b_addr]); + + let converged = pump_until_pair( + &mut node_a, + &mut node_b, + Duration::from_secs(5), + |a, b| { + let a_key = PublicKey::from_bytes(&a.node_id().0).unwrap(); + let b_key = PublicKey::from_bytes(&b.node_id().0).unwrap(); + sees_alive(a, &b_key) && sees_alive(b, &a_key) + }, + ); + + assert!(converged, "nodes did not converge within timeout"); + assert_eq!(node_a.snapshot().alive_count, 1, "node_a should see 1 alive peer"); + assert_eq!(node_b.snapshot().alive_count, 1, "node_b should see 1 alive peer"); + + node_a.shutdown(); + node_b.shutdown(); +} + +#[test] +fn two_nodes_form_cluster_via_mutual_join() { + let mut node_a = make_driver(); + let mut node_b = make_driver(); + + let b_addr = node_b.endpoint_addr(); + let a_addr = node_a.endpoint_addr(); + + node_a.join(&[b_addr]); + node_b.join(&[a_addr]); + + let converged = pump_until_pair( + &mut node_a, + &mut node_b, + Duration::from_secs(5), + |a, b| { + let a_key = PublicKey::from_bytes(&a.node_id().0).unwrap(); + let b_key = PublicKey::from_bytes(&b.node_id().0).unwrap(); + sees_alive(a, &b_key) && sees_alive(b, &a_key) + }, + ); + + assert!(converged, "nodes did not converge within timeout (mutual join)"); + assert_eq!(node_a.snapshot().alive_count, 1); + assert_eq!(node_b.snapshot().alive_count, 1); + + node_a.shutdown(); + node_b.shutdown(); +} + +#[test] +fn two_nodes_form_cluster_with_peer_auth() { + let auth_a = Arc::new(Mutex::new(PeerAllowList::open())); + let auth_b = Arc::new(Mutex::new(PeerAllowList::open())); + + let mut node_a = make_driver_with_auth(auth_a.clone()); + let mut node_b = make_driver_with_auth(auth_b.clone()); + + let a_id = node_a.node_id(); + let b_id = node_b.node_id(); + let b_addr = node_b.endpoint_addr(); + + // Switch to restrictive mode by adding each other + auth_a.lock().unwrap().add_peer(b_id, "node-b".into()); + auth_b.lock().unwrap().add_peer(a_id, "node-a".into()); + + node_a.join(&[b_addr]); + + let converged = pump_until_pair( + &mut node_a, + &mut node_b, + Duration::from_secs(5), + |a, b| { + let a_key = PublicKey::from_bytes(&a.node_id().0).unwrap(); + let b_key = PublicKey::from_bytes(&b.node_id().0).unwrap(); + sees_alive(a, &b_key) && sees_alive(b, &a_key) + }, + ); + + assert!(converged, "nodes with peer auth did not converge within timeout"); + assert_eq!(node_a.snapshot().alive_count, 1); + assert_eq!(node_b.snapshot().alive_count, 1); + + node_a.shutdown(); + node_b.shutdown(); +} + +#[test] +fn peer_auth_prevents_unauthorized_join() { + let mut node_a = make_driver(); + + // Node B has auth with only a dummy peer — node_a is NOT authorized + let auth_b = Arc::new(Mutex::new(PeerAllowList::open())); + let dummy_id = distribution::types::NodeId([0xAA; 32]); + auth_b.lock().unwrap().add_peer(dummy_id, "dummy".into()); + let mut node_b = make_driver_with_auth(auth_b); + + let b_addr = node_b.endpoint_addr(); + node_a.join(&[b_addr]); + + let converged = pump_until_pair( + &mut node_a, + &mut node_b, + Duration::from_secs(3), + |_a, b| b.snapshot().alive_count > 0, + ); + + assert!(!converged, "unauthorized peer should NOT have joined"); + assert_eq!(node_b.snapshot().alive_count, 0, "node_b should have no alive peers"); + + node_a.shutdown(); + node_b.shutdown(); +} + +// ─── Multi-node tests ─────────────────────────────────────────────────── + +#[test] +fn three_nodes_converge_via_star_join() { + let mut cluster = IrohTestCluster::star(3); + + let converged = cluster.pump_until(Duration::from_secs(10), |drivers| { + drivers.iter().all(|d| d.snapshot().alive_count == 2) + }); + + assert!(converged, "3-node star did not converge"); + cluster.shutdown(); +} diff --git a/crates/local-runner/Cargo.toml b/crates/local-runner/Cargo.toml deleted file mode 100644 index 5164e74..0000000 --- a/crates/local-runner/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "local-runner" -version = "0.1.0" -edition = "2024" - -[[bin]] -name = "local-runner" -path = "src/main.rs" - -[dependencies] -swactor = { path = "../..", features = ["serde"] } -swactor-ci = { path = "../ci", features = ["local"] } -runtime-dashboard = { path = "../runtime-dashboard", features = ["ci"] } -clap = { version = "4", features = ["derive"] } -ctrlc = "3" -iroh = "0.96" -tokio = { version = "1", features = ["rt-multi-thread"] } -serde_json = "1" diff --git a/crates/runtime-dashboard/src/datastore_collector.rs b/crates/runtime-dashboard/src/datastore_collector.rs deleted file mode 100644 index 47ec74d..0000000 --- a/crates/runtime-dashboard/src/datastore_collector.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! Datastore stats provider for the runtime dashboard. -//! -//! The trait returns a pre-serialized JSON string so that `runtime-dashboard` -//! has no compile-time dependency on `swactor-datastore` (which would create a -//! circular dependency since `swactor-datastore[node]` depends on us). -//! -//! The `swactor-datastore` crate implements this trait in its `node` feature. - -/// Trait for providing datastore stats to the dashboard. -/// -/// Implementations capture a point-in-time snapshot as serialized JSON. -/// The dashboard polls this every ~200ms via SSE. -pub trait DatastoreStatsProvider: Send + Sync { - /// Return a JSON-serialized datastore snapshot, or `None` if unavailable. - fn snapshot_json(&self) -> Option; -} diff --git a/crates/runtime-dashboard/src/datastore_html.rs b/crates/runtime-dashboard/src/datastore_html.rs deleted file mode 100644 index 3afd976..0000000 --- a/crates/runtime-dashboard/src/datastore_html.rs +++ /dev/null @@ -1,286 +0,0 @@ -pub const DATASTORE_HTML: &str = r##" - - - - -Swactor Runtime – Datastore - - - -
-
-

- Swactor Runtime Dashboard - -

- -
-
- Waiting for data... -
-
- -
- -
-

Datastore Stats

-
-
0
Objects
-
0
Total Size
-
0
Puts
-
0
Gets
-
0
Deletes
-
-
- - -
-

Event Timeline

-
-
- - -
-

Objects

-
- - - -
HashNameSize
-
-
- - -
-

Active Transfers

-
-
-
- - - - -"##; diff --git a/crates/runtime-dashboard/src/server.rs b/crates/runtime-dashboard/src/server.rs deleted file mode 100644 index d7d369d..0000000 --- a/crates/runtime-dashboard/src/server.rs +++ /dev/null @@ -1,727 +0,0 @@ -use std::io::{self, Read as IoRead}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{mpsc, Arc, Mutex}; -use std::thread; -use std::time::{Duration, Instant}; - -use std::collections::HashMap; - -use swactor::runtime::Runtime; - -use crate::actor_detail_html::ACTOR_DETAIL_HTML; -use crate::actors_html::ACTORS_HTML; -use crate::collector::StatsCollector; -use crate::dashboard_html::DASHBOARD_HTML; -use crate::history::DashboardHistory; -use crate::layer::EventStore; -use crate::topology; -use crate::topology_html::TOPOLOGY_HTML; -use crate::trace::RuntimeTrace; -use crate::warnings::{WarningConfig, WarningDetector}; - -#[cfg(feature = "distribution")] -use crate::distribution_collector::DistributionStatsProvider; -#[cfg(feature = "distribution")] -use crate::distribution_html::DISTRIBUTION_HTML; - -use crate::datastore_collector::DatastoreStatsProvider; -use crate::datastore_html::DATASTORE_HTML; - -#[cfg(feature = "ci")] -use crate::ci_collector::CiStatsProvider; - -/// Format a server-sent event. -fn format_sse(event: &str, data: &str) -> Vec { - format!("event: {event}\ndata: {data}\n\n").into_bytes() -} - -/// Adapts an `mpsc::Receiver>` to `std::io::Read` for tiny_http streaming. -struct ChannelReader { - rx: mpsc::Receiver>, - buf: Vec, - pos: usize, -} - -impl ChannelReader { - fn new(rx: mpsc::Receiver>) -> Self { - Self { - rx, - buf: Vec::new(), - pos: 0, - } - } -} - -impl IoRead for ChannelReader { - fn read(&mut self, out: &mut [u8]) -> io::Result { - // Drain current buffer first. - if self.pos < self.buf.len() { - let n = std::cmp::min(out.len(), self.buf.len() - self.pos); - out[..n].copy_from_slice(&self.buf[self.pos..self.pos + n]); - self.pos += n; - return Ok(n); - } - - // Wait for next chunk. - match self.rx.recv() { - Ok(data) => { - if data.is_empty() { - return Ok(0); // EOF signal - } - let n = std::cmp::min(out.len(), data.len()); - out[..n].copy_from_slice(&data[..n]); - if n < data.len() { - self.buf = data; - self.pos = n; - } else { - self.buf.clear(); - self.pos = 0; - } - Ok(n) - } - Err(_) => Ok(0), // channel closed - } - } -} - -fn make_sse_response( - rx: mpsc::Receiver>, -) -> tiny_http::Response> { - let reader = ChannelReader::new(rx); - tiny_http::Response::new( - tiny_http::StatusCode(200), - vec![ - "Content-Type: text/event-stream" - .parse::() - .unwrap(), - "Cache-Control: no-cache" - .parse::() - .unwrap(), - "Connection: keep-alive" - .parse::() - .unwrap(), - ], - Box::new(reader) as Box, - None, - None, - ) -} - -fn respond_html(request: tiny_http::Request, html_template: &str, mode: &str) { - let html = html_template.replace("__DASHBOARD_MODE__", mode); - let response = tiny_http::Response::from_string(html).with_header( - "Content-Type: text/html; charset=utf-8" - .parse::() - .unwrap(), - ); - let _ = request.respond(response); -} - -fn respond_404(request: tiny_http::Request) { - let response = tiny_http::Response::from_string("Not Found").with_status_code(404); - let _ = request.respond(response); -} - -// ── Live server ───────────────────────────────────────────────────────── - -/// Start the live HTTP server with a pool of handler threads. -pub(crate) fn spawn_http_server( - store: Arc, - runtime: Arc>>>, - collector: Arc>>>, - shutdown: Arc, - history: Arc, - port: u16, - #[cfg(feature = "distribution")] - distribution: Arc>>>, - datastore: Arc>>>, - #[cfg(feature = "ci")] - ci: Arc>>>, -) { - let addr = format!("0.0.0.0:{port}"); - let server = tiny_http::Server::http(&addr).expect("failed to bind HTTP server"); - let server = Arc::new(server); - let cmd_router = Arc::new(crate::command::CommandRouter::with_builtins()); - - for _ in 0..4 { - let server = Arc::clone(&server); - let store = Arc::clone(&store); - let runtime = Arc::clone(&runtime); - let collector = Arc::clone(&collector); - let shutdown = Arc::clone(&shutdown); - let history = Arc::clone(&history); - let cmd_router = Arc::clone(&cmd_router); - #[cfg(feature = "distribution")] - let distribution = Arc::clone(&distribution); - let datastore = Arc::clone(&datastore); - #[cfg(feature = "ci")] - let ci = Arc::clone(&ci); - thread::spawn(move || { - loop { - let request = match server.recv() { - Ok(r) => r, - Err(_) => break, - }; - - let url = request.url().to_string(); - let path = url.split('?').next().unwrap_or(&url); - match path { - "/" => respond_html(request, DASHBOARD_HTML, "live"), - "/actors" => respond_html(request, ACTORS_HTML, "live"), - "/topology" => respond_html(request, TOPOLOGY_HTML, "live"), - #[cfg(feature = "distribution")] - "/distribution" => respond_html(request, DISTRIBUTION_HTML, "live"), - "/datastore" => respond_html(request, DATASTORE_HTML, "live"), - "/events" => { - handle_live_sse( - request, - Arc::clone(&store), - Arc::clone(&runtime), - Arc::clone(&collector), - Arc::clone(&shutdown), - Arc::clone(&history), - #[cfg(feature = "distribution")] - Arc::clone(&distribution), - Arc::clone(&datastore), - #[cfg(feature = "ci")] - Arc::clone(&ci), - ); - } - "/api/stats" => { - handle_stats_api( - request, - Arc::clone(&runtime), - Arc::clone(&collector), - ); - } - "/api/history" => { - handle_history_api(request, Arc::clone(&history)); - } - "/api/topology" => { - handle_topology_api( - request, - Arc::clone(&runtime), - Arc::clone(&collector), - ); - } - "/api/investigate" => { - handle_investigate_api( - request, - &url, - Arc::clone(&runtime), - Arc::clone(&collector), - Arc::clone(&cmd_router), - ); - } - #[cfg(feature = "distribution")] - "/api/distribution" => { - handle_distribution_api( - request, - Arc::clone(&distribution), - ); - } - "/api/datastore" => { - handle_datastore_api( - request, - Arc::clone(&datastore), - ); - } - "/api/logs" => { - handle_logs_api(request, &url, Arc::clone(&store)); - } - #[cfg(feature = "ci")] - _ if path.starts_with("/api/ci/") => { - handle_ci_api(request, path, Arc::clone(&ci)); - } - _ if path.starts_with("/actor/") => { - let hex = &path[7..]; // strip "/actor/" - respond_actor_detail(request, hex); - } - _ => respond_404(request), - } - } - }); - } -} - -fn respond_actor_detail(request: tiny_http::Request, hex_addr: &str) { - let html = ACTOR_DETAIL_HTML - .replace("__DASHBOARD_MODE__", "live") - .replace("__ACTOR_ADDR__", hex_addr); - let response = tiny_http::Response::from_string(html).with_header( - "Content-Type: text/html; charset=utf-8" - .parse::() - .unwrap(), - ); - let _ = request.respond(response); -} - -fn handle_live_sse( - request: tiny_http::Request, - store: Arc, - runtime: Arc>>>, - collector: Arc>>>, - shutdown: Arc, - history: Arc, - #[cfg(feature = "distribution")] - distribution: Arc>>>, - datastore: Arc>>>, - #[cfg(feature = "ci")] - ci: Arc>>>, -) { - let (tx, rx) = mpsc::channel::>(); - let response = make_sse_response(rx); - - // Spawn producer thread - thread::spawn(move || { - let mut cursor: u64 = 0; - let mut warning_detector = WarningDetector::new(WarningConfig::default()); - let mut tick_count: u64 = 0; - - // Send initial history snapshot so sparklines render immediately - if history.sample_count() > 0 { - let json = history.worker_history_json(); - let _ = tx.send(format_sse("history", &json)); - } - - loop { - // Send stats if runtime is available - { - let maybe_rt = runtime.lock().unwrap().clone(); - if let Some(rt) = maybe_rt { - let mut stats = rt.stats(); - if let Some(col) = collector.lock().unwrap().as_ref() { - col.enrich(&mut stats); - } - history.record(&stats); - - // Run warning detection - let warnings = warning_detector.check(&stats); - if !warnings.is_empty() { - if let Ok(wjson) = serde_json::to_string(&warnings) { - if tx.send(format_sse("warnings", &wjson)).is_err() { - return; - } - } - } - - let json = serde_json::to_string(&stats).unwrap(); - if tx.send(format_sse("stats", &json)).is_err() { - return; - } - - // Send topology every 5th tick (~1/sec) - tick_count += 1; - if tick_count % 5 == 0 { - let topo = topology::worker_topology(&stats); - if let Ok(tjson) = serde_json::to_string(&topo) { - if tx.send(format_sse("topology", &tjson)).is_err() { - return; - } - } - } - } - } - - // Send distribution snapshot if provider is attached - #[cfg(feature = "distribution")] - { - let maybe_dist = distribution.lock().unwrap().clone(); - if let Some(provider) = maybe_dist { - if let Some(snapshot) = provider.snapshot() { - if let Ok(json) = serde_json::to_string(&snapshot) { - if tx.send(format_sse("distribution", &json)).is_err() { - return; - } - } - } - } - } - - // Send datastore snapshot if provider is attached - { - let maybe_ds = datastore.lock().unwrap().clone(); - if let Some(provider) = maybe_ds { - if let Some(json) = provider.snapshot_json() { - if tx.send(format_sse("datastore", &json)).is_err() { - return; - } - } - } - } - - // Send CI snapshot if provider is attached - #[cfg(feature = "ci")] - { - let maybe_ci = ci.lock().unwrap().clone(); - if let Some(provider) = maybe_ci { - let snapshot = provider.snapshot(); - if let Ok(json) = serde_json::to_string(&snapshot) { - if tx.send(format_sse("ci", &json)).is_err() { - return; - } - } - } - } - - // Send new activity events - let (batch, new_cursor) = store.read_from(cursor); - if !batch.is_empty() { - let json = serde_json::to_string(&batch).unwrap(); - if tx.send(format_sse("activity", &json)).is_err() { - return; - } - cursor = new_cursor; - } - - if shutdown.load(Ordering::Relaxed) { - let _ = tx.send(format_sse("done", "{}")); - let _ = tx.send(Vec::new()); // EOF - return; - } - - thread::sleep(Duration::from_millis(200)); - } - }); - - // Blocks until connection closes - let _ = request.respond(response); -} - -fn handle_stats_api( - request: tiny_http::Request, - runtime: Arc>>>, - collector: Arc>>>, -) { - let maybe_rt = runtime.lock().unwrap().clone(); - let json = match maybe_rt { - Some(rt) => { - let mut stats = rt.stats(); - if let Some(col) = collector.lock().unwrap().as_ref() { - col.enrich(&mut stats); - } - serde_json::to_string(&stats).unwrap() - } - None => "{}".to_string(), - }; - let response = tiny_http::Response::from_string(json).with_header( - "Content-Type: application/json" - .parse::() - .unwrap(), - ); - let _ = request.respond(response); -} - -fn handle_investigate_api( - request: tiny_http::Request, - url: &str, - runtime: Arc>>>, - collector: Arc>>>, - cmd_router: Arc, -) { - let params = parse_query_string(url); - - let maybe_rt = runtime.lock().unwrap().clone(); - let maybe_col = collector.lock().unwrap().clone(); - - let json = match (maybe_rt, maybe_col) { - (Some(rt), Some(col)) => { - let ctx = crate::command::CommandContext::with_enricher(rt, col); - let req = crate::command::from_query_params(¶ms); - cmd_router.dispatch(&req, &ctx).to_json_line() - } - (Some(rt), None) => { - let ctx = crate::command::CommandContext::new(rt); - let req = crate::command::from_query_params(¶ms); - cmd_router.dispatch(&req, &ctx).to_json_line() - } - _ => { - let cmd = params.get("cmd").map(|s| s.as_str()).unwrap_or("help"); - serde_json::json!({ - "ok": false, - "command": cmd, - "error": "runtime not attached yet" - }) - .to_string() - } - }; - - let response = tiny_http::Response::from_string(json).with_header( - "Content-Type: application/json" - .parse::() - .unwrap(), - ); - let _ = request.respond(response); -} - -#[cfg(feature = "distribution")] -fn handle_distribution_api( - request: tiny_http::Request, - distribution: Arc>>>, -) { - let json = match distribution.lock().unwrap().as_ref() { - Some(provider) => match provider.snapshot() { - Some(snapshot) => serde_json::to_string(&snapshot).unwrap_or_else(|_| "{}".into()), - None => "{}".to_string(), - }, - None => serde_json::json!({ - "error": "distribution provider not attached" - }) - .to_string(), - }; - - let response = tiny_http::Response::from_string(json).with_header( - "Content-Type: application/json" - .parse::() - .unwrap(), - ); - let _ = request.respond(response); -} - -fn handle_datastore_api( - request: tiny_http::Request, - datastore: Arc>>>, -) { - let json = match datastore.lock().unwrap().as_ref() { - Some(provider) => provider.snapshot_json().unwrap_or_else(|| "{}".into()), - None => serde_json::json!({ - "error": "datastore provider not attached" - }) - .to_string(), - }; - - let response = tiny_http::Response::from_string(json).with_header( - "Content-Type: application/json" - .parse::() - .unwrap(), - ); - let _ = request.respond(response); -} - -#[cfg(feature = "ci")] -fn handle_ci_api( - request: tiny_http::Request, - path: &str, - ci: Arc>>>, -) { - use crate::ci_collector; - - let route = ci_collector::parse_route(path); - let json = match ci.lock().unwrap().as_ref() { - Some(provider) => { - let snapshot = provider.snapshot(); - ci_collector::handle_route(&route, &snapshot) - .unwrap_or_else(|| r#"{"error":"not found"}"#.to_string()) - } - None => serde_json::json!({ - "error": "CI provider not attached" - }) - .to_string(), - }; - - let response = tiny_http::Response::from_string(json).with_header( - "Content-Type: application/json" - .parse::() - .unwrap(), - ); - let _ = request.respond(response); -} - -fn handle_topology_api( - request: tiny_http::Request, - runtime: Arc>>>, - collector: Arc>>>, -) { - let maybe_rt = runtime.lock().unwrap().clone(); - let json = match maybe_rt { - Some(rt) => { - let mut stats = rt.stats(); - if let Some(col) = collector.lock().unwrap().as_ref() { - col.enrich(&mut stats); - } - let topo = topology::worker_topology(&stats); - serde_json::to_string(&topo).unwrap_or_else(|_| "{}".into()) - } - None => "{}".to_string(), - }; - let response = tiny_http::Response::from_string(json).with_header( - "Content-Type: application/json" - .parse::() - .unwrap(), - ); - let _ = request.respond(response); -} - -fn handle_logs_api(request: tiny_http::Request, url: &str, store: Arc) { - let params = parse_query_string(url); - let actor = params.get("actor").cloned().unwrap_or_default(); - let limit: usize = params - .get("limit") - .and_then(|s| s.parse().ok()) - .unwrap_or(200); - let level = params.get("level").cloned(); - - let mut events = store.read_for_actor(&actor, limit); - - // Filter by level if specified - if let Some(ref lvl) = level { - let lvl_upper = lvl.to_uppercase(); - events.retain(|e| e.level == lvl_upper); - } - - let json = serde_json::to_string(&events).unwrap_or_else(|_| "[]".into()); - let response = tiny_http::Response::from_string(json).with_header( - "Content-Type: application/json" - .parse::() - .unwrap(), - ); - let _ = request.respond(response); -} - -fn handle_history_api(request: tiny_http::Request, history: Arc) { - let json = history.worker_history_json(); - let response = tiny_http::Response::from_string(json).with_header( - "Content-Type: application/json" - .parse::() - .unwrap(), - ); - let _ = request.respond(response); -} - -fn parse_query_string(url: &str) -> HashMap { - let mut params = HashMap::new(); - if let Some(qs) = url.split('?').nth(1) { - for pair in qs.split('&') { - let mut kv = pair.splitn(2, '='); - if let (Some(k), Some(v)) = (kv.next(), kv.next()) { - params.insert(k.to_string(), v.to_string()); - } - } - } - params -} - -// ── Replay server ─────────────────────────────────────────────────────── - -/// Start a replay HTTP server that serves a pre-recorded trace. -pub(crate) fn spawn_replay_server(trace: Arc, port: u16, speed: f64) { - let addr = format!("0.0.0.0:{port}"); - let server = tiny_http::Server::http(&addr).expect("failed to bind HTTP server"); - let server = Arc::new(server); - - for _ in 0..4 { - let server = Arc::clone(&server); - let trace = Arc::clone(&trace); - thread::spawn(move || { - loop { - let request = match server.recv() { - Ok(r) => r, - Err(_) => break, - }; - - let url = request.url().to_string(); - match url.as_str() { - "/" => respond_html(request, DASHBOARD_HTML, "replay"), - "/actors" => respond_html(request, ACTORS_HTML, "replay"), - "/events" => { - handle_replay_sse(request, Arc::clone(&trace), speed); - } - _ => respond_404(request), - } - } - }); - } -} - -fn handle_replay_sse(request: tiny_http::Request, trace: Arc, speed: f64) { - let (tx, rx) = mpsc::channel::>(); - let response = make_sse_response(rx); - - thread::spawn(move || { - // Send replay metadata - let meta = serde_json::json!({ - "total_events": trace.events.len(), - "total_stats": trace.stats_timeline.len(), - "speed": speed, - }); - if tx.send(format_sse("replay_meta", &meta.to_string())).is_err() { - return; - } - - // Find the earliest timestamp across events and stats - let base_time = trace - .events - .first() - .map(|e| e.timestamp_ms) - .into_iter() - .chain(trace.stats_timeline.first().map(|s| s.timestamp_ms)) - .min() - .unwrap_or(0); - - let playback_start = Instant::now(); - let mut event_idx = 0; - let mut stats_idx = 0; - - loop { - let elapsed_ms = (playback_start.elapsed().as_millis() as f64 * speed) as u64; - let virtual_time = base_time + elapsed_ms; - - // Batch events up to virtual_time - let mut batch = Vec::new(); - while event_idx < trace.events.len() - && trace.events[event_idx].timestamp_ms <= virtual_time - { - batch.push(trace.events[event_idx].clone()); - event_idx += 1; - } - if !batch.is_empty() { - let json = serde_json::to_string(&batch).unwrap(); - if tx.send(format_sse("activity", &json)).is_err() { - return; - } - } - - // Send stats snapshots up to virtual_time - while stats_idx < trace.stats_timeline.len() - && trace.stats_timeline[stats_idx].timestamp_ms <= virtual_time - { - let json = - serde_json::to_string(&trace.stats_timeline[stats_idx].stats).unwrap(); - if tx.send(format_sse("stats", &json)).is_err() { - return; - } - stats_idx += 1; - } - - // Send progress - let total = trace.events.len() + trace.stats_timeline.len(); - let done_count = event_idx + stats_idx; - let progress = if total > 0 { - done_count as f64 / total as f64 - } else { - 1.0 - }; - let progress_json = serde_json::json!({ "progress": progress }); - if tx - .send(format_sse("replay_progress", &progress_json.to_string())) - .is_err() - { - return; - } - - // Check if replay is complete - if event_idx >= trace.events.len() - && stats_idx >= trace.stats_timeline.len() - { - let _ = tx.send(format_sse("done", "{}")); - let _ = tx.send(Vec::new()); // EOF - return; - } - - thread::sleep(Duration::from_millis(50)); - } - }); - - // Blocks until connection closes - let _ = request.respond(response); -} diff --git a/crates/simulation/src/distribution/properties.rs b/crates/simulation/src/distribution/properties.rs index 5ef4ee8..697451e 100644 --- a/crates/simulation/src/distribution/properties.rs +++ b/crates/simulation/src/distribution/properties.rs @@ -496,3 +496,268 @@ pub fn check_cache_bounded( description: "Cache never exceeds configured capacity".into(), } } + +// ─── Deployment Topology Property Checks ──────────────────────────────────── + +/// Check convergence within a specific group of nodes (not the whole cluster). +/// +/// Passes if there exists a round after `after_round` where all alive nodes +/// in `group_indices` have member_count within `tolerance` of each other. +pub fn check_group_convergence( + trace: &DistTrace, + group_indices: &[usize], + after_round: usize, + tolerance: usize, +) -> crate::properties::PropertyResult { + let converged = trace + .snapshots_per_round + .iter() + .skip(after_round) + .any(|round_snaps| { + let counts: Vec = group_indices + .iter() + .filter_map(|&idx| { + if idx < round_snaps.len() { + let (_, s) = &round_snaps[idx]; + if s.is_alive { Some(s.member_count) } else { None } + } else { + None + } + }) + .collect(); + if counts.is_empty() { + return true; + } + let min = *counts.iter().min().unwrap(); + let max = *counts.iter().max().unwrap(); + max - min <= tolerance + }); + + crate::properties::PropertyResult { + name: "group_convergence".into(), + category: "Deployment Topology".into(), + passed: converged, + expected: format!( + "group {:?} converges (spread ≤ {tolerance}) after round {after_round}", + group_indices + ), + actual: if converged { + "converged".into() + } else { + let final_counts: Vec = group_indices + .iter() + .filter_map(|&idx| { + trace.snapshots_per_round.last().and_then(|r| { + if idx < r.len() { + let (_, s) = &r[idx]; + if s.is_alive { Some(s.member_count) } else { None } + } else { + None + } + }) + }) + .collect(); + format!("final group member_counts: {final_counts:?}") + }, + description: "Membership views converge within a node group".into(), + } +} + +/// Detects membership oscillation (suspect→dead→alive cycling). +/// +/// For each alive node, counts how many times `member_count` changes direction +/// (increase→decrease or vice versa) after `after_round`. Fails if any node +/// exceeds `max_flips`. +pub fn check_membership_stability( + trace: &DistTrace, + after_round: usize, + max_flips: usize, +) -> crate::properties::PropertyResult { + let mut worst_node = String::new(); + let mut worst_flips = 0usize; + + let num_nodes = trace.node_names.len(); + for node_idx in 0..num_nodes { + let rounds: Vec<(usize, bool)> = trace + .snapshots_per_round + .iter() + .skip(after_round) + .map(|round_snaps| { + let (_, s) = &round_snaps[node_idx]; + (s.member_count, s.is_alive) + }) + .collect(); + + let mut flips = 0usize; + // Track direction: +1 = increasing, -1 = decreasing, 0 = no change yet + let mut direction: i32 = 0; + let mut prev_count: Option = None; + + for (count, is_alive) in &rounds { + if !is_alive { + prev_count = None; + direction = 0; + continue; + } + if let Some(prev) = prev_count { + let new_dir = if *count > prev { + 1 + } else if *count < prev { + -1 + } else { + direction // no change keeps previous direction + }; + if direction != 0 && new_dir != 0 && new_dir != direction { + flips += 1; + } + if new_dir != 0 { + direction = new_dir; + } + } + prev_count = Some(*count); + } + + if flips > worst_flips { + worst_flips = flips; + worst_node = trace.node_names[node_idx].clone(); + } + } + + crate::properties::PropertyResult { + name: "membership_stability".into(), + category: "Topology Adversarial".into(), + passed: worst_flips <= max_flips, + expected: format!("≤{max_flips} direction flips per node after round {after_round}"), + actual: format!("{worst_node} had {worst_flips} flips"), + description: "Membership count does not oscillate excessively".into(), + } +} + +/// Checks that the spread (max - min) of `member_count` across alive nodes +/// stays within `max_spread` for at least one round after `after_round`. +/// +/// Asymmetric relay links cause some nodes to see the full cluster while others +/// see a reduced view — this detects that divergence. +pub fn check_view_asymmetry( + trace: &DistTrace, + after_round: usize, + max_spread: usize, +) -> crate::properties::PropertyResult { + let within_spread = trace + .snapshots_per_round + .iter() + .skip(after_round) + .any(|round_snaps| { + let counts: Vec = round_snaps + .iter() + .filter(|(_, s)| s.is_alive) + .map(|(_, s)| s.member_count) + .collect(); + if counts.is_empty() { + return true; + } + let min = *counts.iter().min().unwrap(); + let max = *counts.iter().max().unwrap(); + max - min <= max_spread + }); + + let final_spread = trace + .snapshots_per_round + .last() + .map(|round_snaps| { + let counts: Vec = round_snaps + .iter() + .filter(|(_, s)| s.is_alive) + .map(|(_, s)| s.member_count) + .collect(); + if counts.is_empty() { + return (0, Vec::new()); + } + let min = *counts.iter().min().unwrap(); + let max = *counts.iter().max().unwrap(); + (max - min, counts) + }) + .unwrap_or((0, Vec::new())); + + crate::properties::PropertyResult { + name: "view_asymmetry".into(), + category: "Topology Adversarial".into(), + passed: within_spread, + expected: format!("member_count spread ≤{max_spread} for at least one round after {after_round}"), + actual: format!("final spread={}, counts={:?}", final_spread.0, final_spread.1), + description: "Membership views across alive nodes do not diverge excessively".into(), + } +} + +/// Detect total convergence failure: all alive nodes have member_count == 0 +/// for every round after `after_round`. This catches the deploy auth race +/// failure mode where peer introductions happen but SWIM joins never complete. +pub fn check_zero_convergence( + trace: &DistTrace, + after_round: usize, +) -> crate::properties::PropertyResult { + let all_zero = trace + .snapshots_per_round + .iter() + .skip(after_round) + .all(|round_snaps| { + let alive: Vec<_> = round_snaps.iter().filter(|(_, s)| s.is_alive).collect(); + !alive.is_empty() && alive.iter().all(|(_, s)| s.member_count == 0) + }); + + crate::properties::PropertyResult { + name: "zero_convergence".into(), + category: "Cluster Formation".into(), + passed: !all_zero, + expected: format!("at least one alive node has member_count > 0 after round {after_round}"), + actual: if all_zero { + "all alive nodes stuck at member_count=0".into() + } else { + "membership progressing".into() + }, + description: "Detects total SWIM convergence failure (auth race / join never completed)".into(), + } +} + +/// Check that staggered-join nodes eventually reach min_members by a deadline. +/// +/// Passes if by `by_round`, at least `min_members` of the `expected_joined` nodes +/// are alive and have member_count >= 1. +pub fn check_staggered_join( + trace: &DistTrace, + expected_joined: &[usize], + min_members: usize, + by_round: usize, +) -> crate::properties::PropertyResult { + let joined_count = trace + .snapshots_per_round + .iter() + .take(by_round) + .last() + .map(|round_snaps| { + expected_joined + .iter() + .filter(|&&idx| { + if idx < round_snaps.len() { + let (_, s) = &round_snaps[idx]; + s.is_alive && s.member_count >= 1 + } else { + false + } + }) + .count() + }) + .unwrap_or(0); + + crate::properties::PropertyResult { + name: "staggered_join".into(), + category: "Deployment Topology".into(), + passed: joined_count >= min_members, + expected: format!( + "≥{min_members} of {:?} joined with ≥1 member by round {by_round}", + expected_joined + ), + actual: format!("{joined_count} nodes joined"), + description: "Staggered-join nodes reach membership by deadline".into(), + } +} diff --git a/crates/simulation/src/distribution/sim.rs b/crates/simulation/src/distribution/sim.rs index 3de07df..85472ce 100644 --- a/crates/simulation/src/distribution/sim.rs +++ b/crates/simulation/src/distribution/sim.rs @@ -1,4 +1,4 @@ -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use distribution::node::{DistributedNode, DistributedNodeConfig, ResolveResult}; use distribution::swim::node::NodeAction; @@ -10,6 +10,26 @@ use crate::trace::{Event, SimulationTrace}; use super::trace::{DistributionEventKind, DistributionSnapshot}; +/// Network location of a simulated node. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum NodeLocation { + /// Publicly reachable (e.g. cloud VPS). Can receive inbound from anyone. + Public, + /// Behind NAT. Can only receive inbound from same LAN group or via relay. + Nat { group: String }, + /// Completely firewalled — no inbound or outbound. + Firewalled, +} + +/// Network topology describing NAT/firewall/relay placement. +#[derive(Debug, Clone)] +pub struct NetworkTopology { + /// Per-node location (indexed by node_idx). Length must equal num_nodes. + pub locations: Vec, + /// Node indices that act as relay forwarders for cross-NAT traffic. + pub relay_nodes: Vec, +} + /// A network partition between two sets of nodes. /// Nodes in `side_a` cannot communicate with nodes in `side_b`. #[derive(Debug, Clone)] @@ -31,6 +51,10 @@ pub enum SimAction { UnregisterName { node_idx: usize, name: String }, /// Graceful leave — node announces its own death before being removed. GracefulLeave { node_idx: usize }, + /// Mid-simulation join: node_idx sends a join request to seed_idx. + Join { node_idx: usize, seed_idx: usize }, + /// Bidirectional introduction (models POST /api/peers/add from deploy script). + Introduce { node_a: usize, node_b: usize }, } /// Schedule entry for network faults. @@ -42,6 +66,10 @@ pub enum NetworkFault { Heal { round: usize }, /// Set message drop rate (0.0 = no drops, 1.0 = drop all). SetDropRate { round: usize, rate: f64 }, + /// Per-link drop rate. rate=0.0 clears the fault. + LinkFault { round: usize, from: usize, to: usize, rate: f64, bidirectional: bool }, + /// Relay penalty — extra drop probability for relay-routed messages. + SetRelayPenalty { round: usize, rate: f64 }, } /// Configuration for a distribution simulation run. @@ -67,6 +95,10 @@ pub struct DistributionSimConfig { pub registry_tombstone_ttl: Option, pub registry_gc_interval: Option, pub registry_dissemination_lambda: Option, + /// Network topology for NAT/firewall simulation. None = full connectivity. + pub topology: Option, + /// Node indices that skip the initial join phase (must be joined via SimAction). + pub deferred_join: Vec, } impl Default for DistributionSimConfig { @@ -92,6 +124,8 @@ impl Default for DistributionSimConfig { registry_tombstone_ttl: None, registry_gc_interval: None, registry_dissemination_lambda: None, + topology: None, + deferred_join: Vec::new(), } } } @@ -104,6 +138,14 @@ struct NetworkState { drop_rate: f64, /// Simple counter-based deterministic "random" for drop decisions. drop_counter: u64, + /// Optional NAT/firewall topology. + topology: Option, + /// Per-node alive status (indexed by node_idx). + alive: Vec, + /// Per-link drop rates (from, to) -> rate. + link_drop_rates: HashMap<(usize, usize), f64>, + /// Extra drop probability for relay-routed messages. + relay_penalty: f64, } impl NetworkState { @@ -111,7 +153,29 @@ impl NetworkState { Self { blocked: HashSet::new(), drop_rate: 0.0, - drop_counter: 0x853c49e6748fea9b, // Non-zero seed for better distribution + drop_counter: 0x853c49e6748fea9b, + topology: None, + alive: Vec::new(), + link_drop_rates: HashMap::new(), + relay_penalty: 0.0, + } + } + + fn new_with_topology(topology: Option, num_nodes: usize) -> Self { + Self { + blocked: HashSet::new(), + drop_rate: 0.0, + drop_counter: 0x853c49e6748fea9b, + topology, + alive: vec![true; num_nodes], + link_drop_rates: HashMap::new(), + relay_penalty: 0.0, + } + } + + fn set_alive(&mut self, idx: usize, alive: bool) { + if idx < self.alive.len() { + self.alive[idx] = alive; } } @@ -135,18 +199,109 @@ impl NetworkState { NetworkFault::SetDropRate { rate, .. } => { self.drop_rate = rate.clamp(0.0, 1.0); } + NetworkFault::LinkFault { from, to, rate, bidirectional, .. } => { + let rate = rate.clamp(0.0, 1.0); + if rate == 0.0 { + self.link_drop_rates.remove(&(*from, *to)); + if *bidirectional { + self.link_drop_rates.remove(&(*to, *from)); + } + } else { + self.link_drop_rates.insert((*from, *to), rate); + if *bidirectional { + self.link_drop_rates.insert((*to, *from), rate); + } + } + } + NetworkFault::SetRelayPenalty { rate, .. } => { + self.relay_penalty = rate.clamp(0.0, 1.0); + } } } + /// Check if `from` can directly initiate a connection to `to`. + fn directly_reachable(&self, from: usize, to: usize) -> bool { + let topo = match &self.topology { + Some(t) => t, + None => return true, // No topology = full connectivity + }; + if from >= topo.locations.len() || to >= topo.locations.len() { + return true; + } + match (&topo.locations[from], &topo.locations[to]) { + (_, NodeLocation::Firewalled) => false, + (NodeLocation::Firewalled, _) => false, + (_, NodeLocation::Public) => true, // Anyone can reach public + (NodeLocation::Public, NodeLocation::Nat { .. }) => false, // Can't initiate inbound to NAT + (NodeLocation::Nat { group: g1 }, NodeLocation::Nat { group: g2 }) => g1 == g2, // Same LAN + } + } + + /// Check if two nodes can communicate (bidirectional once established). + /// Either direct reachability in either direction, or via a relay. + fn can_reach(&self, from: usize, to: usize) -> bool { + let topo = match &self.topology { + Some(t) => t, + None => return true, + }; + // Direct: if either side can initiate, the connection is bidirectional + if self.directly_reachable(from, to) || self.directly_reachable(to, from) { + return true; + } + // Relay path: any alive relay R where both endpoints can bidirectionally reach R + for &r in &topo.relay_nodes { + if r == from || r == to { + continue; + } + if !self.alive.get(r).copied().unwrap_or(false) { + continue; + } + let from_reaches_r = self.directly_reachable(from, r) || self.directly_reachable(r, from); + let to_reaches_r = self.directly_reachable(to, r) || self.directly_reachable(r, to); + if from_reaches_r && to_reaches_r { + return true; + } + } + false + } + + /// Returns true when neither direction is directly reachable but a relay path exists. + fn requires_relay(&self, from: usize, to: usize) -> bool { + if self.topology.is_none() { + return false; + } + if self.directly_reachable(from, to) || self.directly_reachable(to, from) { + return false; + } + self.can_reach(from, to) + } + /// Returns true if this message should be delivered. fn should_deliver(&mut self, from_idx: usize, to_idx: usize) -> bool { + // 1. Check partition blocks if self.blocked.contains(&(from_idx, to_idx)) { return false; } - if self.drop_rate > 0.0 { + // 2. Check NAT reachability (only if topology is set) + if self.topology.is_some() && !self.can_reach(from_idx, to_idx) { + return false; + } + // 3. Determine effective drop rate: per-link if set, else global + let base_rate = self.link_drop_rates + .get(&(from_idx, to_idx)) + .copied() + .unwrap_or(self.drop_rate); + // 4. Compose relay penalty if applicable + let effective_rate = if self.relay_penalty > 0.0 && self.requires_relay(from_idx, to_idx) { + 1.0 - (1.0 - base_rate) * (1.0 - self.relay_penalty) + } else { + base_rate + }; + // 5. Apply effective rate via LCG PRNG + if effective_rate > 0.0 { self.drop_counter = self.drop_counter.wrapping_mul(6364136223846793005).wrapping_add(1); let r = (self.drop_counter >> 33) as f64 / (u32::MAX as f64); - if r < self.drop_rate { + if r < effective_rate { return false; } } @@ -198,9 +353,12 @@ fn run_simulation_inner(config: DistributionSimConfig) -> (DistTrace, Vec