Compare commits
3 commits
23c829a9ba
...
553347a8f7
| Author | SHA1 | Date | |
|---|---|---|---|
| 553347a8f7 | |||
| c7d9c28e2b | |||
| 319ac1b8b8 |
186 changed files with 27312 additions and 9340 deletions
|
|
@ -1,3 +1,6 @@
|
|||
[build]
|
||||
rustc-workspace-wrapper = "tools/actor-control-flow-lint/rustc-wrapper.py"
|
||||
|
||||
[alias]
|
||||
xtask = "run --package xtask --"
|
||||
myelin-chat = "run --package xtask -- myelin-chat"
|
||||
|
|
|
|||
119
.github/workflows/myelin-properties.yml
vendored
Normal file
119
.github/workflows/myelin-properties.yml
vendored
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
name: Myelin properties
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
schedule:
|
||||
- cron: "17 3 * * *"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
component-properties:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Install pinned Rust toolchain
|
||||
run: rustup toolchain install nightly-2026-02-07 --profile minimal --component rustfmt --component rustc-dev --component rust-src --component llvm-tools-preview
|
||||
- name: Check formatting
|
||||
run: cargo fmt --all -- --check
|
||||
- name: Check default workspace members
|
||||
run: cargo check
|
||||
- name: Check compiler-policy contracts
|
||||
run: cargo test -p actor-control-flow-lint-tests --test contracts compiler_policy_contracts -- --exact
|
||||
- name: Engine scheduling properties
|
||||
run: |
|
||||
cargo test -p swactor-engine --test engine_unit generated_actor_timers_and_completion_are_bounded -- --exact
|
||||
cargo test -p swactor-engine --test engine_unit lifecycle_invariant_detects_injected_duplicate_completion -- --exact
|
||||
cargo test -p swactor-engine --test engine_unit lifecycle_invariant_detects_injected_uncancelled_periodic_timer -- --exact
|
||||
- name: Process adapter properties
|
||||
run: |
|
||||
cargo test -p swactor-process --lib operations::properties::generated_stream_observations_close_once_and_stay_closed -- --exact
|
||||
cargo test -p swactor-process --lib operations::properties::generated_lifecycle_actions_make_stop_idempotent_and_exit_terminal -- --exact
|
||||
cargo test -p swactor-process --lib operations::properties::generated_stop_notifications_are_delivered_at_most_once -- --exact
|
||||
cargo test -p swactor-process --lib operations::properties::generated_stdin_commands_and_eof_notify_once -- --exact
|
||||
cargo test -p swactor-process --lib operations::properties::property_invariants_reject_controlled_defects -- --exact
|
||||
cargo test -p swactor-process --lib operations::properties::trivial_real_child_exit_has_a_hard_timeout -- --exact
|
||||
- name: Myelin component properties (exclude job and reconciler tests)
|
||||
run: |
|
||||
cargo test -p myelin --lib orchestration::manual_control::tests::aggressive_random_event_stream_preserves_control_invariants -- --exact
|
||||
cargo test -p myelin --lib orchestration::manual_control::tests::rental_free_end_to_end_sequences_converge -- --exact
|
||||
cargo test -p myelin --lib orchestration::manual_control::tests::manual_actor_generated_public_actions_and_callbacks_are_bounded -- --exact
|
||||
cargo test -p myelin --lib orchestration::manual_control::tests::fixed_helper_cardinality_invariant_detects_controlled_extra_spawn -- --exact
|
||||
cargo test -p myelin --lib orchestration::manual_control::tests::callback_panic_reports_typed_failure_without_poisoning_work_actor -- --exact
|
||||
cargo test -p myelin --lib orchestration::manual_control::tests::callback_panic_invariant_detects_controlled_unguarded_panic -- --exact
|
||||
cargo test -p myelin --lib provisioning::tests::mock_vastai_handle_state_survives_random_create_and_stop_sequences -- --exact
|
||||
cargo test -p myelin --lib provisioning::tests::docker_generated_attempt_lifecycles_are_idempotent_and_bounded -- --exact
|
||||
cargo test -p myelin --lib provisioning::tests::docker_duplicate_resource_detector_rejects_controlled_fault -- --exact
|
||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::offer_status_classes_are_offers_or_typed_rejections -- --exact
|
||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::malformed_offer_bodies_are_typed_rejections -- --exact
|
||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::wrong_or_missing_offer_fields_are_typed_rejections -- --exact
|
||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::duplicate_offer_records_remain_explicit_values -- --exact
|
||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::provider_monitor_preserves_contract_identity_and_cardinality -- --exact
|
||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::provider_monitor_terminal_polling_stops_after_one_typed_outcome -- --exact
|
||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::provider_monitor_poll_stop_orderings_cease_polling -- --exact
|
||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::duplicate_terminal_detector_rejects_controlled_fault -- --exact
|
||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_output_lines_preserve_stream_and_protocol -- --exact
|
||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_malformed_protocol_is_data_not_poison -- --exact
|
||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_eof_orderings_stop_relay_and_actor -- --exact
|
||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_child_failures_have_typed_attempt_outcomes -- --exact
|
||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_timeout_is_typed_and_stops_polling -- --exact
|
||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_stop_orderings_emit_one_terminal_and_stop_all_actors -- --exact
|
||||
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::runtime_actors_generated_transitions_complete_once_on_one_worker -- --exact
|
||||
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::runtime_invariant_checker_rejects_duplicate_readiness_publication -- --exact
|
||||
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::helper_wait_generated_terminal_sequences_complete_once_on_one_worker -- --exact
|
||||
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::helper_invariant_checker_rejects_expected_output_after_terminal_error -- --exact
|
||||
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::stage_fetch_generated_observations_complete_once_on_one_worker -- --exact
|
||||
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::stage_invariant_checker_rejects_wrong_terminal_classification -- --exact
|
||||
cargo test -p myelin --lib orchestration::app::serve_cluster_properties::serve_cluster_production_transitions_converge_once_without_growth -- --exact
|
||||
cargo test -p myelin --lib orchestration::app::serve_cluster_properties::serve_cluster_lifecycle_invariants_reject_injected_duplicate_and_growth -- --exact
|
||||
cargo test -p myelin --lib orchestration::control::properties::generated_http_bridge_sequences_terminate_without_control_actor_growth -- --exact
|
||||
cargo test -p myelin --lib orchestration::control::properties::generated_duplicate_control_replies_deliver_first_once_and_remove_observer -- --exact
|
||||
cargo test -p myelin --lib orchestration::control::properties::reply_observer_disappearance_returns_a_bounded_terminal_http_response -- --exact
|
||||
cargo test -p myelin --lib orchestration::control::properties::http_bridge_invariant_rejects_a_controlled_duplicate_forward -- --exact
|
||||
- name: Controlled E2E oracle contract
|
||||
run: cargo test -p myelin --features test-support --test stateful_vastai e2e_oracle_rejects_controlled_lifecycle_faults -- --exact
|
||||
- name: Dashboard bridge properties
|
||||
run: |
|
||||
cargo test -p dashboard --features demo-control --lib control::properties::generated_concurrent_bridge_commands_forward_once_and_shutdown -- --exact
|
||||
cargo test -p dashboard --features demo-control --lib control::properties::bridge_invariant_rejects_a_controlled_duplicate_delivery -- --exact
|
||||
cargo test -p dashboard --features demo-control --lib server::tests::generated_control_http_sequences_are_bounded_and_typed -- --exact
|
||||
cargo test -p dashboard --features demo-control --lib server::tests::control_http_invariant_rejects_a_controlled_server_error -- --exact
|
||||
- name: Demo actor properties
|
||||
run: |
|
||||
cargo test -p xtask --bin xtask demo::control::properties::generated_control_commands_forward_only_after_supervisor_registration -- --exact
|
||||
cargo test -p xtask --bin xtask demo::control::properties::control_transition_oracle_rejects_duplicate_forwarding -- --exact
|
||||
cargo test -p xtask --bin xtask demo::feed::properties::generated_supervisor_transitions_are_once_only_nonblocking_and_clean -- --exact
|
||||
cargo test -p xtask --bin xtask demo::feed::properties::supervisor_transition_oracle_rejects_duplicate_identity_resources -- --exact
|
||||
cargo test -p xtask --bin xtask demo::node::properties::generated_node_runtime_transitions_emit_heartbeats_and_stop_once -- --exact
|
||||
cargo test -p xtask --bin xtask demo::node::properties::node_transition_oracle_rejects_duplicate_resources -- --exact
|
||||
cargo test -p xtask --bin xtask demo::provider::properties::generated_process_reports_complete_exit_watchers_once_and_preserve_last_state -- --exact
|
||||
cargo test -p xtask --bin xtask demo::provider::properties::process_relay_oracle_rejects_lost_exit -- --exact
|
||||
cargo test -p xtask --bin xtask demo::properties::direct_binary_signal_smoke_has_a_hard_timeout -- --exact
|
||||
|
||||
process-e2e:
|
||||
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
||||
needs: component-properties
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Install pinned Rust toolchain
|
||||
run: rustup toolchain install nightly-2026-02-07 --profile minimal --component rustc-dev --component rust-src --component llvm-tools-preview
|
||||
- name: Stateful VastAI process E2E
|
||||
env:
|
||||
PROPTEST_CASES: "4"
|
||||
run: |
|
||||
set -o pipefail
|
||||
mkdir -p artifacts
|
||||
cargo test -p myelin --features test-support --test stateful_vastai stateful_vastai_dashboard_control_survives_restarts -- --ignored --exact --nocapture 2>&1 | tee artifacts/stateful-vastai.log
|
||||
- name: Preserve E2E failure artifacts
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: stateful-vastai-failure-${{ github.run_id }}
|
||||
path: |
|
||||
artifacts/stateful-vastai.log
|
||||
apps/myelin/proptest-regressions/tests/e2e_vastai.txt
|
||||
if-no-files-found: warn
|
||||
22
Cargo.lock
generated
22
Cargo.lock
generated
|
|
@ -2,6 +2,10 @@
|
|||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "actor-control-flow-lint-tests"
|
||||
version = "0.1.0"
|
||||
|
||||
[[package]]
|
||||
name = "adler2"
|
||||
version = "2.0.1"
|
||||
|
|
@ -900,12 +904,15 @@ version = "0.1.0"
|
|||
dependencies = [
|
||||
"axum",
|
||||
"parking_lot",
|
||||
"proptest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"swactor",
|
||||
"swactor-engine",
|
||||
"telemetry",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tower",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1131,6 +1138,7 @@ dependencies = [
|
|||
"serde",
|
||||
"serde_json",
|
||||
"swactor",
|
||||
"swactor-engine",
|
||||
"swactor-transport",
|
||||
"telemetry",
|
||||
"uuid",
|
||||
|
|
@ -2478,6 +2486,7 @@ dependencies = [
|
|||
name = "myelin"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"axum",
|
||||
"blake3",
|
||||
"dashboard",
|
||||
"data-plane",
|
||||
|
|
@ -2486,10 +2495,10 @@ dependencies = [
|
|||
"iroh-driver",
|
||||
"libc",
|
||||
"parking_lot",
|
||||
"proptest",
|
||||
"provisioning",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"signal-hook",
|
||||
"swactor",
|
||||
"swactor-engine",
|
||||
"swactor-job-runner",
|
||||
|
|
@ -2501,7 +2510,6 @@ dependencies = [
|
|||
"tokio",
|
||||
"toml 0.8.23",
|
||||
"ureq",
|
||||
"wiremock",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3266,6 +3274,7 @@ dependencies = [
|
|||
"serde_json",
|
||||
"swactor",
|
||||
"swactor-engine",
|
||||
"swactor-process",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3337,6 +3346,7 @@ version = "0.1.0"
|
|||
dependencies = [
|
||||
"pyo3",
|
||||
"swactor",
|
||||
"swactor-engine",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -4305,6 +4315,7 @@ name = "swactor-engine"
|
|||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"parking_lot",
|
||||
"proptest",
|
||||
"swactor",
|
||||
"tokio",
|
||||
]
|
||||
|
|
@ -4331,11 +4342,16 @@ version = "0.1.0"
|
|||
dependencies = [
|
||||
"crossbeam-queue",
|
||||
"libc",
|
||||
"parking_lot",
|
||||
"proptest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_yaml",
|
||||
"signal-hook",
|
||||
"swactor",
|
||||
"swactor-engine",
|
||||
"telemetry",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -5179,6 +5195,7 @@ name = "wasm-runtime"
|
|||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"swactor",
|
||||
"swactor-engine",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
|
|
@ -5773,6 +5790,7 @@ dependencies = [
|
|||
"iroh-driver",
|
||||
"libc",
|
||||
"parking_lot",
|
||||
"proptest",
|
||||
"provisioning",
|
||||
"serde",
|
||||
"serde_json",
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ members = [
|
|||
"apps/myelin",
|
||||
"xtask",
|
||||
"tools/vastai",
|
||||
"tools/actor-control-flow-lint",
|
||||
]
|
||||
default-members = [
|
||||
".",
|
||||
|
|
|
|||
|
|
@ -10,12 +10,13 @@ autobins = false
|
|||
[features]
|
||||
default = ["dashboard"]
|
||||
dashboard = []
|
||||
test-support = ["swactor-vastai/test-support"]
|
||||
|
||||
[dependencies]
|
||||
telemetry = { path = "../../crates/telemetry" }
|
||||
data-plane = { path = "../../crates/data-plane" }
|
||||
provisioning = { path = "../../crates/provisioning" }
|
||||
dashboard = { path = "../../crates/dashboard", features = ["demo-control"] }
|
||||
dashboard = { path = "../../crates/dashboard" }
|
||||
serde_json = "1"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
swactor = { path = "../..", features = ["serde", "transport"] }
|
||||
|
|
@ -32,14 +33,20 @@ parking_lot = "0.12"
|
|||
blake3 = "1"
|
||||
toml = "0.8"
|
||||
ureq = "2"
|
||||
axum = "0.8"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
wiremock = "0.6"
|
||||
swactor-vastai = { path = "../../tools/vastai", features = ["test-support"] }
|
||||
proptest = "1"
|
||||
|
||||
[[test]]
|
||||
name = "stateful_vastai"
|
||||
path = "tests/stateful_vastai.rs"
|
||||
required-features = ["test-support"]
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
libc = "0.2"
|
||||
signal-hook = "0.3"
|
||||
|
||||
[[bin]]
|
||||
name = "myelin-worker"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,7 @@
|
|||
# Seeds for failure cases proptest has generated in the past. It is
|
||||
# automatically read and these particular cases re-run before any
|
||||
# novel cases are generated.
|
||||
#
|
||||
# It is recommended to check this file in to source control so that
|
||||
# everyone who runs the test benefits from these saved cases.
|
||||
cc be434e1d0bf8073f32e4072d9093dd2df03537b9d1bf759872e760fcf74a06cb # shrinks to actions = [], extra_ticks = 1
|
||||
7
apps/myelin/proptest-regressions/orchestration/app.txt
Normal file
7
apps/myelin/proptest-regressions/orchestration/app.txt
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
# Seeds for failure cases proptest has generated in the past. It is
|
||||
# automatically read and these particular cases re-run before any
|
||||
# novel cases are generated.
|
||||
#
|
||||
# It is recommended to check this file in to source control so that
|
||||
# everyone who runs the test benefits from these saved cases.
|
||||
cc ed6cf4326f625c499a6752aa3630891a4242ed8aa4cfe59148d901030efd4ff1 # shrinks to actions = []
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
# Seeds for failure cases proptest has generated in the past. It is
|
||||
# automatically read and these particular cases re-run before any
|
||||
# novel cases are generated.
|
||||
#
|
||||
# It is recommended to check this file in to source control so that
|
||||
# everyone who runs the test benefits from these saved cases.
|
||||
cc a36a5c2d2904ecfb2cf31f3e446db588a7591e074994fc61142f37aad33272c8 # shrinks to actions = [Provision(163), Drive, Configure(240), StaleSearch(108), Configure(85), Search(84), Flush, Kill(76), Configure(190), StaleValidation(217), Provision(209), StaleSearch(18), StaleValidation(70), Provision(97), Drive, Flush, StaleValidation(81), Search(121), Query, Flush, StaleValidation(91), StaleValidation(110)]
|
||||
cc aa67d0e8e7fd15f0b8cad5ac675b6fcee25fc5246a6841d141054b6932f0db34 # shrinks to actions = [Search(0), Search(0), Provision(86)]
|
||||
cc ad4aa3d6008d92f09c315c680c258a2b3776719bca98ff49d72668542f0c4772 # shrinks to actions = [Provision(17), Flush(81), OfferSearchFinished(200), PersistenceFinished(145), Provision(202), ProviderTerminalFailure(29), Kill(222), Provision(21), Provision(117), Query(131), Provision(126), Rejoin(27), Provision(89), OfferSearchFinished(148), Query(54), Kill(77)]
|
||||
cc be1220292a8604bca5d52ed98402b55bfd81ca591ee47c4139829365a5c1eddc # shrinks to actions = [ProviderValidated(17), Flush(231), OfferSearchFinished(238), Drive, Provision(32), Configure(101), Configure(194), ProviderValidated(226), Provision(152), Configure(235), Provision(208), Kill(68), ProviderValidated(50)]
|
||||
cc fbc45330b5c55a483c910567ed49b8c2f40e4d9b8ad17f9a88f666fc120d4305 # shrinks to actions = [PersistenceFinished(73), Flush(190), EffectFinished(224), Query(58), Kill(182), Rejoin(82), Flush(137), OfferSearchFinished(173), Drive, Provision(204), Kill(137), EffectFinished(187), Configure(80), Configure(104), Drive, PersistenceFinished(92), Flush(180), Search(86), Configure(246), Query(33), Search(43)]
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
# Seeds for failure cases proptest has generated in the past. It is
|
||||
# automatically read and these particular cases re-run before any
|
||||
# novel cases are generated.
|
||||
#
|
||||
# It is recommended to check this file in to source control so that
|
||||
# everyone who runs the test benefits from these saved cases.
|
||||
cc 38b940d757de89f824d145dc409f7f0d3b6d757840eff473ed9acac3eef65d21 # shrinks to actions = [Create(61), Create(253)]
|
||||
cc 502380a923217c7ea1912e0a54392f1389eaafa4e35efeac26cdeb5ac2a0067e # shrinks to actions = [Create(53), Start(0), Adopt(221)]
|
||||
5
apps/myelin/proptest-regressions/tests/e2e_vastai.txt
Normal file
5
apps/myelin/proptest-regressions/tests/e2e_vastai.txt
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
# Seeds for stateful VastAI E2E failures. Proptest replays these before generated cases.
|
||||
cc 26c3cd5944cb25de70a984389373e7380a0c6e6ba1c8f3e6eff5a0f78dbef168 # current-strategy replay; E2eCase is derived from e2e_case()
|
||||
cc 5efb0a6c5c348475c859a2c4267d5177aaecd5adefcc5e8b7fc2e6d728deecd1 # shrinks to case = E2eCase { seed: 7099259626237328177, node_seed: 245, kill_mask: 24, offer_offset: 3, actions: [ConcurrentQueries, Search { count: 2 }, Restart { mode: FlushSafeAbrupt }, Query, Kill { node_slot: 218, command_slot: 5 }, Query, Provision { command_slot: 5, use_searched_offers: true }, EndpointProbe { node_slot: 217 }, Restart { mode: Graceful }, Flush, Kill { node_slot: 217, command_slot: 5 }] }
|
||||
cc 6d005687858520ed65af20ec5b2de056efcda15771b6420d7f67e3cb6c246d5b # shrinks to case = E2eCase { seed: 17649392557414661864, node_seed: 249, kill_mask: 159, offer_offset: 5, actions: [Kill { node_slot: 88, command_slot: 242 }, Flush, ConcurrentQueries, EndpointProbe { node_slot: 88 }, Query, Search { count: 2 }, Provision { command_slot: 242, use_searched_offers: true }, Restart { mode: FlushSafeAbrupt }, Kill { node_slot: 89, command_slot: 242 }, Query, Restart { mode: Graceful }] }
|
||||
cc 2e36f51e7cb347b571ac838f81638748f8414a32ec91cb9f4b2730363d72be61 # shrinks to case = E2eCase { seed: 13301618846512983428, node_seed: 99, kill_mask: 30, offer_offset: 2, actions: [Query, Restart { mode: FlushSafeAbrupt }, Kill { node_slot: 136, command_slot: 187 }, ConcurrentQueries, Restart { mode: Graceful }, Query, EndpointProbe { node_slot: 136 }, Kill { node_slot: 137, command_slot: 187 }, Provision { command_slot: 187, use_searched_offers: false }, Flush, Search { count: 0 }] }
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
# Myelin Fleet-Control Daemon
|
||||
|
||||
Myelin is a persistent, dashboard-first control plane for manually managed compute nodes. It boots an empty fleet, accepts explicit operator commands, records intent and observations, and never performs hidden replacement or teardown.
|
||||
|
||||
## Runtime shape
|
||||
|
||||
```text
|
||||
myelin-orchestrator
|
||||
-> load stable iroh identity and cluster snapshot
|
||||
-> start engine, iroh endpoint, telemetry collector, and dashboard
|
||||
-> adopt provider resources carrying this daemon's stable label
|
||||
-> idle event loop
|
||||
- pump telemetry and membership
|
||||
- dispatch add / kill / destroy commands
|
||||
- atomically persist every state transition
|
||||
-> on exit, detach provider handles without destroying resources
|
||||
```
|
||||
|
||||
There is no prompt RPC, chat loop, desired-shape reconciler, or automatic node replacement. GGUF pipeline code remains dormant for historical compatibility. Job submission and a node re-join handshake are deferred.
|
||||
|
||||
## Starting the daemon
|
||||
|
||||
From the workspace root:
|
||||
|
||||
```sh
|
||||
cargo run -p myelin
|
||||
```
|
||||
|
||||
The local default uses the process provider. Use Docker explicitly when required:
|
||||
|
||||
```sh
|
||||
cargo run -p myelin -- --provider docker
|
||||
```
|
||||
|
||||
`MYELIN_DASHBOARD_PORT` selects the dashboard port. The dashboard root is the fleet-control view. Its control routes dispatch the same provider-neutral commands intended for a future CLI:
|
||||
|
||||
- `Provision { command_id, count }`: add exactly `count` nodes, one transaction at a time.
|
||||
- `Kill { command_id, node }`: stop a node but retain its dead snapshot record.
|
||||
- `Remove { command_id, count }`: destroy the highest numbered managed nodes and remove their snapshot records.
|
||||
- `EstablishEdge { command_id, node }`: rejected; workload topology is not part of fleet control.
|
||||
|
||||
Every request carries a caller-generated command id. The daemon persists that id before any provider mutation; retries are ignored across restarts. This is deliberately at-most-once: a crash after acceptance may require a new operator command, but can never double-rent or double-destroy a resource. Node ids are monotonic and never reused.
|
||||
|
||||
## Durable state
|
||||
|
||||
The state directory contains:
|
||||
|
||||
- `identity.key`: 32-byte iroh secret key. Preserving it keeps the daemon endpoint stable across restarts.
|
||||
- `cluster.json`: schema-versioned snapshot containing the stable provider label, run id, next node id, accepted command ids, node specs, provider references, runtime facts, and observed status.
|
||||
|
||||
Writes use a temporary file plus rename. A corrupt identity or snapshot is a hard startup error. `--reset-state` explicitly clears both files; startup never treats corruption as an empty fleet.
|
||||
|
||||
Provider state is ground truth during adoption:
|
||||
|
||||
- snapshot + provider resource: adopt and observe it;
|
||||
- snapshot only: mark dead;
|
||||
- provider resource only: report as an orphan and take no action.
|
||||
|
||||
The current restart limitation is deliberate: an adopted node still has the prior orchestrator actor address in its environment. Provider monitoring and telemetry collection can resume, but actor-address re-join requires the deferred node handshake.
|
||||
|
||||
## Lifecycle policy
|
||||
|
||||
Graceful shutdown leaves Docker containers and Vast.ai leases running so a later daemon can adopt them. Local process children are different: they cannot be adopted, so Ctrl-C stops them and clears their snapshot records. They also exit when their daemon-owned stdin supervision pipe closes, preventing an abrupt daemon crash from leaving invisible local workers.
|
||||
|
||||
Destruction of durable provider resources occurs only through an explicit dashboard command or the development-only `--destroy-on-exit` flag. The default process path re-enters the running orchestrator executable in an internal worker mode, so `cargo run -p myelin` never depends on a separately built or stale `myelin-worker` binary.
|
||||
|
||||
## Node image contract
|
||||
|
||||
The standard image contains a uniform Myelin agent, SSH bootstrap, and the CUDA runtime. It does not contain tinygrad, NumPy, PyTorch, vLLM, or model weights. Frameworks and application dependencies belong to job payload images. Nodes launched by this daemon set `MYELIN_AGENT_ONLY=1`, so the agent joins membership, announces readiness, and exports telemetry without starting an inference helper.
|
||||
As part of runtime-ready bootstrap, the daemon dials the node's advertised iroh endpoint on `TELEMETRY_ALPN`, requests all telemetry channels, and retains that pull stream for the node's lifetime. The node serves the pull locally; it never needs to reverse-dial the dashboard. Pulled stream descriptors and frames feed both the Fleet view and the live telemetry explorer.
|
||||
|
||||
The retired chat/GGUF specification is archived at `archive/MYELIN_CHAT_SPEC.md`.
|
||||
|
|
@ -7,5 +7,6 @@
|
|||
pub(crate) fn register_myelin_actor_codecs(registry: &mut swactor_transport::CodecRegistry) {
|
||||
crate::node_actor::register_codecs(registry);
|
||||
crate::orchestration::actor::register_codecs(registry);
|
||||
crate::orchestration::manual_control::register_codecs(registry);
|
||||
swactor_job_runner::register_job_codecs(registry);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,10 +13,11 @@ use std::sync::atomic::{AtomicBool, Ordering};
|
|||
use std::time::{Duration, Instant};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use swactor::actor::ActorAddress;
|
||||
use swactor_engine::{Engine, EngineHandle, TokioBackend, TokioConfig};
|
||||
use swactor::actor::{ActorAddress, ActorInterface};
|
||||
use swactor::runtime::{Ctx, ExternalSender};
|
||||
use swactor_engine::{ActorCompletion, Engine, EngineHandle, TokioBackend, TokioConfig};
|
||||
use swactor_job_runner::{
|
||||
JobDone, NodeJobActor, OUTPUTS_EDGE_ID, OrchestratorJobActor, OrchestratorJobMsg,
|
||||
Job, JobDone, NodeJobActor, OUTPUTS_EDGE_ID, OrchestratorJobActor, OrchestratorJobMsg,
|
||||
WORKSPACE_EDGE_ID, register_job_codecs,
|
||||
};
|
||||
use swactor_transport::hex_encode;
|
||||
|
|
@ -100,35 +101,104 @@ pub(crate) fn build_composition() -> Result<JobComposition, String> {
|
|||
Ok((engine, driver, stack))
|
||||
}
|
||||
|
||||
fn identity_for(driver: &IrohDriver, actor: ActorAddress) -> Result<NodeIdentity, String> {
|
||||
let endpoint = advertised_endpoint_for(driver)?;
|
||||
Ok(NodeIdentity {
|
||||
endpoint,
|
||||
actor_hex: hex_encode(&actor.0),
|
||||
})
|
||||
struct ResolveIdentityActor {
|
||||
driver: Option<IrohDriver>,
|
||||
actor: ActorAddress,
|
||||
mask: EndpointAddrMask,
|
||||
started: Instant,
|
||||
engine: EngineHandle,
|
||||
sender: ExternalSender,
|
||||
completion: ActorCompletion<Result<(IrohDriver, NodeIdentity), String>>,
|
||||
}
|
||||
|
||||
fn advertised_endpoint_for(driver: &IrohDriver) -> Result<EndpointAddr, String> {
|
||||
let mask = endpoint_addr_mask_from_env()?;
|
||||
if !mask.requires_relay() {
|
||||
return advertised_endpoint(driver.endpoint_addr(), mask);
|
||||
#[derive(Clone)]
|
||||
enum ResolveIdentityMsg {
|
||||
Check,
|
||||
}
|
||||
|
||||
impl ResolveIdentityActor {
|
||||
fn finish(&mut self, ctx: &swactor::runtime::Ctx, result: Result<NodeIdentity, String>) {
|
||||
let result = result.map(|identity| {
|
||||
(
|
||||
self.driver
|
||||
.take()
|
||||
.expect("identity resolver owns driver until completion"),
|
||||
identity,
|
||||
)
|
||||
});
|
||||
assert!(
|
||||
self.completion.complete(result).is_ok(),
|
||||
"identity resolver completed twice"
|
||||
);
|
||||
ctx.stop_self();
|
||||
}
|
||||
|
||||
let started = Instant::now();
|
||||
loop {
|
||||
let endpoint = driver.endpoint_addr();
|
||||
if endpoint.relay_urls().next().is_some() {
|
||||
return advertised_endpoint(endpoint, mask);
|
||||
}
|
||||
if started.elapsed() >= RELAY_WAIT_DEADLINE {
|
||||
return Err(format!(
|
||||
"relay-only endpoint address mask did not observe a relay URL within {RELAY_WAIT_DEADLINE:?}; last endpoint={endpoint:?}"
|
||||
));
|
||||
}
|
||||
std::thread::sleep(POLL);
|
||||
fn schedule_check(&self, ctx: &swactor::runtime::Ctx) {
|
||||
self.engine.send_after(
|
||||
POLL,
|
||||
self.sender.clone(),
|
||||
ctx.self_addr(),
|
||||
ResolveIdentityMsg::Check,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
impl ActorInterface for ResolveIdentityActor {
|
||||
type Incoming = ResolveIdentityMsg;
|
||||
type Response = ();
|
||||
|
||||
fn on_start(&mut self, ctx: &swactor::runtime::Ctx) {
|
||||
let _ = ctx.send(ctx.self_addr(), ResolveIdentityMsg::Check);
|
||||
}
|
||||
|
||||
fn handle(&mut self, ctx: &swactor::runtime::Ctx, _message: Self::Incoming) {
|
||||
let driver = self
|
||||
.driver
|
||||
.as_ref()
|
||||
.expect("identity resolver handles messages only while live");
|
||||
let endpoint = driver.endpoint_addr();
|
||||
if !self.mask.requires_relay() || endpoint.relay_urls().next().is_some() {
|
||||
let identity = advertised_endpoint(endpoint, self.mask)
|
||||
.map(|endpoint| NodeIdentity {
|
||||
endpoint,
|
||||
actor_hex: hex_encode(&self.actor.0),
|
||||
})
|
||||
.map_err(|error| error.to_string());
|
||||
self.finish(ctx, identity);
|
||||
} else if self.started.elapsed() >= RELAY_WAIT_DEADLINE {
|
||||
self.finish(
|
||||
ctx,
|
||||
Err(format!(
|
||||
"relay-only endpoint address mask did not observe a relay URL within {RELAY_WAIT_DEADLINE:?}; last endpoint={endpoint:?}"
|
||||
)),
|
||||
);
|
||||
} else {
|
||||
self.schedule_check(ctx);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_identity(
|
||||
driver: IrohDriver,
|
||||
actor: ActorAddress,
|
||||
stack: &DistributionRuntimeStack,
|
||||
) -> Result<(IrohDriver, NodeIdentity), String> {
|
||||
let completion = ActorCompletion::new();
|
||||
stack
|
||||
.runtime
|
||||
.spawn(ResolveIdentityActor {
|
||||
driver: Some(driver),
|
||||
actor,
|
||||
mask: endpoint_addr_mask_from_env()?,
|
||||
started: Instant::now(),
|
||||
engine: stack.engine.clone(),
|
||||
sender: stack.runtime.create_sender(),
|
||||
completion: completion.clone(),
|
||||
})
|
||||
.map_err(|error| format!("spawn endpoint identity resolver: {error}"))?;
|
||||
completion.wait()
|
||||
}
|
||||
|
||||
fn endpoint_addr_mask_from_env() -> Result<EndpointAddrMask, String> {
|
||||
match env_optional(MVP_IROH_ENDPOINT_ADDR_MASK_ENV) {
|
||||
Some(mask) => EndpointAddrMask::parse(&mask),
|
||||
|
|
@ -174,6 +244,94 @@ pub(crate) fn parse_actor(hex: &str) -> Result<ActorAddress, String> {
|
|||
Ok(ActorAddress(arr))
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
enum WorkerLifecycleMsg {
|
||||
CheckConnection,
|
||||
Stop,
|
||||
}
|
||||
|
||||
struct WorkerLifecycleActor {
|
||||
driver: IrohDriver,
|
||||
_stack: DistributionRuntimeStack,
|
||||
orchestrator_endpoint: EndpointAddr,
|
||||
orchestrator_node: swactor_transport::NodeId,
|
||||
output_sink: Arc<Mutex<Option<Box<dyn swactor_job_runner::JobEdgeSink>>>>,
|
||||
started: Instant,
|
||||
armed: bool,
|
||||
engine: EngineHandle,
|
||||
sender: ExternalSender,
|
||||
completion: ActorCompletion<Result<(), String>>,
|
||||
}
|
||||
|
||||
impl WorkerLifecycleActor {
|
||||
fn schedule_check(&self, ctx: &Ctx) {
|
||||
self.engine.send_after(
|
||||
POLL,
|
||||
self.sender.clone(),
|
||||
ctx.self_addr(),
|
||||
WorkerLifecycleMsg::CheckConnection,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
impl ActorInterface for WorkerLifecycleActor {
|
||||
type Incoming = WorkerLifecycleMsg;
|
||||
type Response = ();
|
||||
|
||||
fn on_start(&mut self, ctx: &Ctx) {
|
||||
let _ = ctx.send(ctx.self_addr(), WorkerLifecycleMsg::CheckConnection);
|
||||
}
|
||||
|
||||
fn handle(&mut self, ctx: &Ctx, message: Self::Incoming) {
|
||||
match message {
|
||||
WorkerLifecycleMsg::CheckConnection if !self.armed => {
|
||||
if self.driver.has_active_connection(&self.orchestrator_node)
|
||||
|| self.started.elapsed() >= CONVERGE_DEADLINE
|
||||
{
|
||||
match self
|
||||
.driver
|
||||
.spawn_edge_send_pump(self.orchestrator_endpoint.clone(), OUTPUTS_EDGE_ID)
|
||||
{
|
||||
Ok(handle) => {
|
||||
*self.output_sink.lock() = Some(Box::new(IrohEdgeSink(handle)));
|
||||
eprintln!("job-worker: output edge sink armed");
|
||||
}
|
||||
Err(error) => {
|
||||
eprintln!("job-worker: failed to arm output edge sink: {error}")
|
||||
}
|
||||
}
|
||||
self.armed = true;
|
||||
} else {
|
||||
self.schedule_check(ctx);
|
||||
}
|
||||
}
|
||||
WorkerLifecycleMsg::CheckConnection => {}
|
||||
WorkerLifecycleMsg::Stop => {
|
||||
assert!(
|
||||
self.completion.complete(Ok(())).is_ok(),
|
||||
"worker lifecycle completed twice"
|
||||
);
|
||||
ctx.stop_self();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct WorkerStopForwarder {
|
||||
sender: ExternalSender,
|
||||
worker: ActorAddress,
|
||||
}
|
||||
|
||||
impl ActorInterface for WorkerStopForwarder {
|
||||
type Incoming = ();
|
||||
type Response = ();
|
||||
|
||||
fn handle(&mut self, ctx: &Ctx, (): Self::Incoming) {
|
||||
let _ = self.sender.send_to(self.worker, WorkerLifecycleMsg::Stop);
|
||||
ctx.stop_self();
|
||||
}
|
||||
}
|
||||
|
||||
/// Worker: connect to the orchestrator, expose a `NodeJobActor`, run jobs it
|
||||
/// sends over iroh. Prints this node's identity as JSON on stdout, then runs
|
||||
/// until killed. Bulk bytes travel over EDGE_ALPN: the orchestrator pushes the
|
||||
|
|
@ -185,42 +343,38 @@ pub fn run_worker(orch_identity_json: String, workdir: PathBuf) -> Result<(), St
|
|||
.map_err(|e| format!("parse orch identity: {e}"))?;
|
||||
let orch_actor = parse_actor(&orch.actor_hex)?;
|
||||
let orch_endpoint = orch.endpoint.clone();
|
||||
let (_engine, driver, stack) = build_composition()?;
|
||||
let (engine, driver, stack) = build_composition()?;
|
||||
let sender = stack.runtime.create_sender();
|
||||
|
||||
// Edge workspace: the orchestrator pushes the workspace tar over EDGE_ALPN
|
||||
// before submitting the job. A background thread drains those bytes,
|
||||
// extracts them into `workdir`, and signals readiness; the node actor waits
|
||||
// on that flag before announcing the workspace materialized.
|
||||
let workspace_ready = Arc::new(AtomicBool::new(false));
|
||||
let ws_events = driver.edge_events_handle();
|
||||
let ws_workdir = workdir.clone();
|
||||
let ws_ready = workspace_ready.clone();
|
||||
std::thread::Builder::new()
|
||||
.name("job-worker-ws-edge".to_owned())
|
||||
.spawn(move || {
|
||||
drain_workspace_edge(ws_events, ws_workdir, ws_ready);
|
||||
stack
|
||||
.runtime
|
||||
.spawn(WorkspaceEdgeActor {
|
||||
engine: stack.engine.clone(),
|
||||
sender: stack.runtime.create_sender(),
|
||||
events: driver.edge_events_handle(),
|
||||
workdir: workdir.clone(),
|
||||
ready: workspace_ready.clone(),
|
||||
buf: Vec::new(),
|
||||
started: Instant::now(),
|
||||
})
|
||||
.map_err(|e| format!("spawn workspace edge thread: {e}"))?;
|
||||
.map_err(|error| format!("spawn workspace edge actor: {error}"))?;
|
||||
|
||||
// Edge outputs: a slot the main thread fills with an EDGE_ALPN sink to the
|
||||
// orchestrator once the iroh connection is up. The node actor ships
|
||||
// collected outputs through it.
|
||||
let output_sink_slot: Arc<Mutex<Option<Box<dyn swactor_job_runner::JobEdgeSink>>>> =
|
||||
Arc::new(Mutex::new(None));
|
||||
|
||||
let job_actor = stack
|
||||
.runtime
|
||||
.spawn(
|
||||
NodeJobActor::new(orch_actor, workdir, sender, 0)
|
||||
.with_workspace_ready(workspace_ready.clone())
|
||||
.with_actor_timers(engine.handle())
|
||||
.with_workspace_ready(workspace_ready)
|
||||
.with_output_sink_slot(output_sink_slot.clone()),
|
||||
)
|
||||
.map_err(|e| format!("spawn node job actor: {e}"))?;
|
||||
stack.register_local_actor(driver.register_actor(job_actor, 1));
|
||||
driver.join(std::slice::from_ref(&orch.endpoint));
|
||||
|
||||
let id = identity_for(&driver, job_actor)?;
|
||||
let (driver, id) = resolve_identity(driver, job_actor, &stack)?;
|
||||
println!(
|
||||
"JOB_WORKER_IDENTITY {}",
|
||||
serde_json::to_string(&id).map_err(|e| e.to_string())?
|
||||
|
|
@ -231,27 +385,33 @@ pub fn run_worker(orch_identity_json: String, workdir: PathBuf) -> Result<(), St
|
|||
id.actor_hex, id.endpoint
|
||||
);
|
||||
|
||||
// Wait for the iroh connection to the orchestrator, then arm the output
|
||||
// edge sink so it is ready before a CollectOutputs command can arrive.
|
||||
let orch_node = swactor_transport::NodeId(*orch_endpoint.id.as_bytes());
|
||||
let conn_started = Instant::now();
|
||||
while !driver.has_active_connection(&orch_node) {
|
||||
if conn_started.elapsed() >= CONVERGE_DEADLINE {
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(POLL);
|
||||
}
|
||||
match driver.spawn_edge_send_pump(orch_endpoint.clone(), OUTPUTS_EDGE_ID) {
|
||||
Ok(handle) => {
|
||||
*output_sink_slot.lock() = Some(Box::new(IrohEdgeSink(handle)));
|
||||
eprintln!("job-worker: output edge sink armed");
|
||||
}
|
||||
Err(e) => eprintln!("job-worker: failed to arm output edge sink: {e}"),
|
||||
}
|
||||
|
||||
loop {
|
||||
std::thread::sleep(Duration::from_secs(3600));
|
||||
}
|
||||
let runtime = stack.runtime.clone();
|
||||
let completion = ActorCompletion::new();
|
||||
let lifecycle = runtime
|
||||
.spawn(WorkerLifecycleActor {
|
||||
driver,
|
||||
_stack: stack,
|
||||
orchestrator_node: swactor_transport::NodeId(*orch_endpoint.id.as_bytes()),
|
||||
orchestrator_endpoint: orch_endpoint,
|
||||
output_sink: output_sink_slot,
|
||||
started: Instant::now(),
|
||||
armed: false,
|
||||
engine: engine.handle(),
|
||||
sender: runtime.create_sender(),
|
||||
completion: completion.clone(),
|
||||
})
|
||||
.map_err(|error| format!("spawn job worker lifecycle actor: {error}"))?;
|
||||
let stop_forwarder = runtime
|
||||
.spawn(WorkerStopForwarder {
|
||||
sender: runtime.create_sender(),
|
||||
worker: lifecycle,
|
||||
})
|
||||
.map_err(|error| format!("spawn job worker stop forwarder: {error}"))?;
|
||||
#[cfg(target_os = "linux")]
|
||||
swactor_process::spawn_os_stop_signal_wait(runtime.create_sender(), stop_forwarder);
|
||||
let result = completion.wait();
|
||||
drop(engine);
|
||||
result
|
||||
}
|
||||
|
||||
/// Starts the operator-side job actor and publishes enough identity for a
|
||||
|
|
@ -268,7 +428,7 @@ pub(crate) fn start_orchestrator(landing: PathBuf) -> Result<JobOrchestratorSess
|
|||
.map_err(|e| format!("spawn orchestrator: {e}"))?;
|
||||
stack.register_local_actor(driver.register_actor(orch, 1));
|
||||
|
||||
let identity = identity_for(&driver, orch)?;
|
||||
let (driver, identity) = resolve_identity(driver, orch, &stack)?;
|
||||
Ok(JobOrchestratorSession {
|
||||
_engine: engine,
|
||||
driver,
|
||||
|
|
@ -280,6 +440,257 @@ pub(crate) fn start_orchestrator(landing: PathBuf) -> Result<JobOrchestratorSess
|
|||
})
|
||||
}
|
||||
|
||||
pub(crate) struct JobRunStateMachine {
|
||||
session: JobOrchestratorSession,
|
||||
job: Option<Job>,
|
||||
worker: NodeIdentity,
|
||||
node_actor: ActorAddress,
|
||||
phase: JobRunPhase,
|
||||
}
|
||||
|
||||
enum JobRunPhase {
|
||||
Created,
|
||||
Directory {
|
||||
deadline: Instant,
|
||||
},
|
||||
Connection {
|
||||
started: Instant,
|
||||
deadline: Instant,
|
||||
},
|
||||
Running {
|
||||
deadline: Instant,
|
||||
output_buf: Vec<u8>,
|
||||
outputs_ended: bool,
|
||||
pending_done: Option<JobDone>,
|
||||
},
|
||||
Finished,
|
||||
}
|
||||
|
||||
impl JobRunStateMachine {
|
||||
pub(crate) fn new(
|
||||
session: JobOrchestratorSession,
|
||||
job: Job,
|
||||
worker: NodeIdentity,
|
||||
) -> Result<Self, String> {
|
||||
let node_actor = parse_actor(&worker.actor_hex)?;
|
||||
Ok(Self {
|
||||
session,
|
||||
job: Some(job),
|
||||
worker,
|
||||
node_actor,
|
||||
phase: JobRunPhase::Created,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn start(&mut self, now: Instant) {
|
||||
self.session
|
||||
.driver
|
||||
.join(std::slice::from_ref(&self.worker.endpoint));
|
||||
self.phase = JobRunPhase::Directory {
|
||||
deadline: now + CONVERGE_DEADLINE,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) fn advance(&mut self, now: Instant) -> Option<Result<JobDone, String>> {
|
||||
let phase = std::mem::replace(&mut self.phase, JobRunPhase::Finished);
|
||||
match phase {
|
||||
JobRunPhase::Created => {
|
||||
self.phase = JobRunPhase::Created;
|
||||
None
|
||||
}
|
||||
JobRunPhase::Directory { deadline } => {
|
||||
let converged = self
|
||||
.session
|
||||
.stack
|
||||
.route_view
|
||||
.read()
|
||||
.map(|view| view.contains_key(&self.node_actor))
|
||||
.unwrap_or(false);
|
||||
if converged {
|
||||
eprintln!(
|
||||
"job-orch: directory converged; waiting for iroh connection to worker"
|
||||
);
|
||||
self.phase = JobRunPhase::Connection {
|
||||
started: now,
|
||||
deadline: now + CONVERGE_DEADLINE,
|
||||
};
|
||||
None
|
||||
} else if now >= deadline {
|
||||
Some(Err(
|
||||
"directory did not converge: orchestrator never learned the worker actor"
|
||||
.into(),
|
||||
))
|
||||
} else {
|
||||
self.phase = JobRunPhase::Directory { deadline };
|
||||
None
|
||||
}
|
||||
}
|
||||
JobRunPhase::Connection { started, deadline } => {
|
||||
let worker_node = swactor_transport::NodeId(*self.worker.endpoint.id.as_bytes());
|
||||
let connected = self.session.driver.has_active_connection(&worker_node);
|
||||
if !connected && now < deadline {
|
||||
self.phase = JobRunPhase::Connection { started, deadline };
|
||||
return None;
|
||||
}
|
||||
if connected {
|
||||
eprintln!(
|
||||
"job-orch: iroh connection to worker established after {:?}",
|
||||
now.saturating_duration_since(started)
|
||||
);
|
||||
eprintln!("job-orch: submitting job");
|
||||
} else {
|
||||
eprintln!(
|
||||
"job-orch: no iroh connection to worker after {CONVERGE_DEADLINE:?}; join_statuses={:?}; submitting best-effort",
|
||||
self.session.driver.join_statuses()
|
||||
);
|
||||
}
|
||||
match self.submit(now) {
|
||||
Ok(()) => None,
|
||||
Err(error) => Some(Err(error)),
|
||||
}
|
||||
}
|
||||
JobRunPhase::Running {
|
||||
deadline,
|
||||
mut output_buf,
|
||||
mut outputs_ended,
|
||||
mut pending_done,
|
||||
} => {
|
||||
let edge_events = self.session.driver.edge_events_handle();
|
||||
let drained: Vec<WireEvent> = edge_events.lock().drain(..).collect();
|
||||
for event in drained {
|
||||
match event {
|
||||
WireEvent::BytesRead { edge_id, bytes, .. }
|
||||
if edge_id.0 == OUTPUTS_EDGE_ID =>
|
||||
{
|
||||
output_buf.extend_from_slice(&bytes);
|
||||
}
|
||||
WireEvent::StreamEnded { edge_id, .. } if edge_id.0 == OUTPUTS_EDGE_ID => {
|
||||
if !output_buf.is_empty() {
|
||||
if let Err(error) = swactor_job_runner::extract_tar(
|
||||
&output_buf,
|
||||
&self.session.landing,
|
||||
) {
|
||||
eprintln!("job-orch: untar edge outputs failed: {error}");
|
||||
}
|
||||
output_buf.clear();
|
||||
}
|
||||
outputs_ended = true;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if pending_done.is_none() {
|
||||
pending_done = self.session.done.try_recv();
|
||||
}
|
||||
if let Some(done) = pending_done.as_ref() {
|
||||
let need_outputs = done.exit_code.is_some() && !outputs_ended;
|
||||
if !need_outputs || now >= deadline {
|
||||
let done = pending_done
|
||||
.take()
|
||||
.expect("pending job result was observed");
|
||||
if need_outputs {
|
||||
eprintln!("job-orch: output edge stream did not land before deadline");
|
||||
}
|
||||
return Some(Ok(done));
|
||||
}
|
||||
}
|
||||
if now >= deadline {
|
||||
return Some(Err("job did not complete within deadline".into()));
|
||||
}
|
||||
self.phase = JobRunPhase::Running {
|
||||
deadline,
|
||||
output_buf,
|
||||
outputs_ended,
|
||||
pending_done,
|
||||
};
|
||||
None
|
||||
}
|
||||
JobRunPhase::Finished => {
|
||||
Some(Err("job state machine advanced after completion".into()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn submit(&mut self, now: Instant) -> Result<(), String> {
|
||||
let job = self
|
||||
.job
|
||||
.take()
|
||||
.ok_or_else(|| "job was already submitted".to_owned())?;
|
||||
let workspace_bytes =
|
||||
swactor_job_runner::pack_workspace(&job).map_err(|e| format!("pack workspace: {e}"))?;
|
||||
if !workspace_bytes.is_empty() {
|
||||
let pump = self
|
||||
.session
|
||||
.driver
|
||||
.spawn_edge_send_pump(self.worker.endpoint.clone(), WORKSPACE_EDGE_ID)
|
||||
.map_err(|e| format!("workspace edge pump: {e}"))?;
|
||||
for record in workspace_bytes.chunks(swactor_job_runner::EDGE_RECORD_SIZE) {
|
||||
pump.send(record.to_vec())
|
||||
.map_err(|e| format!("workspace edge send: {e}"))?;
|
||||
}
|
||||
drop(pump);
|
||||
eprintln!("job-orch: workspace pushed over EDGE_ALPN");
|
||||
}
|
||||
self.session
|
||||
.stack
|
||||
.runtime
|
||||
.send_to(
|
||||
self.session.orch,
|
||||
OrchestratorJobMsg::Submit {
|
||||
job,
|
||||
node_actor: self.node_actor,
|
||||
},
|
||||
)
|
||||
.map_err(|e| format!("submit: {e}"))?;
|
||||
self.phase = JobRunPhase::Running {
|
||||
deadline: now + JOB_DEADLINE,
|
||||
output_buf: Vec::new(),
|
||||
outputs_ended: false,
|
||||
pending_done: None,
|
||||
};
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct JobRunTick;
|
||||
|
||||
struct JobRunActor {
|
||||
machine: JobRunStateMachine,
|
||||
engine: EngineHandle,
|
||||
sender: ExternalSender,
|
||||
completion: ActorCompletion<Result<JobDone, String>>,
|
||||
}
|
||||
|
||||
impl JobRunActor {
|
||||
fn schedule(&self, ctx: &Ctx) {
|
||||
self.engine
|
||||
.send_after(POLL, self.sender.clone(), ctx.self_addr(), JobRunTick);
|
||||
}
|
||||
}
|
||||
|
||||
impl ActorInterface for JobRunActor {
|
||||
type Incoming = JobRunTick;
|
||||
type Response = ();
|
||||
|
||||
fn on_start(&mut self, ctx: &Ctx) {
|
||||
self.machine.start(Instant::now());
|
||||
let _ = ctx.send(ctx.self_addr(), JobRunTick);
|
||||
}
|
||||
|
||||
fn handle(&mut self, ctx: &Ctx, _message: Self::Incoming) {
|
||||
if let Some(result) = self.machine.advance(Instant::now()) {
|
||||
assert!(
|
||||
self.completion.complete(result).is_ok(),
|
||||
"job lifecycle completed twice"
|
||||
);
|
||||
ctx.stop_self();
|
||||
} else {
|
||||
self.schedule(ctx);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl JobOrchestratorSession {
|
||||
pub(crate) fn identity_json(&self) -> Result<String, String> {
|
||||
serde_json::to_string(&self.identity).map_err(|e| e.to_string())
|
||||
|
|
@ -294,151 +705,29 @@ impl JobOrchestratorSession {
|
|||
}
|
||||
|
||||
pub(crate) fn run_to_completion(
|
||||
&mut self,
|
||||
job: swactor_job_runner::Job,
|
||||
self,
|
||||
job: Job,
|
||||
worker: NodeIdentity,
|
||||
) -> Result<JobDone, String> {
|
||||
let node_actor = parse_actor(&worker.actor_hex)?;
|
||||
self.driver.join(std::slice::from_ref(&worker.endpoint));
|
||||
|
||||
let started = Instant::now();
|
||||
while started.elapsed() < CONVERGE_DEADLINE {
|
||||
if self
|
||||
.stack
|
||||
.route_view
|
||||
.read()
|
||||
.map(|v| v.contains_key(&node_actor))
|
||||
.unwrap_or(false)
|
||||
{
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(POLL);
|
||||
}
|
||||
if !self
|
||||
.stack
|
||||
.route_view
|
||||
.read()
|
||||
.map(|v| v.contains_key(&node_actor))
|
||||
.unwrap_or(false)
|
||||
{
|
||||
return Err(
|
||||
"directory did not converge: orchestrator never learned the worker actor".into(),
|
||||
);
|
||||
}
|
||||
eprintln!("job-orch: directory converged; waiting for iroh connection to worker");
|
||||
let worker_node = swactor_transport::NodeId(*worker.endpoint.id.as_bytes());
|
||||
let conn_started = Instant::now();
|
||||
while !self.driver.has_active_connection(&worker_node) {
|
||||
if conn_started.elapsed() >= CONVERGE_DEADLINE {
|
||||
eprintln!(
|
||||
"job-orch: no iroh connection to worker after {CONVERGE_DEADLINE:?}; join_statuses={:?}; submitting best-effort",
|
||||
self.driver.join_statuses()
|
||||
);
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(POLL);
|
||||
}
|
||||
if self.driver.has_active_connection(&worker_node) {
|
||||
eprintln!(
|
||||
"job-orch: iroh connection to worker established after {:?}",
|
||||
conn_started.elapsed()
|
||||
);
|
||||
eprintln!("job-orch: submitting job");
|
||||
} else {
|
||||
eprintln!("job-orch: submitting job (no confirmed connection)");
|
||||
}
|
||||
|
||||
// EDGE: push the workspace tar over EDGE_ALPN before submitting. Small
|
||||
// commands/events still travel as actor messages; only bulk bytes move
|
||||
// onto the edge transport so they survive relay (NAT) traversal.
|
||||
let edge_events = self.driver.edge_events_handle();
|
||||
let workspace_bytes =
|
||||
swactor_job_runner::pack_workspace(&job).map_err(|e| format!("pack workspace: {e}"))?;
|
||||
if !workspace_bytes.is_empty() {
|
||||
let pump = self
|
||||
.driver
|
||||
.spawn_edge_send_pump(worker.endpoint.clone(), WORKSPACE_EDGE_ID)
|
||||
.map_err(|e| format!("workspace edge pump: {e}"))?;
|
||||
for record in workspace_bytes.chunks(swactor_job_runner::EDGE_RECORD_SIZE) {
|
||||
pump.send(record.to_vec())
|
||||
.map_err(|e| format!("workspace edge send: {e}"))?;
|
||||
}
|
||||
drop(pump); // finish the edge stream → receiver observes end-of-stream
|
||||
eprintln!("job-orch: workspace pushed over EDGE_ALPN");
|
||||
}
|
||||
|
||||
self.stack
|
||||
.runtime
|
||||
.send_to(self.orch, OrchestratorJobMsg::Submit { job, node_actor })
|
||||
.map_err(|e| format!("submit: {e}"))?;
|
||||
|
||||
// Drive lifecycle (actor messages) while draining the output edge stream.
|
||||
let started = Instant::now();
|
||||
let mut output_buf: Vec<u8> = Vec::new();
|
||||
let mut outputs_ended = false;
|
||||
// The orchestrator actor reports `JobDone` exactly once; hold it here
|
||||
// while we wait for the output edge stream to land so it is not lost.
|
||||
let mut pending_done: Option<JobDone> = None;
|
||||
loop {
|
||||
// Drain output edge bytes; extract the tar as soon as the stream ends
|
||||
// (release the edge-event lock before the potentially slow untar).
|
||||
let drained: Vec<WireEvent> = edge_events.lock().drain(..).collect();
|
||||
for ev in drained {
|
||||
match ev {
|
||||
WireEvent::BytesRead { edge_id, bytes, .. } if edge_id.0 == OUTPUTS_EDGE_ID => {
|
||||
output_buf.extend_from_slice(&bytes);
|
||||
}
|
||||
WireEvent::StreamEnded { edge_id, .. } if edge_id.0 == OUTPUTS_EDGE_ID => {
|
||||
if !output_buf.is_empty() {
|
||||
if let Err(e) =
|
||||
swactor_job_runner::extract_tar(&output_buf, &self.landing)
|
||||
{
|
||||
eprintln!("job-orch: untar edge outputs failed: {e}");
|
||||
}
|
||||
output_buf.clear();
|
||||
}
|
||||
outputs_ended = true;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
if pending_done.is_none() {
|
||||
pending_done = self.done.try_recv();
|
||||
}
|
||||
|
||||
// A job that ran (exit code observed) collected outputs over edge —
|
||||
// wait for that stream to land before returning so the landing dir is
|
||||
// populated. A pre-run fault (no exit code) ships no outputs.
|
||||
let ready = match &pending_done {
|
||||
Some(done) => {
|
||||
let need_outputs = done.exit_code.is_some() && !outputs_ended;
|
||||
!need_outputs || started.elapsed() >= JOB_DEADLINE
|
||||
}
|
||||
None => false,
|
||||
};
|
||||
if ready {
|
||||
let done = pending_done
|
||||
.take()
|
||||
.expect("pending_done observed Some in ready branch");
|
||||
if done.exit_code.is_some() && !outputs_ended {
|
||||
eprintln!("job-orch: output edge stream did not land before deadline");
|
||||
}
|
||||
return Ok(done);
|
||||
}
|
||||
|
||||
if started.elapsed() >= JOB_DEADLINE {
|
||||
return Err("job did not complete within deadline".into());
|
||||
}
|
||||
std::thread::sleep(POLL);
|
||||
}
|
||||
let runtime = self.stack.runtime.clone();
|
||||
let engine = self.stack.engine.clone();
|
||||
let completion = ActorCompletion::new();
|
||||
runtime
|
||||
.spawn(JobRunActor {
|
||||
machine: JobRunStateMachine::new(self, job, worker)?,
|
||||
engine,
|
||||
sender: runtime.create_sender(),
|
||||
completion: completion.clone(),
|
||||
})
|
||||
.map_err(|error| format!("spawn job lifecycle actor: {error}"))?;
|
||||
completion.wait()
|
||||
}
|
||||
}
|
||||
|
||||
/// Orchestrator: expose an `OrchestratorJobActor`, print its identity, read the
|
||||
/// worker identity from stdin, drive the job to completion over iroh.
|
||||
pub fn run_serve(job: swactor_job_runner::Job, landing: PathBuf) -> Result<JobDone, String> {
|
||||
let mut session = start_orchestrator(landing)?;
|
||||
let session = start_orchestrator(landing)?;
|
||||
println!("JOB_ORCH_IDENTITY {}", session.identity_json()?);
|
||||
let _ = std::io::Write::flush(&mut std::io::stdout());
|
||||
eprintln!("job-orch: published identity; waiting for worker identity on stdin...");
|
||||
|
|
@ -473,40 +762,64 @@ impl swactor_job_runner::JobEdgeSink for IrohEdgeSink {
|
|||
/// without the readiness flag being set).
|
||||
const WORKSPACE_EDGE_WAIT: Duration = Duration::from_secs(60 * 30);
|
||||
|
||||
/// Drain EDGE_ALPN workspace bytes (edge id `WORKSPACE_EDGE_ID`) the orchestrator
|
||||
/// pushed, extract the tar into `workdir`, then signal readiness. Runs on a
|
||||
/// background worker thread; the driver auto-accepts EDGE_ALPN connections and
|
||||
/// pushes their bytes into the shared event queue drained here.
|
||||
fn drain_workspace_edge(
|
||||
#[derive(Clone)]
|
||||
struct WorkspaceEdgePoll;
|
||||
|
||||
struct WorkspaceEdgeActor {
|
||||
engine: EngineHandle,
|
||||
sender: ExternalSender,
|
||||
events: Arc<Mutex<Vec<WireEvent>>>,
|
||||
workdir: PathBuf,
|
||||
ready: Arc<AtomicBool>,
|
||||
) {
|
||||
let mut buf = Vec::new();
|
||||
let started = Instant::now();
|
||||
loop {
|
||||
let drained: Vec<WireEvent> = events.lock().drain(..).collect();
|
||||
for ev in drained {
|
||||
match ev {
|
||||
buf: Vec<u8>,
|
||||
started: Instant,
|
||||
}
|
||||
|
||||
impl WorkspaceEdgeActor {
|
||||
fn schedule(&self, ctx: &Ctx, delay: Duration) {
|
||||
self.engine.send_after(
|
||||
delay,
|
||||
self.sender.clone(),
|
||||
ctx.self_addr(),
|
||||
WorkspaceEdgePoll,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
impl ActorInterface for WorkspaceEdgeActor {
|
||||
type Incoming = WorkspaceEdgePoll;
|
||||
type Response = ();
|
||||
|
||||
fn on_start(&mut self, ctx: &Ctx) {
|
||||
self.schedule(ctx, Duration::ZERO);
|
||||
}
|
||||
|
||||
fn handle(&mut self, ctx: &Ctx, _message: Self::Incoming) {
|
||||
let drained: Vec<WireEvent> = self.events.lock().drain(..).collect();
|
||||
for event in drained {
|
||||
match event {
|
||||
WireEvent::BytesRead { edge_id, bytes, .. } if edge_id.0 == WORKSPACE_EDGE_ID => {
|
||||
buf.extend_from_slice(&bytes);
|
||||
self.buf.extend_from_slice(&bytes);
|
||||
}
|
||||
WireEvent::StreamEnded { edge_id, .. } if edge_id.0 == WORKSPACE_EDGE_ID => {
|
||||
if !buf.is_empty() {
|
||||
if let Err(e) = swactor_job_runner::extract_tar(&buf, &workdir) {
|
||||
eprintln!("job-worker: untar workspace failed: {e}");
|
||||
}
|
||||
if !self.buf.is_empty()
|
||||
&& let Err(error) =
|
||||
swactor_job_runner::extract_tar(&self.buf, &self.workdir)
|
||||
{
|
||||
eprintln!("job-worker: untar workspace failed: {error}");
|
||||
}
|
||||
ready.store(true, Ordering::Release);
|
||||
self.ready.store(true, Ordering::Release);
|
||||
ctx.stop_self();
|
||||
return;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if started.elapsed() >= WORKSPACE_EDGE_WAIT {
|
||||
if self.started.elapsed() >= WORKSPACE_EDGE_WAIT {
|
||||
eprintln!("job-worker: workspace edge stream did not arrive");
|
||||
ctx.stop_self();
|
||||
return;
|
||||
}
|
||||
std::thread::sleep(POLL);
|
||||
self.schedule(ctx, POLL);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,8 +10,6 @@
|
|||
extern crate self as myelin;
|
||||
|
||||
const DEFAULT_PIPELINE_CACHED_MODEL_FILE: &str = "SmolLM2-135M-Instruct.Q4_0.gguf";
|
||||
const DEFAULT_PIPELINE_CACHED_MODEL_REPO: &str = "QuantFactory/SmolLM2-135M-Instruct-GGUF";
|
||||
const DEFAULT_PIPELINE_CACHED_MODEL_ID: &str = "smollm2-135m-instruct-q4";
|
||||
#[doc(hidden)]
|
||||
pub const ORCHESTRATOR_WORKER_MODE_ARG: &str = "--myelin-worker-node";
|
||||
pub fn run_orchestrator_from_args<I>(args: I) -> Result<(), String>
|
||||
|
|
|
|||
|
|
@ -145,6 +145,10 @@ pub(crate) enum NodeAgentMsg {
|
|||
stage_index: u32,
|
||||
readiness_id: u64,
|
||||
},
|
||||
RebindOrchestrator {
|
||||
orchestrator_actor: ActorAddress,
|
||||
control_generation: u64,
|
||||
},
|
||||
MarkWeightsReady {
|
||||
run_id: u64,
|
||||
node_id: u64,
|
||||
|
|
@ -331,6 +335,7 @@ pub(crate) struct NodeAgentActor {
|
|||
core: stage::StageController,
|
||||
orchestrator: ActorAddress,
|
||||
report_to: Option<ActorAddress>,
|
||||
control_generation: u64,
|
||||
inbound_edge: Option<StageInboundEdgeWire>,
|
||||
outbound_edge: Option<StageOutboundEdgeWire>,
|
||||
command_cursor: usize,
|
||||
|
|
@ -348,6 +353,7 @@ impl NodeAgentActor {
|
|||
core: stage::StageController::new(local_node_id),
|
||||
orchestrator,
|
||||
report_to,
|
||||
control_generation: 0,
|
||||
inbound_edge: None,
|
||||
outbound_edge: None,
|
||||
command_cursor: 0,
|
||||
|
|
@ -430,6 +436,17 @@ impl NodeAgentActor {
|
|||
}
|
||||
|
||||
fn observe(&mut self, ctx: &Ctx, msg: NodeAgentMsg) {
|
||||
if let NodeAgentMsg::RebindOrchestrator {
|
||||
orchestrator_actor,
|
||||
control_generation,
|
||||
} = msg
|
||||
{
|
||||
if control_generation >= self.control_generation {
|
||||
self.orchestrator = orchestrator_actor;
|
||||
self.control_generation = control_generation;
|
||||
}
|
||||
return;
|
||||
}
|
||||
let Some(msg) = self.forward_prompt_or_snapshot(ctx, msg) else {
|
||||
return;
|
||||
};
|
||||
|
|
@ -586,6 +603,9 @@ impl NodeAgentActor {
|
|||
| NodeAgentMsg::EncodePrompt { .. }
|
||||
| NodeAgentMsg::DecodeTokens { .. }
|
||||
| NodeAgentMsg::Snapshot { .. } => unreachable!("prompt messages returned early"),
|
||||
NodeAgentMsg::RebindOrchestrator { .. } => {
|
||||
unreachable!("rebind messages returned early")
|
||||
}
|
||||
}
|
||||
self.drain_outputs(ctx);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,4 +4,4 @@
|
|||
//! only wire entrypoints into it.
|
||||
|
||||
pub(super) mod prompt_wire;
|
||||
pub(super) mod worker_node_runtime;
|
||||
pub(crate) mod worker_node_runtime;
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,245 +0,0 @@
|
|||
//! Dashboard view over Myelin cluster provisioning and lifecycle telemetry records.
|
||||
|
||||
use std::collections::{BTreeMap, VecDeque};
|
||||
|
||||
use dashboard::FrameEvent;
|
||||
use dashboard::view::DashboardView;
|
||||
use serde::Serialize;
|
||||
use serde_json::{Value, json};
|
||||
use std::sync::RwLock;
|
||||
use telemetry::Record;
|
||||
use telemetry::frame::{Frame, StreamId};
|
||||
|
||||
use crate::observability::lifecycle as obs;
|
||||
use crate::observability::telemetry::{
|
||||
MYELIN_LIFECYCLE, MYELIN_PROVISIONING_EVENTS, MYELIN_PROVISIONING_LOGS, MyelinLifecycleRecord,
|
||||
MyelinProvisionEventRecord, MyelinProvisionLogRecord,
|
||||
};
|
||||
use crate::provisioning::{ProvisionEventKind, ProvisionLogStream};
|
||||
|
||||
const CHANNELS: &[&str] = &[];
|
||||
const PROVISIONING_LOG_PREFIX: &str = "myelin.provisioning.logs.node.";
|
||||
const EVENT_LOG_CAP: usize = 256;
|
||||
const LOG_TAIL_CAP: usize = 128;
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct MyelinClusterDashboardView {
|
||||
state: RwLock<MyelinClusterDashboardState>,
|
||||
}
|
||||
|
||||
impl MyelinClusterDashboardView {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default, Serialize)]
|
||||
struct MyelinClusterDashboardState {
|
||||
events: VecDeque<DashboardEventEntry>,
|
||||
nodes: BTreeMap<u64, ProvisionNodeView>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Serialize)]
|
||||
struct DashboardEventEntry {
|
||||
channel: String,
|
||||
position: u64,
|
||||
label: String,
|
||||
run_id: Option<u64>,
|
||||
node_id: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Default, Serialize)]
|
||||
struct ProvisionNodeView {
|
||||
node_id: u64,
|
||||
phase: String,
|
||||
last_message: Option<String>,
|
||||
stdout_tail: VecDeque<String>,
|
||||
stderr_tail: VecDeque<String>,
|
||||
provider_tail: VecDeque<String>,
|
||||
}
|
||||
|
||||
impl MyelinClusterDashboardState {
|
||||
fn push_event(&mut self, entry: DashboardEventEntry) {
|
||||
if self.events.len() == EVENT_LOG_CAP {
|
||||
self.events.pop_front();
|
||||
}
|
||||
self.events.push_back(entry);
|
||||
}
|
||||
|
||||
fn apply_provision_event(&mut self, frame: &Frame, record: MyelinProvisionEventRecord) {
|
||||
let phase = provision_kind_label(record.event.kind);
|
||||
let node = self
|
||||
.nodes
|
||||
.entry(record.event.node_id)
|
||||
.or_insert_with(|| ProvisionNodeView {
|
||||
node_id: record.event.node_id,
|
||||
..ProvisionNodeView::default()
|
||||
});
|
||||
node.phase = phase.to_owned();
|
||||
node.last_message = record.event.message.clone();
|
||||
self.push_event(DashboardEventEntry {
|
||||
channel: MYELIN_PROVISIONING_EVENTS.to_owned(),
|
||||
position: frame.position.0,
|
||||
label: phase.to_owned(),
|
||||
run_id: Some(record.event.run_id),
|
||||
node_id: Some(record.event.node_id),
|
||||
});
|
||||
}
|
||||
|
||||
fn apply_provision_log(&mut self, record: MyelinProvisionLogRecord) {
|
||||
let node = self
|
||||
.nodes
|
||||
.entry(record.line.node_id)
|
||||
.or_insert_with(|| ProvisionNodeView {
|
||||
node_id: record.line.node_id,
|
||||
..ProvisionNodeView::default()
|
||||
});
|
||||
let tail = match record.line.stream {
|
||||
ProvisionLogStream::Stdout => &mut node.stdout_tail,
|
||||
ProvisionLogStream::Stderr => &mut node.stderr_tail,
|
||||
ProvisionLogStream::Provider => &mut node.provider_tail,
|
||||
};
|
||||
if tail.len() == LOG_TAIL_CAP {
|
||||
tail.pop_front();
|
||||
}
|
||||
tail.push_back(record.line.line);
|
||||
}
|
||||
|
||||
fn apply_lifecycle(&mut self, frame: &Frame, record: MyelinLifecycleRecord) {
|
||||
let (run_id, node_id) = lifecycle_ids(&record.event);
|
||||
self.push_event(DashboardEventEntry {
|
||||
channel: MYELIN_LIFECYCLE.to_owned(),
|
||||
position: frame.position.0,
|
||||
label: format!("{:?}", record.event.kind()),
|
||||
run_id,
|
||||
node_id,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
impl DashboardView for MyelinClusterDashboardView {
|
||||
fn id(&self) -> &'static str {
|
||||
"myelin/cluster"
|
||||
}
|
||||
|
||||
fn title(&self) -> &'static str {
|
||||
"Myelin cluster"
|
||||
}
|
||||
|
||||
fn channels(&self) -> &'static [&'static str] {
|
||||
CHANNELS
|
||||
}
|
||||
|
||||
fn ingest(&self, _stream: &StreamId, frame: &Frame, event: &FrameEvent) {
|
||||
let mut state = self.state.write().expect("Myelin dashboard state poisoned");
|
||||
match event.channel.as_str() {
|
||||
MYELIN_PROVISIONING_EVENTS => {
|
||||
if let Ok(record) = MyelinProvisionEventRecord::decode(&event.payload) {
|
||||
state.apply_provision_event(frame, record);
|
||||
}
|
||||
}
|
||||
channel
|
||||
if channel == MYELIN_PROVISIONING_LOGS
|
||||
|| channel.starts_with(PROVISIONING_LOG_PREFIX) =>
|
||||
{
|
||||
if let Ok(record) = MyelinProvisionLogRecord::decode(&event.payload) {
|
||||
state.apply_provision_log(record);
|
||||
}
|
||||
}
|
||||
MYELIN_LIFECYCLE => {
|
||||
if let Ok(record) = MyelinLifecycleRecord::decode(&event.payload) {
|
||||
state.apply_lifecycle(frame, record);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn snapshot_json(&self) -> Value {
|
||||
let state = self.state.read().expect("Myelin dashboard state poisoned");
|
||||
json!({
|
||||
"events": state.events,
|
||||
"nodes": state.nodes.values().collect::<Vec<_>>(),
|
||||
})
|
||||
}
|
||||
|
||||
fn html(&self) -> Option<&'static str> {
|
||||
Some(MYELIN_CLUSTER_HTML)
|
||||
}
|
||||
}
|
||||
|
||||
fn provision_kind_label(kind: ProvisionEventKind) -> &'static str {
|
||||
match kind {
|
||||
ProvisionEventKind::ProvisionStart => "provision_start",
|
||||
ProvisionEventKind::NodeLive => "node_live",
|
||||
ProvisionEventKind::ProvisionFailed => "provision_failed",
|
||||
ProvisionEventKind::NodeStopped => "node_stopped",
|
||||
}
|
||||
}
|
||||
|
||||
fn lifecycle_ids(event: &obs::Event) -> (Option<u64>, Option<u64>) {
|
||||
match event {
|
||||
obs::Event::RunScoped { run_id, .. } | obs::Event::StageScoped { run_id, .. } => {
|
||||
(Some(run_id.0), None)
|
||||
}
|
||||
obs::Event::NodeScoped { node_id, .. } | obs::Event::NodeFaulted { node_id, .. } => {
|
||||
(None, Some(node_id.0))
|
||||
}
|
||||
obs::Event::EdgeScoped { .. }
|
||||
| obs::Event::RingScoped { .. }
|
||||
| obs::Event::ObjectScoped { .. }
|
||||
| obs::Event::StepScoped { .. }
|
||||
| obs::Event::WorkerScoped { .. } => (None, None),
|
||||
}
|
||||
}
|
||||
|
||||
const MYELIN_CLUSTER_HTML: &str = r#"<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>Myelin cluster</title>
|
||||
<style>
|
||||
body { font: 13px system-ui, sans-serif; margin: 1rem; background: #0f1115; color: #e8e8e8; }
|
||||
table { border-collapse: collapse; width: 100%; margin-bottom: 1rem; }
|
||||
th, td { border-bottom: 1px solid #30343d; padding: .35rem .5rem; text-align: left; vertical-align: top; }
|
||||
th { color: #aab; font-weight: 600; }
|
||||
pre { white-space: pre-wrap; margin: 0; max-height: 12rem; overflow: auto; }
|
||||
.ok { color: #7ee787; }
|
||||
.bad { color: #ff7b72; }
|
||||
.muted { color: #8b949e; }
|
||||
.grid { display: grid; grid-template-columns: 1fr 1fr; gap: 1rem; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<!--swactor:nav-->
|
||||
<h1>Myelin cluster</h1>
|
||||
<p id="status" class="muted">loading…</p>
|
||||
<h2>Provisioned nodes</h2>
|
||||
<table><thead><tr><th>node</th><th>phase</th><th>message</th></tr></thead><tbody id="nodes"></tbody></table>
|
||||
<div class="grid">
|
||||
<section><h2>Event log</h2><table><thead><tr><th>pos</th><th>channel</th><th>run</th><th>node</th><th>event</th></tr></thead><tbody id="events"></tbody></table></section>
|
||||
<section><h2>Selected node logs</h2><div id="logs" class="muted">select a node row</div></section>
|
||||
</div>
|
||||
<script>
|
||||
let selected = null;
|
||||
async function refresh() {
|
||||
const data = await fetch('/api/view/myelin/cluster').then(r => r.json());
|
||||
document.getElementById('status').textContent = `${data.nodes.length} node(s), ${data.events.length} event(s)`;
|
||||
const nodes = document.getElementById('nodes');
|
||||
nodes.innerHTML = data.nodes.map(n => `<tr data-node="${n.node_id}"><td>${n.node_id}</td><td class="${n.phase === 'provision_failed' ? 'bad' : 'ok'}">${n.phase || ''}</td><td>${n.last_message || ''}</td></tr>`).join('') || '<tr><td colspan="3" class="muted">No provisioning records yet.</td></tr>';
|
||||
for (const row of nodes.querySelectorAll('tr[data-node]')) row.onclick = () => { selected = Number(row.dataset.node); renderLogs(data); };
|
||||
const events = document.getElementById('events');
|
||||
events.innerHTML = data.events.slice().reverse().map(e => `<tr><td>${e.position}</td><td>${e.channel}</td><td>${e.run_id ?? ''}</td><td>${e.node_id ?? ''}</td><td>${e.label}</td></tr>`).join('');
|
||||
renderLogs(data);
|
||||
}
|
||||
function renderLogs(data) {
|
||||
if (selected == null) return;
|
||||
const n = data.nodes.find(n => n.node_id === selected);
|
||||
if (!n) return;
|
||||
document.getElementById('logs').innerHTML = `<h3>node ${n.node_id}</h3><h4>stdout</h4><pre>${esc(n.stdout_tail.join('\n'))}</pre><h4>stderr</h4><pre>${esc(n.stderr_tail.join('\n'))}</pre><h4>provider</h4><pre>${esc(n.provider_tail.join('\n'))}</pre>`;
|
||||
}
|
||||
function esc(s) { return s.replace(/[&<>]/g, c => ({'&':'&','<':'<','>':'>'}[c])); }
|
||||
refresh(); setInterval(refresh, 1000);
|
||||
</script>
|
||||
</body>
|
||||
</html>"#;
|
||||
|
|
@ -1,20 +1,16 @@
|
|||
//! Frame collection and load-progress extraction for the control loop.
|
||||
//!
|
||||
//! [`FrameCollector`] wraps the mpsc channel that buffers telemetry frames
|
||||
//! drained from the iroh driver. It exposes two drain methods that forward
|
||||
//! drained from the iroh driver. It exposes drain methods that forward
|
||||
//! queued frames to sinks (dashboard/archive) via closures; the control loop
|
||||
//! never names [`Frame`] or [`TelemetryEvent`] directly — frames reach sinks
|
||||
//! only through these closures. Load-progress extraction
|
||||
//! ([`StageLoadProgress`]) is co-located here because it is the one legitimate
|
||||
//! read of telemetry content for control decisions (weight-load liveness).
|
||||
//! only through these closures.
|
||||
|
||||
use iroh::EndpointAddr;
|
||||
use iroh_driver::{IrohDriver, TelemetryQuicHeader, spawn_pull_collector};
|
||||
use iroh_driver::{IrohDriver, PullCollectorHandle, TelemetryQuicHeader, spawn_pull_collector};
|
||||
use parking_lot::Mutex;
|
||||
use serde_json::Value;
|
||||
use std::collections::BTreeMap;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::sync::{Arc, mpsc};
|
||||
use std::time::Instant;
|
||||
use swactor_engine::EngineHandle;
|
||||
use telemetry::frame::{ChannelRef, Frame, StreamId, TelemetryEvent};
|
||||
use telemetry::{
|
||||
|
|
@ -32,37 +28,6 @@ struct CollectedTelemetryFrame {
|
|||
frame: Frame,
|
||||
}
|
||||
|
||||
/// Per-node load progress distilled from telemetry frames (control input).
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub(crate) struct StageLoadProgress {
|
||||
pub(crate) node_id: u64,
|
||||
pub(crate) stage_index: Option<u32>,
|
||||
pub(crate) phase: Option<String>,
|
||||
pub(crate) bytes_done: Option<u64>,
|
||||
pub(crate) bytes_total: Option<u64>,
|
||||
pub(crate) last_progress: Option<Instant>,
|
||||
pub(crate) last_worker_event: Option<String>,
|
||||
pub(crate) failure_reason: Option<String>,
|
||||
pub(crate) host_gpu_samples: u64,
|
||||
}
|
||||
|
||||
impl StageLoadProgress {
|
||||
pub(crate) fn to_json(&self) -> Value {
|
||||
serde_json::json!({
|
||||
"node_id": self.node_id,
|
||||
"stage_index": self.stage_index,
|
||||
"phase": self.phase.as_deref().unwrap_or("unknown"),
|
||||
"bytes_done": self.bytes_done,
|
||||
"bytes_total": self.bytes_total,
|
||||
"last_progress_age_ms": self.last_progress.map(|at| at.elapsed().as_millis()),
|
||||
"last_worker_event": self.last_worker_event,
|
||||
"failure_reason": self.failure_reason,
|
||||
"host_gpu_samples": self.host_gpu_samples,
|
||||
"host_gpu_missing": self.host_gpu_samples == 0,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Buffers telemetry frames drained from the driver and forwards them to sinks.
|
||||
pub(crate) struct FrameCollector {
|
||||
tx: mpsc::Sender<CollectedTelemetryFrame>,
|
||||
|
|
@ -73,6 +38,8 @@ pub(crate) struct FrameCollector {
|
|||
pull_header_rx: mpsc::Receiver<TelemetryQuicHeader>,
|
||||
pull_channels: Mutex<BTreeMap<ChannelRef, String>>,
|
||||
pull_streams: Mutex<BTreeMap<StreamId, StreamDescriptor>>,
|
||||
pull_stream_owners: Mutex<BTreeMap<StreamId, (u64, u64)>>,
|
||||
pull_collectors: Mutex<BTreeMap<(u64, u64), PullCollectorHandle>>,
|
||||
}
|
||||
|
||||
impl FrameCollector {
|
||||
|
|
@ -96,6 +63,8 @@ impl FrameCollector {
|
|||
pull_header_rx,
|
||||
pull_channels: Mutex::new(BTreeMap::new()),
|
||||
pull_streams: Mutex::new(BTreeMap::new()),
|
||||
pull_stream_owners: Mutex::new(BTreeMap::new()),
|
||||
pull_collectors: Mutex::new(BTreeMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -111,7 +80,7 @@ impl FrameCollector {
|
|||
let mut flow_id = [0_u8; 16];
|
||||
flow_id[..8].copy_from_slice(&run_id.to_le_bytes());
|
||||
flow_id[8..].copy_from_slice(&node_id.to_le_bytes());
|
||||
spawn_pull_collector(
|
||||
let collector = spawn_pull_collector(
|
||||
engine,
|
||||
endpoint,
|
||||
peer,
|
||||
|
|
@ -121,6 +90,36 @@ impl FrameCollector {
|
|||
Arc::clone(&self.pull_fanout),
|
||||
self.pull_header_tx.clone(),
|
||||
);
|
||||
if let Some(previous) = self
|
||||
.pull_collectors
|
||||
.lock()
|
||||
.insert((run_id, node_id), collector)
|
||||
{
|
||||
previous.cancel();
|
||||
}
|
||||
}
|
||||
/// Stop retaining and reconnecting a telemetry subscription for a terminal node.
|
||||
pub(crate) fn unsubscribe_node(&self, run_id: u64, node_id: u64) {
|
||||
if let Some(collector) = self.pull_collectors.lock().remove(&(run_id, node_id)) {
|
||||
collector.cancel();
|
||||
}
|
||||
let mut ended = BTreeSet::new();
|
||||
self.pull_stream_owners.lock().retain(|stream, owner| {
|
||||
if *owner == (run_id, node_id) {
|
||||
ended.insert(stream.clone());
|
||||
false
|
||||
} else {
|
||||
true
|
||||
}
|
||||
});
|
||||
if !ended.is_empty() {
|
||||
self.pull_streams
|
||||
.lock()
|
||||
.retain(|stream, _| !ended.contains(stream));
|
||||
self.pull_channels
|
||||
.lock()
|
||||
.retain(|channel, _| !ended.contains(&channel.stream));
|
||||
}
|
||||
}
|
||||
|
||||
fn pump_pulls(&self) {
|
||||
|
|
@ -128,6 +127,14 @@ impl FrameCollector {
|
|||
self.pull_streams
|
||||
.lock()
|
||||
.insert(header.stream.stream.clone(), header.stream.clone());
|
||||
let mut run_id = [0_u8; 8];
|
||||
run_id.copy_from_slice(&header.flow_id[..8]);
|
||||
let mut node_id = [0_u8; 8];
|
||||
node_id.copy_from_slice(&header.flow_id[8..]);
|
||||
self.pull_stream_owners.lock().insert(
|
||||
header.stream.stream.clone(),
|
||||
(u64::from_le_bytes(run_id), u64::from_le_bytes(node_id)),
|
||||
);
|
||||
let mut channels = self.pull_channels.lock();
|
||||
for descriptor in header.channels {
|
||||
channels.insert(
|
||||
|
|
@ -180,6 +187,10 @@ impl FrameCollector {
|
|||
}
|
||||
TelemetryEvent::StreamEnded(stream) => {
|
||||
self.pull_streams.lock().remove(&stream);
|
||||
self.pull_stream_owners.lock().remove(&stream);
|
||||
self.pull_channels
|
||||
.lock()
|
||||
.retain(|channel, _| channel.stream != stream);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -206,165 +217,6 @@ impl FrameCollector {
|
|||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Drain queued frames, extracting load progress and forwarding each via the closure.
|
||||
pub(crate) fn drain_with_progress<F>(
|
||||
&self,
|
||||
progress: &mut BTreeMap<u64, StageLoadProgress>,
|
||||
forward: F,
|
||||
) where
|
||||
F: FnMut(&StreamId, Option<&StreamDescriptor>, &str, &Frame),
|
||||
{
|
||||
let mut forward = forward;
|
||||
let now = Instant::now();
|
||||
while let Ok(collected) = self.rx.try_recv() {
|
||||
update_load_progress_from_frame(progress, &collected, now);
|
||||
forward(
|
||||
&collected.stream,
|
||||
collected.descriptor.as_ref(),
|
||||
&collected.channel_name,
|
||||
&collected.frame,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn update_load_progress_from_frame(
|
||||
progress: &mut BTreeMap<u64, StageLoadProgress>,
|
||||
collected: &CollectedTelemetryFrame,
|
||||
now: Instant,
|
||||
) {
|
||||
let stream_node_id = collected.stream.node.as_str().parse::<u64>().ok();
|
||||
if collected.channel_name == "host.gpu" {
|
||||
if let Some(node_id) = stream_node_id {
|
||||
let entry = progress
|
||||
.entry(node_id)
|
||||
.or_insert_with(|| StageLoadProgress {
|
||||
node_id,
|
||||
..StageLoadProgress::default()
|
||||
});
|
||||
entry.host_gpu_samples = entry.host_gpu_samples.saturating_add(1);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let Ok(value) = serde_json::from_slice::<Value>(&collected.frame.payload) else {
|
||||
return;
|
||||
};
|
||||
if value.get("type").and_then(Value::as_str) == Some("NodeEvent") {
|
||||
update_load_progress_from_node_event(progress, &value, now);
|
||||
return;
|
||||
}
|
||||
if collected.channel_name == "myelin.worker.weights" {
|
||||
let Some(node_id) = stream_node_id else {
|
||||
return;
|
||||
};
|
||||
update_load_progress_from_worker_event(progress, node_id, None, &value, now);
|
||||
}
|
||||
}
|
||||
|
||||
fn update_load_progress_from_node_event(
|
||||
progress: &mut BTreeMap<u64, StageLoadProgress>,
|
||||
value: &Value,
|
||||
now: Instant,
|
||||
) {
|
||||
let Some(node_id) = numeric_json_field(value, "node_id") else {
|
||||
return;
|
||||
};
|
||||
let stage_index =
|
||||
numeric_json_field(value, "stage_index").and_then(|stage| u32::try_from(stage).ok());
|
||||
let phase = value.get("phase").and_then(Value::as_str);
|
||||
let status = value.get("status").and_then(Value::as_str);
|
||||
let detail = value.get("detail").unwrap_or(&Value::Null);
|
||||
if phase == Some("load_weights") {
|
||||
let load_phase = match status {
|
||||
Some("started") => Some("loading_weights"),
|
||||
Some("ready") => Some("weights_loaded"),
|
||||
Some("failed") => Some("failed"),
|
||||
_ => None,
|
||||
};
|
||||
if let Some(load_phase) = load_phase {
|
||||
let entry = progress
|
||||
.entry(node_id)
|
||||
.or_insert_with(|| StageLoadProgress {
|
||||
node_id,
|
||||
..StageLoadProgress::default()
|
||||
});
|
||||
entry.stage_index = stage_index.or(entry.stage_index);
|
||||
entry.phase = Some(load_phase.to_owned());
|
||||
entry.last_progress = Some(now);
|
||||
if status == Some("failed") {
|
||||
entry.failure_reason = detail
|
||||
.get("error")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_owned);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(worker_event) = detail.get("event") {
|
||||
update_load_progress_from_worker_event(progress, node_id, stage_index, worker_event, now);
|
||||
}
|
||||
}
|
||||
|
||||
fn update_load_progress_from_worker_event(
|
||||
progress: &mut BTreeMap<u64, StageLoadProgress>,
|
||||
node_id: u64,
|
||||
stage_index: Option<u32>,
|
||||
event: &Value,
|
||||
now: Instant,
|
||||
) {
|
||||
let Some(event_type) = event.get("type").and_then(Value::as_str) else {
|
||||
return;
|
||||
};
|
||||
let Some(phase) = load_phase_for_worker_event(event_type) else {
|
||||
return;
|
||||
};
|
||||
let entry = progress
|
||||
.entry(node_id)
|
||||
.or_insert_with(|| StageLoadProgress {
|
||||
node_id,
|
||||
..StageLoadProgress::default()
|
||||
});
|
||||
entry.stage_index = stage_index.or(entry.stage_index);
|
||||
entry.phase = Some(phase.to_owned());
|
||||
entry.last_worker_event = Some(event_type.to_owned());
|
||||
entry.last_progress = Some(now);
|
||||
if let Some(bytes_done) =
|
||||
numeric_json_field(event, "bytes_done").or_else(|| numeric_json_field(event, "bytes"))
|
||||
{
|
||||
entry.bytes_done = Some(bytes_done);
|
||||
}
|
||||
if let Some(bytes_total) = numeric_json_field(event, "bytes_total") {
|
||||
entry.bytes_total = Some(bytes_total);
|
||||
}
|
||||
}
|
||||
|
||||
fn numeric_json_field(value: &Value, field: &str) -> Option<u64> {
|
||||
value
|
||||
.get(field)
|
||||
.and_then(|value| value.as_u64().or_else(|| value.as_str()?.parse().ok()))
|
||||
}
|
||||
|
||||
fn load_phase_for_worker_event(event_type: &str) -> Option<&'static str> {
|
||||
match event_type {
|
||||
"GgufDownloadStarted" | "GgufDownloadProgress" => Some("prefetching_model"),
|
||||
"GgufCacheReady" => Some("cache_ready"),
|
||||
"StageShardFetchStarted"
|
||||
| "StageShardRangeFetchStarted"
|
||||
| "StageShardRangeFetchReady"
|
||||
| "StageShardTensorFetchStarted"
|
||||
| "StageShardTensorFetchReady" => Some("fetching_stage_shard"),
|
||||
"StageShardCacheReady" => Some("stage_shard_cache_ready"),
|
||||
"StageShardReady" => Some("stage_shard_ready"),
|
||||
"StageShardFetchFailed" => Some("failed"),
|
||||
"PipelineStageFromGgufStarted" => Some("constructing_stage"),
|
||||
"PipelineStageFromGgufReady" => Some("stage_constructed"),
|
||||
"TokenizerBuildStarted" => Some("building_tokenizer"),
|
||||
"TokenizerBuildReady" => Some("tokenizer_ready"),
|
||||
"WeightsLoaded" => Some("weights_loaded"),
|
||||
"WorkerFatal" => Some("failed"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn drain_telemetry_connections(
|
||||
|
|
@ -445,9 +297,10 @@ pub(crate) fn ingest_dashboard_frame(
|
|||
stream: &StreamId,
|
||||
channel: &str,
|
||||
frame: &Frame,
|
||||
descriptor: Option<&StreamDescriptor>,
|
||||
) {
|
||||
if let Some(dashboard) = dashboard {
|
||||
dashboard.publish_frame(stream, channel, frame);
|
||||
dashboard.publish_frame(stream, descriptor, channel, frame);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -511,5 +364,69 @@ mod tests {
|
|||
assert_eq!(observed_channel, "host.net");
|
||||
assert_eq!(observed_frame.position, Position(11));
|
||||
assert_eq!(observed_frame.payload, br#"{"rx":1}"#);
|
||||
collector
|
||||
.pull_fanout
|
||||
.publish(TelemetryEvent::StreamEnded(stream.clone()));
|
||||
collector.pump_pulls();
|
||||
assert!(!collector.pull_streams.lock().contains_key(&stream));
|
||||
assert!(
|
||||
collector
|
||||
.pull_channels
|
||||
.lock()
|
||||
.keys()
|
||||
.all(|channel| channel.stream != stream),
|
||||
"ended stream retained channel descriptors"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn node_unsubscribe_releases_abrupt_stream_metadata() {
|
||||
let collector = FrameCollector::new();
|
||||
let stream = StreamId::new(NodeId::new("node-9"), Lifetime(4));
|
||||
let descriptor = StreamDescriptor {
|
||||
stream: stream.clone(),
|
||||
label: Some("worker nine".to_owned()),
|
||||
origin: StreamOrigin::RemoteNode,
|
||||
};
|
||||
let channel = ChannelDescriptor {
|
||||
stream: stream.clone(),
|
||||
id: ChannelId(3),
|
||||
name: "runtime.actors".to_owned(),
|
||||
label: None,
|
||||
content: ChannelContent::JsonRecord { schema: None },
|
||||
};
|
||||
let mut flow_id = [0_u8; 16];
|
||||
flow_id[..8].copy_from_slice(&5_u64.to_le_bytes());
|
||||
flow_id[8..].copy_from_slice(&9_u64.to_le_bytes());
|
||||
collector
|
||||
.pull_header_tx
|
||||
.send(TelemetryQuicHeader::new(
|
||||
flow_id,
|
||||
Vec::new(),
|
||||
descriptor,
|
||||
vec![channel],
|
||||
))
|
||||
.unwrap();
|
||||
collector.pump_pulls();
|
||||
assert!(collector.pull_streams.lock().contains_key(&stream));
|
||||
assert!(
|
||||
collector
|
||||
.pull_channels
|
||||
.lock()
|
||||
.keys()
|
||||
.any(|channel| channel.stream == stream)
|
||||
);
|
||||
|
||||
collector.unsubscribe_node(5, 9);
|
||||
|
||||
assert!(!collector.pull_streams.lock().contains_key(&stream));
|
||||
assert!(
|
||||
collector
|
||||
.pull_channels
|
||||
.lock()
|
||||
.keys()
|
||||
.all(|channel| channel.stream != stream),
|
||||
"abrupt node stop retained channel descriptors"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
//! Myelin observability public surface.
|
||||
|
||||
pub(crate) mod benchmark;
|
||||
#[cfg(feature = "dashboard")]
|
||||
pub(crate) mod dashboard_view;
|
||||
pub(crate) mod frame_archive;
|
||||
pub(crate) mod frame_collector;
|
||||
// Lifecycle event schema is currently consumed only by the test harness
|
||||
// (mock nodes + guarantee tests); the production binary never constructs it.
|
||||
#[cfg(test)]
|
||||
pub(crate) mod lifecycle;
|
||||
pub(crate) mod orch_telemetry;
|
||||
pub(crate) mod provisioning_logs;
|
||||
|
|
|
|||
|
|
@ -16,8 +16,6 @@ use telemetry::{
|
|||
};
|
||||
|
||||
use crate::observability::benchmark;
|
||||
#[cfg(feature = "dashboard")]
|
||||
use crate::observability::dashboard_view::MyelinClusterDashboardView;
|
||||
use crate::observability::frame_archive::FrameArchive;
|
||||
use crate::observability::frame_collector::ingest_dashboard_frame;
|
||||
use crate::observability::telemetry::{
|
||||
|
|
@ -37,12 +35,12 @@ pub(crate) const MYELIN_SWIM_MEMBERSHIP: &str = "myelin.swim.membership";
|
|||
pub(crate) const MYELIN_STAGE_ROUTE: &str = "myelin.orch.stage_route";
|
||||
|
||||
pub(crate) struct OrchTelemetry {
|
||||
stream: StreamId,
|
||||
endpoint: TelemetryEndpoint,
|
||||
producer: TelemetryProducer,
|
||||
channels: BTreeMap<String, ChannelId>,
|
||||
channel_names: BTreeMap<ChannelId, String>,
|
||||
archive: Option<FrameArchive>,
|
||||
descriptor: StreamDescriptor,
|
||||
}
|
||||
|
||||
impl OrchTelemetry {
|
||||
|
|
@ -58,8 +56,12 @@ impl OrchTelemetry {
|
|||
1024,
|
||||
);
|
||||
let producer = endpoint.producer();
|
||||
let descriptor = StreamDescriptor {
|
||||
stream: stream.clone(),
|
||||
label: Some("myelin orchestrator".to_owned()),
|
||||
origin: StreamOrigin::Orchestrator,
|
||||
};
|
||||
let mut out = Self {
|
||||
stream,
|
||||
endpoint,
|
||||
producer,
|
||||
channels: BTreeMap::new(),
|
||||
|
|
@ -67,6 +69,7 @@ impl OrchTelemetry {
|
|||
archive: frame_log
|
||||
.map(|p| FrameArchive::open_with_label(p, "telemetry frame log"))
|
||||
.transpose()?,
|
||||
descriptor,
|
||||
};
|
||||
for name in [
|
||||
MYELIN_PROVISIONING_EVENTS,
|
||||
|
|
@ -184,43 +187,6 @@ impl OrchTelemetry {
|
|||
self.emit_bytes(dashboard, channel, payload);
|
||||
}
|
||||
|
||||
pub(crate) fn emit_prompt(
|
||||
&mut self,
|
||||
dashboard: Option<&DashboardSupport>,
|
||||
run_id: u64,
|
||||
node_id: u64,
|
||||
request_id: u64,
|
||||
phase: &str,
|
||||
status: &str,
|
||||
detail: Value,
|
||||
) {
|
||||
let benchmark = benchmark::stamp("myelin-orchestrator");
|
||||
let payload = serde_json::to_vec(&json!({
|
||||
"schema_version": benchmark["schema_version"].clone(),
|
||||
"type":"OrchPromptEvent",
|
||||
"event_type":"OrchPromptEvent",
|
||||
"event_name":phase,
|
||||
"phase":phase,
|
||||
"status":status,
|
||||
"run_id":run_id,
|
||||
"node_id":node_id,
|
||||
"request_id":request_id,
|
||||
"producer_component":benchmark["producer_component"].clone(),
|
||||
"producer_instance_id":benchmark["producer_instance_id"].clone(),
|
||||
"producer_process_id":benchmark["producer_process_id"].clone(),
|
||||
"producer_sequence":benchmark["producer_sequence"].clone(),
|
||||
"wall_clock_unix_ms":benchmark["wall_clock_unix_ms"].clone(),
|
||||
"monotonic_ms":benchmark["monotonic_ms"].clone(),
|
||||
"clock_source":benchmark["clock_source"].clone(),
|
||||
"span_id":format!("myelin-orchestrator:{run_id}:{request_id}:{}:{phase}", benchmark["producer_sequence"]),
|
||||
"parent_span_id":format!("request:{request_id}"),
|
||||
"benchmark":benchmark,
|
||||
"detail":detail,
|
||||
}))
|
||||
.expect("serialize orch prompt event");
|
||||
self.emit_bytes(dashboard, MYELIN_ORCH_PROMPT, payload);
|
||||
}
|
||||
|
||||
pub(crate) fn emit_record<R: Record>(
|
||||
&mut self,
|
||||
dashboard: Option<&DashboardSupport>,
|
||||
|
|
@ -253,14 +219,14 @@ impl OrchTelemetry {
|
|||
}
|
||||
|
||||
pub(crate) fn flush(&mut self, dashboard: Option<&DashboardSupport>, source: &str) {
|
||||
let stream = self.stream.clone();
|
||||
let stream = self.descriptor.stream.clone();
|
||||
for frame in self.endpoint.mux().drain() {
|
||||
let channel = self
|
||||
.channel_names
|
||||
.get(&frame.channel)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| format!("channel#{}", frame.channel.0));
|
||||
ingest_dashboard_frame(dashboard, &stream, &channel, &frame);
|
||||
ingest_dashboard_frame(dashboard, &stream, &channel, &frame, Some(&self.descriptor));
|
||||
self.archive_frame(source, &stream, &channel, &frame);
|
||||
}
|
||||
}
|
||||
|
|
@ -306,30 +272,26 @@ impl DashboardSupport {
|
|||
.transpose()
|
||||
}
|
||||
|
||||
pub(crate) fn start(enabled: bool, engine: &EngineHandle) -> Result<Option<Self>, String> {
|
||||
pub(crate) fn start_with_plugins(
|
||||
enabled: bool,
|
||||
engine: &EngineHandle,
|
||||
plugins: Vec<dashboard::DashboardPlugin>,
|
||||
) -> Result<Option<Self>, String> {
|
||||
if !enabled {
|
||||
return Ok(None);
|
||||
}
|
||||
let handle = dashboard::DashboardHandle::new(Self::config()?);
|
||||
handle.register_view(Arc::new(MyelinClusterDashboardView::new()));
|
||||
engine.spawn(handle.http_server());
|
||||
let mut config = Self::config()?;
|
||||
config
|
||||
.page_script_urls
|
||||
.push(crate::orchestration::control::FLEET_CONTROL_SCRIPT_URL.to_owned());
|
||||
let handle = dashboard::DashboardHandle::new(config);
|
||||
handle.spawn_with_plugins(engine, plugins);
|
||||
Ok(Some(Self { handle }))
|
||||
}
|
||||
|
||||
pub(crate) fn publish_frame(&self, stream: &StreamId, channel: &str, frame: &Frame) {
|
||||
self.handle.publish(dashboard::FrameEvent {
|
||||
stream: dashboard::StreamEvent {
|
||||
node: stream.node.as_str().to_string(),
|
||||
life: stream.life.0,
|
||||
origin: None,
|
||||
label: None,
|
||||
},
|
||||
channel: channel.to_owned(),
|
||||
position: frame.position.0,
|
||||
payload: frame.payload.clone(),
|
||||
});
|
||||
}
|
||||
pub(crate) fn publish_collected_frame(
|
||||
/// Publish a frame, classifying the stream by its descriptor's origin so
|
||||
/// the fleet view can privilege the orchestrator card.
|
||||
pub(crate) fn publish_frame(
|
||||
&self,
|
||||
stream: &StreamId,
|
||||
descriptor: Option<&StreamDescriptor>,
|
||||
|
|
@ -373,7 +335,11 @@ impl DashboardSupport {
|
|||
Ok(None)
|
||||
}
|
||||
|
||||
pub(crate) fn start(enabled: bool, _engine: &EngineHandle) -> Result<Option<Self>, String> {
|
||||
pub(crate) fn start_with_plugins(
|
||||
enabled: bool,
|
||||
_engine: &EngineHandle,
|
||||
_plugins: Vec<dashboard::DashboardPlugin>,
|
||||
) -> Result<Option<Self>, String> {
|
||||
if enabled {
|
||||
return Err(
|
||||
"MYELIN_DASHBOARD requires building myelin-system with feature dashboard"
|
||||
|
|
@ -383,8 +349,7 @@ impl DashboardSupport {
|
|||
Ok(None)
|
||||
}
|
||||
|
||||
pub(crate) fn publish_frame(&self, _stream: &StreamId, _channel: &str, _frame: &Frame) {}
|
||||
pub(crate) fn publish_collected_frame(
|
||||
pub(crate) fn publish_frame(
|
||||
&self,
|
||||
_stream: &StreamId,
|
||||
_descriptor: Option<&StreamDescriptor>,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,3 @@
|
|||
use std::io::{BufRead, BufReader, Read};
|
||||
use std::thread::{self, JoinHandle};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use telemetry::{ChannelContent, Lifetime, NodeId, StreamId, TelemetryProducer};
|
||||
|
|
@ -74,66 +71,6 @@ impl BootstrapTelemetryBridge {
|
|||
});
|
||||
}
|
||||
|
||||
// provider log capture is out of scope (ENGINE_SPEC.md §2)
|
||||
#[allow(clippy::disallowed_methods)]
|
||||
pub(crate) fn spawn_stdout_reader<R>(&self, stdout: R) -> JoinHandle<()>
|
||||
where
|
||||
R: Read + Send + 'static,
|
||||
{
|
||||
let bridge = self.clone();
|
||||
thread::spawn(move || bridge.read_stdout(stdout))
|
||||
}
|
||||
|
||||
// provider log capture is out of scope (ENGINE_SPEC.md §2)
|
||||
#[allow(clippy::disallowed_methods)]
|
||||
pub(crate) fn spawn_stderr_reader<R>(&self, stderr: R) -> JoinHandle<()>
|
||||
where
|
||||
R: Read + Send + 'static,
|
||||
{
|
||||
let bridge = self.clone();
|
||||
thread::spawn(move || bridge.read_stderr(stderr))
|
||||
}
|
||||
|
||||
fn read_stdout<R>(&self, stdout: R)
|
||||
where
|
||||
R: Read,
|
||||
{
|
||||
let reader = BufReader::new(stdout);
|
||||
for next in reader.lines() {
|
||||
match next {
|
||||
Ok(line) => self.observe_stdout_line(line),
|
||||
Err(error) => {
|
||||
self.sink.observe(PluginObservation::Failed {
|
||||
run_id: self.spec.run_id,
|
||||
node_id: self.spec.node_id,
|
||||
reason: format!("read stdout: {error}"),
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn read_stderr<R>(&self, stderr: R)
|
||||
where
|
||||
R: Read,
|
||||
{
|
||||
let reader = BufReader::new(stderr);
|
||||
for next in reader.lines() {
|
||||
match next {
|
||||
Ok(line) => self.observe_stderr_line(line),
|
||||
Err(error) => {
|
||||
self.sink.observe(PluginObservation::Failed {
|
||||
run_id: self.spec.run_id,
|
||||
node_id: self.spec.node_id,
|
||||
reason: format!("read stderr: {error}"),
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn submit_log(&self, stream: ProvisionLogStream, line: &str) {
|
||||
let Some(producer) = &self.producer else {
|
||||
return;
|
||||
|
|
|
|||
|
|
@ -3,26 +3,13 @@
|
|||
use serde::{Deserialize, Serialize};
|
||||
use telemetry::Record;
|
||||
|
||||
use crate::observability::lifecycle as obs;
|
||||
use crate::provisioning::{self, ProvisionLogStream};
|
||||
|
||||
/// Structured Myelin lifecycle facts: run, node, stage, edge, ring, object, step, and worker events.
|
||||
pub(crate) const MYELIN_LIFECYCLE: &str = "myelin.lifecycle";
|
||||
/// Structured node provisioning milestones emitted before a remote swactor runtime is live.
|
||||
pub(crate) const MYELIN_PROVISIONING_EVENTS: &str = "myelin.provisioning.events";
|
||||
|
||||
/// Raw provider/process stream lines captured during provisioning.
|
||||
pub(crate) const MYELIN_PROVISIONING_LOGS: &str = "myelin.provisioning.logs";
|
||||
|
||||
/// Telemetry payload for the Myelin lifecycle channel.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub(crate) struct MyelinLifecycleRecord {
|
||||
pub event: obs::Event,
|
||||
}
|
||||
|
||||
impl Record for MyelinLifecycleRecord {
|
||||
const CHANNEL: &'static str = MYELIN_LIFECYCLE;
|
||||
}
|
||||
/// Telemetry payload for provisioning lifecycle events.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub(crate) struct MyelinProvisionEventRecord {
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ use swactor::actor::{ActorAddress, ActorInterface};
|
|||
use swactor::runtime::Ctx;
|
||||
use swactor_transport::{CodecRegistry, NetworkMessage};
|
||||
|
||||
use crate::orchestration::manual_control::{ManualActorControl, ManualControlMsg};
|
||||
use crate::run_fsm as core;
|
||||
|
||||
use swactor_transport::JsonCodec;
|
||||
|
|
@ -13,8 +14,7 @@ pub(crate) struct StageRefWire {
|
|||
pub stage_index: u32,
|
||||
pub node_id: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub(crate) enum OrchestratorMsg {
|
||||
ObservePoolReady {
|
||||
nodes: Vec<u64>,
|
||||
|
|
@ -78,6 +78,7 @@ pub(crate) enum OrchestratorMsg {
|
|||
Snapshot {
|
||||
reply_to: ActorAddress,
|
||||
},
|
||||
Manual(ManualControlMsg),
|
||||
}
|
||||
|
||||
impl NetworkMessage for OrchestratorMsg {
|
||||
|
|
@ -193,6 +194,7 @@ pub(crate) struct OrchestratorActor {
|
|||
report_to: Option<ActorAddress>,
|
||||
command_cursor: usize,
|
||||
event_cursor: usize,
|
||||
manual: Option<ManualActorControl>,
|
||||
}
|
||||
|
||||
impl OrchestratorActor {
|
||||
|
|
@ -202,9 +204,15 @@ impl OrchestratorActor {
|
|||
report_to,
|
||||
command_cursor: 0,
|
||||
event_cursor: 0,
|
||||
manual: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn with_manual_control(mut self, manual: ManualActorControl) -> Self {
|
||||
self.manual = Some(manual);
|
||||
self
|
||||
}
|
||||
|
||||
fn observe(&mut self, msg: OrchestratorMsg) {
|
||||
match msg {
|
||||
OrchestratorMsg::ObservePoolReady { nodes } => {
|
||||
|
|
@ -235,7 +243,8 @@ impl OrchestratorActor {
|
|||
OrchestratorMsg::ObserveNodeRuntimeReady { .. }
|
||||
| OrchestratorMsg::ObserveNodeRuntimeReadyAck { .. }
|
||||
| OrchestratorMsg::ObserveWeightsReady { .. }
|
||||
| OrchestratorMsg::Snapshot { .. } => {}
|
||||
| OrchestratorMsg::Snapshot { .. }
|
||||
| OrchestratorMsg::Manual(_) => {}
|
||||
OrchestratorMsg::ObserveTokenInEndpointReady => {
|
||||
self.core.observe(core::RunEvent::TokenInEndpointReady)
|
||||
}
|
||||
|
|
@ -316,7 +325,18 @@ impl ActorInterface for OrchestratorActor {
|
|||
type Incoming = OrchestratorMsg;
|
||||
type Response = ();
|
||||
|
||||
fn on_start(&mut self, ctx: &Ctx) {
|
||||
if let Some(manual) = self.manual.as_mut() {
|
||||
manual.start(ctx.self_addr());
|
||||
}
|
||||
}
|
||||
fn handle(&mut self, ctx: &Ctx, msg: Self::Incoming) {
|
||||
if let OrchestratorMsg::Manual(manual_msg) = msg.clone() {
|
||||
if let Some(manual) = self.manual.as_mut() {
|
||||
manual.handle(ctx, manual_msg);
|
||||
}
|
||||
return;
|
||||
}
|
||||
match msg.clone() {
|
||||
OrchestratorMsg::ObserveNodeRuntimeReady {
|
||||
run_id,
|
||||
|
|
@ -326,6 +346,27 @@ impl ActorInterface for OrchestratorActor {
|
|||
node_actor,
|
||||
readiness_id,
|
||||
} => {
|
||||
if let Some(manual) = self.manual.as_mut() {
|
||||
manual.observe_runtime_ready(
|
||||
ctx.self_addr(),
|
||||
node_id,
|
||||
crate::orchestration::daemon::RuntimeFacts {
|
||||
run_id,
|
||||
attempt_id: manual
|
||||
.read_model()
|
||||
.nodes
|
||||
.iter()
|
||||
.find(|node| node.logical_node_id == node_id)
|
||||
.and_then(|node| node.spec.as_ref())
|
||||
.map_or(0, |spec| spec.attempt_id),
|
||||
endpoint: serde_json::to_string(&endpoint).unwrap_or_default(),
|
||||
node_actor,
|
||||
swim_node_id: distribution::types::NodeId(*endpoint.id.as_bytes()),
|
||||
stage_index,
|
||||
readiness_id,
|
||||
},
|
||||
);
|
||||
}
|
||||
if let Some(report_to) = self.report_to {
|
||||
let _ = ctx.send(
|
||||
report_to,
|
||||
|
|
@ -347,6 +388,9 @@ impl ActorInterface for OrchestratorActor {
|
|||
stage_index,
|
||||
readiness_id,
|
||||
} => {
|
||||
if let Some(manual) = self.manual.as_mut() {
|
||||
manual.observe_node_ack(ctx.self_addr(), node_id, readiness_id);
|
||||
}
|
||||
if let Some(report_to) = self.report_to {
|
||||
let _ = ctx.send(
|
||||
report_to,
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,7 +1,9 @@
|
|||
//! Myelin integration for the provider-neutral cluster reconciler.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet, VecDeque};
|
||||
use std::sync::mpsc::{self, Receiver, Sender, TryRecvError};
|
||||
#[cfg(test)]
|
||||
use std::sync::mpsc::TryRecvError;
|
||||
use std::sync::mpsc::{self, Receiver, Sender};
|
||||
use std::sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard};
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
|
|
@ -12,7 +14,9 @@ use provisioning::{
|
|||
NodeManagerCommand, NodeObservation, NodeStage, OperationOutcome, PlannedEffect,
|
||||
ProviderLeaseId, RetryPolicy, SshEndpoint, SwactorId,
|
||||
};
|
||||
use swactor_engine::EngineHandle;
|
||||
use swactor::actor::{ActorAddress, ActorInterface};
|
||||
use swactor::runtime::{Ctx, ExternalSender, Runtime};
|
||||
use swactor_engine::{BlockingWorkSender, EngineHandle};
|
||||
|
||||
use crate::provisioning::{
|
||||
NodeProvisionSpec, PluginNodeHandle, PluginObservation, PluginObservationSink, PluginSink,
|
||||
|
|
@ -219,23 +223,6 @@ impl MyelinEffectBackend {
|
|||
(external_id, logical_id)
|
||||
}
|
||||
|
||||
fn detach_all(&self) {
|
||||
let nodes = lock_nodes_read(&self.nodes)
|
||||
.values()
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
for effects in nodes {
|
||||
let mut effects = lock_node(&effects);
|
||||
if let Some(live) = effects.live.take() {
|
||||
live.failure_sink.discard();
|
||||
}
|
||||
effects.plugin.detach_all();
|
||||
if let Some(staged) = effects.staged.as_mut() {
|
||||
staged.plugin.detach_all();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn stop_all(&self) -> Result<(), String> {
|
||||
let mut first_error = None;
|
||||
let nodes = lock_nodes_read(&self.nodes)
|
||||
|
|
@ -464,12 +451,14 @@ impl EffectBackend for MyelinEffectBackend {
|
|||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct EngineEffectSpawner {
|
||||
engine: EngineHandle,
|
||||
blocking: BlockingWorkSender,
|
||||
}
|
||||
|
||||
impl EngineEffectSpawner {
|
||||
fn new(engine: EngineHandle) -> Self {
|
||||
Self { engine }
|
||||
fn new(engine: &EngineHandle) -> Self {
|
||||
Self {
|
||||
blocking: engine.blocking_work_sender(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -480,11 +469,9 @@ impl BlockingEffectSpawner for EngineEffectSpawner {
|
|||
&self,
|
||||
work: provisioning::BlockingEffectWork,
|
||||
) -> Result<(), Self::SpawnError> {
|
||||
if !self.engine.capabilities().blocking {
|
||||
return Err("engine blocking work capability is unavailable".to_owned());
|
||||
}
|
||||
self.engine.spawn_blocking(work);
|
||||
Ok(())
|
||||
self.blocking
|
||||
.submit(work)
|
||||
.map_err(|_| "engine stopped before provider effect submission".to_owned())
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -493,14 +480,79 @@ enum ControllerWake {
|
|||
Periodic,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
enum ControllerTimerMsg {
|
||||
ScheduleDeadline(Option<SystemTime>),
|
||||
DeadlineElapsed(SystemTime),
|
||||
PeriodicElapsed,
|
||||
}
|
||||
|
||||
struct ControllerTimerActor {
|
||||
engine: EngineHandle,
|
||||
sender: ExternalSender,
|
||||
wake: Sender<ControllerWake>,
|
||||
scheduled_deadline: Option<SystemTime>,
|
||||
}
|
||||
|
||||
impl ControllerTimerActor {
|
||||
fn schedule_periodic(&self, ctx: &Ctx) {
|
||||
self.engine.send_after(
|
||||
PERIODIC_RECONCILE,
|
||||
self.sender.clone(),
|
||||
ctx.self_addr(),
|
||||
ControllerTimerMsg::PeriodicElapsed,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
impl ActorInterface for ControllerTimerActor {
|
||||
type Incoming = ControllerTimerMsg;
|
||||
type Response = ();
|
||||
|
||||
fn on_start(&mut self, ctx: &Ctx) {
|
||||
self.schedule_periodic(ctx);
|
||||
}
|
||||
|
||||
fn handle(&mut self, ctx: &Ctx, message: Self::Incoming) {
|
||||
match message {
|
||||
ControllerTimerMsg::ScheduleDeadline(deadline) => {
|
||||
self.scheduled_deadline = deadline;
|
||||
if let Some(deadline) = deadline {
|
||||
self.engine.send_after(
|
||||
deadline
|
||||
.duration_since(SystemTime::now())
|
||||
.unwrap_or(Duration::ZERO),
|
||||
self.sender.clone(),
|
||||
ctx.self_addr(),
|
||||
ControllerTimerMsg::DeadlineElapsed(deadline),
|
||||
);
|
||||
}
|
||||
}
|
||||
ControllerTimerMsg::DeadlineElapsed(deadline) => {
|
||||
if self.scheduled_deadline == Some(deadline) {
|
||||
self.scheduled_deadline = None;
|
||||
let _ = self.wake.send(ControllerWake::Deadline(deadline));
|
||||
}
|
||||
}
|
||||
ControllerTimerMsg::PeriodicElapsed => {
|
||||
if self.wake.send(ControllerWake::Periodic).is_ok() {
|
||||
self.schedule_periodic(ctx);
|
||||
} else {
|
||||
ctx.stop_self();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct ProvisionedClusterGuard {
|
||||
driver: ClusterDriver,
|
||||
executor: IdempotentEffectExecutor<MyelinEffectBackend, EngineEffectSpawner>,
|
||||
external_nodes: BTreeMap<u64, LogicalNodeId>,
|
||||
failure_rx: Receiver<TaggedFailure>,
|
||||
deferred_failures: VecDeque<TaggedFailure>,
|
||||
engine: EngineHandle,
|
||||
wake_tx: Sender<ControllerWake>,
|
||||
runtime: Runtime,
|
||||
timer_actor: ActorAddress,
|
||||
wake_rx: Receiver<ControllerWake>,
|
||||
scheduled_deadline: Option<SystemTime>,
|
||||
stopped: bool,
|
||||
|
|
@ -512,6 +564,7 @@ impl ProvisionedClusterGuard {
|
|||
bindings: Vec<ReconcilerNodeBinding>,
|
||||
retry: RetryPolicy,
|
||||
engine: EngineHandle,
|
||||
runtime: Runtime,
|
||||
sink: PluginSink,
|
||||
) -> Result<Self, String> {
|
||||
let expanded = desired.expand().map_err(|error| error.to_string())?;
|
||||
|
|
@ -537,19 +590,25 @@ impl ProvisionedClusterGuard {
|
|||
}
|
||||
let (failure_tx, failure_rx) = mpsc::channel();
|
||||
let (backend, external_nodes) = MyelinEffectBackend::new(bindings, sink, failure_tx)?;
|
||||
let executor =
|
||||
IdempotentEffectExecutor::new(backend, EngineEffectSpawner::new(engine.clone()));
|
||||
let executor = IdempotentEffectExecutor::new(backend, EngineEffectSpawner::new(&engine));
|
||||
let driver = ClusterDriver::new(desired, retry).map_err(|error| error.to_string())?;
|
||||
let (wake_tx, wake_rx) = mpsc::channel();
|
||||
spawn_periodic_wake(&engine, wake_tx.clone());
|
||||
let timer_actor = runtime
|
||||
.spawn(ControllerTimerActor {
|
||||
engine: engine.clone(),
|
||||
sender: runtime.create_sender(),
|
||||
wake: wake_tx,
|
||||
scheduled_deadline: None,
|
||||
})
|
||||
.map_err(|error| format!("spawn cluster timer actor: {error}"))?;
|
||||
Ok(Self {
|
||||
driver,
|
||||
executor,
|
||||
external_nodes,
|
||||
failure_rx,
|
||||
deferred_failures: VecDeque::new(),
|
||||
engine,
|
||||
wake_tx,
|
||||
runtime,
|
||||
timer_actor,
|
||||
wake_rx,
|
||||
scheduled_deadline: None,
|
||||
stopped: false,
|
||||
|
|
@ -626,6 +685,9 @@ impl ProvisionedClusterGuard {
|
|||
.map(|node| node.attempt)
|
||||
}
|
||||
|
||||
// Readiness predicate: exercised only by reconciler guarantee tests today;
|
||||
// no production caller yet.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn awaiting_runtime(&self) -> bool {
|
||||
!self.driver.state().nodes.is_empty()
|
||||
&& self.driver.state().nodes.values().all(|node| {
|
||||
|
|
@ -671,7 +733,7 @@ impl ProvisionedClusterGuard {
|
|||
Ok(submitted)
|
||||
}
|
||||
|
||||
fn begin_shutdown(&mut self) -> Result<(), String> {
|
||||
pub(crate) fn begin_shutdown(&mut self) -> Result<(), String> {
|
||||
if self.stopped {
|
||||
return Ok(());
|
||||
}
|
||||
|
|
@ -687,39 +749,46 @@ impl ProvisionedClusterGuard {
|
|||
self.driver.state().nodes.is_empty()
|
||||
}
|
||||
|
||||
// Synchronous orchestration waits while all provider work remains engine-hosted.
|
||||
#[allow(clippy::disallowed_methods)]
|
||||
pub(crate) fn finish_shutdown(&mut self) -> Result<(), String> {
|
||||
if !self.stopped {
|
||||
self.executor.backend().stop_all()?;
|
||||
self.stopped = true;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn stop(&mut self) -> Result<(), String> {
|
||||
self.begin_shutdown()?;
|
||||
while !self.is_stopped() {
|
||||
self.poll(SystemTime::now())
|
||||
.map_err(|error| error.to_string())?;
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
self.wait_for_work(Duration::from_millis(10));
|
||||
}
|
||||
self.executor.backend().stop_all()?;
|
||||
self.stopped = true;
|
||||
Ok(())
|
||||
self.finish_shutdown()
|
||||
}
|
||||
|
||||
/// Releases the guard without touching any provider resource: no
|
||||
/// shutdown shape, no stop_all. The daemon exits; nodes keep running and
|
||||
/// are adopted by spec on the next boot.
|
||||
pub(crate) fn detach(&mut self) {
|
||||
self.executor.backend().detach_all();
|
||||
self.stopped = true;
|
||||
#[cfg(test)]
|
||||
pub(crate) fn wait_for_work(&mut self, timeout: Duration) {
|
||||
if let Ok(wake) = self.wake_rx.recv_timeout(timeout) {
|
||||
self.apply_wake(wake, SystemTime::now());
|
||||
}
|
||||
}
|
||||
|
||||
fn drain_wakes(&mut self, now: SystemTime) {
|
||||
loop {
|
||||
match self.wake_rx.try_recv() {
|
||||
Ok(ControllerWake::Periodic) => self.driver.trigger(),
|
||||
Ok(ControllerWake::Deadline(deadline)) => {
|
||||
if self.scheduled_deadline == Some(deadline) {
|
||||
self.scheduled_deadline = None;
|
||||
}
|
||||
self.driver.trigger_if_due(now);
|
||||
while let Ok(wake) = self.wake_rx.try_recv() {
|
||||
self.apply_wake(wake, now);
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_wake(&mut self, wake: ControllerWake, now: SystemTime) {
|
||||
match wake {
|
||||
ControllerWake::Periodic => self.driver.trigger(),
|
||||
ControllerWake::Deadline(deadline) => {
|
||||
if self.scheduled_deadline == Some(deadline) {
|
||||
self.scheduled_deadline = None;
|
||||
}
|
||||
Err(TryRecvError::Empty | TryRecvError::Disconnected) => return,
|
||||
self.driver.trigger_if_due(now);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -820,20 +889,19 @@ impl ProvisionedClusterGuard {
|
|||
})
|
||||
{
|
||||
self.scheduled_deadline = None;
|
||||
let _ = self
|
||||
.runtime
|
||||
.send_to(self.timer_actor, ControllerTimerMsg::ScheduleDeadline(None));
|
||||
return;
|
||||
}
|
||||
if deadline.is_none() || deadline == self.scheduled_deadline {
|
||||
if deadline == self.scheduled_deadline {
|
||||
return;
|
||||
}
|
||||
let deadline = deadline.expect("checked deadline");
|
||||
self.scheduled_deadline = Some(deadline);
|
||||
let delay = deadline.duration_since(now).unwrap_or(Duration::ZERO);
|
||||
let timer = self.engine.timer(delay);
|
||||
let wake = self.wake_tx.clone();
|
||||
self.engine.spawn(async move {
|
||||
timer.await;
|
||||
let _ = wake.send(ControllerWake::Deadline(deadline));
|
||||
});
|
||||
self.scheduled_deadline = deadline;
|
||||
let _ = self.runtime.send_to(
|
||||
self.timer_actor,
|
||||
ControllerTimerMsg::ScheduleDeadline(deadline),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -845,18 +913,6 @@ impl Drop for ProvisionedClusterGuard {
|
|||
}
|
||||
}
|
||||
|
||||
fn spawn_periodic_wake(engine: &EngineHandle, wake: Sender<ControllerWake>) {
|
||||
let mut interval = engine.interval(PERIODIC_RECONCILE);
|
||||
engine.spawn(async move {
|
||||
loop {
|
||||
(&mut interval).await;
|
||||
if wake.send(ControllerWake::Periodic).is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn lock_node(node: &Mutex<NodeEffects>) -> MutexGuard<'_, NodeEffects> {
|
||||
node.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
|
|
@ -1375,7 +1431,6 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
#[allow(clippy::disallowed_methods)]
|
||||
fn engine_hosted_controller_converges_and_cleans_up_end_to_end() {
|
||||
let stats = Arc::new(PluginStats::default());
|
||||
let desired_node = desired();
|
||||
|
|
@ -1413,6 +1468,7 @@ mod tests {
|
|||
}),
|
||||
};
|
||||
let parts = swactor::runtime::RuntimeParts::new(swactor::config::RuntimeConfig::default());
|
||||
let runtime = parts.runtime().clone();
|
||||
let backend =
|
||||
swactor_engine::TokioBackend::new(swactor_engine::TokioConfig::default()).unwrap();
|
||||
let engine = swactor_engine::Engine::new(parts, backend).unwrap();
|
||||
|
|
@ -1422,6 +1478,7 @@ mod tests {
|
|||
vec![binding],
|
||||
RetryPolicy::default(),
|
||||
engine.handle(),
|
||||
runtime,
|
||||
sink,
|
||||
)
|
||||
.unwrap();
|
||||
|
|
@ -1525,7 +1582,6 @@ mod tests {
|
|||
assert_eq!(scaled_stats.stops.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
#[test]
|
||||
#[allow(clippy::disallowed_methods)]
|
||||
fn ambiguous_create_timeout_retries_by_adoption_and_discards_late_success() {
|
||||
let stats = Arc::new(PluginStats::default());
|
||||
let (entered_tx, entered_rx) = mpsc::channel();
|
||||
|
|
@ -1567,6 +1623,7 @@ mod tests {
|
|||
}),
|
||||
};
|
||||
let parts = swactor::runtime::RuntimeParts::new(swactor::config::RuntimeConfig::default());
|
||||
let runtime = parts.runtime().clone();
|
||||
let backend =
|
||||
swactor_engine::TokioBackend::new(swactor_engine::TokioConfig::default()).unwrap();
|
||||
let engine = swactor_engine::Engine::new(parts, backend).unwrap();
|
||||
|
|
@ -1578,6 +1635,7 @@ mod tests {
|
|||
..RetryPolicy::default()
|
||||
},
|
||||
engine.handle(),
|
||||
runtime,
|
||||
PluginSink::new(Arc::new(NullSink)),
|
||||
)
|
||||
.unwrap();
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ pub(crate) const DEFAULT_CONFIG_PATH: &str = ".config/config.toml";
|
|||
#[derive(Clone, Debug, Default, Deserialize, PartialEq)]
|
||||
#[serde(default)]
|
||||
pub(crate) struct VastAiConfig {
|
||||
pub provisioning_mode: Option<String>,
|
||||
pub api_key: Option<String>,
|
||||
pub image: Option<String>,
|
||||
pub relay_url: Option<String>,
|
||||
|
|
@ -28,65 +29,6 @@ pub(crate) struct VastAiConfig {
|
|||
pub ssh_identity: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub(crate) struct ResolvedVastAiConfig {
|
||||
pub api_key: String,
|
||||
pub relay_url: String,
|
||||
pub image: String,
|
||||
pub bootstrap_command: String,
|
||||
pub disk_gb: Option<u32>,
|
||||
pub gpu_name: Option<String>,
|
||||
pub min_gpu_ram_mb: Option<u64>,
|
||||
pub min_down_mbps: Option<f64>,
|
||||
pub min_up_mbps: Option<f64>,
|
||||
pub max_dph_total: Option<f64>,
|
||||
pub min_reliability: Option<f64>,
|
||||
pub require_verified: Option<bool>,
|
||||
pub blacklist_hosts: Vec<u64>,
|
||||
pub onstart: Option<String>,
|
||||
pub ssh_identity: Option<String>,
|
||||
}
|
||||
|
||||
impl ResolvedVastAiConfig {
|
||||
pub(crate) fn validate(self) -> Result<Self, String> {
|
||||
require_non_empty("VAST_API_KEY", &self.api_key)?;
|
||||
require_non_empty("relay.url", &self.relay_url)?;
|
||||
require_non_empty("vastai.image", &self.image)?;
|
||||
require_non_empty("vastai.bootstrap_command", &self.bootstrap_command)?;
|
||||
if let Some(identity) = &self.ssh_identity {
|
||||
require_non_empty("vastai.ssh_identity", identity)?;
|
||||
}
|
||||
if !looks_remote_image(&self.image) {
|
||||
return Err(format!(
|
||||
"vastai.image {:?} must include a registry namespace",
|
||||
self.image
|
||||
));
|
||||
}
|
||||
Ok(self)
|
||||
}
|
||||
}
|
||||
|
||||
fn require_non_empty(label: &str, value: &str) -> Result<(), String> {
|
||||
if value.trim().is_empty() {
|
||||
Err(format!("missing required {label}"))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn looks_remote_image(image: &str) -> bool {
|
||||
let repository = image.split('@').next().unwrap_or(image);
|
||||
let last_slash = repository.rfind('/');
|
||||
let tag_separator = repository
|
||||
.rfind(':')
|
||||
.filter(|separator| last_slash.is_some_and(|slash| *separator > slash));
|
||||
let repository = tag_separator.map_or(repository, |separator| &repository[..separator]);
|
||||
let Some((host, _)) = repository.split_once('/') else {
|
||||
return false;
|
||||
};
|
||||
host == "localhost" || host.contains('.') || host.contains(':')
|
||||
}
|
||||
|
||||
/// Shared overlay for daemon and worker configuration parsing. Unknown
|
||||
/// workload-specific fields remain available to dormant pipeline tooling.
|
||||
#[derive(Clone, Debug, Default, Deserialize, PartialEq)]
|
||||
|
|
|
|||
924
apps/myelin/src/orchestration/control.rs
Normal file
924
apps/myelin/src/orchestration/control.rs
Normal file
|
|
@ -0,0 +1,924 @@
|
|||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use axum::extract::{Path, State};
|
||||
use axum::http::{StatusCode, header};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::routing::{get, post};
|
||||
use axum::{Json, Router};
|
||||
use serde::Serialize;
|
||||
use swactor::actor::{ActorAddress, ActorInterface};
|
||||
use swactor::runtime::{Ctx, ExternalSender, Runtime};
|
||||
use swactor_engine::EngineHandle;
|
||||
use swactor_vastai::VastClient;
|
||||
|
||||
use crate::orchestration::actor::OrchestratorMsg;
|
||||
use crate::orchestration::manual_control::{
|
||||
KillRequest, ManualControlMsg, ManualControlReply, OfferSearchRequest,
|
||||
ProviderConfigurationRequest, ProvisionRequest,
|
||||
};
|
||||
|
||||
const CONTROL_REPLY_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
const OFFER_SEARCH_REPLY_MARGIN: Duration = Duration::from_secs(5);
|
||||
const OFFER_SEARCH_REPLY_TIMEOUT: Duration =
|
||||
VastClient::REQUEST_TIMEOUT.saturating_add(OFFER_SEARCH_REPLY_MARGIN);
|
||||
struct ControlReplyObserver {
|
||||
reply: Arc<Mutex<Option<tokio::sync::oneshot::Sender<ManualControlReply>>>>,
|
||||
engine: EngineHandle,
|
||||
sender: ExternalSender,
|
||||
timeout: Duration,
|
||||
}
|
||||
|
||||
impl ActorInterface for ControlReplyObserver {
|
||||
type Incoming = ManualControlReply;
|
||||
type Response = ();
|
||||
|
||||
fn on_start(&mut self, ctx: &Ctx) {
|
||||
self.engine.send_after(
|
||||
self.timeout,
|
||||
self.sender.clone(),
|
||||
ctx.self_addr(),
|
||||
ManualControlReply::TimedOut,
|
||||
);
|
||||
}
|
||||
|
||||
fn handle(&mut self, ctx: &Ctx, reply: Self::Incoming) {
|
||||
if let Some(response) = self
|
||||
.reply
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.take()
|
||||
{
|
||||
let _ = response.send(reply);
|
||||
}
|
||||
ctx.stop_self();
|
||||
}
|
||||
}
|
||||
const PROVISION_PAGE: &str = include_str!("provision_page.html");
|
||||
const FLEET_CONTROL_SCRIPT: &str = include_str!("fleet_control.js");
|
||||
pub(crate) const FLEET_CONTROL_SCRIPT_URL: &str = "/assets/myelin-fleet-control.js";
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ControlHttpState {
|
||||
runtime: Runtime,
|
||||
engine: EngineHandle,
|
||||
orchestrator: ActorAddress,
|
||||
}
|
||||
|
||||
pub(crate) fn plugin(
|
||||
runtime: Runtime,
|
||||
engine: EngineHandle,
|
||||
orchestrator: ActorAddress,
|
||||
) -> dashboard::DashboardPlugin {
|
||||
let state = ControlHttpState {
|
||||
runtime,
|
||||
engine,
|
||||
orchestrator,
|
||||
};
|
||||
let routes = Router::new()
|
||||
.route(FLEET_CONTROL_SCRIPT_URL, get(fleet_control_script))
|
||||
.route("/api/control/status", get(status))
|
||||
.route("/api/control/actors", get(actor_stats))
|
||||
.route("/api/control/provision", post(provision))
|
||||
.route("/api/control/kill", post(kill))
|
||||
.route("/api/control/provider", post(configure_provider))
|
||||
.route("/api/control/offers", post(search_offers))
|
||||
.route("/api/control/flush", post(flush))
|
||||
.route("/api/control/nodes/{logical_node_id}/kill", post(kill_path))
|
||||
.with_state(state);
|
||||
dashboard::DashboardPlugin::new(routes).with_page(dashboard::PluginPage::new(
|
||||
"provision",
|
||||
"Provision",
|
||||
"/provision",
|
||||
PROVISION_PAGE,
|
||||
))
|
||||
}
|
||||
|
||||
async fn fleet_control_script() -> impl IntoResponse {
|
||||
(
|
||||
[(header::CONTENT_TYPE, "text/javascript; charset=utf-8")],
|
||||
FLEET_CONTROL_SCRIPT,
|
||||
)
|
||||
}
|
||||
|
||||
async fn provision(
|
||||
State(state): State<ControlHttpState>,
|
||||
Json(request): Json<ProvisionRequest>,
|
||||
) -> Response {
|
||||
route_mutation(
|
||||
&state,
|
||||
ManualControlMsg::Provision {
|
||||
request,
|
||||
reply_to: None,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
async fn kill(State(state): State<ControlHttpState>, Json(request): Json<KillRequest>) -> Response {
|
||||
route_mutation(
|
||||
&state,
|
||||
ManualControlMsg::Kill {
|
||||
request,
|
||||
reply_to: None,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct KillPathRequest {
|
||||
command_id: String,
|
||||
}
|
||||
|
||||
async fn kill_path(
|
||||
Path(logical_node_id): Path<u64>,
|
||||
State(state): State<ControlHttpState>,
|
||||
Json(request): Json<KillPathRequest>,
|
||||
) -> Response {
|
||||
route_mutation(
|
||||
&state,
|
||||
ManualControlMsg::Kill {
|
||||
request: KillRequest {
|
||||
command_id: request.command_id,
|
||||
logical_node_id,
|
||||
},
|
||||
reply_to: None,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
async fn configure_provider(
|
||||
State(state): State<ControlHttpState>,
|
||||
Json(request): Json<ProviderConfigurationRequest>,
|
||||
) -> Response {
|
||||
route_mutation(
|
||||
&state,
|
||||
ManualControlMsg::Configure {
|
||||
request,
|
||||
reply_to: None,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
async fn status(State(state): State<ControlHttpState>) -> Response {
|
||||
request_reply(&state, CONTROL_REPLY_TIMEOUT, |reply_to| {
|
||||
ManualControlMsg::Query { reply_to }
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn actor_stats(State(state): State<ControlHttpState>) -> impl IntoResponse {
|
||||
Json(state.runtime.stats())
|
||||
}
|
||||
|
||||
async fn search_offers(
|
||||
State(state): State<ControlHttpState>,
|
||||
Json(request): Json<OfferSearchRequest>,
|
||||
) -> Response {
|
||||
request_reply(&state, OFFER_SEARCH_REPLY_TIMEOUT, |reply_to| {
|
||||
ManualControlMsg::SearchOffers { request, reply_to }
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn flush(State(state): State<ControlHttpState>) -> Response {
|
||||
request_reply(&state, CONTROL_REPLY_TIMEOUT, |reply_to| {
|
||||
ManualControlMsg::Flush { reply_to }
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
fn route_mutation(state: &ControlHttpState, msg: ManualControlMsg) -> Response {
|
||||
match state
|
||||
.runtime
|
||||
.send_to(state.orchestrator, OrchestratorMsg::Manual(msg))
|
||||
{
|
||||
Ok(()) => StatusCode::ACCEPTED.into_response(),
|
||||
Err(error) => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(ErrorResponse {
|
||||
error: format!("orchestrator control actor unavailable: {error}"),
|
||||
}),
|
||||
)
|
||||
.into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn request_reply(
|
||||
state: &ControlHttpState,
|
||||
timeout: Duration,
|
||||
build: impl FnOnce(ActorAddress) -> ManualControlMsg,
|
||||
) -> Response {
|
||||
let response_rx = match begin_request_reply(state, timeout, build) {
|
||||
Ok(response_rx) => response_rx,
|
||||
Err(response) => return response,
|
||||
};
|
||||
|
||||
match response_rx.await {
|
||||
Ok(ManualControlReply::Rejected(error)) => {
|
||||
(StatusCode::CONFLICT, Json(ErrorResponse { error })).into_response()
|
||||
}
|
||||
Ok(ManualControlReply::TimedOut) => (
|
||||
StatusCode::GATEWAY_TIMEOUT,
|
||||
Json(ErrorResponse {
|
||||
error: "orchestrator control reply timed out".to_owned(),
|
||||
}),
|
||||
)
|
||||
.into_response(),
|
||||
Ok(reply) => Json(reply).into_response(),
|
||||
Err(error) => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(ErrorResponse {
|
||||
error: format!("control reply observer stopped: {error}"),
|
||||
}),
|
||||
)
|
||||
.into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
fn begin_request_reply(
|
||||
state: &ControlHttpState,
|
||||
timeout: Duration,
|
||||
build: impl FnOnce(ActorAddress) -> ManualControlMsg,
|
||||
) -> Result<tokio::sync::oneshot::Receiver<ManualControlReply>, Response> {
|
||||
let (response_tx, response_rx) = tokio::sync::oneshot::channel();
|
||||
let response_tx = Arc::new(Mutex::new(Some(response_tx)));
|
||||
let reply_to = state
|
||||
.runtime
|
||||
.spawn(ControlReplyObserver {
|
||||
reply: response_tx,
|
||||
engine: state.engine.clone(),
|
||||
sender: state.runtime.create_sender(),
|
||||
timeout,
|
||||
})
|
||||
.map_err(|error| {
|
||||
(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(ErrorResponse {
|
||||
error: format!("create control reply observer: {error}"),
|
||||
}),
|
||||
)
|
||||
.into_response()
|
||||
})?;
|
||||
if let Err(error) = state
|
||||
.runtime
|
||||
.send_to(state.orchestrator, OrchestratorMsg::Manual(build(reply_to)))
|
||||
{
|
||||
let _ = state.runtime.stop_actor(reply_to);
|
||||
return Err((
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(ErrorResponse {
|
||||
error: format!("orchestrator control actor unavailable: {error}"),
|
||||
}),
|
||||
)
|
||||
.into_response());
|
||||
}
|
||||
Ok(response_rx)
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ErrorResponse {
|
||||
error: String,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod properties {
|
||||
use proptest::prelude::*;
|
||||
use swactor::config::RuntimeConfig;
|
||||
use swactor::runtime::RuntimeParts;
|
||||
use swactor_engine::{Engine, SteppingBackend};
|
||||
|
||||
use super::*;
|
||||
use crate::tests::fuzz_support::{
|
||||
actor_census, assert_actor_delta_at_most, assert_mailboxes_drained, assert_no_poison,
|
||||
drive_steps,
|
||||
};
|
||||
|
||||
const STEP_BUDGET: usize = 64;
|
||||
|
||||
fn reply(code: u8) -> ManualControlReply {
|
||||
match code % 3 {
|
||||
0 => ManualControlReply::Flushed,
|
||||
1 => ManualControlReply::Rejected(format!("rejected-{code}")),
|
||||
_ => ManualControlReply::TimedOut,
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
enum HttpAction {
|
||||
Provision { command_slot: u8, count: u8 },
|
||||
Kill { command_slot: u8, node: u8 },
|
||||
KillPath { command_slot: u8, node: u8 },
|
||||
Configure { value: u8 },
|
||||
Status,
|
||||
ActorStats,
|
||||
Flush,
|
||||
SearchOffers,
|
||||
}
|
||||
|
||||
impl HttpAction {
|
||||
fn from_raw(kind: u8, command_slot: u8, value: u8) -> Self {
|
||||
match kind % 8 {
|
||||
0 => Self::Provision {
|
||||
command_slot,
|
||||
count: value,
|
||||
},
|
||||
1 => Self::Kill {
|
||||
command_slot,
|
||||
node: value,
|
||||
},
|
||||
2 => Self::KillPath {
|
||||
command_slot,
|
||||
node: value,
|
||||
},
|
||||
3 => Self::Configure { value },
|
||||
4 => Self::Status,
|
||||
5 => Self::ActorStats,
|
||||
6 => Self::Flush,
|
||||
_ => Self::SearchOffers,
|
||||
}
|
||||
}
|
||||
|
||||
fn command_id(command_slot: u8) -> String {
|
||||
format!("repeated-{}", command_slot % 4)
|
||||
}
|
||||
|
||||
fn expected_bridge_observation(&self) -> Option<String> {
|
||||
match self {
|
||||
Self::Provision { command_slot, .. } => {
|
||||
Some(format!("Provision:{}", Self::command_id(*command_slot)))
|
||||
}
|
||||
Self::Kill { command_slot, .. } | Self::KillPath { command_slot, .. } => {
|
||||
Some(format!("Kill:{}", Self::command_id(*command_slot)))
|
||||
}
|
||||
Self::Configure { .. } => Some("Configure".to_owned()),
|
||||
Self::Status => Some("Status".to_owned()),
|
||||
Self::ActorStats => None,
|
||||
Self::Flush => Some("Flush".to_owned()),
|
||||
Self::SearchOffers => Some("SearchOffers".to_owned()),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_mutation(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Self::Provision { .. }
|
||||
| Self::Kill { .. }
|
||||
| Self::KillPath { .. }
|
||||
| Self::Configure { .. }
|
||||
)
|
||||
}
|
||||
|
||||
fn uses_reply_observer(&self) -> bool {
|
||||
matches!(self, Self::Status | Self::Flush | Self::SearchOffers)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct HttpObservation {
|
||||
index: usize,
|
||||
action: HttpAction,
|
||||
status: StatusCode,
|
||||
}
|
||||
|
||||
struct HttpBridgeProbe {
|
||||
observations: Arc<Mutex<Vec<String>>>,
|
||||
disappear_reply_observers: bool,
|
||||
}
|
||||
|
||||
impl HttpBridgeProbe {
|
||||
fn finish_reply(&self, ctx: &Ctx, reply_to: ActorAddress, reply: ManualControlReply) {
|
||||
if self.disappear_reply_observers {
|
||||
let _ = ctx.stop_actor(reply_to);
|
||||
} else {
|
||||
let _ = ctx.send(reply_to, reply);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ActorInterface for HttpBridgeProbe {
|
||||
type Incoming = OrchestratorMsg;
|
||||
type Response = ();
|
||||
|
||||
fn handle(&mut self, ctx: &Ctx, message: Self::Incoming) {
|
||||
let OrchestratorMsg::Manual(message) = message else {
|
||||
return;
|
||||
};
|
||||
match message {
|
||||
ManualControlMsg::Provision { request, .. } => self
|
||||
.observations
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.push(format!("Provision:{}", request.command_id)),
|
||||
ManualControlMsg::Kill { request, .. } => self
|
||||
.observations
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.push(format!("Kill:{}", request.command_id)),
|
||||
ManualControlMsg::Configure { .. } => self
|
||||
.observations
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.push("Configure".to_owned()),
|
||||
ManualControlMsg::Query { reply_to } => {
|
||||
self.observations
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.push("Status".to_owned());
|
||||
self.finish_reply(ctx, reply_to, ManualControlReply::Flushed);
|
||||
}
|
||||
ManualControlMsg::Flush { reply_to } => {
|
||||
self.observations
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.push("Flush".to_owned());
|
||||
self.finish_reply(ctx, reply_to, ManualControlReply::Flushed);
|
||||
}
|
||||
ManualControlMsg::SearchOffers { reply_to, .. } => {
|
||||
self.observations
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.push("SearchOffers".to_owned());
|
||||
self.finish_reply(ctx, reply_to, ManualControlReply::Offers(Vec::new()));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum PendingHttpObservation {
|
||||
Ready(HttpObservation),
|
||||
Reply {
|
||||
index: usize,
|
||||
action: HttpAction,
|
||||
receiver: tokio::sync::oneshot::Receiver<ManualControlReply>,
|
||||
},
|
||||
}
|
||||
|
||||
fn begin_http_action(
|
||||
state: &ControlHttpState,
|
||||
index: usize,
|
||||
action: HttpAction,
|
||||
) -> PendingHttpObservation {
|
||||
let immediate = |status| {
|
||||
PendingHttpObservation::Ready(HttpObservation {
|
||||
index,
|
||||
action: action.clone(),
|
||||
status,
|
||||
})
|
||||
};
|
||||
let reply = |result: Result<_, Response>| match result {
|
||||
Ok(receiver) => PendingHttpObservation::Reply {
|
||||
index,
|
||||
action: action.clone(),
|
||||
receiver,
|
||||
},
|
||||
Err(response) => immediate(response.status()),
|
||||
};
|
||||
|
||||
match &action {
|
||||
HttpAction::Provision {
|
||||
command_slot,
|
||||
count,
|
||||
} => immediate(
|
||||
route_mutation(
|
||||
state,
|
||||
ManualControlMsg::Provision {
|
||||
request: ProvisionRequest {
|
||||
command_id: HttpAction::command_id(*command_slot),
|
||||
count: u32::from(*count),
|
||||
selected_offer_ids: Vec::new(),
|
||||
},
|
||||
reply_to: None,
|
||||
},
|
||||
)
|
||||
.status(),
|
||||
),
|
||||
HttpAction::Kill { command_slot, node }
|
||||
| HttpAction::KillPath { command_slot, node } => immediate(
|
||||
route_mutation(
|
||||
state,
|
||||
ManualControlMsg::Kill {
|
||||
request: KillRequest {
|
||||
command_id: HttpAction::command_id(*command_slot),
|
||||
logical_node_id: u64::from(*node),
|
||||
},
|
||||
reply_to: None,
|
||||
},
|
||||
)
|
||||
.status(),
|
||||
),
|
||||
HttpAction::Configure { value } => immediate(
|
||||
route_mutation(
|
||||
state,
|
||||
ManualControlMsg::Configure {
|
||||
request: ProviderConfigurationRequest {
|
||||
api_key: Some(format!("generated-key-{value}")),
|
||||
ssh_identity: None,
|
||||
bootstrap_command: None,
|
||||
},
|
||||
reply_to: None,
|
||||
},
|
||||
)
|
||||
.status(),
|
||||
),
|
||||
HttpAction::Status => reply(begin_request_reply(
|
||||
state,
|
||||
CONTROL_REPLY_TIMEOUT,
|
||||
|reply_to| ManualControlMsg::Query { reply_to },
|
||||
)),
|
||||
HttpAction::ActorStats => immediate(StatusCode::OK),
|
||||
HttpAction::Flush => reply(begin_request_reply(
|
||||
state,
|
||||
CONTROL_REPLY_TIMEOUT,
|
||||
|reply_to| ManualControlMsg::Flush { reply_to },
|
||||
)),
|
||||
HttpAction::SearchOffers => reply(begin_request_reply(
|
||||
state,
|
||||
OFFER_SEARCH_REPLY_TIMEOUT,
|
||||
|reply_to| ManualControlMsg::SearchOffers {
|
||||
request: OfferSearchRequest::default(),
|
||||
reply_to,
|
||||
},
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn finish_http_action(observation: PendingHttpObservation) -> Result<HttpObservation, String> {
|
||||
let PendingHttpObservation::Reply {
|
||||
index,
|
||||
action,
|
||||
mut receiver,
|
||||
} = observation
|
||||
else {
|
||||
let PendingHttpObservation::Ready(observation) = observation else {
|
||||
unreachable!("pending HTTP observation variant changed")
|
||||
};
|
||||
return Ok(observation);
|
||||
};
|
||||
let status = match receiver.try_recv() {
|
||||
Ok(ManualControlReply::Rejected(_)) => StatusCode::CONFLICT,
|
||||
Ok(ManualControlReply::TimedOut) => StatusCode::GATEWAY_TIMEOUT,
|
||||
Ok(_) => StatusCode::OK,
|
||||
Err(tokio::sync::oneshot::error::TryRecvError::Closed) => {
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
}
|
||||
Err(tokio::sync::oneshot::error::TryRecvError::Empty) => {
|
||||
return Err(format!(
|
||||
"control reply remained pending after fixed step budget; \
|
||||
index={index}, action={action:?}"
|
||||
));
|
||||
}
|
||||
};
|
||||
Ok(HttpObservation {
|
||||
index,
|
||||
action,
|
||||
status,
|
||||
})
|
||||
}
|
||||
|
||||
fn http_bridge_invariant_failure(
|
||||
actions: &[HttpAction],
|
||||
responses: &[HttpObservation],
|
||||
observed: &[String],
|
||||
state: &ControlHttpState,
|
||||
baseline_actors: usize,
|
||||
disappear_reply_observers: bool,
|
||||
) -> Option<String> {
|
||||
let mut expected_bridge = actions
|
||||
.iter()
|
||||
.filter_map(HttpAction::expected_bridge_observation)
|
||||
.collect::<Vec<_>>();
|
||||
expected_bridge.sort();
|
||||
let mut actual_bridge = observed.to_vec();
|
||||
actual_bridge.sort();
|
||||
let statuses_valid = responses.iter().all(|response| {
|
||||
let expected = if response.action.is_mutation() {
|
||||
StatusCode::ACCEPTED
|
||||
} else if disappear_reply_observers && response.action.uses_reply_observer() {
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
} else {
|
||||
StatusCode::OK
|
||||
};
|
||||
response.status == expected
|
||||
});
|
||||
let stats = state.runtime.stats();
|
||||
let panics = stats
|
||||
.workers
|
||||
.iter()
|
||||
.map(|worker| worker.panics)
|
||||
.sum::<u64>();
|
||||
let mailbox_depth = stats
|
||||
.workers
|
||||
.iter()
|
||||
.map(|worker| worker.mailbox_depth)
|
||||
.sum::<usize>()
|
||||
+ stats
|
||||
.actor_details
|
||||
.iter()
|
||||
.map(|actor| actor.mailbox_depth)
|
||||
.sum::<usize>();
|
||||
if responses.len() == actions.len()
|
||||
&& statuses_valid
|
||||
&& actual_bridge == expected_bridge
|
||||
&& stats.actors.len() == baseline_actors
|
||||
&& stats.actor_details.iter().all(|actor| !actor.poisoned)
|
||||
&& panics == 0
|
||||
&& mailbox_depth == 0
|
||||
{
|
||||
None
|
||||
} else {
|
||||
Some(format!(
|
||||
"responses={responses:?}, expected_bridge={expected_bridge:?}, \
|
||||
observed_bridge={actual_bridge:?}, expected_actor_count={baseline_actors}, \
|
||||
mailbox_depth={mailbox_depth}, actor_census=\n{}",
|
||||
actor_census(&state.runtime),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#![proptest_config(ProptestConfig {
|
||||
cases: 128,
|
||||
max_shrink_iters: 2_000,
|
||||
..ProptestConfig::default()
|
||||
})]
|
||||
|
||||
#[test]
|
||||
fn generated_http_bridge_sequences_terminate_without_control_actor_growth(
|
||||
raw_actions in prop::collection::vec(
|
||||
(any::<u8>(), any::<u8>(), any::<u8>()),
|
||||
0..=32,
|
||||
),
|
||||
concurrent in any::<bool>(),
|
||||
disappear_reply_observers in any::<bool>(),
|
||||
) {
|
||||
let actions = raw_actions
|
||||
.into_iter()
|
||||
.map(|(kind, command_slot, value)| {
|
||||
HttpAction::from_raw(kind, command_slot, value)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let mut config = RuntimeConfig::default();
|
||||
config.worker_count = 1;
|
||||
let parts = RuntimeParts::new(config);
|
||||
let runtime = parts.runtime().clone();
|
||||
let backend = SteppingBackend::new();
|
||||
let engine =
|
||||
Engine::new(parts, backend.clone()).expect("control HTTP stepping engine");
|
||||
let observations = Arc::new(Mutex::new(Vec::new()));
|
||||
let orchestrator = runtime
|
||||
.spawn(HttpBridgeProbe {
|
||||
observations: Arc::clone(&observations),
|
||||
disappear_reply_observers,
|
||||
})
|
||||
.expect("spawn control HTTP bridge probe");
|
||||
drive_steps(&backend, STEP_BUDGET);
|
||||
let baseline_actors = runtime.stats().actors.len();
|
||||
let baseline_tasks = backend.pending_task_count();
|
||||
let state = ControlHttpState {
|
||||
runtime: runtime.clone(),
|
||||
engine: engine.handle(),
|
||||
orchestrator,
|
||||
};
|
||||
let outcome = (|| {
|
||||
let mut responses = Vec::with_capacity(actions.len());
|
||||
if concurrent {
|
||||
let pending = actions
|
||||
.iter()
|
||||
.cloned()
|
||||
.enumerate()
|
||||
.map(|(index, action)| begin_http_action(&state, index, action))
|
||||
.collect::<Vec<_>>();
|
||||
drive_steps(&backend, STEP_BUDGET);
|
||||
drive_steps(&backend, STEP_BUDGET);
|
||||
for observation in pending {
|
||||
responses.push(finish_http_action(observation)?);
|
||||
}
|
||||
} else {
|
||||
for (index, action) in actions.iter().cloned().enumerate() {
|
||||
let pending = begin_http_action(&state, index, action);
|
||||
drive_steps(&backend, STEP_BUDGET);
|
||||
drive_steps(&backend, STEP_BUDGET);
|
||||
responses.push(finish_http_action(pending)?);
|
||||
}
|
||||
}
|
||||
responses.sort_by_key(|response| response.index);
|
||||
Ok::<_, String>(responses)
|
||||
})();
|
||||
prop_assert!(
|
||||
outcome.is_ok(),
|
||||
"control HTTP request did not terminate; actions={:?}; error={:?}; \
|
||||
responses=[]; actor_census=\n{}",
|
||||
actions,
|
||||
outcome.as_ref().err(),
|
||||
actor_census(&runtime),
|
||||
);
|
||||
let responses = outcome.expect("outcome checked above");
|
||||
drive_steps(&backend, STEP_BUDGET);
|
||||
let observed = observations
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.clone();
|
||||
let failure = http_bridge_invariant_failure(
|
||||
&actions,
|
||||
&responses,
|
||||
&observed,
|
||||
&state,
|
||||
baseline_actors,
|
||||
disappear_reply_observers,
|
||||
);
|
||||
prop_assert!(
|
||||
failure.is_none(),
|
||||
"control HTTP/bridge invariant failed; actions={:?}; responses={:?}; failure={}",
|
||||
actions,
|
||||
responses,
|
||||
failure.unwrap_or_default(),
|
||||
);
|
||||
|
||||
backend.advance_time(OFFER_SEARCH_REPLY_TIMEOUT);
|
||||
drive_steps(&backend, STEP_BUDGET);
|
||||
prop_assert_eq!(
|
||||
backend.pending_task_count(),
|
||||
baseline_tasks,
|
||||
"control reply timers survived their fixed drain budget; actions={:?}; \
|
||||
responses={:?}; actor_census=\n{}",
|
||||
actions,
|
||||
responses,
|
||||
actor_census(&runtime),
|
||||
);
|
||||
runtime
|
||||
.stop_actor(orchestrator)
|
||||
.expect("stop control HTTP bridge probe");
|
||||
drive_steps(&backend, STEP_BUDGET);
|
||||
assert_no_poison(&runtime);
|
||||
assert_actor_delta_at_most(&runtime, 0, 0);
|
||||
assert_mailboxes_drained(&runtime);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generated_duplicate_control_replies_deliver_first_once_and_remove_observer(
|
||||
replies in prop::collection::vec(any::<u8>(), 0..=16)
|
||||
) {
|
||||
let mut config = RuntimeConfig::default();
|
||||
config.worker_count = 1;
|
||||
let parts = RuntimeParts::new(config);
|
||||
let runtime = parts.runtime().clone();
|
||||
let backend = SteppingBackend::new();
|
||||
let engine =
|
||||
Engine::new(parts, backend.clone()).expect("control reply stepping engine");
|
||||
let baseline_actors = runtime.stats().actors.len();
|
||||
let baseline_tasks = backend.pending_task_count();
|
||||
let (response_tx, mut response_rx) = tokio::sync::oneshot::channel();
|
||||
let observer = runtime
|
||||
.spawn(ControlReplyObserver {
|
||||
reply: Arc::new(Mutex::new(Some(response_tx))),
|
||||
engine: engine.handle(),
|
||||
sender: runtime.create_sender(),
|
||||
timeout: Duration::from_millis(1),
|
||||
})
|
||||
.expect("spawn control reply observer");
|
||||
for code in &replies {
|
||||
runtime
|
||||
.send_to(observer, reply(*code))
|
||||
.expect("send generated control reply");
|
||||
}
|
||||
drive_steps(&backend, 32);
|
||||
if replies.is_empty() {
|
||||
backend.advance_time(Duration::from_millis(1));
|
||||
drive_steps(&backend, 32);
|
||||
}
|
||||
|
||||
let observed = response_rx
|
||||
.try_recv()
|
||||
.expect("control reply observer produced a terminal reply");
|
||||
let expected = replies
|
||||
.first()
|
||||
.map(|code| reply(*code))
|
||||
.unwrap_or(ManualControlReply::TimedOut);
|
||||
prop_assert_eq!(
|
||||
observed,
|
||||
expected,
|
||||
"control reply observer accepted a stale duplicate; replies={:?}; \
|
||||
actor_census=\n{}",
|
||||
replies,
|
||||
actor_census(&runtime),
|
||||
);
|
||||
|
||||
backend.advance_time(Duration::from_millis(1));
|
||||
drive_steps(&backend, 32);
|
||||
prop_assert_eq!(backend.pending_task_count(), baseline_tasks);
|
||||
assert_no_poison(&runtime);
|
||||
assert_actor_delta_at_most(&runtime, baseline_actors, 0);
|
||||
assert_mailboxes_drained(&runtime);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disappeared_orchestrator_removes_control_reply_actor() {
|
||||
let parts = RuntimeParts::new(RuntimeConfig::default());
|
||||
let runtime = parts.runtime().clone();
|
||||
let backend = SteppingBackend::new();
|
||||
let engine = Engine::new(parts, backend.clone()).expect("control failure stepping engine");
|
||||
let baseline_tasks = backend.pending_task_count();
|
||||
let state = ControlHttpState {
|
||||
runtime: runtime.clone(),
|
||||
engine: engine.handle(),
|
||||
orchestrator: ActorAddress::default(),
|
||||
};
|
||||
let response = match begin_request_reply(&state, Duration::from_millis(1), |reply_to| {
|
||||
ManualControlMsg::Query { reply_to }
|
||||
}) {
|
||||
Ok(_) => panic!("missing orchestrator unexpectedly accepted a control query"),
|
||||
Err(response) => response,
|
||||
};
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
drive_steps(&backend, STEP_BUDGET);
|
||||
backend.advance_time(Duration::from_millis(1));
|
||||
drive_steps(&backend, STEP_BUDGET);
|
||||
assert_eq!(backend.pending_task_count(), baseline_tasks);
|
||||
assert_no_poison(&runtime);
|
||||
assert_actor_delta_at_most(&runtime, 0, 0);
|
||||
assert_mailboxes_drained(&runtime);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reply_observer_disappearance_returns_a_bounded_terminal_http_response() {
|
||||
let mut config = RuntimeConfig::default();
|
||||
config.worker_count = 1;
|
||||
let parts = RuntimeParts::new(config);
|
||||
let runtime = parts.runtime().clone();
|
||||
let backend = SteppingBackend::new();
|
||||
let engine =
|
||||
Engine::new(parts, backend.clone()).expect("reply disappearance stepping engine");
|
||||
let observations = Arc::new(Mutex::new(Vec::new()));
|
||||
let orchestrator = runtime
|
||||
.spawn(HttpBridgeProbe {
|
||||
observations,
|
||||
disappear_reply_observers: true,
|
||||
})
|
||||
.expect("spawn disappearing-reply bridge");
|
||||
drive_steps(&backend, STEP_BUDGET);
|
||||
let baseline_actors = runtime.stats().actors.len();
|
||||
let baseline_tasks = backend.pending_task_count();
|
||||
let state = ControlHttpState {
|
||||
runtime: runtime.clone(),
|
||||
engine: engine.handle(),
|
||||
orchestrator,
|
||||
};
|
||||
let pending = begin_http_action(&state, 0, HttpAction::Status);
|
||||
drive_steps(&backend, STEP_BUDGET);
|
||||
drive_steps(&backend, STEP_BUDGET);
|
||||
let response = finish_http_action(pending).unwrap_or_else(|error| {
|
||||
panic!(
|
||||
"{error}; responses=[]; actor_census=\n{}",
|
||||
actor_census(&runtime),
|
||||
)
|
||||
});
|
||||
assert_eq!(
|
||||
response.status,
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"reply observer disappearance returned {}; actor_census=\n{}",
|
||||
response.status,
|
||||
actor_census(&runtime),
|
||||
);
|
||||
drive_steps(&backend, STEP_BUDGET);
|
||||
assert_actor_delta_at_most(&runtime, baseline_actors, 0);
|
||||
backend.advance_time(CONTROL_REPLY_TIMEOUT);
|
||||
drive_steps(&backend, STEP_BUDGET);
|
||||
assert_eq!(backend.pending_task_count(), baseline_tasks);
|
||||
runtime
|
||||
.stop_actor(orchestrator)
|
||||
.expect("stop disappearing-reply bridge");
|
||||
drive_steps(&backend, STEP_BUDGET);
|
||||
assert_no_poison(&runtime);
|
||||
assert_actor_delta_at_most(&runtime, 0, 0);
|
||||
assert_mailboxes_drained(&runtime);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn http_bridge_invariant_rejects_a_controlled_duplicate_forward() {
|
||||
let parts = RuntimeParts::new(RuntimeConfig::default());
|
||||
let runtime = parts.runtime().clone();
|
||||
let backend = SteppingBackend::new();
|
||||
let engine =
|
||||
Engine::new(parts, backend.clone()).expect("control invariant stepping engine");
|
||||
let state = ControlHttpState {
|
||||
runtime,
|
||||
engine: engine.handle(),
|
||||
orchestrator: ActorAddress::default(),
|
||||
};
|
||||
let actions = vec![HttpAction::Status];
|
||||
let responses = vec![HttpObservation {
|
||||
index: 0,
|
||||
action: HttpAction::Status,
|
||||
status: StatusCode::OK,
|
||||
}];
|
||||
let duplicated = vec!["Status".to_owned(), "Status".to_owned()];
|
||||
|
||||
assert!(
|
||||
http_bridge_invariant_failure(&actions, &responses, &duplicated, &state, 0, false,)
|
||||
.is_some(),
|
||||
"control HTTP/bridge invariant accepted a controlled duplicate forward"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -7,38 +7,58 @@
|
|||
//! intent and facts so a restarted daemon can adopt what still exists and
|
||||
//! never silently re-provisions.
|
||||
|
||||
#[cfg(test)]
|
||||
use std::collections::BTreeMap;
|
||||
use std::collections::BTreeSet;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use crate::orchestration::manual_control::{CommandKind, CommandRecord, CommandState, NodePhase};
|
||||
use crate::provisioning::NodeProvisionSpec;
|
||||
use distribution::types::NodeId as DistNodeId;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use swactor::actor::ActorAddress;
|
||||
|
||||
pub(crate) const SNAPSHOT_SCHEMA_VERSION: u32 = 1;
|
||||
pub(crate) const SNAPSHOT_SCHEMA_VERSION: u32 = 2;
|
||||
pub(crate) const IDENTITY_FILE: &str = "identity.key";
|
||||
pub(crate) const SNAPSHOT_FILE: &str = "cluster.json";
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub(crate) enum NodeStatus {
|
||||
/// Provider resource exists and the runtime joined (or is expected to).
|
||||
enum LegacyNodeStatus {
|
||||
Running,
|
||||
/// Tracked by the snapshot but gone from the provider.
|
||||
Dead,
|
||||
/// Exists at the provider under this daemon's label but was never added
|
||||
/// through this daemon's command surface.
|
||||
Orphan,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct LegacySnapshotNode {
|
||||
logical_node_id: u64,
|
||||
spec: Option<NodeProvisionSpec>,
|
||||
provider_ref: Option<String>,
|
||||
status: LegacyNodeStatus,
|
||||
runtime: Option<RuntimeFacts>,
|
||||
last_seen_unix_ms: u64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct LegacyClusterSnapshot {
|
||||
schema_version: u32,
|
||||
run_id: u64,
|
||||
label: String,
|
||||
next_node_id: u64,
|
||||
#[serde(default)]
|
||||
accepted_command_ids: BTreeSet<String>,
|
||||
nodes: Vec<LegacySnapshotNode>,
|
||||
}
|
||||
|
||||
/// Join/readiness facts captured when a node announced itself. Persisted so a
|
||||
/// restarted daemon can re-subscribe telemetry once routes recover.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub(crate) struct RuntimeFacts {
|
||||
#[serde(default)]
|
||||
pub run_id: u64,
|
||||
#[serde(default)]
|
||||
pub attempt_id: u64,
|
||||
pub endpoint: String,
|
||||
pub node_actor: ActorAddress,
|
||||
pub swim_node_id: DistNodeId,
|
||||
|
|
@ -46,17 +66,21 @@ pub(crate) struct RuntimeFacts {
|
|||
pub readiness_id: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub(crate) struct SnapshotNode {
|
||||
pub logical_node_id: u64,
|
||||
/// Provision intent for nodes added through this daemon. Absent for
|
||||
/// orphans (discovered, not managed).
|
||||
pub spec: Option<NodeProvisionSpec>,
|
||||
/// Provider-side address (e.g. docker container name) for records that
|
||||
/// exist without a full spec.
|
||||
/// Exact operator selection. Only Vast.ai nodes set this.
|
||||
#[serde(default)]
|
||||
pub selected_offer_id: Option<u64>,
|
||||
/// Provider-side address (container name, contract label, or process id).
|
||||
pub provider_ref: Option<String>,
|
||||
pub status: NodeStatus,
|
||||
pub phase: NodePhase,
|
||||
pub runtime: Option<RuntimeFacts>,
|
||||
#[serde(default)]
|
||||
pub last_error: Option<String>,
|
||||
pub last_seen_unix_ms: u64,
|
||||
}
|
||||
|
||||
|
|
@ -66,11 +90,9 @@ pub(crate) struct ClusterSnapshot {
|
|||
pub run_id: u64,
|
||||
pub label: String,
|
||||
pub next_node_id: u64,
|
||||
/// Durable at-most-once ledger for dashboard requests. A command id is
|
||||
/// recorded before provider mutation, so retrying after a timeout or crash
|
||||
/// cannot create or destroy a second resource.
|
||||
/// Durable at-most-once command ledger, including terminal outcomes.
|
||||
#[serde(default)]
|
||||
pub accepted_command_ids: BTreeSet<String>,
|
||||
pub commands: BTreeMap<String, CommandRecord>,
|
||||
pub nodes: Vec<SnapshotNode>,
|
||||
}
|
||||
|
||||
|
|
@ -81,7 +103,7 @@ impl ClusterSnapshot {
|
|||
run_id,
|
||||
label: label.into(),
|
||||
next_node_id: 1,
|
||||
accepted_command_ids: BTreeSet::new(),
|
||||
commands: BTreeMap::new(),
|
||||
nodes: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
|
@ -98,12 +120,6 @@ impl ClusterSnapshot {
|
|||
.find(|node| node.logical_node_id == logical_node_id)
|
||||
}
|
||||
|
||||
pub(crate) fn running_nodes(&self) -> impl Iterator<Item = &SnapshotNode> {
|
||||
self.nodes
|
||||
.iter()
|
||||
.filter(|node| node.status == NodeStatus::Running)
|
||||
}
|
||||
|
||||
/// Allocates the next logical node id. Ids are monotonic and never reused.
|
||||
pub(crate) fn allocate_node_id(&mut self) -> u64 {
|
||||
let id = self.next_node_id;
|
||||
|
|
@ -129,85 +145,6 @@ impl ClusterSnapshot {
|
|||
None => self.nodes.push(node),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn remove_node(&mut self, logical_node_id: u64) -> Option<SnapshotNode> {
|
||||
let index = self
|
||||
.nodes
|
||||
.iter()
|
||||
.position(|node| node.logical_node_id == logical_node_id)?;
|
||||
Some(self.nodes.remove(index))
|
||||
}
|
||||
|
||||
/// Makes orphan records exactly match provider-only resources. Orphans are
|
||||
/// keyed by provider reference because they intentionally have no logical
|
||||
/// node id or provision spec.
|
||||
pub(crate) fn sync_orphans(
|
||||
&mut self,
|
||||
provider_refs: impl IntoIterator<Item = String>,
|
||||
) -> Vec<String> {
|
||||
let provider_refs = provider_refs
|
||||
.into_iter()
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
self.nodes.retain(|node| {
|
||||
node.status != NodeStatus::Orphan
|
||||
|| node
|
||||
.provider_ref
|
||||
.as_ref()
|
||||
.is_some_and(|provider_ref| provider_refs.contains(provider_ref))
|
||||
});
|
||||
let known = self
|
||||
.nodes
|
||||
.iter()
|
||||
.filter(|node| node.status == NodeStatus::Orphan)
|
||||
.filter_map(|node| node.provider_ref.clone())
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
let added = provider_refs
|
||||
.difference(&known)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
let now = unix_ms_now();
|
||||
for provider_ref in &added {
|
||||
self.nodes.push(SnapshotNode {
|
||||
logical_node_id: 0,
|
||||
spec: None,
|
||||
provider_ref: Some(provider_ref.clone()),
|
||||
status: NodeStatus::Orphan,
|
||||
runtime: None,
|
||||
last_seen_unix_ms: now,
|
||||
});
|
||||
}
|
||||
added
|
||||
}
|
||||
|
||||
pub(crate) fn accept_command(&mut self, command_id: &str) -> Result<bool, String> {
|
||||
let command_id = command_id.trim();
|
||||
if command_id.is_empty() {
|
||||
return Err("dashboard command_id must not be empty".to_owned());
|
||||
}
|
||||
Ok(self.accepted_command_ids.insert(command_id.to_owned()))
|
||||
}
|
||||
}
|
||||
|
||||
/// Operator commands accepted by the dispatcher. The dashboard (and later the
|
||||
/// CLI) is a transport into this surface.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub(crate) enum DaemonCommand {
|
||||
/// Provision exactly one node using the configured provider + selection
|
||||
/// policy. Fails (does not retry) if bring-up fails.
|
||||
AddNode,
|
||||
/// Terminate a node's provider resource; the record stays (status Dead).
|
||||
Kill { logical_node_id: u64 },
|
||||
/// Terminate and forget a node.
|
||||
Destroy { logical_node_id: u64 },
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub(crate) enum DaemonOutcome {
|
||||
Added { logical_node_id: u64 },
|
||||
Killed { logical_node_id: u64 },
|
||||
Destroyed { logical_node_id: u64 },
|
||||
NoSuchNode { logical_node_id: u64 },
|
||||
Failed { command: String, reason: String },
|
||||
}
|
||||
|
||||
/// Where the daemon keeps `identity.key` and `cluster.json`.
|
||||
|
|
@ -221,10 +158,6 @@ impl StateDir {
|
|||
Self { root: root.into() }
|
||||
}
|
||||
|
||||
pub(crate) fn root(&self) -> &Path {
|
||||
&self.root
|
||||
}
|
||||
|
||||
fn identity_path(&self) -> PathBuf {
|
||||
self.root.join(IDENTITY_FILE)
|
||||
}
|
||||
|
|
@ -232,6 +165,9 @@ impl StateDir {
|
|||
fn snapshot_path(&self) -> PathBuf {
|
||||
self.root.join(SNAPSHOT_FILE)
|
||||
}
|
||||
pub(crate) fn process_registry_path(&self) -> PathBuf {
|
||||
self.root.join("process-nodes.json")
|
||||
}
|
||||
|
||||
/// Loads the persisted iroh secret key, creating it on first boot. The
|
||||
/// endpoint address baked into every launched node's env stays valid
|
||||
|
|
@ -263,23 +199,44 @@ impl StateDir {
|
|||
let path = self.snapshot_path();
|
||||
match fs::read_to_string(&path) {
|
||||
Ok(content) => {
|
||||
let snapshot: ClusterSnapshot = serde_json::from_str(&content).map_err(|e| {
|
||||
format!(
|
||||
"cluster snapshot {} is corrupt ({e}); inspect it or remove it with \
|
||||
--reset-state — refusing to silently re-provision",
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
if snapshot.schema_version != SNAPSHOT_SCHEMA_VERSION {
|
||||
return Err(format!(
|
||||
"cluster snapshot {} has unsupported schema_version {} (expected {}); \
|
||||
let schema_version = serde_json::from_str::<serde_json::Value>(&content)
|
||||
.ok()
|
||||
.and_then(|value| value.get("schema_version").and_then(|value| value.as_u64()))
|
||||
.and_then(|value| u32::try_from(value).ok())
|
||||
.ok_or_else(|| {
|
||||
format!(
|
||||
"cluster snapshot {} is corrupt (missing schema_version); inspect it or remove it with \
|
||||
--reset-state — refusing to silently re-provision",
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
match schema_version {
|
||||
SNAPSHOT_SCHEMA_VERSION => serde_json::from_str(&content).map_err(|error| {
|
||||
format!(
|
||||
"cluster snapshot {} is corrupt ({error}); inspect it or remove it with \
|
||||
--reset-state — refusing to silently re-provision",
|
||||
path.display()
|
||||
)
|
||||
}),
|
||||
1 => {
|
||||
let legacy: LegacyClusterSnapshot =
|
||||
serde_json::from_str(&content).map_err(|error| {
|
||||
format!(
|
||||
"cluster snapshot {} schema v1 is corrupt ({error}); inspect it or remove it with \
|
||||
--reset-state — refusing to silently re-provision",
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
Ok(migrate_v1(legacy))
|
||||
}
|
||||
unsupported => Err(format!(
|
||||
"cluster snapshot {} has unsupported schema_version {} (expected {} or migratable v1); \
|
||||
migrate or remove it with --reset-state",
|
||||
path.display(),
|
||||
snapshot.schema_version,
|
||||
unsupported,
|
||||
SNAPSHOT_SCHEMA_VERSION
|
||||
));
|
||||
)),
|
||||
}
|
||||
Ok(snapshot)
|
||||
}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||
// Caller decides the run id/label for a fresh snapshot.
|
||||
|
|
@ -288,8 +245,8 @@ impl StateDir {
|
|||
run_id: 0,
|
||||
label: String::new(),
|
||||
next_node_id: 1,
|
||||
commands: BTreeMap::new(),
|
||||
nodes: Vec::new(),
|
||||
accepted_command_ids: BTreeSet::new(),
|
||||
})
|
||||
}
|
||||
Err(error) => Err(format!("read cluster snapshot {}: {error}", path.display())),
|
||||
|
|
@ -317,6 +274,62 @@ impl StateDir {
|
|||
}
|
||||
}
|
||||
|
||||
fn migrate_v1(legacy: LegacyClusterSnapshot) -> ClusterSnapshot {
|
||||
debug_assert_eq!(legacy.schema_version, 1);
|
||||
let commands = legacy
|
||||
.accepted_command_ids
|
||||
.into_iter()
|
||||
.map(|command_id| {
|
||||
(
|
||||
command_id.clone(),
|
||||
CommandRecord {
|
||||
command_id,
|
||||
kind: CommandKind::Migrated,
|
||||
state: CommandState::Failed,
|
||||
node_ids: Vec::new(),
|
||||
error: Some(
|
||||
"migrated schema-v1 command; outcome was not recorded and will not be replayed"
|
||||
.to_owned(),
|
||||
),
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
let nodes = legacy
|
||||
.nodes
|
||||
.into_iter()
|
||||
.map(|legacy_node| {
|
||||
let mut runtime = legacy_node.runtime;
|
||||
if let (Some(spec), Some(facts)) = (legacy_node.spec.as_ref(), runtime.as_mut()) {
|
||||
facts.run_id = spec.run_id;
|
||||
facts.attempt_id = spec.attempt_id;
|
||||
}
|
||||
SnapshotNode {
|
||||
logical_node_id: legacy_node.logical_node_id,
|
||||
spec: legacy_node.spec,
|
||||
selected_offer_id: None,
|
||||
provider_ref: legacy_node.provider_ref,
|
||||
phase: match legacy_node.status {
|
||||
LegacyNodeStatus::Running => NodePhase::Running,
|
||||
LegacyNodeStatus::Dead => NodePhase::Stopped,
|
||||
LegacyNodeStatus::Orphan => NodePhase::Orphan,
|
||||
},
|
||||
runtime,
|
||||
last_error: None,
|
||||
last_seen_unix_ms: legacy_node.last_seen_unix_ms,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
ClusterSnapshot {
|
||||
schema_version: SNAPSHOT_SCHEMA_VERSION,
|
||||
run_id: legacy.run_id,
|
||||
label: legacy.label,
|
||||
next_node_id: legacy.next_node_id,
|
||||
commands,
|
||||
nodes,
|
||||
}
|
||||
}
|
||||
|
||||
fn write_atomic(path: &Path, bytes: &[u8]) -> Result<(), String> {
|
||||
let tmp = path.with_extension("tmp");
|
||||
fs::write(&tmp, bytes).map_err(|error| format!("write {}: {error}", tmp.display()))?;
|
||||
|
|
@ -333,321 +346,79 @@ pub(crate) fn unix_ms_now() -> u64 {
|
|||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Joins snapshot intent with provider ground truth at boot. Never takes a
|
||||
/// lifecycle action: live nodes are adopted for observation, missing ones are
|
||||
/// marked dead, provider-only resources are recorded as orphans.
|
||||
pub(crate) struct BootJoin {
|
||||
pub snapshot: ClusterSnapshot,
|
||||
}
|
||||
|
||||
pub(crate) struct JoinOutcome {
|
||||
pub adopted: Vec<u64>,
|
||||
pub dead: Vec<u64>,
|
||||
pub orphans: Vec<String>,
|
||||
}
|
||||
|
||||
impl BootJoin {
|
||||
/// `labeled` lists provider resources carrying this daemon's label that
|
||||
/// the snapshot does not account for.
|
||||
pub(crate) fn apply_provider_truthtable(
|
||||
&mut self,
|
||||
live_specs: &BTreeMap<u64, bool>,
|
||||
labeled_orphans: Vec<String>,
|
||||
) -> JoinOutcome {
|
||||
let mut adopted = Vec::new();
|
||||
let mut dead = Vec::new();
|
||||
let now = unix_ms_now();
|
||||
for node in &mut self.snapshot.nodes {
|
||||
if node.status == NodeStatus::Orphan {
|
||||
continue;
|
||||
}
|
||||
match live_specs.get(&node.logical_node_id) {
|
||||
Some(true) => {
|
||||
node.status = NodeStatus::Running;
|
||||
node.last_seen_unix_ms = now;
|
||||
adopted.push(node.logical_node_id);
|
||||
}
|
||||
_ => {
|
||||
node.status = NodeStatus::Dead;
|
||||
dead.push(node.logical_node_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
self.snapshot.sync_orphans(labeled_orphans.clone());
|
||||
JoinOutcome {
|
||||
adopted,
|
||||
dead,
|
||||
orphans: labeled_orphans,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
static NEXT_TEST_DIR: AtomicU64 = AtomicU64::new(1);
|
||||
#[test]
|
||||
fn schema_v1_migrates_without_replaying_accepted_commands() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let state = StateDir::new(temp.path());
|
||||
fs::write(
|
||||
temp.path().join(SNAPSHOT_FILE),
|
||||
serde_json::json!({
|
||||
"schema_version": 1,
|
||||
"run_id": 7,
|
||||
"label": "legacy",
|
||||
"next_node_id": 2,
|
||||
"accepted_command_ids": ["already-accepted"],
|
||||
"nodes": [{
|
||||
"logical_node_id": 1,
|
||||
"spec": null,
|
||||
"provider_ref": "legacy-resource",
|
||||
"status": "orphan",
|
||||
"runtime": null,
|
||||
"last_seen_unix_ms": 4
|
||||
}]
|
||||
})
|
||||
.to_string(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
fn test_dir(name: &str) -> PathBuf {
|
||||
std::env::temp_dir().join(format!(
|
||||
"myelin-{name}-{}-{}",
|
||||
std::process::id(),
|
||||
NEXT_TEST_DIR.fetch_add(1, Ordering::Relaxed)
|
||||
))
|
||||
}
|
||||
|
||||
fn node(id: u64, status: NodeStatus) -> SnapshotNode {
|
||||
SnapshotNode {
|
||||
logical_node_id: id,
|
||||
spec: None,
|
||||
provider_ref: Some(format!("container-{id}")),
|
||||
status,
|
||||
runtime: None,
|
||||
last_seen_unix_ms: 0,
|
||||
}
|
||||
let migrated = state.load_snapshot().unwrap();
|
||||
assert_eq!(migrated.schema_version, SNAPSHOT_SCHEMA_VERSION);
|
||||
assert_eq!(migrated.nodes[0].phase, NodePhase::Orphan);
|
||||
assert_eq!(
|
||||
migrated.commands["already-accepted"].state,
|
||||
CommandState::Failed
|
||||
);
|
||||
assert_eq!(
|
||||
migrated.commands["already-accepted"].kind,
|
||||
CommandKind::Migrated
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn node_ids_allocate_monotonically_and_never_reuse() {
|
||||
let mut snapshot = ClusterSnapshot::fresh(1, "test");
|
||||
fn unsupported_and_corrupt_snapshots_are_hard_errors() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let state = StateDir::new(temp.path());
|
||||
fs::write(temp.path().join(SNAPSHOT_FILE), r#"{"schema_version":99}"#).unwrap();
|
||||
assert!(state.load_snapshot().unwrap_err().contains("unsupported"));
|
||||
fs::write(temp.path().join(SNAPSHOT_FILE), "{broken").unwrap();
|
||||
assert!(state.load_snapshot().unwrap_err().contains("corrupt"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn atomic_round_trip_preserves_monotonic_ids_and_commands() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let state = StateDir::new(temp.path());
|
||||
let mut snapshot = ClusterSnapshot::fresh(9, "roundtrip");
|
||||
assert_eq!(snapshot.allocate_node_id(), 1);
|
||||
assert_eq!(snapshot.allocate_node_id(), 2);
|
||||
snapshot.next_node_id = u64::MAX;
|
||||
assert!(
|
||||
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||
snapshot.allocate_node_id()
|
||||
}))
|
||||
.is_err()
|
||||
snapshot.commands.insert(
|
||||
"done".to_owned(),
|
||||
CommandRecord {
|
||||
command_id: "done".to_owned(),
|
||||
kind: CommandKind::Kill,
|
||||
state: CommandState::Succeeded,
|
||||
node_ids: vec![1],
|
||||
error: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remove_then_allocate_does_not_reuse_ids() {
|
||||
let mut snapshot = ClusterSnapshot::fresh(1, "test");
|
||||
snapshot.upsert_node(node(1, NodeStatus::Running));
|
||||
assert!(snapshot.remove_node(1).is_some());
|
||||
assert_eq!(snapshot.allocate_node_id(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn join_marks_live_missing_and_orphans_without_actions() {
|
||||
let mut join = BootJoin {
|
||||
snapshot: ClusterSnapshot::fresh(1, "test"),
|
||||
};
|
||||
join.snapshot.upsert_node(node(1, NodeStatus::Running));
|
||||
join.snapshot.upsert_node(node(2, NodeStatus::Running));
|
||||
let live = BTreeMap::from([(1_u64, true), (2_u64, false)]);
|
||||
let outcome = join.apply_provider_truthtable(&live, vec!["orphan-a".to_owned()]);
|
||||
assert_eq!(outcome.adopted, vec![1]);
|
||||
assert_eq!(outcome.dead, vec![2]);
|
||||
assert_eq!(outcome.orphans, vec!["orphan-a".to_owned()]);
|
||||
assert_eq!(join.snapshot.node(1).unwrap().status, NodeStatus::Running);
|
||||
assert_eq!(join.snapshot.node(2).unwrap().status, NodeStatus::Dead);
|
||||
assert_eq!(
|
||||
join.snapshot
|
||||
.nodes
|
||||
.iter()
|
||||
.find(|node| node.provider_ref.as_deref() == Some("orphan-a"))
|
||||
.unwrap()
|
||||
.status,
|
||||
NodeStatus::Orphan
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snapshot_round_trips_through_disk() {
|
||||
let dir = test_dir("snapshot");
|
||||
let state = StateDir::new(&dir);
|
||||
let mut snapshot = ClusterSnapshot::fresh(7, "label");
|
||||
snapshot.upsert_node(SnapshotNode {
|
||||
logical_node_id: 1,
|
||||
spec: Some(NodeProvisionSpec {
|
||||
run_id: 7,
|
||||
node_id: 1,
|
||||
attempt_id: 0,
|
||||
stage_index: Some(0),
|
||||
image: "myelin-node:latest".to_owned(),
|
||||
env: vec![("A".to_owned(), "B".to_owned())],
|
||||
args: vec![],
|
||||
mounts: vec![],
|
||||
}),
|
||||
provider_ref: Some("container-1".to_owned()),
|
||||
status: NodeStatus::Running,
|
||||
runtime: None,
|
||||
last_seen_unix_ms: 42,
|
||||
});
|
||||
state.save_snapshot(&snapshot).unwrap();
|
||||
let loaded = state.load_snapshot().unwrap();
|
||||
assert_eq!(loaded.run_id, 7);
|
||||
assert_eq!(loaded.nodes.len(), 1);
|
||||
assert_eq!(
|
||||
loaded.nodes[0].spec.as_ref().unwrap().image,
|
||||
"myelin-node:latest"
|
||||
);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corrupt_snapshot_is_a_hard_error() {
|
||||
let dir = test_dir("corrupt");
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
std::fs::write(dir.join(SNAPSHOT_FILE), b"{ not json").unwrap();
|
||||
let state = StateDir::new(&dir);
|
||||
let error = state.load_snapshot().unwrap_err();
|
||||
assert!(error.contains("corrupt"), "unexpected error: {error}");
|
||||
assert!(error.contains("refusing"), "unexpected error: {error}");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identity_key_is_stable_across_loads() {
|
||||
let dir = test_dir("identity");
|
||||
let state = StateDir::new(&dir);
|
||||
let first = state.load_or_create_identity().unwrap();
|
||||
let second = state.load_or_create_identity().unwrap();
|
||||
assert_eq!(first.to_bytes(), second.to_bytes());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn command_ids_are_durable_at_most_once_tokens() {
|
||||
let dir = test_dir("commands");
|
||||
let state = StateDir::new(&dir);
|
||||
let mut snapshot = ClusterSnapshot::fresh(1, "test");
|
||||
assert!(snapshot.accept_command("request-a").unwrap());
|
||||
assert!(!snapshot.accept_command("request-a").unwrap());
|
||||
assert!(snapshot.accept_command("request-b").unwrap());
|
||||
assert!(snapshot.accept_command(" ").is_err());
|
||||
state.save_snapshot(&snapshot).unwrap();
|
||||
let mut loaded = state.load_snapshot().unwrap();
|
||||
assert!(!loaded.accept_command("request-a").unwrap());
|
||||
assert!(!loaded.accept_command("request-b").unwrap());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn orphan_records_exactly_follow_provider_ground_truth() {
|
||||
let mut snapshot = ClusterSnapshot::fresh(1, "test");
|
||||
snapshot.sync_orphans(["b".to_owned(), "a".to_owned()]);
|
||||
snapshot.sync_orphans(["b".to_owned(), "c".to_owned()]);
|
||||
let refs = snapshot
|
||||
.nodes
|
||||
.iter()
|
||||
.filter(|node| node.status == NodeStatus::Orphan)
|
||||
.filter_map(|node| node.provider_ref.clone())
|
||||
.collect::<BTreeSet<_>>();
|
||||
assert_eq!(refs, BTreeSet::from(["b".to_owned(), "c".to_owned()]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_temp_write_never_replaces_last_snapshot() {
|
||||
let dir = test_dir("atomic");
|
||||
let state = StateDir::new(&dir);
|
||||
let mut snapshot = ClusterSnapshot::fresh(7, "stable");
|
||||
snapshot.allocate_node_id();
|
||||
state.save_snapshot(&snapshot).unwrap();
|
||||
std::fs::write(dir.join("cluster.tmp"), b"{partial").unwrap();
|
||||
let loaded = state.load_snapshot().unwrap();
|
||||
assert_eq!(loaded.run_id, 7);
|
||||
assert_eq!(loaded.next_node_id, 2);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identity_changes_only_after_explicit_reset() {
|
||||
let dir = test_dir("identity-reset");
|
||||
let state = StateDir::new(&dir);
|
||||
let first = state.load_or_create_identity().unwrap();
|
||||
assert_eq!(
|
||||
first.to_bytes(),
|
||||
state.load_or_create_identity().unwrap().to_bytes()
|
||||
);
|
||||
state.reset().unwrap();
|
||||
let replacement = state.load_or_create_identity().unwrap();
|
||||
assert_ne!(first.to_bytes(), replacement.to_bytes());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fuzzed_manual_interleavings_preserve_snapshot_invariants() {
|
||||
let dir = test_dir("state-fuzz");
|
||||
let state = StateDir::new(&dir);
|
||||
let mut snapshot = ClusterSnapshot::fresh(9, "fuzz");
|
||||
let mut model_nodes = BTreeMap::<u64, NodeStatus>::new();
|
||||
let mut provider_orphans = BTreeSet::<String>::new();
|
||||
let mut command_ids = BTreeSet::<String>::new();
|
||||
let mut rng = 0x6a09_e667_f3bc_c909_u64;
|
||||
|
||||
for step in 0..2_000_u64 {
|
||||
rng = rng
|
||||
.wrapping_mul(6_364_136_223_846_793_005)
|
||||
.wrapping_add(1_442_695_040_888_963_407);
|
||||
match rng % 7 {
|
||||
0 => {
|
||||
let id = snapshot.allocate_node_id();
|
||||
snapshot.upsert_node(node(id, NodeStatus::Running));
|
||||
model_nodes.insert(id, NodeStatus::Running);
|
||||
}
|
||||
1 => {
|
||||
let id = 1 + rng.rotate_left(17) % snapshot.next_node_id.max(2);
|
||||
if let Some(status) = model_nodes.get_mut(&id) {
|
||||
*status = NodeStatus::Dead;
|
||||
snapshot.node_mut(id).unwrap().status = NodeStatus::Dead;
|
||||
}
|
||||
}
|
||||
2 => {
|
||||
let id = 1 + rng.rotate_right(11) % snapshot.next_node_id.max(2);
|
||||
model_nodes.remove(&id);
|
||||
snapshot.remove_node(id);
|
||||
}
|
||||
3 => {
|
||||
provider_orphans.insert(format!("orphan-{}", rng % 19));
|
||||
}
|
||||
4 => {
|
||||
provider_orphans.remove(&format!("orphan-{}", rng % 19));
|
||||
}
|
||||
5 => {
|
||||
let command_id = format!("command-{}", rng % 31);
|
||||
let expected = command_ids.insert(command_id.clone());
|
||||
assert_eq!(snapshot.accept_command(&command_id).unwrap(), expected);
|
||||
}
|
||||
_ => {
|
||||
state.save_snapshot(&snapshot).unwrap();
|
||||
snapshot = state.load_snapshot().unwrap();
|
||||
}
|
||||
}
|
||||
snapshot.sync_orphans(provider_orphans.iter().cloned());
|
||||
|
||||
let managed_ids = snapshot
|
||||
.nodes
|
||||
.iter()
|
||||
.filter(|node| node.status != NodeStatus::Orphan)
|
||||
.map(|node| node.logical_node_id)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
managed_ids.iter().copied().collect::<BTreeSet<_>>().len(),
|
||||
managed_ids.len(),
|
||||
"duplicate managed id after step {step}"
|
||||
);
|
||||
for (id, expected) in &model_nodes {
|
||||
assert_eq!(
|
||||
snapshot.node(*id).map(|node| &node.status),
|
||||
Some(expected),
|
||||
"node model diverged after step {step}"
|
||||
);
|
||||
}
|
||||
assert!(managed_ids.iter().all(|id| *id < snapshot.next_node_id));
|
||||
let observed_orphans = snapshot
|
||||
.nodes
|
||||
.iter()
|
||||
.filter(|node| node.status == NodeStatus::Orphan)
|
||||
.filter_map(|node| node.provider_ref.clone())
|
||||
.collect::<BTreeSet<_>>();
|
||||
assert_eq!(observed_orphans, provider_orphans);
|
||||
assert_eq!(snapshot.accepted_command_ids, command_ids);
|
||||
}
|
||||
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
assert_eq!(loaded.next_node_id, 3);
|
||||
assert_eq!(loaded.commands["done"].state, CommandState::Succeeded);
|
||||
assert!(!temp.path().join("cluster.tmp").exists());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ use std::time::{Duration, Instant};
|
|||
|
||||
use swactor::actor::{ActorAddress, ActorInterface};
|
||||
use swactor::config::RuntimeConfig;
|
||||
use swactor::runtime::{Ctx, Runtime, RuntimeParts};
|
||||
use swactor::runtime::{Ctx, ExternalSender, Runtime, RuntimeParts};
|
||||
use swactor::stats::StatsHook;
|
||||
use swactor::std::StdExtension;
|
||||
use swactor_engine::EngineHandle;
|
||||
|
|
@ -25,7 +25,7 @@ use distribution::messages::{
|
|||
};
|
||||
use distribution::node::DistributedNodeConfig;
|
||||
use distribution::node_metadata_actor::{MetadataActor, MetadataIn};
|
||||
use distribution::registry_actor::{RegistryActor, RegistryIn};
|
||||
use distribution::registry_actor::{RegistryActor, RegistryIn, RegistryView};
|
||||
use distribution::swim::actor::{MembershipChanged, SwimActor, SwimIn};
|
||||
use distribution::swim::member_list::MemberList;
|
||||
use distribution::swim::probe::SwimConfig;
|
||||
|
|
@ -37,6 +37,49 @@ use distribution::transport_bridge::{
|
|||
};
|
||||
use distribution::types::{DirectoryEntry, MemberState, NodeId};
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ProtocolTick;
|
||||
|
||||
struct ProtocolTicker {
|
||||
runtime: Runtime,
|
||||
engine: EngineHandle,
|
||||
sender: ExternalSender,
|
||||
period: Duration,
|
||||
swim: ActorAddress,
|
||||
registry: ActorAddress,
|
||||
metadata: ActorAddress,
|
||||
directory: ActorAddress,
|
||||
}
|
||||
|
||||
impl ProtocolTicker {
|
||||
fn schedule(&self, ctx: &Ctx) {
|
||||
self.engine.send_after(
|
||||
self.period,
|
||||
self.sender.clone(),
|
||||
ctx.self_addr(),
|
||||
ProtocolTick,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
impl ActorInterface for ProtocolTicker {
|
||||
type Incoming = ProtocolTick;
|
||||
type Response = ();
|
||||
|
||||
fn on_start(&mut self, ctx: &Ctx) {
|
||||
self.schedule(ctx);
|
||||
}
|
||||
|
||||
fn handle(&mut self, ctx: &Ctx, _message: Self::Incoming) {
|
||||
let now = self.engine.now().to_instant();
|
||||
let _ = self.runtime.send_to(self.swim, SwimIn::Tick { now });
|
||||
let _ = self.runtime.send_to(self.registry, RegistryIn::Tick);
|
||||
let _ = self.runtime.send_to(self.metadata, MetadataIn::Tick);
|
||||
let _ = self.runtime.send_to(self.directory, DirectoryIn::Tick);
|
||||
self.schedule(ctx);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct DistributionActorAddrs {
|
||||
pub swim: ActorAddress,
|
||||
|
|
@ -56,6 +99,7 @@ pub(crate) struct DistributionRuntimeStack {
|
|||
pub outbox: Outbox,
|
||||
pub relay_mirror: RelayMirror,
|
||||
pub route_view: RouteView,
|
||||
pub registry_view: RegistryView,
|
||||
pub membership_mirror: Arc<Mutex<MemberList>>,
|
||||
pub swim_telemetry: Arc<SwimTelemetry>,
|
||||
pub swim_config: SwimConfig,
|
||||
|
|
@ -118,6 +162,7 @@ impl DistributionRuntimeStack {
|
|||
));
|
||||
let swim_config = config.swim.clone();
|
||||
let swim_telemetry = SwimTelemetry::new();
|
||||
let registry_view = Arc::new(RwLock::new(Default::default()));
|
||||
|
||||
let swim_addr = runtime
|
||||
.spawn(
|
||||
|
|
@ -131,11 +176,10 @@ impl DistributionRuntimeStack {
|
|||
)
|
||||
.expect("spawn SwimActor");
|
||||
let registry_addr = runtime
|
||||
.spawn(RegistryActor::new(
|
||||
node_id,
|
||||
config.registry.clone(),
|
||||
peer_directory.clone(),
|
||||
))
|
||||
.spawn(
|
||||
RegistryActor::new(node_id, config.registry.clone(), peer_directory.clone())
|
||||
.with_view(Arc::clone(®istry_view)),
|
||||
)
|
||||
.expect("spawn RegistryActor");
|
||||
let metadata_addr = runtime
|
||||
.spawn(MetadataActor::new(
|
||||
|
|
@ -188,6 +232,7 @@ impl DistributionRuntimeStack {
|
|||
relay_mirror,
|
||||
route_view,
|
||||
membership_mirror,
|
||||
registry_view,
|
||||
swim_telemetry,
|
||||
swim_config,
|
||||
actors: DistributionActorAddrs {
|
||||
|
|
@ -221,29 +266,20 @@ impl DistributionRuntimeStack {
|
|||
routes
|
||||
}
|
||||
|
||||
/// Spawn an engine-hosted interval task that injects protocol Tick messages
|
||||
/// (SWIM, registry, metadata, directory), replacing the manual tick
|
||||
/// injection previously done by the application pump loop
|
||||
/// (ENGINE_SPEC.md). The engine owns protocol progression; the
|
||||
/// application loop no longer calls tick or core-driving methods.
|
||||
/// Spawn the actor that owns periodic distribution protocol ticks.
|
||||
pub(crate) fn spawn_protocol_ticker(&self, period: Duration) {
|
||||
let runtime = self.runtime.clone();
|
||||
let swim = self.actors.swim;
|
||||
let registry = self.actors.registry;
|
||||
let metadata = self.actors.metadata;
|
||||
let directory = self.actors.directory;
|
||||
let engine = self.engine.clone();
|
||||
engine.clone().spawn(async move {
|
||||
let mut interval = engine.interval(period);
|
||||
loop {
|
||||
(&mut interval).await;
|
||||
let now = engine.now().to_instant();
|
||||
let _ = runtime.send_to(swim, SwimIn::Tick { now });
|
||||
let _ = runtime.send_to(registry, RegistryIn::Tick);
|
||||
let _ = runtime.send_to(metadata, MetadataIn::Tick);
|
||||
let _ = runtime.send_to(directory, DirectoryIn::Tick);
|
||||
}
|
||||
});
|
||||
self.runtime
|
||||
.spawn(ProtocolTicker {
|
||||
runtime: self.runtime.clone(),
|
||||
engine: self.engine.clone(),
|
||||
sender: self.runtime.create_sender(),
|
||||
period,
|
||||
swim: self.actors.swim,
|
||||
registry: self.actors.registry,
|
||||
metadata: self.actors.metadata,
|
||||
directory: self.actors.directory,
|
||||
})
|
||||
.expect("spawn distribution protocol ticker actor");
|
||||
}
|
||||
|
||||
pub(crate) fn register_local_actor(&self, entry: DirectoryEntry) {
|
||||
|
|
|
|||
|
|
@ -1,251 +0,0 @@
|
|||
use crate::run_plan::RunId;
|
||||
|
||||
use super::error::EngineBuildError;
|
||||
use super::planner::{
|
||||
FixedLinearPipelinePlanner, RoleAssignment, RoleAssignmentPlan, RoleKind, RolePlannerInput,
|
||||
};
|
||||
use super::pool::{ModelSpec, NodeFacts, StaticPoolProvider};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum EngineEvent {
|
||||
PoolAcquired,
|
||||
ClusterConverged,
|
||||
RoleAssigned(RoleKind),
|
||||
EngineReady,
|
||||
}
|
||||
|
||||
fn launch_node(facts: &NodeFacts) -> StaticNodeControl {
|
||||
StaticNodeControl {
|
||||
facts: facts.clone(),
|
||||
booted: false,
|
||||
stopped: false,
|
||||
assigned_roles: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
struct StaticNodeControl {
|
||||
facts: NodeFacts,
|
||||
booted: bool,
|
||||
stopped: bool,
|
||||
assigned_roles: Vec<RoleAssignment>,
|
||||
}
|
||||
|
||||
impl StaticNodeControl {
|
||||
fn node_id(&self) -> u64 {
|
||||
self.facts.node_id.0
|
||||
}
|
||||
}
|
||||
|
||||
impl StaticNodeControl {
|
||||
fn wait_boot_ready(&mut self) -> Result<NodeFacts, EngineBuildError> {
|
||||
if self.stopped {
|
||||
return Err(EngineBuildError::Stopped {
|
||||
node_id: self.node_id(),
|
||||
});
|
||||
}
|
||||
self.booted = true;
|
||||
Ok(self.facts.clone())
|
||||
}
|
||||
|
||||
fn wait_cluster_converged(&mut self, expected_alive: usize) -> Result<(), EngineBuildError> {
|
||||
if self.stopped {
|
||||
return Err(EngineBuildError::Stopped {
|
||||
node_id: self.node_id(),
|
||||
});
|
||||
}
|
||||
if !self.booted {
|
||||
return Err(EngineBuildError::NotBooted {
|
||||
node_id: self.node_id(),
|
||||
});
|
||||
}
|
||||
if expected_alive == 0 {
|
||||
return Err(EngineBuildError::Backend(
|
||||
"expected_alive must be greater than zero",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn assign_role(&mut self, assignment: RoleAssignment) -> Result<(), EngineBuildError> {
|
||||
if self.stopped {
|
||||
return Err(EngineBuildError::Stopped {
|
||||
node_id: self.node_id(),
|
||||
});
|
||||
}
|
||||
if !self.booted {
|
||||
return Err(EngineBuildError::NotBooted {
|
||||
node_id: self.node_id(),
|
||||
});
|
||||
}
|
||||
let role_node_id = assignment.node_id().0;
|
||||
if role_node_id != self.node_id() {
|
||||
return Err(EngineBuildError::RoleNodeMismatch {
|
||||
node_id: self.node_id(),
|
||||
role_node_id,
|
||||
});
|
||||
}
|
||||
self.assigned_roles.push(assignment);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn shutdown(&mut self) -> Result<(), EngineBuildError> {
|
||||
if self.stopped {
|
||||
return Ok(());
|
||||
}
|
||||
self.stopped = true;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct ClusterBuilder {
|
||||
cluster_id: String,
|
||||
run_id: RunId,
|
||||
model: ModelSpec,
|
||||
pool_provider: Option<StaticPoolProvider>,
|
||||
planner: Option<FixedLinearPipelinePlanner>,
|
||||
}
|
||||
|
||||
impl ClusterBuilder {
|
||||
pub(crate) fn new(cluster_id: impl Into<String>, model: ModelSpec) -> Self {
|
||||
Self {
|
||||
cluster_id: cluster_id.into(),
|
||||
run_id: RunId(1),
|
||||
model,
|
||||
pool_provider: None,
|
||||
planner: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn run_id(mut self, run_id: impl Into<RunId>) -> Self {
|
||||
self.run_id = run_id.into();
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn pool_provider(mut self, provider: StaticPoolProvider) -> Self {
|
||||
self.pool_provider = Some(provider);
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn planner(mut self, planner: FixedLinearPipelinePlanner) -> Self {
|
||||
self.planner = Some(planner);
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn launch(mut self) -> Result<ClusterHandle, EngineBuildError> {
|
||||
let pool_provider = self
|
||||
.pool_provider
|
||||
.take()
|
||||
.ok_or(EngineBuildError::MissingComponent("pool_provider"))?;
|
||||
let planner = self
|
||||
.planner
|
||||
.take()
|
||||
.ok_or(EngineBuildError::MissingComponent("planner"))?;
|
||||
|
||||
let mut events = Vec::new();
|
||||
let leases = pool_provider.acquire_pool(planner.required_node_count())?;
|
||||
if leases.is_empty() {
|
||||
return Err(EngineBuildError::EmptyPool);
|
||||
}
|
||||
events.push(EngineEvent::PoolAcquired);
|
||||
|
||||
let mut nodes = Vec::with_capacity(leases.len());
|
||||
let mut iter = leases.into_iter();
|
||||
let coordinator_lease = iter.next().ok_or(EngineBuildError::EmptyPool)?;
|
||||
let mut coordinator = launch_node(&coordinator_lease);
|
||||
let coordinator_facts = coordinator.wait_boot_ready()?;
|
||||
nodes.push(EngineNode::new(coordinator, coordinator_facts));
|
||||
|
||||
for lease in iter {
|
||||
let mut node = launch_node(&lease);
|
||||
let facts = node.wait_boot_ready()?;
|
||||
nodes.push(EngineNode::new(node, facts));
|
||||
}
|
||||
|
||||
let expected_alive = nodes.len();
|
||||
for node in &mut nodes {
|
||||
node.control.wait_cluster_converged(expected_alive)?;
|
||||
}
|
||||
events.push(EngineEvent::ClusterConverged);
|
||||
|
||||
let plan = planner.plan(RolePlannerInput {
|
||||
run_id: self.run_id,
|
||||
model: self.model,
|
||||
nodes: nodes.iter().map(|node| node.facts.clone()).collect(),
|
||||
})?;
|
||||
|
||||
assign_role(
|
||||
&mut nodes,
|
||||
RoleAssignment::Coordinator(plan.coordinator.clone()),
|
||||
&mut events,
|
||||
)?;
|
||||
for stage in &plan.stages {
|
||||
assign_role(
|
||||
&mut nodes,
|
||||
RoleAssignment::StageWorker(stage.clone()),
|
||||
&mut events,
|
||||
)?;
|
||||
}
|
||||
events.push(EngineEvent::EngineReady);
|
||||
|
||||
Ok(ClusterHandle {
|
||||
nodes,
|
||||
plan,
|
||||
events,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct ClusterHandle {
|
||||
nodes: Vec<EngineNode>,
|
||||
plan: RoleAssignmentPlan,
|
||||
events: Vec<EngineEvent>,
|
||||
}
|
||||
|
||||
impl ClusterHandle {
|
||||
pub(crate) fn role_plan(&self) -> &RoleAssignmentPlan {
|
||||
&self.plan
|
||||
}
|
||||
|
||||
pub(crate) fn events(&self) -> &[EngineEvent] {
|
||||
&self.events
|
||||
}
|
||||
|
||||
pub(crate) fn shutdown(mut self) -> Result<(), EngineBuildError> {
|
||||
for node in &mut self.nodes {
|
||||
node.control.shutdown()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
struct EngineNode {
|
||||
facts: NodeFacts,
|
||||
roles: Vec<RoleAssignment>,
|
||||
control: StaticNodeControl,
|
||||
}
|
||||
|
||||
impl EngineNode {
|
||||
fn new(control: StaticNodeControl, facts: NodeFacts) -> Self {
|
||||
Self {
|
||||
facts,
|
||||
roles: Vec::new(),
|
||||
control,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn assign_role(
|
||||
nodes: &mut [EngineNode],
|
||||
assignment: RoleAssignment,
|
||||
events: &mut Vec<EngineEvent>,
|
||||
) -> Result<(), EngineBuildError> {
|
||||
let node_id = assignment.node_id();
|
||||
let node = nodes
|
||||
.iter_mut()
|
||||
.find(|node| node.facts.node_id == node_id)
|
||||
.ok_or(EngineBuildError::RoleTargetMissing { node_id: node_id.0 })?;
|
||||
node.control.assign_role(assignment.clone())?;
|
||||
let role = assignment.kind();
|
||||
node.roles.push(assignment);
|
||||
events.push(EngineEvent::RoleAssigned(role));
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
use crate::run_plan;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum EngineBuildError {
|
||||
MissingComponent(&'static str),
|
||||
EmptyPool,
|
||||
RoleTargetMissing { node_id: u64 },
|
||||
InsufficientNodes { requested: usize, available: usize },
|
||||
NotBooted { node_id: u64 },
|
||||
Stopped { node_id: u64 },
|
||||
RoleNodeMismatch { node_id: u64, role_node_id: u64 },
|
||||
Backend(&'static str),
|
||||
DuplicateNodeId { node_id: u64 },
|
||||
NoCoordinatorCandidate,
|
||||
InsufficientWorkers { required: usize, available: usize },
|
||||
ModelRejected(run_plan::PlanRejectionKind),
|
||||
StageProjection(run_plan::ProjectionRejection),
|
||||
}
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
//! Pool-based engine/node builder primitives.
|
||||
//!
|
||||
//! This module owns topology construction: acquire a role-neutral node pool,
|
||||
//! launch the same node image everywhere, wait for node/cluster readiness, map a
|
||||
//! model onto discovered nodes, assign roles, and return a live cluster handle.
|
||||
//! Workload semantics stay outside this module; see [`WorkloadAdapter`].
|
||||
|
||||
pub(crate) mod engine;
|
||||
pub(crate) mod error;
|
||||
pub(crate) mod planner;
|
||||
pub(crate) mod pool;
|
||||
|
||||
pub(crate) use crate::run_plan::{DTypeFamily, NodeId};
|
||||
pub(crate) use engine::{ClusterBuilder, EngineEvent};
|
||||
pub(crate) use planner::{FixedLinearPipelinePlanner, RoleKind};
|
||||
pub(crate) use pool::{ModelSpec, NodeCapability, NodeFacts, StaticPoolProvider};
|
||||
|
|
@ -1,194 +0,0 @@
|
|||
use std::collections::BTreeSet;
|
||||
|
||||
use crate::run_plan::{self, NodeId, RunId};
|
||||
|
||||
use super::error::EngineBuildError;
|
||||
use super::pool::{ModelSpec, NodeCapability, NodeFacts};
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct CoordinatorAssignment {
|
||||
pub node_id: NodeId,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct StageAssignment {
|
||||
pub provision: run_plan::ProvisionStage,
|
||||
}
|
||||
|
||||
impl StageAssignment {
|
||||
pub(crate) fn node_id(&self) -> NodeId {
|
||||
self.provision.node_id
|
||||
}
|
||||
|
||||
pub(crate) fn stage_index(&self) -> u32 {
|
||||
self.provision.stage_index
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum RoleAssignment {
|
||||
Coordinator(CoordinatorAssignment),
|
||||
StageWorker(StageAssignment),
|
||||
}
|
||||
|
||||
impl RoleAssignment {
|
||||
pub(crate) fn node_id(&self) -> NodeId {
|
||||
match self {
|
||||
Self::Coordinator(assignment) => assignment.node_id,
|
||||
Self::StageWorker(assignment) => assignment.node_id(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn kind(&self) -> RoleKind {
|
||||
match self {
|
||||
Self::Coordinator(_) => RoleKind::Coordinator,
|
||||
Self::StageWorker(assignment) => RoleKind::StageWorker {
|
||||
stage_index: assignment.stage_index(),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum RoleKind {
|
||||
Coordinator,
|
||||
StageWorker { stage_index: u32 },
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct RolePlannerInput {
|
||||
pub run_id: RunId,
|
||||
pub model: ModelSpec,
|
||||
pub nodes: Vec<NodeFacts>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct RoleAssignmentPlan {
|
||||
pub coordinator: CoordinatorAssignment,
|
||||
pub stages: Vec<StageAssignment>,
|
||||
pub run_plan: run_plan::RunPlan,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct FixedLinearPipelinePlanner {
|
||||
pub stage_count: u32,
|
||||
pub runtime: run_plan::RuntimeConfig,
|
||||
pub activation_ring: run_plan::RingSpec,
|
||||
pub token_ring: run_plan::RingSpec,
|
||||
}
|
||||
|
||||
impl FixedLinearPipelinePlanner {
|
||||
pub(crate) fn new(stage_count: u32) -> Self {
|
||||
Self {
|
||||
stage_count,
|
||||
runtime: run_plan::RuntimeConfig {
|
||||
max_tokens: 4,
|
||||
sampling: run_plan::SamplingPolicy {
|
||||
temperature_millis: 0,
|
||||
top_k: 1,
|
||||
},
|
||||
},
|
||||
activation_ring: run_plan::RingSpec {
|
||||
data_capacity: 1 << 20,
|
||||
alignment: 64,
|
||||
direction: run_plan::RingDirection::Egress,
|
||||
host_pinning: run_plan::HostPinning::Pageable,
|
||||
wake_coalescing: run_plan::WakeCoalescing::PendingBit,
|
||||
},
|
||||
token_ring: run_plan::RingSpec {
|
||||
data_capacity: 4096,
|
||||
alignment: 8,
|
||||
direction: run_plan::RingDirection::Egress,
|
||||
host_pinning: run_plan::HostPinning::Pageable,
|
||||
wake_coalescing: run_plan::WakeCoalescing::PendingBit,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn runtime(mut self, runtime: run_plan::RuntimeConfig) -> Self {
|
||||
self.runtime = runtime;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl FixedLinearPipelinePlanner {
|
||||
pub(crate) fn required_node_count(&self) -> usize {
|
||||
self.stage_count as usize + 1
|
||||
}
|
||||
|
||||
pub(crate) fn plan(
|
||||
&self,
|
||||
input: RolePlannerInput,
|
||||
) -> Result<RoleAssignmentPlan, EngineBuildError> {
|
||||
reject_duplicate_nodes(&input.nodes)?;
|
||||
let coordinator = input
|
||||
.nodes
|
||||
.iter()
|
||||
.find(|node| node.capabilities.contains(&NodeCapability::Coordinator))
|
||||
.ok_or(EngineBuildError::NoCoordinatorCandidate)?;
|
||||
let workers = input
|
||||
.nodes
|
||||
.iter()
|
||||
.filter(|node| {
|
||||
node.node_id != coordinator.node_id
|
||||
&& node.capabilities.contains(&NodeCapability::Worker)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let required = self.stage_count as usize;
|
||||
if workers.len() < required {
|
||||
return Err(EngineBuildError::InsufficientWorkers {
|
||||
required,
|
||||
available: workers.len(),
|
||||
});
|
||||
}
|
||||
|
||||
let placements = workers
|
||||
.iter()
|
||||
.take(required)
|
||||
.enumerate()
|
||||
.map(|(stage_index, node)| run_plan::StagePlacement {
|
||||
stage_index: stage_index as u32,
|
||||
node_id: node.node_id,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let candidate_pool = workers.iter().map(|node| node.node_id).collect::<Vec<_>>();
|
||||
let run_plan = run_plan::plan_run(run_plan::PlannerInput {
|
||||
run_id: input.run_id,
|
||||
orchestrator_node_id: coordinator.node_id,
|
||||
model: input.model.to_run_plan_facts(),
|
||||
runtime: self.runtime.clone(),
|
||||
candidate_pool,
|
||||
stage_count: self.stage_count,
|
||||
placement: run_plan::PlacementInput::FixedLinear(placements),
|
||||
activation_ring: self.activation_ring,
|
||||
token_ring: self.token_ring,
|
||||
})
|
||||
.map_err(|err| EngineBuildError::ModelRejected(err.kind()))?;
|
||||
|
||||
let mut stages = Vec::with_capacity(self.stage_count as usize);
|
||||
for stage_index in 0..self.stage_count {
|
||||
let provision = run_plan::derive_stage_provision(&run_plan, stage_index)
|
||||
.map_err(EngineBuildError::StageProjection)?;
|
||||
stages.push(StageAssignment { provision });
|
||||
}
|
||||
|
||||
Ok(RoleAssignmentPlan {
|
||||
coordinator: CoordinatorAssignment {
|
||||
node_id: coordinator.node_id,
|
||||
},
|
||||
stages,
|
||||
run_plan,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn reject_duplicate_nodes(nodes: &[NodeFacts]) -> Result<(), EngineBuildError> {
|
||||
let mut seen = BTreeSet::<NodeId>::new();
|
||||
for node in nodes {
|
||||
if !seen.insert(node.node_id) {
|
||||
return Err(EngineBuildError::DuplicateNodeId {
|
||||
node_id: node.node_id.0,
|
||||
});
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -1,91 +0,0 @@
|
|||
use crate::run_plan::{self, NodeId};
|
||||
|
||||
use super::error::EngineBuildError;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct NodeFacts {
|
||||
pub node_id: NodeId,
|
||||
pub capabilities: Vec<NodeCapability>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum NodeCapability {
|
||||
Coordinator,
|
||||
Worker,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct StaticPoolProvider {
|
||||
nodes: Vec<NodeFacts>,
|
||||
}
|
||||
|
||||
impl StaticPoolProvider {
|
||||
pub(crate) fn new(nodes: Vec<NodeFacts>) -> Self {
|
||||
Self { nodes }
|
||||
}
|
||||
pub(crate) fn acquire_pool(
|
||||
&self,
|
||||
min_nodes: usize,
|
||||
) -> Result<Vec<NodeFacts>, EngineBuildError> {
|
||||
if self.nodes.len() < min_nodes {
|
||||
return Err(EngineBuildError::InsufficientNodes {
|
||||
requested: min_nodes,
|
||||
available: self.nodes.len(),
|
||||
});
|
||||
}
|
||||
Ok(self.nodes.clone())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct ModelSpec {
|
||||
pub model_id: String,
|
||||
pub gguf_source: run_plan::GgufSource,
|
||||
pub tokenizer: run_plan::TokenizerSource,
|
||||
pub num_layers: u32,
|
||||
pub hidden_dim: u64,
|
||||
pub dtype_family: run_plan::DTypeFamily,
|
||||
pub dtype_width_bytes: u64,
|
||||
pub max_seq_len: u64,
|
||||
pub eos_token_id: u32,
|
||||
}
|
||||
|
||||
impl ModelSpec {
|
||||
pub(crate) fn pipelined_causal_llm(
|
||||
model_id: impl Into<String>,
|
||||
gguf_source: run_plan::GgufSource,
|
||||
num_layers: u32,
|
||||
hidden_dim: u64,
|
||||
dtype_family: run_plan::DTypeFamily,
|
||||
dtype_width_bytes: u64,
|
||||
max_seq_len: u64,
|
||||
eos_token_id: u32,
|
||||
tokenizer: run_plan::TokenizerSource,
|
||||
) -> Self {
|
||||
Self {
|
||||
model_id: model_id.into(),
|
||||
gguf_source,
|
||||
tokenizer,
|
||||
num_layers,
|
||||
hidden_dim,
|
||||
dtype_family,
|
||||
dtype_width_bytes,
|
||||
max_seq_len,
|
||||
eos_token_id,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn to_run_plan_facts(&self) -> run_plan::ModelFacts {
|
||||
run_plan::ModelFacts {
|
||||
model_id: self.model_id.clone(),
|
||||
gguf_source: self.gguf_source.clone(),
|
||||
num_layers: self.num_layers,
|
||||
hidden_dim: self.hidden_dim,
|
||||
dtype_family: self.dtype_family,
|
||||
dtype_width_bytes: self.dtype_width_bytes,
|
||||
max_seq_len: self.max_seq_len,
|
||||
eos_token_id: self.eos_token_id,
|
||||
tokenizer: self.tokenizer.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
78
apps/myelin/src/orchestration/fleet_control.js
Normal file
78
apps/myelin/src/orchestration/fleet_control.js
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
(() => {
|
||||
const CONTROL_ID = 'myelin-fleet-control';
|
||||
|
||||
async function syncControl() {
|
||||
const nodeView = document.querySelector('.node-view[data-node]');
|
||||
if (!nodeView) return;
|
||||
const rawNodeId = nodeView.getAttribute('data-node') || '';
|
||||
if (!/^\d+$/.test(rawNodeId)) return;
|
||||
const logicalNodeId = Number(rawNodeId);
|
||||
|
||||
let model;
|
||||
try {
|
||||
const response = await fetch('/api/control/status', { cache: 'no-store' });
|
||||
if (!response.ok) return;
|
||||
const payload = await response.json();
|
||||
model = payload.Status;
|
||||
} catch (_) {
|
||||
return;
|
||||
}
|
||||
const node = model?.nodes?.find(candidate => candidate.logical_node_id === logicalNodeId);
|
||||
if (!node) return;
|
||||
|
||||
const heading = nodeView.querySelector('section.panel > h2');
|
||||
if (!heading) return;
|
||||
let control = document.getElementById(CONTROL_ID);
|
||||
if (!control) {
|
||||
control = document.createElement('div');
|
||||
control.id = CONTROL_ID;
|
||||
control.style.cssText = 'display:flex;align-items:center;gap:10px;margin:0 0 12px';
|
||||
const button = document.createElement('button');
|
||||
button.type = 'button';
|
||||
button.dataset.action = 'kill';
|
||||
button.style.cssText = 'border:1px solid #ef4444;background:transparent;color:#ef4444;border-radius:3px;padding:6px 10px;font:600 12px ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;cursor:pointer';
|
||||
const message = document.createElement('span');
|
||||
message.dataset.message = '';
|
||||
message.style.cssText = 'font:12px ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;color:var(--muted)';
|
||||
control.append(button, message);
|
||||
heading.after(control);
|
||||
}
|
||||
|
||||
const button = control.querySelector('[data-action="kill"]');
|
||||
const message = control.querySelector('[data-message]');
|
||||
const terminal = node.phase === 'stopped' || node.phase === 'orphan';
|
||||
const pending = node.phase === 'kill_requested' || node.phase === 'stopping';
|
||||
button.hidden = terminal;
|
||||
button.disabled = pending;
|
||||
button.textContent = pending ? 'Kill requested' : 'Kill';
|
||||
button.style.opacity = pending ? '.55' : '1';
|
||||
message.textContent = terminal ? `managed node ${logicalNodeId}: ${node.phase}` : '';
|
||||
|
||||
button.onclick = async () => {
|
||||
if (!window.confirm(`Kill managed node ${logicalNodeId}? Vast.ai contracts are destroyed and billing stops.`)) return;
|
||||
button.disabled = true;
|
||||
message.textContent = 'submitting…';
|
||||
const commandId = `fleet-kill-${globalThis.crypto?.randomUUID?.() || Date.now()}`;
|
||||
try {
|
||||
const response = await fetch(`/api/control/nodes/${logicalNodeId}/kill`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ command_id: commandId }),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const body = await response.json().catch(() => ({}));
|
||||
throw new Error(body.error || `HTTP ${response.status}`);
|
||||
}
|
||||
message.textContent = 'Kill accepted';
|
||||
} catch (error) {
|
||||
button.disabled = false;
|
||||
message.textContent = error.message;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
const page = document.getElementById('page');
|
||||
if (page) new MutationObserver(syncControl).observe(page, { childList: true, subtree: true });
|
||||
syncControl();
|
||||
setInterval(syncControl, 1000);
|
||||
})();
|
||||
|
|
@ -14,10 +14,13 @@ use provisioning::{
|
|||
BootSpec, ClusterShape, DesiredNodeShape, LogicalNodeId, NodeAttemptId, NodeGroupId,
|
||||
ProviderKind, RetryPolicy, RoleId, RunId, RunNodeGroupSpec, SwactorId, SwarmJoinTemplate,
|
||||
};
|
||||
use swactor::actor::ActorInterface;
|
||||
use swactor::runtime::{Ctx, ExternalSender};
|
||||
use swactor_engine::{ActorCompletion, EngineHandle};
|
||||
use swactor_job_runner::{Job, JobDone};
|
||||
use swactor_vastai::{LifecyclePolicy, SelectionPolicy};
|
||||
use swactor_vastai::SelectionPolicy;
|
||||
|
||||
use crate::job_deploy::{self, NodeIdentity};
|
||||
use crate::job_deploy::{self, JobRunStateMachine, NodeIdentity};
|
||||
use crate::orchestration::app::{
|
||||
derive_ssh_public_key, ensure_vastai_account_ssh_key, resolve_vastai_ssh_identity,
|
||||
ssh_public_key_fingerprint,
|
||||
|
|
@ -124,7 +127,7 @@ pub(crate) fn run_vastai_job(
|
|||
validate_non_empty("worker workdir", &options.worker_workdir)?;
|
||||
validate_non_empty("endpoint address mask", &options.endpoint_addr_mask)?;
|
||||
|
||||
let mut session = job_deploy::start_orchestrator(landing)?;
|
||||
let session = job_deploy::start_orchestrator(landing)?;
|
||||
let orch_json = session.identity_json()?;
|
||||
println!("JOB_ORCH_IDENTITY {orch_json}");
|
||||
let _ = std::io::Write::flush(&mut std::io::stdout());
|
||||
|
|
@ -134,102 +137,235 @@ pub(crate) fn run_vastai_job(
|
|||
);
|
||||
|
||||
let (sink, observations) = observation_channel();
|
||||
let provisioner = build_vastai_provisioner(&api_key, &options, session.runtime())?;
|
||||
let runtime = session.runtime();
|
||||
let engine = session.engine_handle();
|
||||
let provisioner =
|
||||
build_vastai_provisioner(&api_key, &options, runtime.clone(), engine.clone())?;
|
||||
let spec = job_node_spec(&options, image, &orch_json)?;
|
||||
let mut cluster = build_cluster(&options, spec, provisioner, session.engine_handle(), sink)?;
|
||||
let cluster = build_cluster(
|
||||
&options,
|
||||
spec,
|
||||
provisioner,
|
||||
engine.clone(),
|
||||
runtime.clone(),
|
||||
sink,
|
||||
)?;
|
||||
let completion = ActorCompletion::new();
|
||||
runtime
|
||||
.spawn(ReconciledJobActor {
|
||||
cluster,
|
||||
observations,
|
||||
session: Some(session),
|
||||
job: Some(job),
|
||||
phase: ReconciledJobPhase::Provisioning {
|
||||
deadline: Instant::now() + options.provision_timeout,
|
||||
},
|
||||
node_id: options.node_id,
|
||||
engine,
|
||||
sender: runtime.create_sender(),
|
||||
completion: completion.clone(),
|
||||
})
|
||||
.map_err(|error| format!("spawn reconciled job actor: {error}"))?;
|
||||
completion.wait()
|
||||
}
|
||||
|
||||
let result = run_with_cluster(job, &mut session, &mut cluster, &observations, &options);
|
||||
let stop_result = cluster.stop();
|
||||
match (result, stop_result) {
|
||||
(Ok(done), Ok(())) => Ok(done),
|
||||
(Ok(_), Err(cleanup)) => Err(format!(
|
||||
#[derive(Clone)]
|
||||
struct ReconciledJobTick;
|
||||
|
||||
enum ReconciledJobPhase {
|
||||
Provisioning {
|
||||
deadline: Instant,
|
||||
},
|
||||
Converging {
|
||||
deadline: Instant,
|
||||
worker: NodeIdentity,
|
||||
},
|
||||
Running(JobRunStateMachine),
|
||||
Stopping {
|
||||
result: Result<JobDone, String>,
|
||||
},
|
||||
Finished,
|
||||
}
|
||||
|
||||
struct ReconciledJobActor {
|
||||
cluster: ProvisionedClusterGuard,
|
||||
observations: mpsc::Receiver<PluginObservation>,
|
||||
session: Option<job_deploy::JobOrchestratorSession>,
|
||||
job: Option<Job>,
|
||||
phase: ReconciledJobPhase,
|
||||
node_id: u64,
|
||||
engine: EngineHandle,
|
||||
sender: ExternalSender,
|
||||
completion: ActorCompletion<Result<JobDone, String>>,
|
||||
}
|
||||
|
||||
impl ReconciledJobActor {
|
||||
fn schedule(&self, ctx: &Ctx) {
|
||||
self.engine.send_after(
|
||||
POLL,
|
||||
self.sender.clone(),
|
||||
ctx.self_addr(),
|
||||
ReconciledJobTick,
|
||||
);
|
||||
}
|
||||
|
||||
fn begin_stop(&mut self, result: Result<JobDone, String>) {
|
||||
let result = match self.cluster.begin_shutdown() {
|
||||
Ok(()) => result,
|
||||
Err(cleanup) => merge_cleanup(result, cleanup),
|
||||
};
|
||||
self.phase = ReconciledJobPhase::Stopping { result };
|
||||
}
|
||||
|
||||
fn complete(
|
||||
&mut self,
|
||||
ctx: &Ctx,
|
||||
result: Result<JobDone, String>,
|
||||
cleanup: Result<(), String>,
|
||||
) {
|
||||
let result = match cleanup {
|
||||
Ok(()) => result,
|
||||
Err(cleanup) => merge_cleanup(result, cleanup),
|
||||
};
|
||||
assert!(
|
||||
self.completion.complete(result).is_ok(),
|
||||
"reconciled job completed twice"
|
||||
);
|
||||
self.phase = ReconciledJobPhase::Finished;
|
||||
ctx.stop_self();
|
||||
}
|
||||
}
|
||||
|
||||
impl ActorInterface for ReconciledJobActor {
|
||||
type Incoming = ReconciledJobTick;
|
||||
type Response = ();
|
||||
|
||||
fn on_start(&mut self, ctx: &Ctx) {
|
||||
let _ = ctx.send(ctx.self_addr(), ReconciledJobTick);
|
||||
}
|
||||
|
||||
fn handle(&mut self, ctx: &Ctx, _message: Self::Incoming) {
|
||||
if let Err(error) = self.cluster.poll(SystemTime::now()) {
|
||||
let cleanup = self.cluster.finish_shutdown();
|
||||
self.complete(ctx, Err(format!("job reconciler poll: {error}")), cleanup);
|
||||
return;
|
||||
}
|
||||
|
||||
let phase = std::mem::replace(&mut self.phase, ReconciledJobPhase::Finished);
|
||||
match phase {
|
||||
ReconciledJobPhase::Provisioning { deadline } => {
|
||||
let mut worker = None;
|
||||
if let Err(error) = drain_observations(&self.observations, &mut worker) {
|
||||
self.begin_stop(Err(error));
|
||||
} else if let Some(worker) = worker {
|
||||
eprintln!("job-reconcile: worker identity observed through reconciler stdout");
|
||||
let result = job_deploy::parse_actor(&worker.actor_hex).and_then(|actor| {
|
||||
let attempt = self.cluster.current_attempt(self.node_id).ok_or_else(|| {
|
||||
format!(
|
||||
"reconciler has no active attempt for node {}",
|
||||
self.node_id
|
||||
)
|
||||
})?;
|
||||
self.cluster
|
||||
.observe_runtime_ready(
|
||||
self.node_id,
|
||||
NodeAttemptId(attempt.0),
|
||||
SwactorId(format!("{actor:?}")),
|
||||
SystemTime::now(),
|
||||
)
|
||||
.then_some(())
|
||||
.ok_or_else(|| {
|
||||
format!(
|
||||
"reconciler rejected runtime-ready observation for node {} attempt {}",
|
||||
self.node_id, attempt.0
|
||||
)
|
||||
})
|
||||
});
|
||||
match result {
|
||||
Ok(()) => {
|
||||
self.phase = ReconciledJobPhase::Converging {
|
||||
deadline: Instant::now() + RUNTIME_CONVERGENCE_TIMEOUT,
|
||||
worker,
|
||||
};
|
||||
}
|
||||
Err(error) => self.begin_stop(Err(error)),
|
||||
}
|
||||
} else if Instant::now() >= deadline {
|
||||
self.begin_stop(Err(format!(
|
||||
"timed out waiting for reconciled job worker identity"
|
||||
)));
|
||||
} else {
|
||||
self.phase = ReconciledJobPhase::Provisioning { deadline };
|
||||
}
|
||||
}
|
||||
ReconciledJobPhase::Converging { deadline, worker } => {
|
||||
let mut ignored = None;
|
||||
if let Err(error) = drain_observations(&self.observations, &mut ignored) {
|
||||
self.begin_stop(Err(error));
|
||||
} else if self.cluster.is_converged() {
|
||||
eprintln!("job-reconcile: reconciler accepted runtime-ready worker");
|
||||
let machine = self
|
||||
.session
|
||||
.take()
|
||||
.zip(self.job.take())
|
||||
.ok_or_else(|| "reconciled job lost session state".to_owned())
|
||||
.and_then(|(session, job)| JobRunStateMachine::new(session, job, worker));
|
||||
match machine {
|
||||
Ok(mut machine) => {
|
||||
machine.start(Instant::now());
|
||||
self.phase = ReconciledJobPhase::Running(machine);
|
||||
}
|
||||
Err(error) => self.begin_stop(Err(error)),
|
||||
}
|
||||
} else if Instant::now() >= deadline {
|
||||
self.begin_stop(Err(format!(
|
||||
"timed out after {RUNTIME_CONVERGENCE_TIMEOUT:?} waiting for reconciler convergence"
|
||||
)));
|
||||
} else {
|
||||
self.phase = ReconciledJobPhase::Converging { deadline, worker };
|
||||
}
|
||||
}
|
||||
ReconciledJobPhase::Running(mut machine) => {
|
||||
let mut ignored = None;
|
||||
if let Err(error) = drain_observations(&self.observations, &mut ignored) {
|
||||
self.begin_stop(Err(error));
|
||||
} else if let Some(result) = machine.advance(Instant::now()) {
|
||||
self.begin_stop(result);
|
||||
} else {
|
||||
self.phase = ReconciledJobPhase::Running(machine);
|
||||
}
|
||||
}
|
||||
ReconciledJobPhase::Stopping { result } => {
|
||||
if self.cluster.is_stopped() {
|
||||
let cleanup = self.cluster.finish_shutdown();
|
||||
self.complete(ctx, result, cleanup);
|
||||
return;
|
||||
}
|
||||
self.phase = ReconciledJobPhase::Stopping { result };
|
||||
}
|
||||
ReconciledJobPhase::Finished => {
|
||||
ctx.stop_self();
|
||||
return;
|
||||
}
|
||||
}
|
||||
self.schedule(ctx);
|
||||
}
|
||||
}
|
||||
|
||||
fn merge_cleanup(result: Result<JobDone, String>, cleanup: String) -> Result<JobDone, String> {
|
||||
match result {
|
||||
Ok(_) => Err(format!(
|
||||
"job completed but reconciler cleanup failed: {cleanup}"
|
||||
)),
|
||||
(Err(error), Ok(())) => Err(error),
|
||||
(Err(error), Err(cleanup)) => Err(format!("{error}; reconciler cleanup failed: {cleanup}")),
|
||||
Err(error) => Err(format!("{error}; reconciler cleanup failed: {cleanup}")),
|
||||
}
|
||||
}
|
||||
|
||||
fn run_with_cluster(
|
||||
job: Job,
|
||||
session: &mut job_deploy::JobOrchestratorSession,
|
||||
cluster: &mut ProvisionedClusterGuard,
|
||||
observations: &mpsc::Receiver<PluginObservation>,
|
||||
options: &VastAiJobOptions,
|
||||
) -> Result<JobDone, String> {
|
||||
let worker = wait_for_worker_identity(cluster, observations, options)?;
|
||||
let actor = job_deploy::parse_actor(&worker.actor_hex)?;
|
||||
let attempt = cluster.current_attempt(options.node_id).ok_or_else(|| {
|
||||
format!(
|
||||
"reconciler has no active attempt for node {}",
|
||||
options.node_id
|
||||
)
|
||||
})?;
|
||||
if !cluster.observe_runtime_ready(
|
||||
options.node_id,
|
||||
NodeAttemptId(attempt.0),
|
||||
SwactorId(format!("{actor:?}")),
|
||||
SystemTime::now(),
|
||||
) {
|
||||
return Err(format!(
|
||||
"reconciler rejected runtime-ready observation for node {} attempt {}",
|
||||
options.node_id, attempt.0
|
||||
));
|
||||
}
|
||||
wait_for_cluster_convergence(cluster, observations)?;
|
||||
session.run_to_completion(job, worker)
|
||||
}
|
||||
|
||||
fn wait_for_worker_identity(
|
||||
cluster: &mut ProvisionedClusterGuard,
|
||||
observations: &mpsc::Receiver<PluginObservation>,
|
||||
options: &VastAiJobOptions,
|
||||
) -> Result<NodeIdentity, String> {
|
||||
let started = Instant::now();
|
||||
let mut worker = None;
|
||||
while started.elapsed() < options.provision_timeout {
|
||||
cluster
|
||||
.poll(SystemTime::now())
|
||||
.map_err(|error| format!("job reconciler poll: {error}"))?;
|
||||
drain_observations(observations, &mut worker)?;
|
||||
if let Some(worker) = worker.take() {
|
||||
eprintln!("job-reconcile: worker identity observed through reconciler stdout");
|
||||
return Ok(worker);
|
||||
}
|
||||
std::thread::sleep(POLL);
|
||||
}
|
||||
Err(format!(
|
||||
"timed out after {:?} waiting for reconciled job worker identity",
|
||||
options.provision_timeout
|
||||
))
|
||||
}
|
||||
|
||||
fn wait_for_cluster_convergence(
|
||||
cluster: &mut ProvisionedClusterGuard,
|
||||
observations: &mpsc::Receiver<PluginObservation>,
|
||||
) -> Result<(), String> {
|
||||
let started = Instant::now();
|
||||
let mut ignored = None;
|
||||
while started.elapsed() < RUNTIME_CONVERGENCE_TIMEOUT {
|
||||
cluster
|
||||
.poll(SystemTime::now())
|
||||
.map_err(|error| format!("job reconciler convergence poll: {error}"))?;
|
||||
drain_observations(observations, &mut ignored)?;
|
||||
if cluster.is_converged() {
|
||||
eprintln!("job-reconcile: reconciler accepted runtime-ready worker");
|
||||
return Ok(());
|
||||
}
|
||||
std::thread::sleep(POLL);
|
||||
}
|
||||
Err(format!(
|
||||
"timed out after {RUNTIME_CONVERGENCE_TIMEOUT:?} waiting for reconciler convergence"
|
||||
))
|
||||
}
|
||||
|
||||
fn build_vastai_provisioner(
|
||||
api_key: &str,
|
||||
options: &VastAiJobOptions,
|
||||
runtime: swactor::runtime::Runtime,
|
||||
engine: swactor_engine::EngineHandle,
|
||||
) -> Result<Box<dyn ProvisionPlugin>, String> {
|
||||
let identity = resolve_vastai_ssh_identity(options.ssh_identity.clone())?;
|
||||
if !identity.is_file() {
|
||||
|
|
@ -259,8 +395,9 @@ fn build_vastai_provisioner(
|
|||
}
|
||||
|
||||
Ok(Box::new(VastAiProvisioningPlugin::new(
|
||||
ToolsVastAiLeaseClient::from_api_key(api_key.to_owned())?,
|
||||
SshCommandBootstrapLauncher::new(Some(identity), runtime),
|
||||
ToolsVastAiLeaseClient::from_api_key(api_key.to_owned())?
|
||||
.with_actor_host(runtime.clone(), engine.clone()),
|
||||
SshCommandBootstrapLauncher::new(Some(identity), runtime, engine),
|
||||
config,
|
||||
)))
|
||||
}
|
||||
|
|
@ -320,6 +457,7 @@ fn build_cluster(
|
|||
spec: NodeProvisionSpec,
|
||||
provisioner: Box<dyn ProvisionPlugin>,
|
||||
engine: swactor_engine::EngineHandle,
|
||||
runtime: swactor::runtime::Runtime,
|
||||
sink: PluginSink,
|
||||
) -> Result<ProvisionedClusterGuard, String> {
|
||||
let group_id = NodeGroupId(format!("job-node-{}", spec.node_id));
|
||||
|
|
@ -375,6 +513,7 @@ fn build_cluster(
|
|||
}],
|
||||
retry,
|
||||
engine,
|
||||
runtime,
|
||||
sink,
|
||||
)
|
||||
}
|
||||
|
|
@ -543,6 +682,3 @@ fn default_run_id() -> u64 {
|
|||
fn sessionless_orchestrator_ref() -> String {
|
||||
"job-orchestrator-identity-exchanged-out-of-band".to_owned()
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn _assert_lifecycle_policy_is_linked(_: LifecyclePolicy) {}
|
||||
|
|
|
|||
3975
apps/myelin/src/orchestration/manual_control.rs
Normal file
3975
apps/myelin/src/orchestration/manual_control.rs
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -9,13 +9,11 @@ pub(crate) mod actor;
|
|||
pub(crate) mod app;
|
||||
pub(crate) mod cluster_reconciler;
|
||||
pub(crate) mod config;
|
||||
pub(crate) mod control;
|
||||
pub(crate) mod daemon;
|
||||
pub(crate) mod distribution_stack;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod engine_builder;
|
||||
pub(crate) mod job_reconciler;
|
||||
|
||||
pub(crate) mod node_image;
|
||||
pub(crate) mod manual_control;
|
||||
pub(crate) mod provider_adapters {
|
||||
pub(crate) mod relay;
|
||||
pub(super) mod vastai;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,8 @@
|
|||
// NOTE: This module is currently excluded from compilation (its `mod` entry
|
||||
// was removed from orchestration/mod.rs). Backburner code — deprecated for
|
||||
// now but not yet staged for pruning. Re-wire or delete when node-image
|
||||
// provisioning is revisited.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::fs::{self, File};
|
||||
use std::io::{BufRead, BufReader, Read};
|
||||
|
|
@ -215,11 +220,10 @@ fn prepare_node_image_inner(
|
|||
}
|
||||
|
||||
fn workspace_root() -> Result<PathBuf, String> {
|
||||
let output = Command::new("git")
|
||||
let output = swactor_process::command_output(&mut Command::new("git")
|
||||
.args(["rev-parse", "--show-toplevel"])
|
||||
.current_dir(env!("CARGO_MANIFEST_DIR"))
|
||||
.stdin(Stdio::null())
|
||||
.output()
|
||||
.stdin(Stdio::null()))
|
||||
.map_err(|e| format!("locate repository root with git: {e}"))?;
|
||||
if !output.status.success() {
|
||||
return Err(format!(
|
||||
|
|
@ -246,11 +250,10 @@ fn image_version_tag(root: &Path, image_content_hash: &str) -> Result<String, St
|
|||
}
|
||||
|
||||
fn git_capture(root: &Path, args: &[&str]) -> Result<String, String> {
|
||||
let output = Command::new("git")
|
||||
let output = swactor_process::command_output(&mut Command::new("git")
|
||||
.current_dir(root)
|
||||
.args(args)
|
||||
.stdin(Stdio::null())
|
||||
.output()
|
||||
.stdin(Stdio::null()))
|
||||
.map_err(|e| format!("run git {}: {e}", args.join(" ")))?;
|
||||
if output.status.success() {
|
||||
Ok(String::from_utf8_lossy(&output.stdout).to_string())
|
||||
|
|
@ -608,8 +611,6 @@ enum CommandOutputLine {
|
|||
Stderr(String),
|
||||
}
|
||||
|
||||
// container image build is provisioning infrastructure, out of scope (ENGINE_SPEC.md §2)
|
||||
#[allow(clippy::disallowed_methods)]
|
||||
fn spawn_line_reader<R>(
|
||||
reader: R,
|
||||
to_line: fn(String) -> CommandOutputLine,
|
||||
|
|
@ -649,8 +650,6 @@ fn drain_command_lines(
|
|||
}
|
||||
}
|
||||
|
||||
// container image build is provisioning infrastructure, out of scope (ENGINE_SPEC.md §2)
|
||||
#[allow(clippy::disallowed_methods)]
|
||||
fn run_status_command(
|
||||
root: &Path,
|
||||
program: &str,
|
||||
|
|
@ -662,13 +661,12 @@ fn run_status_command(
|
|||
let args: Vec<String> = args.iter().map(|arg| (*arg).to_owned()).collect();
|
||||
eprintln!("myelin-node-image: {label}");
|
||||
if progress.is_none() {
|
||||
let status = Command::new(program)
|
||||
let status = swactor_process::command_status(&mut Command::new(program)
|
||||
.current_dir(root)
|
||||
.args(&args)
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::inherit())
|
||||
.stderr(Stdio::inherit())
|
||||
.status()
|
||||
.stderr(Stdio::inherit()))
|
||||
.map_err(|e| format!("run {label}: {e}"))?;
|
||||
return if status.success() {
|
||||
Ok(())
|
||||
|
|
@ -688,13 +686,12 @@ fn run_status_command(
|
|||
args: args.to_vec(),
|
||||
},
|
||||
);
|
||||
let mut child = match Command::new(program)
|
||||
let mut child = match swactor_process::command_spawn(&mut Command::new(program)
|
||||
.current_dir(root)
|
||||
.args(&args)
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.stderr(Stdio::piped()))
|
||||
{
|
||||
Ok(child) => child,
|
||||
Err(error) => {
|
||||
|
|
@ -732,7 +729,7 @@ fn run_status_command(
|
|||
drop(tx);
|
||||
|
||||
let status = loop {
|
||||
match child.try_wait() {
|
||||
match swactor_process::child_try_wait(&mut child) {
|
||||
Ok(Some(status)) => break status,
|
||||
Ok(None) => {
|
||||
drain_command_lines(&rx, progress, label, image_ref, started);
|
||||
|
|
@ -780,13 +777,12 @@ fn run_status_command(
|
|||
}
|
||||
|
||||
fn docker_image_exists(root: &Path, image_ref: &str) -> bool {
|
||||
Command::new("docker")
|
||||
swactor_process::command_status(&mut Command::new("docker")
|
||||
.current_dir(root)
|
||||
.args(["image", "inspect", image_ref])
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.status()
|
||||
.stderr(Stdio::null()))
|
||||
.map(|status| status.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
|
@ -795,7 +791,7 @@ fn docker_image_labels(
|
|||
root: &Path,
|
||||
image_ref: &str,
|
||||
) -> Result<Option<BTreeMap<String, String>>, String> {
|
||||
let output = Command::new("docker")
|
||||
let output = swactor_process::command_output(&mut Command::new("docker")
|
||||
.current_dir(root)
|
||||
.args([
|
||||
"image",
|
||||
|
|
@ -804,8 +800,7 @@ fn docker_image_labels(
|
|||
"{{ json .Config.Labels }}",
|
||||
image_ref,
|
||||
])
|
||||
.stdin(Stdio::null())
|
||||
.output()
|
||||
.stdin(Stdio::null()))
|
||||
.map_err(|e| format!("inspect docker image {image_ref}: {e}"))?;
|
||||
if !output.status.success() {
|
||||
return Ok(None);
|
||||
|
|
@ -817,19 +812,18 @@ fn docker_image_labels(
|
|||
}
|
||||
|
||||
fn docker_manifest_exists(root: &Path, image_ref: &str) -> bool {
|
||||
Command::new("docker")
|
||||
swactor_process::command_status(&mut Command::new("docker")
|
||||
.current_dir(root)
|
||||
.args(["manifest", "inspect", image_ref])
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.status()
|
||||
.stderr(Stdio::null()))
|
||||
.map(|status| status.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn docker_image_has_container(root: &Path, image_ref: &str) -> bool {
|
||||
Command::new("docker")
|
||||
swactor_process::command_output(&mut Command::new("docker")
|
||||
.current_dir(root)
|
||||
.args([
|
||||
"ps",
|
||||
|
|
@ -839,14 +833,13 @@ fn docker_image_has_container(root: &Path, image_ref: &str) -> bool {
|
|||
"--format",
|
||||
"{{.ID}}",
|
||||
])
|
||||
.stdin(Stdio::null())
|
||||
.output()
|
||||
.stdin(Stdio::null()))
|
||||
.map(|output| output.status.success() && !output.stdout.is_empty())
|
||||
.unwrap_or(true)
|
||||
}
|
||||
|
||||
fn docker_image_tags(root: &Path, repository: &str) -> Result<Vec<(String, String)>, String> {
|
||||
let output = Command::new("docker")
|
||||
let output = swactor_process::command_output(&mut Command::new("docker")
|
||||
.current_dir(root)
|
||||
.args([
|
||||
"image",
|
||||
|
|
@ -855,8 +848,7 @@ fn docker_image_tags(root: &Path, repository: &str) -> Result<Vec<(String, Strin
|
|||
"{{.Repository}}\t{{.Tag}}",
|
||||
repository,
|
||||
])
|
||||
.stdin(Stdio::null())
|
||||
.output()
|
||||
.stdin(Stdio::null()))
|
||||
.map_err(|error| format!("docker image ls failed: {error}"))?;
|
||||
if !output.status.success() {
|
||||
return Err(format!("docker image ls failed with {}", output.status));
|
||||
|
|
@ -872,13 +864,12 @@ fn docker_image_tags(root: &Path, repository: &str) -> Result<Vec<(String, Strin
|
|||
}
|
||||
|
||||
fn docker_image_remove(root: &Path, image_ref: &str) -> Result<(), String> {
|
||||
let status = Command::new("docker")
|
||||
let status = swactor_process::command_status(&mut Command::new("docker")
|
||||
.current_dir(root)
|
||||
.args(["image", "rm", image_ref])
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.status()
|
||||
.stderr(Stdio::null()))
|
||||
.map_err(|error| format!("docker image rm failed: {error}"))?;
|
||||
if status.success() {
|
||||
Ok(())
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
173
apps/myelin/src/orchestration/provision_page.html
Normal file
173
apps/myelin/src/orchestration/provision_page.html
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>Myelin Provision</title>
|
||||
<style>
|
||||
:root {
|
||||
color-scheme: dark;
|
||||
font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
|
||||
--mono: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
|
||||
--bg: #000000; --panel: #001220; --panel-hover: #001c38; --inset: #00060c;
|
||||
--border: #14406a; --divider: #0d2c4a;
|
||||
--text: #ffffff; --muted: #9db2c4;
|
||||
--amber: #ff9900; --ghost: rgba(255, 153, 0, .16);
|
||||
--ok: #00d400; --bad: #ff4141; --cyan: #5cd5ff;
|
||||
--primary-fill: #ff9900; --primary-ink: #000000;
|
||||
--selected: rgba(255, 153, 0, .10); --selected-edge: #ff9900;
|
||||
--row-hover: #002a47; --r: 2px; --t: 120ms;
|
||||
background: var(--bg); color: var(--text);
|
||||
}
|
||||
:root[data-theme="light"] {
|
||||
color-scheme: light;
|
||||
--bg: #fafafa; --panel: #ffffff; --panel-hover: #f0f0f0; --inset: #f5f5f5;
|
||||
--border: #dcdce0; --divider: #eaeaea;
|
||||
--text: #383a42; --muted: #696c77;
|
||||
--amber: #986801; --ghost: rgba(152, 104, 1, .18);
|
||||
--ok: #50a14f; --bad: #e45649; --cyan: #0184bc;
|
||||
--primary-fill: #0184bc; --primary-ink: #ffffff;
|
||||
--selected: rgba(64, 120, 242, .08); --selected-edge: #4078f2;
|
||||
--row-hover: #eef1f8;
|
||||
}
|
||||
@media (prefers-color-scheme: light) {
|
||||
:root:not([data-theme]) {
|
||||
color-scheme: light;
|
||||
--bg: #fafafa; --panel: #ffffff; --panel-hover: #f0f0f0; --inset: #f5f5f5;
|
||||
--border: #dcdce0; --divider: #eaeaea;
|
||||
--text: #383a42; --muted: #696c77;
|
||||
--amber: #986801; --ghost: rgba(152, 104, 1, .18);
|
||||
--ok: #50a14f; --bad: #e45649; --cyan: #0184bc;
|
||||
--primary-fill: #0184bc; --primary-ink: #ffffff;
|
||||
--selected: rgba(64, 120, 242, .08); --selected-edge: #4078f2;
|
||||
--row-hover: #eef1f8;
|
||||
}
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; padding: 20px; background: var(--bg); color: var(--text); }
|
||||
.page { max-width: 1180px; margin: 0 auto; }
|
||||
h1 { margin: 0; font-size: 20px; }
|
||||
h2 { margin: 0 0 12px; color: var(--amber); font-size: 13px; font-weight: 700; text-transform: uppercase; letter-spacing: .08em; }
|
||||
h3 { color: var(--amber); font-size: 11px; text-transform: uppercase; letter-spacing: .08em; }
|
||||
.grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); gap: 12px; margin-top: 16px; }
|
||||
.card { background: var(--panel); border: 1px solid var(--border); border-radius: var(--r); padding: 16px; }
|
||||
form { display: grid; gap: 10px; }
|
||||
#search-form { grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
||||
label { display: grid; gap: 5px; margin: 0; color: var(--muted); font-size: 12px; }
|
||||
label.row { display: flex; gap: 8px; align-items: center; }
|
||||
input {
|
||||
width: 100%; padding: 7px 9px; background: var(--inset); color: var(--text);
|
||||
border: 1px solid var(--border); border-radius: var(--r); font: 12px var(--mono);
|
||||
transition: border-color var(--t);
|
||||
}
|
||||
input:hover { border-color: var(--cyan); }
|
||||
input[type="checkbox"] { width: auto; accent-color: var(--primary-fill); }
|
||||
button {
|
||||
padding: 6px 12px; background: transparent; color: var(--text);
|
||||
border: 1px solid var(--border); border-radius: var(--r); cursor: pointer;
|
||||
font: 600 13px var(--mono); transition: background var(--t), border-color var(--t);
|
||||
}
|
||||
button:hover:not(:disabled) { background: var(--panel-hover); border-color: var(--cyan); }
|
||||
button:active:not(:disabled) { transform: translateY(1px); }
|
||||
button.primary { background: var(--primary-fill); border-color: var(--primary-fill); color: var(--primary-ink); }
|
||||
button.primary:hover:not(:disabled) { background: var(--primary-fill); border-color: var(--cyan); filter: brightness(1.1); }
|
||||
button:disabled { opacity: .45; cursor: not-allowed; }
|
||||
:is(a, button, input):focus-visible { outline: 2px solid var(--cyan); outline-offset: 2px; }
|
||||
.status { font: 600 12px var(--mono); }
|
||||
.error { color: var(--bad); white-space: pre-wrap; font: 12px var(--mono); }
|
||||
.ok { color: var(--ok); }
|
||||
.muted { color: var(--muted); font-size: 12px; }
|
||||
#simulation-banner { padding: 8px 10px; border: 1px solid var(--ok); background: var(--panel); }
|
||||
table { width: 100%; min-width: 1060px; border-collapse: collapse; font: 12px var(--mono); }
|
||||
th, td { padding: 7px 8px; text-align: left; border-bottom: 1px solid var(--divider); font-variant-numeric: tabular-nums; }
|
||||
th { color: var(--amber); font-size: 10px; font-weight: 600; text-transform: uppercase; letter-spacing: .08em; }
|
||||
tbody tr:hover { background: var(--row-hover); }
|
||||
tbody tr:has(input:checked) { background: var(--selected); box-shadow: inset 2px 0 0 var(--selected-edge); }
|
||||
code { color: var(--cyan); font-family: var(--mono); }
|
||||
.offers-scroll { max-height: min(36rem, 65vh); overflow: auto; border: 1px solid var(--border); border-radius: var(--r); background: var(--inset); }
|
||||
.offers-scroll th { position: sticky; top: 0; z-index: 1; background: var(--panel); white-space: nowrap; }
|
||||
.sort-controls { display: inline-flex; gap: 2px; margin-left: 4px; vertical-align: middle; }
|
||||
.sort-button { position: relative; width: 14px; height: 14px; padding: 0; border-color: transparent; font-size: 0; line-height: 0; }
|
||||
.sort-button::before { content: ""; position: absolute; left: 4px; width: 0; height: 0; border-left: 3px solid transparent; border-right: 3px solid transparent; }
|
||||
.sort-button[data-direction="asc"]::before { top: 3px; border-bottom: 5px solid currentColor; }
|
||||
.sort-button[data-direction="desc"]::before { bottom: 3px; border-top: 5px solid currentColor; }
|
||||
.sort-button:hover:not(:disabled), .sort-button[aria-pressed="true"] { color: var(--cyan); border-color: var(--cyan); background: var(--panel-hover); }
|
||||
@media (max-width: 680px) { #search-form { grid-template-columns: 1fr; } }
|
||||
@media (prefers-reduced-motion: reduce) { * { transition-duration: .01ms !important; } }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<!--swactor:nav-->
|
||||
<main class="page">
|
||||
<h1>Manual node control</h1>
|
||||
<p class="muted">Every provision targets the submitted intent. Vast.ai selections never fall back to another offer.</p>
|
||||
<p id="simulation-banner" class="status ok" hidden>SIMULATED provisioning: offers are live; no Vast.ai contract will be created or destroyed.</p>
|
||||
<div class="grid">
|
||||
<section id="provider-config-card" class="card">
|
||||
<h2>Provider readiness</h2>
|
||||
<p id="readiness" class="status">Loading…</p><p id="readiness-error" class="error"></p>
|
||||
<form id="config-form" autocomplete="off">
|
||||
<label>Vast.ai API key<input name="api_key" type="password" autocomplete="off"></label>
|
||||
<label data-real-vastai>SSH identity path<input name="ssh_identity" placeholder="~/.ssh/id_ed25519"></label>
|
||||
<label data-real-vastai>Bootstrap command<input name="bootstrap_command"></label>
|
||||
<button class="primary">Validate configuration</button>
|
||||
</form>
|
||||
<p class="muted">The API key is sent once and held only in daemon memory; it is never stored in the browser. Non-secret search state stays in this tab.</p>
|
||||
</section>
|
||||
<section id="vast-search-card" class="card">
|
||||
<h2>Search offers</h2>
|
||||
<form id="search-form">
|
||||
<label>GPU model contains<input name="gpu_model" placeholder="4090 or A100"></label>
|
||||
<label>Minimum GPU RAM (MB)<input name="min_gpu_ram_mb" type="number" min="0"></label>
|
||||
<label>Minimum compute capability<input name="min_compute_cap" type="number" min="0"></label>
|
||||
<label>Minimum reliability<input name="min_reliability" type="number" min="0" max="1" step="0.001"></label>
|
||||
<label>Minimum download Mbps<input name="min_download_mbps" type="number" min="0" step="0.1"></label>
|
||||
<label>Minimum upload Mbps<input name="min_upload_mbps" type="number" min="0" step="0.1"></label>
|
||||
<label>Maximum hourly price<input name="max_hourly_price" type="number" min="0" step="0.001"></label>
|
||||
<label>Blacklist host IDs<input name="blacklist_hosts" placeholder="123,456"></label>
|
||||
<label class="row"><input name="require_verified" type="checkbox" style="width:auto"> Verified hosts only</label>
|
||||
<button id="search-button" class="primary">Search without leasing</button>
|
||||
</form>
|
||||
</section>
|
||||
<section id="local-provision-card" class="card" hidden>
|
||||
<h2><span id="local-provider-name"></span> nodes</h2>
|
||||
<p class="muted">Starts the configured Myelin worker directly. No marketplace offer is involved.</p>
|
||||
<form id="local-provision-form">
|
||||
<label>Node count<input name="count" type="number" min="1" max="8" value="1"></label>
|
||||
<button id="local-provision-button" class="primary">Provision nodes</button>
|
||||
</form>
|
||||
<p id="local-provision-error" class="error"></p>
|
||||
</section>
|
||||
</div>
|
||||
<section id="vast-offers-card" class="card" style="margin-top:1rem">
|
||||
<h2>Offers</h2><p id="offer-status" class="muted"></p><p id="offer-error" class="error"></p>
|
||||
<div class="offers-scroll"><table id="offers-table"><thead><tr><th>Select</th><th>Offer<span class="sort-controls"><button type="button" class="sort-button" data-sort="offer_id" data-direction="asc" title="Sort offer ascending" aria-label="Sort offer ascending">↑</button><button type="button" class="sort-button" data-sort="offer_id" data-direction="desc" title="Sort offer descending" aria-label="Sort offer descending">↓</button></span></th><th>Host<span class="sort-controls"><button type="button" class="sort-button" data-sort="host_id" data-direction="asc" title="Sort host ascending" aria-label="Sort host ascending">↑</button><button type="button" class="sort-button" data-sort="host_id" data-direction="desc" title="Sort host descending" aria-label="Sort host descending">↓</button></span></th><th>GPU<span class="sort-controls"><button type="button" class="sort-button" data-sort="gpu_model" data-direction="asc" title="Sort GPU ascending" aria-label="Sort GPU ascending">↑</button><button type="button" class="sort-button" data-sort="gpu_model" data-direction="desc" title="Sort GPU descending" aria-label="Sort GPU descending">↓</button></span></th><th>RAM<span class="sort-controls"><button type="button" class="sort-button" data-sort="gpu_ram_mb" data-direction="asc" title="Sort RAM ascending" aria-label="Sort RAM ascending">↑</button><button type="button" class="sort-button" data-sort="gpu_ram_mb" data-direction="desc" title="Sort RAM descending" aria-label="Sort RAM descending">↓</button></span></th><th>CC<span class="sort-controls"><button type="button" class="sort-button" data-sort="compute_cap" data-direction="asc" title="Sort compute capability ascending" aria-label="Sort compute capability ascending">↑</button><button type="button" class="sort-button" data-sort="compute_cap" data-direction="desc" title="Sort compute capability descending" aria-label="Sort compute capability descending">↓</button></span></th><th>Verified<span class="sort-controls"><button type="button" class="sort-button" data-sort="verification" data-direction="asc" title="Sort verification ascending" aria-label="Sort verification ascending">↑</button><button type="button" class="sort-button" data-sort="verification" data-direction="desc" title="Sort verification descending" aria-label="Sort verification descending">↓</button></span></th><th>Reliability<span class="sort-controls"><button type="button" class="sort-button" data-sort="reliability" data-direction="asc" title="Sort reliability ascending" aria-label="Sort reliability ascending">↑</button><button type="button" class="sort-button" data-sort="reliability" data-direction="desc" title="Sort reliability descending" aria-label="Sort reliability descending">↓</button></span></th><th>Down<span class="sort-controls"><button type="button" class="sort-button" data-sort="download_mbps" data-direction="asc" title="Sort download speed ascending" aria-label="Sort download speed ascending">↑</button><button type="button" class="sort-button" data-sort="download_mbps" data-direction="desc" title="Sort download speed descending" aria-label="Sort download speed descending">↓</button></span></th><th>Up<span class="sort-controls"><button type="button" class="sort-button" data-sort="upload_mbps" data-direction="asc" title="Sort upload speed ascending" aria-label="Sort upload speed ascending">↑</button><button type="button" class="sort-button" data-sort="upload_mbps" data-direction="desc" title="Sort upload speed descending" aria-label="Sort upload speed descending">↓</button></span></th><th>Location<span class="sort-controls"><button type="button" class="sort-button" data-sort="location" data-direction="asc" title="Sort location ascending" aria-label="Sort location ascending">↑</button><button type="button" class="sort-button" data-sort="location" data-direction="desc" title="Sort location descending" aria-label="Sort location descending">↓</button></span></th><th>$/hr<span class="sort-controls"><button type="button" class="sort-button" data-sort="hourly_price" data-direction="asc" title="Sort price ascending" aria-label="Sort price ascending">↑</button><button type="button" class="sort-button" data-sort="hourly_price" data-direction="desc" title="Sort price descending" aria-label="Sort price descending">↓</button></span></th></tr></thead><tbody id="offers"></tbody></table></div>
|
||||
<p>Selected exact offer IDs: <code id="selection">none (0/8)</code></p>
|
||||
<button id="provision-button" class="primary" disabled>Provision selected offers</button>
|
||||
</section>
|
||||
</main>
|
||||
<script>
|
||||
'use strict';
|
||||
const $=id=>document.getElementById(id); const MAX_SELECTED_OFFERS=8; const PAGE_STATE_KEY='myelin.provision.page.v1'; const SEARCH_FIELDS=['gpu_model','min_gpu_ram_mb','min_compute_cap','min_reliability','min_download_mbps','min_upload_mbps','max_hourly_price','blacklist_hosts','require_verified']; let selected=new Set(); let provisioningMode='real'; let currentOffers=[]; let offerSorts=[]; let sortDoubleClickStartedActive=null;
|
||||
const value=(form,name)=>{const raw=new FormData(form).get(name);return typeof raw==='string'&&raw.trim()!==''?raw.trim():null};
|
||||
const number=(form,name)=>{const raw=value(form,name);return raw===null?null:Number(raw)};
|
||||
const escapeHtml=raw=>String(raw??'').replace(/[&<>"']/g,char=>({'&':'&','<':'<','>':'>','"':'"',"'":'''}[char]));
|
||||
async function jsonFetch(url,options={}){const response=await fetch(url,{...options,headers:{'content-type':'application/json',...(options.headers||{})}});const text=await response.text();let body=null;try{body=text?JSON.parse(text):null}catch{body={error:text}}if(!response.ok)throw new Error(body?.error||`${response.status} ${response.statusText}`);return body}
|
||||
function commandId(prefix){return `${prefix}-${crypto.randomUUID()}`}
|
||||
function formSnapshot(form,names){return Object.fromEntries(names.map(name=>{const field=form.elements[name];return [name,field.type==='checkbox'?field.checked:field.value]}))}
|
||||
function restoreForm(form,state){if(!state)return;Object.entries(state).forEach(([name,saved])=>{const field=form.elements[name];if(!field)return;if(field.type==='checkbox')field.checked=Boolean(saved);else field.value=String(saved??'')})}
|
||||
function persistPageState(){try{sessionStorage.setItem(PAGE_STATE_KEY,JSON.stringify({config:formSnapshot($('config-form'),['ssh_identity','bootstrap_command']),search:formSnapshot($('search-form'),SEARCH_FIELDS),local:formSnapshot($('local-provision-form'),['count']),offers:currentOffers,selected:[...selected],sorts:offerSorts}))}catch{}}
|
||||
function restorePageState(){try{const saved=JSON.parse(sessionStorage.getItem(PAGE_STATE_KEY)||'null');if(!saved)return;restoreForm($('config-form'),saved.config);restoreForm($('search-form'),saved.search);restoreForm($('local-provision-form'),saved.local);currentOffers=Array.isArray(saved.offers)?saved.offers:[];const available=new Set(currentOffers.map(offer=>Number(offer.offer_id)));selected=new Set((Array.isArray(saved.selected)?saved.selected:[]).map(Number).filter(id=>available.has(id)).slice(0,MAX_SELECTED_OFFERS));const savedSorts=Array.isArray(saved.sorts)?saved.sorts:(typeof saved.sortKey==='string'?[{key:saved.sortKey,direction:saved.sortDirection}]:[]);const seenSortKeys=new Set();offerSorts=savedSorts.filter(sort=>sort&&typeof sort.key==='string'&&!seenSortKeys.has(sort.key)&&seenSortKeys.add(sort.key)).map(sort=>({key:sort.key,direction:sort.direction==='desc'?'desc':'asc'}));renderOffers();updateSelectionUi();if(currentOffers.length)$('offer-status').textContent=`${currentOffers.length} matching offer${currentOffers.length===1?'':'s'}`}catch{}}
|
||||
function updateSelectionUi(){const count=selected.size;$('selection').textContent=count?`${[...selected].join(', ')} (${count}/${MAX_SELECTED_OFFERS})`:`none (0/${MAX_SELECTED_OFFERS})`;$('provision-button').disabled=count===0;$('provision-button').textContent=count===0?(provisioningMode==='mock'?'Simulate selected offers':'Provision selected offers'):`${provisioningMode==='mock'?'Simulate':'Provision'} ${count} selected offer${count===1?'':'s'}`}
|
||||
async function refresh(){try{const reply=await jsonFetch('/api/control/status');const model=reply.Status;const provider=model.provider.name;provisioningMode=model.provider.provisioning_mode||'real';const vast=provider==='vastai';const simulated=vast&&provisioningMode==='mock';const local=provider==='process'||provider==='docker';$('provider-config-card').hidden=!vast;$('vast-search-card').hidden=!vast;$('vast-offers-card').hidden=!vast;$('local-provision-card').hidden=!local;$('simulation-banner').hidden=!simulated;document.querySelectorAll('[data-real-vastai]').forEach(element=>element.hidden=simulated);$('local-provider-name').textContent=provider;$('readiness').textContent=`${provider} · ${provisioningMode} · ${model.provider.kind}`;$('readiness').className=model.provider.kind==='ready'?'status ok':'status';$('readiness-error').textContent=model.provider.error||'';$('search-button').disabled=!vast||model.provider.kind!=='ready';$('local-provision-button').disabled=!local||model.provider.kind!=='ready';updateSelectionUi()}catch(error){$('readiness-error').textContent=error.message}}
|
||||
$('config-form').addEventListener('submit',async event=>{event.preventDefault();const form=event.currentTarget;try{await jsonFetch('/api/control/provider',{method:'POST',body:JSON.stringify({api_key:value(form,'api_key'),ssh_identity:value(form,'ssh_identity'),bootstrap_command:value(form,'bootstrap_command')})});form.elements.api_key.value='';persistPageState();await refresh()}catch(error){$('readiness-error').textContent=error.message}});
|
||||
$('local-provision-form').addEventListener('submit',async event=>{event.preventDefault();const form=event.currentTarget;const count=number(form,'count')||1;const provider=$('local-provider-name').textContent;if(!confirm(`Provision ${count} ${provider} node${count===1?'':'s'}?`))return;try{await jsonFetch('/api/control/provision',{method:'POST',body:JSON.stringify({command_id:commandId('provision'),count,selected_offer_ids:[]})});$('local-provision-error').textContent='';await refresh()}catch(error){$('local-provision-error').textContent=error.message}});
|
||||
$('search-form').addEventListener('submit',async event=>{event.preventDefault();const form=event.currentTarget;const blacklist=(value(form,'blacklist_hosts')||'').split(',').map(v=>Number(v.trim())).filter(Number.isFinite);$('offer-status').textContent='Searching…';$('offer-error').textContent='';try{const reply=await jsonFetch('/api/control/offers',{method:'POST',body:JSON.stringify({gpu_model:value(form,'gpu_model'),min_gpu_ram_mb:number(form,'min_gpu_ram_mb'),min_compute_cap:number(form,'min_compute_cap'),min_reliability:number(form,'min_reliability'),require_verified:form.elements.require_verified.checked,min_download_mbps:number(form,'min_download_mbps'),min_upload_mbps:number(form,'min_upload_mbps'),max_hourly_price:number(form,'max_hourly_price'),blacklist_hosts:blacklist,count:null})});currentOffers=reply.Offers;selected.clear();renderOffers();updateSelectionUi();$('offer-status').textContent=currentOffers.length===0?'No offers match this search.':`${currentOffers.length} matching offer${currentOffers.length===1?'':'s'}`;persistPageState()}catch(error){$('offer-status').textContent='';$('offer-error').textContent=error.message}});
|
||||
function compareOfferValues(left,right){const leftMissing=left===null||left===undefined||left==='';const rightMissing=right===null||right===undefined||right==='';if(leftMissing||rightMissing)return leftMissing===rightMissing?0:leftMissing?1:-1;if(typeof left==='number'&&typeof right==='number')return left-right;return String(left).localeCompare(String(right),undefined,{numeric:true,sensitivity:'base'})}
|
||||
function renderOffers(){const offers=[...currentOffers];if(offerSorts.length)offers.sort((left,right)=>{for(const sort of offerSorts){const compared=compareOfferValues(left[sort.key],right[sort.key]);if(compared!==0)return compared*(sort.direction==='asc'?1:-1)}return Number(left.offer_id)-Number(right.offer_id)});$('offers').replaceChildren(...offers.map(offer=>{const tr=document.createElement('tr');tr.innerHTML=`<td><input type="checkbox" data-offer="${Number(offer.offer_id)}" style="width:auto"></td><td>${escapeHtml(offer.offer_id)}</td><td>${escapeHtml(offer.host_id)}</td><td>${escapeHtml(offer.gpu_model)}</td><td>${escapeHtml(offer.gpu_ram_mb)}</td><td>${escapeHtml(offer.compute_cap)}</td><td>${escapeHtml(offer.verification)}</td><td>${escapeHtml(offer.reliability)}</td><td>${escapeHtml(offer.download_mbps)}</td><td>${escapeHtml(offer.upload_mbps)}</td><td>${escapeHtml(offer.location)}</td><td>${escapeHtml(Number(offer.hourly_price).toFixed(3))}</td>`;tr.querySelector('[data-offer]').checked=selected.has(Number(offer.offer_id));return tr}));document.querySelectorAll('.sort-button').forEach(button=>button.setAttribute('aria-pressed',String(offerSorts.some(sort=>sort.key===button.dataset.sort&&sort.direction===button.dataset.direction))))}
|
||||
$('offers-table').addEventListener('click',event=>{const button=event.target.closest('.sort-button');if(!button)return;const id=`${button.dataset.sort}:${button.dataset.direction}`;const activeIndex=offerSorts.findIndex(sort=>sort.key===button.dataset.sort&&sort.direction===button.dataset.direction);if(event.detail===1)sortDoubleClickStartedActive=activeIndex>=0?id:null;if(event.detail===2&&sortDoubleClickStartedActive===id){offerSorts.splice(activeIndex,1);sortDoubleClickStartedActive=null}else if(activeIndex<0){const columnIndex=offerSorts.findIndex(sort=>sort.key===button.dataset.sort);const next={key:button.dataset.sort,direction:button.dataset.direction};if(columnIndex>=0)offerSorts.splice(columnIndex,1,next);else offerSorts.push(next)}renderOffers();persistPageState()});
|
||||
$('offers').addEventListener('change',event=>{const id=Number(event.target.dataset.offer);if(!Number.isFinite(id))return;if(event.target.checked){if(selected.size>=MAX_SELECTED_OFFERS){event.target.checked=false;$('offer-error').textContent=`Select at most ${MAX_SELECTED_OFFERS} offers.`;return}selected.add(id)}else selected.delete(id);$('offer-error').textContent='';updateSelectionUi();persistPageState()});
|
||||
$('provision-button').addEventListener('click',async()=>{try{const ids=[...selected];if(ids.length===0)return;const question=provisioningMode==='mock'?`Simulate provisioning exact Vast.ai offer IDs ${ids.join(', ')}? No rental will be created.`:`Provision exact Vast.ai offer IDs ${ids.join(', ')}? This can incur charges.`;if(!confirm(question))return;await jsonFetch('/api/control/provision',{method:'POST',body:JSON.stringify({command_id:commandId('provision'),count:ids.length,selected_offer_ids:ids})});const provisioned=new Set(ids);currentOffers=currentOffers.filter(offer=>!provisioned.has(Number(offer.offer_id)));selected.clear();renderOffers();updateSelectionUi();persistPageState();await refresh()}catch(error){$('offer-error').textContent=error.message}});
|
||||
$('config-form').addEventListener('input',persistPageState);$('search-form').addEventListener('input',persistPageState);$('local-provision-form').addEventListener('input',persistPageState);restorePageState();refresh();setInterval(refresh,1000);
|
||||
</script>
|
||||
</body></html>
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,61 +1,4 @@
|
|||
pub(crate) const MO01_HEADER_BYTES: u64 = 40;
|
||||
const TOKEN_ID_WIDTH_BYTES: u32 = 4;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub(crate) struct RunId(pub(crate) u64);
|
||||
|
||||
impl From<u64> for RunId {
|
||||
fn from(value: u64) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub(crate) struct NodeId(pub(crate) u64);
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub(crate) struct EdgeId(pub(crate) u64);
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct EdgeAllocator {
|
||||
next: u64,
|
||||
}
|
||||
|
||||
impl EdgeAllocator {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self { next: 1 }
|
||||
}
|
||||
|
||||
pub(crate) fn alloc(&mut self) -> EdgeId {
|
||||
let edge_id = EdgeId(self.next);
|
||||
self.next += 1;
|
||||
edge_id
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for EdgeAllocator {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum DTypeFamily {
|
||||
BFloat,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct ModelFacts {
|
||||
pub model_id: String,
|
||||
pub gguf_source: GgufSource,
|
||||
pub num_layers: u32,
|
||||
pub hidden_dim: u64,
|
||||
pub dtype_family: DTypeFamily,
|
||||
pub dtype_width_bytes: u64,
|
||||
pub max_seq_len: u64,
|
||||
pub eos_token_id: u32,
|
||||
pub tokenizer: TokenizerSource,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
pub(crate) enum GgufSource {
|
||||
|
|
@ -72,543 +15,3 @@ pub(crate) enum TokenizerSource {
|
|||
EmbeddedGguf,
|
||||
LocalPath(String),
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct SamplingPolicy {
|
||||
pub temperature_millis: u32,
|
||||
pub top_k: u32,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub(crate) struct RoleId(pub(crate) u64);
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub(crate) struct PortId(pub(crate) String);
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct GgufModelPlan {
|
||||
pub model_id: String,
|
||||
pub gguf_source: GgufSource,
|
||||
pub num_layers: u32,
|
||||
pub hidden_dim: u32,
|
||||
pub dtype_family: DTypeFamily,
|
||||
pub dtype_width_bytes: u32,
|
||||
pub max_seq_len: u32,
|
||||
pub eos_token_id: u32,
|
||||
pub tokenizer: TokenizerSource,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct RuntimeConfig {
|
||||
pub max_tokens: u32,
|
||||
pub sampling: SamplingPolicy,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct StagePlacement {
|
||||
pub stage_index: u32,
|
||||
pub node_id: NodeId,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum PlacementInput {
|
||||
FixedLinear(Vec<StagePlacement>),
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum RingDirection {
|
||||
Ingress,
|
||||
Egress,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum HostPinning {
|
||||
Pageable,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum WakeCoalescing {
|
||||
PendingBit,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct RingSpec {
|
||||
pub data_capacity: u64,
|
||||
pub alignment: u32,
|
||||
pub direction: RingDirection,
|
||||
pub host_pinning: HostPinning,
|
||||
pub wake_coalescing: WakeCoalescing,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct PlannerInput {
|
||||
pub run_id: RunId,
|
||||
pub orchestrator_node_id: NodeId,
|
||||
pub model: ModelFacts,
|
||||
pub runtime: RuntimeConfig,
|
||||
pub candidate_pool: Vec<NodeId>,
|
||||
pub stage_count: u32,
|
||||
pub placement: PlacementInput,
|
||||
pub activation_ring: RingSpec,
|
||||
pub token_ring: RingSpec,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub(crate) enum EdgeKind {
|
||||
TokenIn,
|
||||
Activation,
|
||||
TokenOut,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum ObjectKind {
|
||||
Token,
|
||||
Activation,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum ShapeRule {
|
||||
TokenIds,
|
||||
ActivationRows { max_seq_len: u32, hidden_dim: u32 },
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum LayoutRule {
|
||||
Contiguous,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum SequencePolicy {
|
||||
Ordered,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct ObjectSpec {
|
||||
pub kind: ObjectKind,
|
||||
pub max_extent: u64,
|
||||
pub dtype_family: DTypeFamily,
|
||||
pub dtype_width_bytes: u32,
|
||||
pub shape: ShapeRule,
|
||||
pub layout: LayoutRule,
|
||||
pub alignment: u32,
|
||||
pub sequence_policy: SequencePolicy,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub(crate) enum EdgeEndpoint {
|
||||
Orchestrator { node_id: NodeId },
|
||||
Stage { node_id: NodeId, stage_index: u32 },
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct EdgePlan {
|
||||
pub run_id: RunId,
|
||||
pub edge_id: EdgeId,
|
||||
pub kind: EdgeKind,
|
||||
pub producer: EdgeEndpoint,
|
||||
pub consumer: EdgeEndpoint,
|
||||
pub object_spec: ObjectSpec,
|
||||
pub ring_spec: RingSpec,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct StagePlan {
|
||||
pub run_id: RunId,
|
||||
pub stage_index: u32,
|
||||
pub stage_count: u32,
|
||||
pub node_id: NodeId,
|
||||
pub gguf_source: GgufSource,
|
||||
pub layer_start: u32,
|
||||
pub layer_end_exclusive: u32,
|
||||
pub inbound_edge: EdgeId,
|
||||
pub outbound_edge: EdgeId,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct RunPlan {
|
||||
pub run_id: RunId,
|
||||
pub model: GgufModelPlan,
|
||||
pub sampling: SamplingPolicy,
|
||||
pub stages: Vec<StagePlan>,
|
||||
pub edges: Vec<EdgePlan>,
|
||||
pub max_tokens: u32,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct InboundEdgeProvision {
|
||||
pub edge_id: EdgeId,
|
||||
pub kind: EdgeKind,
|
||||
pub object_spec: ObjectSpec,
|
||||
pub ring_spec: RingSpec,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct OutboundEdgeProvision {
|
||||
pub edge_id: EdgeId,
|
||||
pub kind: EdgeKind,
|
||||
pub consumer_node_id: NodeId,
|
||||
pub object_spec: ObjectSpec,
|
||||
pub ring_spec: RingSpec,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct StageModelFacts {
|
||||
pub model_id: String,
|
||||
pub hidden_dim: u32,
|
||||
pub dtype_family: DTypeFamily,
|
||||
pub dtype_width_bytes: u32,
|
||||
pub max_seq_len: u32,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct StageRuntimeFacts {
|
||||
pub role_id: RoleId,
|
||||
pub input_port: PortId,
|
||||
pub output_port: PortId,
|
||||
pub sampling: Option<SamplingPolicy>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct ProvisionStage {
|
||||
pub run_id: RunId,
|
||||
pub node_id: NodeId,
|
||||
pub stage_index: u32,
|
||||
pub stage_count: u32,
|
||||
pub gguf_source: GgufSource,
|
||||
pub tokenizer: TokenizerSource,
|
||||
pub layer_start: u32,
|
||||
pub layer_end_exclusive: u32,
|
||||
pub inbound: InboundEdgeProvision,
|
||||
pub outbound: OutboundEdgeProvision,
|
||||
pub model: StageModelFacts,
|
||||
pub runtime: StageRuntimeFacts,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum PlanRejectionKind {
|
||||
UnknownNode,
|
||||
DuplicateStageAssignment,
|
||||
MissingStage,
|
||||
InvalidStageCount,
|
||||
ModelStageLayoutMismatch,
|
||||
InvalidObjectSpec,
|
||||
UnsupportedShapeOrLayout,
|
||||
InvalidRingSpec,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct PlanRejection {
|
||||
kind: PlanRejectionKind,
|
||||
}
|
||||
|
||||
impl PlanRejection {
|
||||
pub(crate) fn kind(&self) -> PlanRejectionKind {
|
||||
self.kind
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum ProjectionRejection {
|
||||
UnknownStage,
|
||||
MissingEdge,
|
||||
}
|
||||
|
||||
pub(crate) fn plan_run(input: PlannerInput) -> Result<RunPlan, PlanRejection> {
|
||||
validate_global_input(&input)?;
|
||||
let placements = validated_placements(&input)?;
|
||||
let model = model_plan(&input.model)?;
|
||||
let max_tokens = input.runtime.max_tokens;
|
||||
let gguf_source = model.gguf_source.clone();
|
||||
let hidden_dim = model.hidden_dim;
|
||||
let dtype_width_bytes = model.dtype_width_bytes;
|
||||
let max_seq_len = model.max_seq_len;
|
||||
|
||||
let activation_extent = input
|
||||
.model
|
||||
.max_seq_len
|
||||
.checked_mul(input.model.hidden_dim)
|
||||
.and_then(|value| value.checked_mul(input.model.dtype_width_bytes))
|
||||
.ok_or_else(|| reject(PlanRejectionKind::InvalidObjectSpec))?;
|
||||
if activation_extent == 0 {
|
||||
return Err(reject(PlanRejectionKind::InvalidObjectSpec));
|
||||
}
|
||||
|
||||
let token_extent = input
|
||||
.model
|
||||
.max_seq_len
|
||||
.checked_mul(u64::from(TOKEN_ID_WIDTH_BYTES))
|
||||
.ok_or_else(|| reject(PlanRejectionKind::InvalidObjectSpec))?;
|
||||
if token_extent == 0 {
|
||||
return Err(reject(PlanRejectionKind::InvalidObjectSpec));
|
||||
}
|
||||
let token_spec = ObjectSpec {
|
||||
kind: ObjectKind::Token,
|
||||
max_extent: token_extent,
|
||||
dtype_family: input.model.dtype_family,
|
||||
dtype_width_bytes: TOKEN_ID_WIDTH_BYTES,
|
||||
shape: ShapeRule::TokenIds,
|
||||
layout: LayoutRule::Contiguous,
|
||||
alignment: TOKEN_ID_WIDTH_BYTES,
|
||||
sequence_policy: SequencePolicy::Ordered,
|
||||
};
|
||||
let activation_spec = ObjectSpec {
|
||||
kind: ObjectKind::Activation,
|
||||
max_extent: activation_extent,
|
||||
dtype_family: input.model.dtype_family,
|
||||
dtype_width_bytes,
|
||||
shape: ShapeRule::ActivationRows {
|
||||
max_seq_len,
|
||||
hidden_dim,
|
||||
},
|
||||
layout: LayoutRule::Contiguous,
|
||||
alignment: dtype_width_bytes,
|
||||
sequence_policy: SequencePolicy::Ordered,
|
||||
};
|
||||
let token_data_capacity = MO01_HEADER_BYTES
|
||||
.checked_add(token_extent)
|
||||
.ok_or_else(|| reject(PlanRejectionKind::InvalidObjectSpec))?;
|
||||
let mut token_ring = input.token_ring;
|
||||
token_ring.data_capacity = token_ring.data_capacity.max(token_data_capacity);
|
||||
|
||||
let mut edge_allocator = EdgeAllocator::new();
|
||||
let token_in_edge = edge_allocator.alloc();
|
||||
let mut activation_edges = Vec::with_capacity(input.stage_count.saturating_sub(1) as usize);
|
||||
for _ in 0..input.stage_count.saturating_sub(1) {
|
||||
activation_edges.push(edge_allocator.alloc());
|
||||
}
|
||||
let token_out_edge = edge_allocator.alloc();
|
||||
|
||||
let mut edges = Vec::with_capacity(input.stage_count as usize + 1);
|
||||
edges.push(EdgePlan {
|
||||
run_id: input.run_id,
|
||||
edge_id: token_in_edge,
|
||||
kind: EdgeKind::TokenIn,
|
||||
producer: EdgeEndpoint::Orchestrator {
|
||||
node_id: input.orchestrator_node_id,
|
||||
},
|
||||
consumer: EdgeEndpoint::Stage {
|
||||
node_id: placements[0].node_id,
|
||||
stage_index: 0,
|
||||
},
|
||||
object_spec: token_spec,
|
||||
ring_spec: token_ring,
|
||||
});
|
||||
|
||||
for stage_index in 0..input.stage_count.saturating_sub(1) {
|
||||
edges.push(EdgePlan {
|
||||
run_id: input.run_id,
|
||||
edge_id: activation_edges[stage_index as usize],
|
||||
kind: EdgeKind::Activation,
|
||||
producer: EdgeEndpoint::Stage {
|
||||
node_id: placements[stage_index as usize].node_id,
|
||||
stage_index,
|
||||
},
|
||||
consumer: EdgeEndpoint::Stage {
|
||||
node_id: placements[stage_index as usize + 1].node_id,
|
||||
stage_index: stage_index + 1,
|
||||
},
|
||||
object_spec: activation_spec,
|
||||
ring_spec: input.activation_ring,
|
||||
});
|
||||
}
|
||||
|
||||
edges.push(EdgePlan {
|
||||
run_id: input.run_id,
|
||||
edge_id: token_out_edge,
|
||||
kind: EdgeKind::TokenOut,
|
||||
producer: EdgeEndpoint::Stage {
|
||||
node_id: placements[input.stage_count as usize - 1].node_id,
|
||||
stage_index: input.stage_count - 1,
|
||||
},
|
||||
consumer: EdgeEndpoint::Orchestrator {
|
||||
node_id: input.orchestrator_node_id,
|
||||
},
|
||||
object_spec: token_spec,
|
||||
ring_spec: token_ring,
|
||||
});
|
||||
|
||||
let mut stages = Vec::with_capacity(input.stage_count as usize);
|
||||
for placement in &placements {
|
||||
let stage_index = placement.stage_index;
|
||||
let start = (u64::from(input.model.num_layers) * u64::from(stage_index)
|
||||
/ u64::from(input.stage_count)) as u32;
|
||||
let end = (u64::from(input.model.num_layers) * u64::from(stage_index + 1)
|
||||
/ u64::from(input.stage_count)) as u32;
|
||||
let inbound_edge = if stage_index == 0 {
|
||||
token_in_edge
|
||||
} else {
|
||||
activation_edges[stage_index as usize - 1]
|
||||
};
|
||||
let outbound_edge = if stage_index + 1 == input.stage_count {
|
||||
token_out_edge
|
||||
} else {
|
||||
activation_edges[stage_index as usize]
|
||||
};
|
||||
stages.push(StagePlan {
|
||||
run_id: input.run_id,
|
||||
stage_index,
|
||||
stage_count: input.stage_count,
|
||||
node_id: placement.node_id,
|
||||
gguf_source: gguf_source.clone(),
|
||||
layer_start: start,
|
||||
layer_end_exclusive: end,
|
||||
inbound_edge,
|
||||
outbound_edge,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(RunPlan {
|
||||
run_id: input.run_id,
|
||||
model,
|
||||
sampling: input.runtime.sampling,
|
||||
stages,
|
||||
edges,
|
||||
max_tokens,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn derive_stage_provision(
|
||||
plan: &RunPlan,
|
||||
stage_index: u32,
|
||||
) -> Result<ProvisionStage, ProjectionRejection> {
|
||||
let stage = plan
|
||||
.stages
|
||||
.iter()
|
||||
.find(|stage| stage.stage_index == stage_index)
|
||||
.ok_or(ProjectionRejection::UnknownStage)?;
|
||||
let inbound = plan
|
||||
.edges
|
||||
.iter()
|
||||
.find(|edge| edge.edge_id == stage.inbound_edge)
|
||||
.ok_or(ProjectionRejection::MissingEdge)?;
|
||||
let outbound = plan
|
||||
.edges
|
||||
.iter()
|
||||
.find(|edge| edge.edge_id == stage.outbound_edge)
|
||||
.ok_or(ProjectionRejection::MissingEdge)?;
|
||||
Ok(ProvisionStage {
|
||||
run_id: plan.run_id,
|
||||
node_id: stage.node_id,
|
||||
stage_index,
|
||||
stage_count: stage.stage_count,
|
||||
gguf_source: stage.gguf_source.clone(),
|
||||
tokenizer: plan.model.tokenizer.clone(),
|
||||
layer_start: stage.layer_start,
|
||||
layer_end_exclusive: stage.layer_end_exclusive,
|
||||
inbound: InboundEdgeProvision {
|
||||
edge_id: inbound.edge_id,
|
||||
kind: inbound.kind,
|
||||
object_spec: inbound.object_spec,
|
||||
ring_spec: ring_spec_for_direction(inbound.ring_spec, RingDirection::Ingress),
|
||||
},
|
||||
outbound: OutboundEdgeProvision {
|
||||
edge_id: outbound.edge_id,
|
||||
kind: outbound.kind,
|
||||
consumer_node_id: match &outbound.consumer {
|
||||
EdgeEndpoint::Orchestrator { node_id } | EdgeEndpoint::Stage { node_id, .. } => {
|
||||
*node_id
|
||||
}
|
||||
},
|
||||
object_spec: outbound.object_spec,
|
||||
ring_spec: ring_spec_for_direction(outbound.ring_spec, RingDirection::Egress),
|
||||
},
|
||||
model: StageModelFacts {
|
||||
model_id: plan.model.model_id.clone(),
|
||||
hidden_dim: plan.model.hidden_dim,
|
||||
dtype_family: plan.model.dtype_family,
|
||||
dtype_width_bytes: plan.model.dtype_width_bytes,
|
||||
max_seq_len: plan.model.max_seq_len,
|
||||
},
|
||||
runtime: StageRuntimeFacts {
|
||||
role_id: RoleId(u64::from(stage.stage_index)),
|
||||
input_port: PortId("input".to_owned()),
|
||||
output_port: PortId("output".to_owned()),
|
||||
sampling: if stage.stage_index + 1 == stage.stage_count {
|
||||
Some(plan.sampling)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_global_input(input: &PlannerInput) -> Result<(), PlanRejection> {
|
||||
if input.stage_count == 0 {
|
||||
return Err(reject(PlanRejectionKind::InvalidStageCount));
|
||||
}
|
||||
if input.model.num_layers < input.stage_count {
|
||||
return Err(reject(PlanRejectionKind::ModelStageLayoutMismatch));
|
||||
}
|
||||
if input.model.max_seq_len == 0 || input.model.dtype_width_bytes == 0 {
|
||||
return Err(reject(PlanRejectionKind::InvalidObjectSpec));
|
||||
}
|
||||
if input.model.hidden_dim == 0 {
|
||||
return Err(reject(PlanRejectionKind::UnsupportedShapeOrLayout));
|
||||
}
|
||||
if !valid_ring(input.activation_ring) || !valid_ring(input.token_ring) {
|
||||
return Err(reject(PlanRejectionKind::InvalidRingSpec));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validated_placements(input: &PlannerInput) -> Result<Vec<StagePlacement>, PlanRejection> {
|
||||
let PlacementInput::FixedLinear(stages) = &input.placement;
|
||||
|
||||
let candidate_nodes = input
|
||||
.candidate_pool
|
||||
.iter()
|
||||
.copied()
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
let mut by_stage = std::collections::BTreeMap::new();
|
||||
for placement in stages {
|
||||
if !candidate_nodes.contains(&placement.node_id) {
|
||||
return Err(reject(PlanRejectionKind::UnknownNode));
|
||||
}
|
||||
if by_stage.insert(placement.stage_index, *placement).is_some() {
|
||||
return Err(reject(PlanRejectionKind::DuplicateStageAssignment));
|
||||
}
|
||||
}
|
||||
|
||||
let mut dense = Vec::with_capacity(input.stage_count as usize);
|
||||
for stage_index in 0..input.stage_count {
|
||||
let placement = by_stage
|
||||
.remove(&stage_index)
|
||||
.ok_or_else(|| reject(PlanRejectionKind::MissingStage))?;
|
||||
dense.push(placement);
|
||||
}
|
||||
Ok(dense)
|
||||
}
|
||||
|
||||
fn model_plan(model: &ModelFacts) -> Result<GgufModelPlan, PlanRejection> {
|
||||
let hidden_dim = u32::try_from(model.hidden_dim)
|
||||
.map_err(|_| reject(PlanRejectionKind::UnsupportedShapeOrLayout))?;
|
||||
let dtype_width_bytes = u32::try_from(model.dtype_width_bytes)
|
||||
.map_err(|_| reject(PlanRejectionKind::InvalidObjectSpec))?;
|
||||
let max_seq_len = u32::try_from(model.max_seq_len)
|
||||
.map_err(|_| reject(PlanRejectionKind::InvalidObjectSpec))?;
|
||||
|
||||
Ok(GgufModelPlan {
|
||||
model_id: model.model_id.clone(),
|
||||
gguf_source: model.gguf_source.clone(),
|
||||
num_layers: model.num_layers,
|
||||
hidden_dim,
|
||||
dtype_family: model.dtype_family,
|
||||
dtype_width_bytes,
|
||||
max_seq_len,
|
||||
eos_token_id: model.eos_token_id,
|
||||
tokenizer: model.tokenizer.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
fn ring_spec_for_direction(mut spec: RingSpec, direction: RingDirection) -> RingSpec {
|
||||
spec.direction = direction;
|
||||
spec
|
||||
}
|
||||
|
||||
fn valid_ring(spec: RingSpec) -> bool {
|
||||
spec.data_capacity > 0 && spec.alignment > 0 && spec.alignment.is_power_of_two()
|
||||
}
|
||||
|
||||
fn reject(kind: PlanRejectionKind) -> PlanRejection {
|
||||
PlanRejection { kind }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,165 +1,9 @@
|
|||
use std::collections::BTreeMap;
|
||||
use std::fs::File;
|
||||
use std::io::{Read, Seek, SeekFrom};
|
||||
use std::path::Path;
|
||||
|
||||
use crate::gguf_common::{GgufValueType, read_integer_value, read_u32, read_u64};
|
||||
use crate::run_plan::{self, DTypeFamily, GgufSource, TokenizerSource};
|
||||
use crate::gguf_common::{GgufValueType, read_u64};
|
||||
|
||||
pub(crate) const GGUF_MAGIC: &[u8; 4] = b"GGUF";
|
||||
pub(crate) const SUPPORTED_GGUF_VERSION: u32 = 3;
|
||||
const DEFAULT_EFFECTIVE_CONTEXT: u64 = 512;
|
||||
const MAX_METADATA_STRING_BYTES: u64 = 16 * 1024 * 1024;
|
||||
const MAX_METADATA_KEY_BYTES: u64 = 1024 * 1024;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct GgufPlanningMetadata {
|
||||
pub version: u32,
|
||||
pub architecture: String,
|
||||
pub name: Option<String>,
|
||||
pub num_layers: u32,
|
||||
pub hidden_dim: u64,
|
||||
pub context_length: u64,
|
||||
pub eos_token_id: u32,
|
||||
}
|
||||
|
||||
impl GgufPlanningMetadata {
|
||||
pub(crate) fn to_model_facts(
|
||||
&self,
|
||||
model_id: impl Into<String>,
|
||||
gguf_source: GgufSource,
|
||||
tokenizer: TokenizerSource,
|
||||
max_context: Option<u32>,
|
||||
) -> Result<run_plan::ModelFacts, String> {
|
||||
let requested_context = max_context
|
||||
.map(u64::from)
|
||||
.unwrap_or(DEFAULT_EFFECTIVE_CONTEXT);
|
||||
if requested_context == 0 {
|
||||
return Err("MYELIN_MAX_CONTEXT/--max-context must be greater than 0".to_owned());
|
||||
}
|
||||
let effective_context = requested_context.min(self.context_length);
|
||||
if effective_context == 0 {
|
||||
return Err("GGUF context length must be greater than 0".to_owned());
|
||||
}
|
||||
Ok(run_plan::ModelFacts {
|
||||
model_id: model_id.into(),
|
||||
gguf_source,
|
||||
num_layers: self.num_layers,
|
||||
hidden_dim: self.hidden_dim,
|
||||
dtype_family: DTypeFamily::BFloat,
|
||||
dtype_width_bytes: 2,
|
||||
max_seq_len: effective_context,
|
||||
eos_token_id: self.eos_token_id,
|
||||
tokenizer,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn read_gguf_planning_metadata(path: &Path) -> Result<GgufPlanningMetadata, String> {
|
||||
let file =
|
||||
File::open(path).map_err(|e| format!("open GGUF metadata {}: {e}", path.display()))?;
|
||||
read_gguf_planning_metadata_from_reader(file)
|
||||
.map_err(|e| format!("read GGUF metadata {}: {e}", path.display()))
|
||||
}
|
||||
|
||||
fn read_gguf_planning_metadata_from_reader<R>(mut reader: R) -> Result<GgufPlanningMetadata, String>
|
||||
where
|
||||
R: Read + Seek,
|
||||
{
|
||||
let mut magic = [0; 4];
|
||||
reader
|
||||
.read_exact(&mut magic)
|
||||
.map_err(|e| format!("read magic: {e}"))?;
|
||||
if &magic != GGUF_MAGIC {
|
||||
return Err("invalid GGUF magic".to_owned());
|
||||
}
|
||||
let version = read_u32(&mut reader)?;
|
||||
if version != SUPPORTED_GGUF_VERSION {
|
||||
return Err(format!(
|
||||
"unsupported GGUF version {version}; expected {SUPPORTED_GGUF_VERSION}"
|
||||
));
|
||||
}
|
||||
let _tensor_count = read_u64(&mut reader)?;
|
||||
let metadata_count = read_u64(&mut reader)?;
|
||||
|
||||
let mut strings = BTreeMap::<String, String>::new();
|
||||
let mut integers = BTreeMap::<String, u64>::new();
|
||||
|
||||
for _ in 0..metadata_count {
|
||||
let key = read_gguf_string(&mut reader, MAX_METADATA_KEY_BYTES)?;
|
||||
let value_type = GgufValueType::read(&mut reader, "GGUF metadata value type")?;
|
||||
match value_type {
|
||||
GgufValueType::String if key == "general.architecture" || key == "general.name" => {
|
||||
strings.insert(
|
||||
key,
|
||||
read_gguf_string(&mut reader, MAX_METADATA_STRING_BYTES)?,
|
||||
);
|
||||
}
|
||||
GgufValueType::String => {
|
||||
skip_gguf_string(&mut reader)?;
|
||||
}
|
||||
value_type if value_type.is_integer() => {
|
||||
let value = read_integer_value(
|
||||
&mut reader,
|
||||
value_type,
|
||||
|other| format!("GGUF value type {other:?} is not an integer"),
|
||||
|value| format!("negative integer metadata value {value}"),
|
||||
)?;
|
||||
if key.ends_with(".block_count")
|
||||
|| key.ends_with(".embedding_length")
|
||||
|| key.ends_with(".context_length")
|
||||
|| key == "tokenizer.ggml.eos_token_id"
|
||||
{
|
||||
integers.insert(key, value);
|
||||
}
|
||||
}
|
||||
GgufValueType::Array => skip_array(&mut reader)?,
|
||||
other => skip_scalar(&mut reader, other)?,
|
||||
}
|
||||
}
|
||||
|
||||
let architecture = strings
|
||||
.remove("general.architecture")
|
||||
.ok_or_else(|| "GGUF metadata missing general.architecture".to_owned())?;
|
||||
let name = strings.remove("general.name");
|
||||
let num_layers = required_u32(
|
||||
&integers,
|
||||
&format!("{architecture}.block_count"),
|
||||
"layer count",
|
||||
)?;
|
||||
let hidden_dim = required_u64(
|
||||
&integers,
|
||||
&format!("{architecture}.embedding_length"),
|
||||
"hidden dimension",
|
||||
)?;
|
||||
let context_length = required_u64(
|
||||
&integers,
|
||||
&format!("{architecture}.context_length"),
|
||||
"context length",
|
||||
)?;
|
||||
let eos_token_id = required_u32(&integers, "tokenizer.ggml.eos_token_id", "EOS token id")?;
|
||||
|
||||
Ok(GgufPlanningMetadata {
|
||||
version,
|
||||
architecture,
|
||||
name,
|
||||
num_layers,
|
||||
hidden_dim,
|
||||
context_length,
|
||||
eos_token_id,
|
||||
})
|
||||
}
|
||||
|
||||
fn required_u64(map: &BTreeMap<String, u64>, key: &str, label: &str) -> Result<u64, String> {
|
||||
map.get(key)
|
||||
.copied()
|
||||
.ok_or_else(|| format!("GGUF metadata missing {label} key {key}"))
|
||||
}
|
||||
|
||||
fn required_u32(map: &BTreeMap<String, u64>, key: &str, label: &str) -> Result<u32, String> {
|
||||
let value = required_u64(map, key, label)?;
|
||||
u32::try_from(value).map_err(|_| format!("GGUF metadata {label} key {key} exceeds u32"))
|
||||
}
|
||||
|
||||
pub(crate) fn skip_scalar<R: Read + Seek>(
|
||||
reader: &mut R,
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@ use crate::staging::gguf_metadata::{
|
|||
|
||||
const DEFAULT_ALIGNMENT: u64 = 32;
|
||||
const MAX_STRING_BYTES: u64 = 64 * 1024 * 1024;
|
||||
const STAGE_SHARD_CACHE_FORMAT_VERSION: &str = "stage-shard-cache-v2";
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub(crate) struct ByteRange {
|
||||
|
|
@ -75,89 +74,18 @@ impl StageShardPlan {
|
|||
self.merged_tensor_ranges.len() + if self.metadata_end > 0 { 1 } else { 0 }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct GgufTensorEntry {
|
||||
name: String,
|
||||
dims: Vec<u64>,
|
||||
ggml_type: u32,
|
||||
source_offset: u64,
|
||||
byte_len: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct GgufDirectory {
|
||||
metadata_count: u64,
|
||||
metadata_end: u64,
|
||||
data_start: u64,
|
||||
alignment: u32,
|
||||
total_bytes: u64,
|
||||
tensors: Vec<GgufTensorEntry>,
|
||||
}
|
||||
|
||||
pub(crate) fn plan_stage_shard(
|
||||
planning_gguf: &Path,
|
||||
source: GgufSource,
|
||||
stage_index: u32,
|
||||
stage_count: u32,
|
||||
layer_start: u32,
|
||||
layer_end_exclusive: u32,
|
||||
) -> Result<StageShardPlan, String> {
|
||||
if layer_start >= layer_end_exclusive {
|
||||
return Err(format!(
|
||||
"stage {stage_index} has empty layer range {layer_start}..{layer_end_exclusive}"
|
||||
));
|
||||
}
|
||||
if stage_count == 0 || stage_index >= stage_count {
|
||||
return Err(format!(
|
||||
"invalid stage index/count: stage {stage_index}, count {stage_count}"
|
||||
));
|
||||
}
|
||||
|
||||
let directory = read_gguf_directory(planning_gguf)?;
|
||||
let tensors = select_stage_tensors(
|
||||
&directory.tensors,
|
||||
stage_index,
|
||||
stage_count,
|
||||
layer_start,
|
||||
layer_end_exclusive,
|
||||
)?;
|
||||
let merged_tensor_ranges = merge_tensor_ranges(directory.data_start, &tensors)?;
|
||||
let cache_key = shard_cache_key(
|
||||
&source,
|
||||
stage_index,
|
||||
stage_count,
|
||||
layer_start,
|
||||
layer_end_exclusive,
|
||||
&tensors,
|
||||
);
|
||||
|
||||
Ok(StageShardPlan {
|
||||
source,
|
||||
stage_index,
|
||||
stage_count,
|
||||
layer_start,
|
||||
layer_end_exclusive,
|
||||
metadata_count: directory.metadata_count,
|
||||
metadata_end: directory.metadata_end,
|
||||
data_start: directory.data_start,
|
||||
alignment: directory.alignment,
|
||||
source_total_bytes: directory.total_bytes,
|
||||
tensors: tensors
|
||||
.into_iter()
|
||||
.map(|tensor| StageShardTensor {
|
||||
name: tensor.name,
|
||||
dims: tensor.dims,
|
||||
ggml_type: tensor.ggml_type,
|
||||
source_offset: tensor.source_offset,
|
||||
byte_len: tensor.byte_len,
|
||||
})
|
||||
.collect(),
|
||||
merged_tensor_ranges,
|
||||
cache_key,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn validate_stage_shard_cache(path: &Path, plan: &StageShardPlan) -> Result<(), String> {
|
||||
let directory = read_gguf_directory(path)
|
||||
.map_err(|error| format!("invalid cached stage shard {}: {error}", path.display()))?;
|
||||
|
|
@ -265,9 +193,6 @@ fn read_gguf_directory(path: &Path) -> Result<GgufDirectory, String> {
|
|||
skip_value(&mut file, value_type)?;
|
||||
}
|
||||
}
|
||||
let metadata_end = file
|
||||
.stream_position()
|
||||
.map_err(|e| format!("locate GGUF metadata end: {e}"))?;
|
||||
|
||||
let mut tensor_infos = Vec::new();
|
||||
for _ in 0..tensor_count {
|
||||
|
|
@ -291,14 +216,12 @@ fn read_gguf_directory(path: &Path) -> Result<GgufDirectory, String> {
|
|||
));
|
||||
}
|
||||
|
||||
let mut order = tensor_infos
|
||||
let order = tensor_infos
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, (_, _, _, offset))| (*offset, index))
|
||||
.collect::<Vec<_>>();
|
||||
order.sort_by_key(|(offset, _)| *offset);
|
||||
let mut byte_lens = vec![0_u64; tensor_infos.len()];
|
||||
for (position, (offset, tensor_index)) in order.iter().copied().enumerate() {
|
||||
for (offset, _) in order {
|
||||
let absolute = data_start
|
||||
.checked_add(offset)
|
||||
.ok_or_else(|| format!("tensor offset overflow at {offset}"))?;
|
||||
|
|
@ -308,153 +231,24 @@ fn read_gguf_directory(path: &Path) -> Result<GgufDirectory, String> {
|
|||
path.display()
|
||||
));
|
||||
}
|
||||
let next_absolute = if let Some((next_offset, _)) = order.get(position + 1) {
|
||||
data_start
|
||||
.checked_add(*next_offset)
|
||||
.ok_or_else(|| format!("next tensor offset overflow at {next_offset}"))?
|
||||
} else {
|
||||
total_bytes
|
||||
};
|
||||
if next_absolute < absolute {
|
||||
return Err("GGUF tensor offsets are not monotonic".to_owned());
|
||||
}
|
||||
byte_lens[tensor_index] = next_absolute - absolute;
|
||||
}
|
||||
|
||||
let tensors = tensor_infos
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(
|
||||
|(index, (name, dims, ggml_type, source_offset))| GgufTensorEntry {
|
||||
name,
|
||||
dims,
|
||||
ggml_type,
|
||||
source_offset,
|
||||
byte_len: byte_lens[index],
|
||||
},
|
||||
)
|
||||
.map(|(name, dims, ggml_type, _)| GgufTensorEntry {
|
||||
name,
|
||||
dims,
|
||||
ggml_type,
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(GgufDirectory {
|
||||
metadata_count,
|
||||
metadata_end,
|
||||
data_start,
|
||||
alignment: u32::try_from(alignment)
|
||||
.map_err(|_| format!("GGUF alignment {alignment} exceeds u32"))?,
|
||||
total_bytes,
|
||||
tensors,
|
||||
})
|
||||
}
|
||||
|
||||
fn select_stage_tensors(
|
||||
tensors: &[GgufTensorEntry],
|
||||
stage_index: u32,
|
||||
stage_count: u32,
|
||||
layer_start: u32,
|
||||
layer_end_exclusive: u32,
|
||||
) -> Result<Vec<GgufTensorEntry>, String> {
|
||||
let first_stage = layer_start == 0;
|
||||
let final_stage = stage_index + 1 == stage_count;
|
||||
let has_output_weight = tensors.iter().any(|tensor| tensor.name == "output.weight");
|
||||
let mut selected = Vec::new();
|
||||
for tensor in tensors {
|
||||
if tensor.name == "token_embd.weight" && (first_stage || final_stage && !has_output_weight)
|
||||
{
|
||||
selected.push(tensor.clone());
|
||||
continue;
|
||||
}
|
||||
if final_stage && matches!(tensor.name.as_str(), "output.weight" | "output_norm.weight") {
|
||||
selected.push(tensor.clone());
|
||||
continue;
|
||||
}
|
||||
if let Some(layer) = tensor_layer_index(&tensor.name)
|
||||
&& layer_start <= layer
|
||||
&& layer < layer_end_exclusive
|
||||
{
|
||||
selected.push(tensor.clone());
|
||||
}
|
||||
}
|
||||
if selected.is_empty() {
|
||||
return Err(format!(
|
||||
"stage {stage_index} selected no tensors for layer range {layer_start}..{layer_end_exclusive}"
|
||||
));
|
||||
}
|
||||
selected.sort_by_key(|tensor| tensor.source_offset);
|
||||
Ok(selected)
|
||||
}
|
||||
|
||||
fn tensor_layer_index(name: &str) -> Option<u32> {
|
||||
let rest = name.strip_prefix("blk.")?;
|
||||
let (raw, _) = rest.split_once('.')?;
|
||||
raw.parse().ok()
|
||||
}
|
||||
|
||||
fn merge_tensor_ranges(
|
||||
data_start: u64,
|
||||
tensors: &[GgufTensorEntry],
|
||||
) -> Result<Vec<ByteRange>, String> {
|
||||
let mut ranges = tensors
|
||||
.iter()
|
||||
.map(|tensor| {
|
||||
let start = data_start
|
||||
.checked_add(tensor.source_offset)
|
||||
.ok_or_else(|| format!("range start overflow for tensor {}", tensor.name))?;
|
||||
Ok(ByteRange {
|
||||
start,
|
||||
len: tensor.byte_len,
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>, String>>()?;
|
||||
ranges.sort_by_key(|range| range.start);
|
||||
let mut merged: Vec<ByteRange> = Vec::new();
|
||||
for range in ranges {
|
||||
if range.len == 0 {
|
||||
continue;
|
||||
}
|
||||
let range_end = range
|
||||
.end_exclusive()
|
||||
.ok_or_else(|| format!("range end overflow at {}", range.start))?;
|
||||
if let Some(last) = merged.last_mut() {
|
||||
let last_end = last
|
||||
.end_exclusive()
|
||||
.ok_or_else(|| format!("range end overflow at {}", last.start))?;
|
||||
if range.start <= last_end {
|
||||
last.len = range_end.saturating_sub(last.start).max(last.len);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
merged.push(range);
|
||||
}
|
||||
Ok(merged)
|
||||
}
|
||||
|
||||
fn shard_cache_key(
|
||||
source: &GgufSource,
|
||||
stage_index: u32,
|
||||
stage_count: u32,
|
||||
layer_start: u32,
|
||||
layer_end_exclusive: u32,
|
||||
tensors: &[GgufTensorEntry],
|
||||
) -> String {
|
||||
let mut hasher = blake3::Hasher::new();
|
||||
hasher.update(format!("format:{STAGE_SHARD_CACHE_FORMAT_VERSION}\n").as_bytes());
|
||||
hasher.update(format!("source:{source:?}\n").as_bytes());
|
||||
hasher.update(
|
||||
format!("stage:{stage_index}/{stage_count}:{layer_start}-{layer_end_exclusive}\n")
|
||||
.as_bytes(),
|
||||
);
|
||||
for tensor in tensors {
|
||||
hasher.update(
|
||||
format!(
|
||||
"{}:{}:{}:{:?}\n",
|
||||
tensor.name, tensor.source_offset, tensor.byte_len, tensor.dims
|
||||
)
|
||||
.as_bytes(),
|
||||
);
|
||||
}
|
||||
hasher.finalize().to_hex()[..24].to_owned()
|
||||
}
|
||||
|
||||
pub(crate) fn materialize_stage_shard_http<F>(
|
||||
plan: &StageShardPlan,
|
||||
output_path: &Path,
|
||||
|
|
|
|||
|
|
@ -76,9 +76,6 @@ pub(crate) enum WeightCommand {
|
|||
source: WeightSource,
|
||||
range: LayerRange,
|
||||
},
|
||||
AdvertiseLoadedLayerRange {
|
||||
range: LayerRange,
|
||||
},
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
|
|||
|
|
@ -101,9 +101,7 @@ fn build_composition() -> (Engine, IrohDriver, DistributionRuntimeStack) {
|
|||
(engine, driver, stack)
|
||||
}
|
||||
|
||||
/// Poll an inbox until a value arrives or the deadline elapses. The only
|
||||
/// `thread::sleep` in this module: test observation, not engine work.
|
||||
#[allow(clippy::disallowed_methods)]
|
||||
/// Bounded inbox polling for test observation; not runtime work.
|
||||
fn recv_within<T: Message>(inbox: &Inbox<T>, deadline: Duration) -> Option<T> {
|
||||
let started = Instant::now();
|
||||
loop {
|
||||
|
|
@ -161,7 +159,7 @@ fn dashboard_server_is_scheduled_through_the_engine() {
|
|||
let mut config = dashboard::DashboardConfig::default();
|
||||
config.port = free_port;
|
||||
let handle = dashboard::DashboardHandle::new(config);
|
||||
engine.handle().spawn(handle.http_server());
|
||||
handle.spawn(&engine.handle());
|
||||
|
||||
// Behavioral proof the server future is actually running on the engine:
|
||||
// the bound port accepts a TCP connection. No second runtime is involved.
|
||||
|
|
@ -171,7 +169,6 @@ fn dashboard_server_is_scheduled_through_the_engine() {
|
|||
}
|
||||
|
||||
#[cfg(feature = "dashboard")]
|
||||
#[allow(clippy::disallowed_methods)]
|
||||
fn poll_connect(addr: (&str, u16), deadline: Duration) -> bool {
|
||||
use std::net::TcpStream;
|
||||
let started = Instant::now();
|
||||
|
|
|
|||
92
apps/myelin/src/tests/fuzz_support.rs
Normal file
92
apps/myelin/src/tests/fuzz_support.rs
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
use std::time::Duration;
|
||||
|
||||
use swactor::runtime::Runtime;
|
||||
use swactor_engine::SteppingBackend;
|
||||
|
||||
pub(crate) fn drive_steps(backend: &SteppingBackend, count: usize) {
|
||||
for _ in 0..count {
|
||||
backend.step();
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn advance_and_drive(backend: &SteppingBackend, duration: Duration, count: usize) {
|
||||
backend.advance_time(duration);
|
||||
drive_steps(backend, count);
|
||||
}
|
||||
|
||||
pub(crate) fn actor_census(runtime: &Runtime) -> String {
|
||||
let stats = runtime.stats();
|
||||
if stats.actor_details.is_empty() {
|
||||
return format!("actors={:?}, workers={:?}", stats.actors, stats.workers);
|
||||
}
|
||||
|
||||
stats
|
||||
.actor_details
|
||||
.iter()
|
||||
.map(|actor| {
|
||||
format!(
|
||||
"address={} name={:?} worker={} mailbox={} last={:?} processed={} poisoned={}",
|
||||
actor.address,
|
||||
actor.name,
|
||||
actor.worker_id,
|
||||
actor.mailbox_depth,
|
||||
actor.last_msg_type,
|
||||
actor.messages_processed,
|
||||
actor.poisoned,
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
pub(crate) fn assert_no_poison(runtime: &Runtime) {
|
||||
assert_no_poison_with_context(runtime, "");
|
||||
}
|
||||
|
||||
pub(crate) fn assert_no_poison_with_context(runtime: &Runtime, context: &str) {
|
||||
let stats = runtime.stats();
|
||||
let poisoned = stats
|
||||
.actor_details
|
||||
.iter()
|
||||
.filter(|actor| actor.poisoned)
|
||||
.collect::<Vec<_>>();
|
||||
let panics = stats
|
||||
.workers
|
||||
.iter()
|
||||
.map(|worker| worker.panics)
|
||||
.sum::<u64>();
|
||||
assert!(
|
||||
poisoned.is_empty() && panics == 0,
|
||||
"actor poison/panic detected: poisoned={poisoned:?}, worker_panics={panics}\n{context}\n{}",
|
||||
actor_census(runtime),
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn assert_actor_delta_at_most(runtime: &Runtime, baseline: usize, limit: usize) {
|
||||
let current = runtime.stats().actors.len();
|
||||
assert!(
|
||||
current <= baseline.saturating_add(limit),
|
||||
"actor count grew from {baseline} to {current}, limit={limit}\n{}",
|
||||
actor_census(runtime),
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn assert_mailboxes_drained(runtime: &Runtime) {
|
||||
let stats = runtime.stats();
|
||||
let worker_depth = stats
|
||||
.workers
|
||||
.iter()
|
||||
.map(|worker| worker.mailbox_depth)
|
||||
.sum::<usize>();
|
||||
let actor_depth = stats
|
||||
.actor_details
|
||||
.iter()
|
||||
.map(|actor| actor.mailbox_depth)
|
||||
.sum::<usize>();
|
||||
assert_eq!(
|
||||
worker_depth + actor_depth,
|
||||
0,
|
||||
"mailboxes did not drain\n{}",
|
||||
actor_census(runtime),
|
||||
);
|
||||
}
|
||||
|
|
@ -26,7 +26,6 @@ use crate::orchestration::distribution_stack::DistributionRuntimeStack;
|
|||
const POLL: Duration = Duration::from_millis(15);
|
||||
const DEADLINE: Duration = Duration::from_secs(20);
|
||||
|
||||
#[allow(clippy::disallowed_methods)]
|
||||
fn recv_within<T: Message>(inbox: &Inbox<T>, deadline: Duration) -> Option<T> {
|
||||
let started = Instant::now();
|
||||
loop {
|
||||
|
|
|
|||
|
|
@ -179,7 +179,6 @@ fn driver_a_join(
|
|||
driver_a.join(std::slice::from_ref(&driver_b.endpoint_addr()));
|
||||
}
|
||||
|
||||
#[allow(clippy::disallowed_methods)]
|
||||
fn wait_until(deadline: Duration, mut check: impl FnMut() -> bool) -> bool {
|
||||
let started = Instant::now();
|
||||
loop {
|
||||
|
|
|
|||
|
|
@ -1,244 +0,0 @@
|
|||
//! Local end-to-end behavior guarantee for the Myelin system lifecycle.
|
||||
//!
|
||||
//! This test composes the crate's Myelin contract harnesses through one in-process
|
||||
//! mock environment. It deliberately avoids Docker, real SWIM, real iroh, GGUF,
|
||||
//! and CUDA while still driving the run through planning, provisioning,
|
||||
//! readiness, prompt injection, stage execution, completion, and teardown.
|
||||
|
||||
use myelin::observability::lifecycle as obs;
|
||||
use myelin::orchestration::engine_builder as engine;
|
||||
|
||||
use super::local_mock::{
|
||||
LocalMockCluster, LocalMockConfig, LocalMockOutcome, assert_happy_path_lifecycle,
|
||||
assert_terminal_fault, assert_terminal_success, assert_topology_surface,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn local_mock_two_stage_pipeline_completes_and_tears_down() {
|
||||
let mut cluster = LocalMockCluster::two_stage();
|
||||
let outcome = cluster.run_prompt("hello world");
|
||||
|
||||
assert_happy_path_lifecycle(&outcome);
|
||||
assert_topology_surface(&outcome);
|
||||
assert_engine_builder_surface(&outcome);
|
||||
assert_terminal_success(&outcome);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_mock_pipeline_topologies_complete() {
|
||||
let cases = [
|
||||
(
|
||||
LocalMockConfig {
|
||||
stage_count: 1,
|
||||
max_tokens: 4,
|
||||
eos_after_sequence: 0,
|
||||
},
|
||||
vec![0],
|
||||
),
|
||||
(
|
||||
LocalMockConfig {
|
||||
stage_count: 2,
|
||||
max_tokens: 4,
|
||||
eos_after_sequence: 1,
|
||||
},
|
||||
vec![0, 1],
|
||||
),
|
||||
(
|
||||
LocalMockConfig {
|
||||
stage_count: 3,
|
||||
max_tokens: 3,
|
||||
eos_after_sequence: 99,
|
||||
},
|
||||
vec![0, 1, 2],
|
||||
),
|
||||
];
|
||||
|
||||
for (config, expected_sequences) in cases {
|
||||
let mut cluster = LocalMockCluster::with_config(config);
|
||||
let outcome = cluster.run_prompt("hello world");
|
||||
|
||||
assert_eq!(outcome.injected_sequences, expected_sequences);
|
||||
assert_topology_surface(&outcome);
|
||||
assert_engine_builder_surface(&outcome);
|
||||
assert_terminal_success(&outcome);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_mock_readiness_barrier_gates_prompt_injection() {
|
||||
let mut cluster = LocalMockCluster::with_config(LocalMockConfig {
|
||||
stage_count: 3,
|
||||
max_tokens: 2,
|
||||
eos_after_sequence: 0,
|
||||
});
|
||||
let outcome = cluster.run_prompt_with_delayed_stage_ready("hello world", 1);
|
||||
|
||||
assert_topology_surface(&outcome);
|
||||
assert_terminal_success(&outcome);
|
||||
assert!(
|
||||
stage_position(&outcome, obs::EventKind::StageReady, 1)
|
||||
< first_position(&outcome, obs::EventKind::ReadinessBarrierPassed),
|
||||
"delayed stage must become ready before the readiness barrier passes"
|
||||
);
|
||||
assert!(
|
||||
last_position(&outcome, obs::EventKind::StageReady)
|
||||
< first_position(&outcome, obs::EventKind::PromptInjected),
|
||||
"prompt injection must wait for every stage_ready event"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_mock_terminal_modes_teardown_everything() {
|
||||
let mut max_token_cluster = LocalMockCluster::with_config(LocalMockConfig {
|
||||
stage_count: 2,
|
||||
max_tokens: 3,
|
||||
eos_after_sequence: 99,
|
||||
});
|
||||
let max_token_outcome = max_token_cluster.run_prompt("hello world");
|
||||
assert_eq!(max_token_outcome.injected_sequences, vec![0, 1, 2]);
|
||||
assert_terminal_success(&max_token_outcome);
|
||||
|
||||
let mut provisioning_fault_cluster = LocalMockCluster::with_config(LocalMockConfig {
|
||||
stage_count: 3,
|
||||
max_tokens: 4,
|
||||
eos_after_sequence: 1,
|
||||
});
|
||||
let provisioning_fault = provisioning_fault_cluster.run_with_unauthorized_provision("hello", 2);
|
||||
assert_eq!(
|
||||
count_kind(&provisioning_fault, obs::EventKind::StageFaulted),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
count_kind(&provisioning_fault, obs::EventKind::PromptInjected),
|
||||
0
|
||||
);
|
||||
assert_terminal_fault(&provisioning_fault);
|
||||
|
||||
let mut execution_fault_cluster = LocalMockCluster::with_config(LocalMockConfig {
|
||||
stage_count: 3,
|
||||
max_tokens: 4,
|
||||
eos_after_sequence: 1,
|
||||
});
|
||||
let execution_fault =
|
||||
execution_fault_cluster.run_with_worker_crash_during_execution("hello", 1, 0);
|
||||
assert_eq!(
|
||||
count_kind(&execution_fault, obs::EventKind::PromptInjected),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
count_kind(&execution_fault, obs::EventKind::TokenReceived),
|
||||
0
|
||||
);
|
||||
assert_terminal_fault(&execution_fault);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_mock_rejects_invalid_cross_stage_events() {
|
||||
let mut wrong_edge_cluster = LocalMockCluster::with_config(LocalMockConfig {
|
||||
stage_count: 2,
|
||||
max_tokens: 2,
|
||||
eos_after_sequence: 0,
|
||||
});
|
||||
let wrong_edge = wrong_edge_cluster.run_wrong_edge_object_then_prompt("hello", 0);
|
||||
assert_eq!(
|
||||
count_kind(&wrong_edge, obs::EventKind::ObjectLoaded),
|
||||
wrong_edge.stage_count * wrong_edge.injected_sequences.len() + 1,
|
||||
"wrong-edge object is observed but must not add an execution step"
|
||||
);
|
||||
assert_eq!(
|
||||
count_kind(&wrong_edge, obs::EventKind::ExecuteStepStarted),
|
||||
wrong_edge.stage_count * wrong_edge.injected_sequences.len()
|
||||
);
|
||||
assert_terminal_success(&wrong_edge);
|
||||
|
||||
let mut sequence_violation_cluster = LocalMockCluster::with_config(LocalMockConfig {
|
||||
stage_count: 2,
|
||||
max_tokens: 2,
|
||||
eos_after_sequence: 0,
|
||||
});
|
||||
let sequence_violation = sequence_violation_cluster.run_with_sequence_violation("hello", 0, 0);
|
||||
assert_eq!(
|
||||
count_kind(&sequence_violation, obs::EventKind::ExecuteStepStarted),
|
||||
0,
|
||||
"out-of-order first object must fault before compute admission"
|
||||
);
|
||||
assert_eq!(
|
||||
count_kind(&sequence_violation, obs::EventKind::TokenReceived),
|
||||
0
|
||||
);
|
||||
assert_terminal_fault(&sequence_violation);
|
||||
}
|
||||
|
||||
fn assert_engine_builder_surface(outcome: &LocalMockOutcome) {
|
||||
assert!(
|
||||
outcome
|
||||
.engine_events
|
||||
.iter()
|
||||
.any(|event| matches!(event, engine::EngineEvent::PoolAcquired)),
|
||||
"local mock integration must be built from a neutral engine pool"
|
||||
);
|
||||
assert!(
|
||||
outcome
|
||||
.engine_events
|
||||
.iter()
|
||||
.any(|event| matches!(event, engine::EngineEvent::ClusterConverged)),
|
||||
"local mock integration must pass through the builder convergence barrier"
|
||||
);
|
||||
assert!(
|
||||
outcome
|
||||
.engine_events
|
||||
.iter()
|
||||
.any(|event| matches!(event, engine::EngineEvent::EngineReady)),
|
||||
"local mock integration must return an engine-ready handle before workload IO"
|
||||
);
|
||||
let assigned_stages = outcome
|
||||
.engine_events
|
||||
.iter()
|
||||
.filter(|event| {
|
||||
matches!(
|
||||
event,
|
||||
engine::EngineEvent::RoleAssigned(engine::RoleKind::StageWorker { .. })
|
||||
)
|
||||
})
|
||||
.count();
|
||||
assert_eq!(assigned_stages, outcome.stage_count);
|
||||
}
|
||||
|
||||
fn count_kind(outcome: &LocalMockOutcome, kind: obs::EventKind) -> usize {
|
||||
outcome
|
||||
.trace
|
||||
.iter()
|
||||
.filter(|event| event.kind() == kind)
|
||||
.count()
|
||||
}
|
||||
|
||||
fn first_position(outcome: &LocalMockOutcome, kind: obs::EventKind) -> usize {
|
||||
outcome
|
||||
.trace
|
||||
.iter()
|
||||
.position(|event| event.kind() == kind)
|
||||
.unwrap_or_else(|| panic!("missing event kind {kind:?}"))
|
||||
}
|
||||
|
||||
fn last_position(outcome: &LocalMockOutcome, kind: obs::EventKind) -> usize {
|
||||
outcome
|
||||
.trace
|
||||
.iter()
|
||||
.rposition(|event| event.kind() == kind)
|
||||
.unwrap_or_else(|| panic!("missing event kind {kind:?}"))
|
||||
}
|
||||
|
||||
fn stage_position(outcome: &LocalMockOutcome, kind: obs::EventKind, stage_index: u32) -> usize {
|
||||
outcome
|
||||
.trace
|
||||
.iter()
|
||||
.position(|event| match event {
|
||||
obs::Event::StageScoped {
|
||||
kind: event_kind,
|
||||
stage_index: event_stage_index,
|
||||
..
|
||||
} => *event_kind == kind && event_stage_index.0 == stage_index,
|
||||
_ => false,
|
||||
})
|
||||
.unwrap_or_else(|| panic!("missing stage event {kind:?} for stage {stage_index}"))
|
||||
}
|
||||
|
|
@ -1,283 +0,0 @@
|
|||
use myelin::observability::lifecycle as obs;
|
||||
|
||||
use super::mock_transport::MockObjectKind;
|
||||
|
||||
use super::environment::LocalMockOutcome;
|
||||
|
||||
pub fn assert_happy_path_lifecycle(outcome: &LocalMockOutcome) {
|
||||
assert_eq!(count_kind(outcome, obs::EventKind::PoolReady), 1);
|
||||
assert_eq!(count_kind(outcome, obs::EventKind::RunPlanned), 1);
|
||||
assert_eq!(
|
||||
count_kind(outcome, obs::EventKind::StageProvisionStarted),
|
||||
outcome.stage_count
|
||||
);
|
||||
assert_eq!(
|
||||
count_kind(outcome, obs::EventKind::WeightsLoaded),
|
||||
outcome.stage_count
|
||||
);
|
||||
assert_eq!(
|
||||
count_kind(outcome, obs::EventKind::StageReady),
|
||||
outcome.stage_count
|
||||
);
|
||||
assert_eq!(
|
||||
count_kind(outcome, obs::EventKind::ReadinessBarrierPassed),
|
||||
1
|
||||
);
|
||||
assert_eq!(outcome.injected_sequences, vec![0, 1]);
|
||||
assert_eq!(
|
||||
sequences_for_kind(outcome, obs::EventKind::PromptInjected),
|
||||
outcome.injected_sequences
|
||||
);
|
||||
assert_eq!(
|
||||
sequences_for_kind(outcome, obs::EventKind::TokenReceived),
|
||||
outcome.injected_sequences
|
||||
);
|
||||
assert_eq!(
|
||||
count_kind(outcome, obs::EventKind::ExecuteStepStarted),
|
||||
outcome.stage_count * outcome.injected_sequences.len()
|
||||
);
|
||||
assert_eq!(
|
||||
count_kind(outcome, obs::EventKind::StepCompleted),
|
||||
outcome.stage_count * outcome.injected_sequences.len()
|
||||
);
|
||||
assert_eq!(
|
||||
outcome.transport_delivery_count,
|
||||
(outcome.stage_count + 1) * outcome.injected_sequences.len()
|
||||
);
|
||||
|
||||
assert_order(
|
||||
outcome,
|
||||
obs::EventKind::PoolReady,
|
||||
obs::EventKind::RunPlanned,
|
||||
);
|
||||
assert_order(
|
||||
outcome,
|
||||
obs::EventKind::RunPlanned,
|
||||
obs::EventKind::StageProvisionStarted,
|
||||
);
|
||||
assert!(
|
||||
last_position(outcome, obs::EventKind::StageReady)
|
||||
< first_position(outcome, obs::EventKind::ReadinessBarrierPassed),
|
||||
"readiness barrier must wait for every stage_ready"
|
||||
);
|
||||
assert_order(
|
||||
outcome,
|
||||
obs::EventKind::ReadinessBarrierPassed,
|
||||
obs::EventKind::PromptInjected,
|
||||
);
|
||||
|
||||
for sequence in &outcome.injected_sequences {
|
||||
assert_eq!(
|
||||
object_sequence_count(outcome, obs::EventKind::ObjectLoaded, *sequence),
|
||||
outcome.stage_count,
|
||||
"each stage must load sequence {sequence} once"
|
||||
);
|
||||
assert_eq!(
|
||||
object_sequence_count(outcome, obs::EventKind::ObjectProduced, *sequence),
|
||||
outcome.stage_count,
|
||||
"each stage must produce sequence {sequence} once"
|
||||
);
|
||||
assert_eq!(
|
||||
object_sequence_count(outcome, obs::EventKind::TokenReceived, *sequence),
|
||||
1,
|
||||
"orchestrator must receive sequence {sequence} once"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn assert_topology_surface(outcome: &LocalMockOutcome) {
|
||||
let sequence_count = outcome.injected_sequences.len();
|
||||
assert_eq!(
|
||||
outcome.transport_delivery_count,
|
||||
(outcome.stage_count + 1) * sequence_count
|
||||
);
|
||||
assert_eq!(
|
||||
count_kind(outcome, obs::EventKind::ExecuteStepStarted),
|
||||
outcome.stage_count * sequence_count
|
||||
);
|
||||
assert_eq!(
|
||||
count_kind(outcome, obs::EventKind::StepCompleted),
|
||||
outcome.stage_count * sequence_count
|
||||
);
|
||||
|
||||
for sequence in &outcome.injected_sequences {
|
||||
let deliveries = outcome
|
||||
.transport_deliveries
|
||||
.iter()
|
||||
.filter(|delivery| delivery.sequence == *sequence)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
deliveries.len(),
|
||||
outcome.stage_count + 1,
|
||||
"sequence {sequence} must cross token-in, every stage output, and token-out"
|
||||
);
|
||||
assert_eq!(deliveries[0].kind, MockObjectKind::Token);
|
||||
assert_eq!(deliveries.last().unwrap().kind, MockObjectKind::Token);
|
||||
assert_eq!(
|
||||
deliveries
|
||||
.iter()
|
||||
.map(|delivery| delivery.edge_id)
|
||||
.collect::<Vec<_>>(),
|
||||
outcome.edge_chain,
|
||||
"sequence {sequence} must follow the planned linear edge chain"
|
||||
);
|
||||
assert_eq!(
|
||||
deliveries
|
||||
.iter()
|
||||
.filter(|delivery| delivery.kind == MockObjectKind::Activation)
|
||||
.count(),
|
||||
outcome.stage_count.saturating_sub(1),
|
||||
"only inter-stage edges carry activations"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn assert_terminal_fault(outcome: &LocalMockOutcome) {
|
||||
assert_eq!(count_kind(outcome, obs::EventKind::RunCompleted), 0);
|
||||
assert_eq!(count_kind(outcome, obs::EventKind::RunFaulted), 1);
|
||||
assert_eq!(count_kind(outcome, obs::EventKind::RunTornDown), 1);
|
||||
assert_eq!(
|
||||
count_kind(outcome, obs::EventKind::StopRunSent),
|
||||
outcome.stage_count
|
||||
);
|
||||
assert_eq!(
|
||||
count_kind(outcome, obs::EventKind::StageStopped),
|
||||
outcome.stage_count
|
||||
);
|
||||
assert_resources_released(outcome);
|
||||
assert_only_teardown_after_terminal(outcome, obs::EventKind::RunFaulted);
|
||||
}
|
||||
|
||||
pub fn assert_terminal_success(outcome: &LocalMockOutcome) {
|
||||
assert_eq!(count_kind(outcome, obs::EventKind::RunCompleted), 1);
|
||||
assert_eq!(count_kind(outcome, obs::EventKind::RunFaulted), 0);
|
||||
assert_eq!(count_kind(outcome, obs::EventKind::RunTornDown), 1);
|
||||
assert_eq!(
|
||||
count_kind(outcome, obs::EventKind::StopRunSent),
|
||||
outcome.stage_count
|
||||
);
|
||||
assert_eq!(
|
||||
count_kind(outcome, obs::EventKind::StageStopped),
|
||||
outcome.stage_count
|
||||
);
|
||||
assert_order(
|
||||
outcome,
|
||||
obs::EventKind::RunCompleted,
|
||||
obs::EventKind::StopRunSent,
|
||||
);
|
||||
assert!(
|
||||
last_position(outcome, obs::EventKind::StageStopped)
|
||||
< first_position(outcome, obs::EventKind::RunTornDown),
|
||||
"run_torn_down must wait for stage_stopped events"
|
||||
);
|
||||
|
||||
for stage_index in 0..outcome.stage_count as u32 {
|
||||
assert!(
|
||||
stage_position(outcome, obs::EventKind::StopRunSent, stage_index)
|
||||
< stage_position(outcome, obs::EventKind::StageStopped, stage_index),
|
||||
"stage {stage_index} must stop after StopRun"
|
||||
);
|
||||
}
|
||||
|
||||
assert_resources_released(outcome);
|
||||
assert_only_teardown_after_terminal(outcome, obs::EventKind::RunCompleted);
|
||||
}
|
||||
|
||||
fn assert_resources_released(outcome: &LocalMockOutcome) {
|
||||
assert_eq!(outcome.live_edges, 0, "teardown must release mock edges");
|
||||
assert_eq!(outcome.live_rings, 0, "teardown must release mock rings");
|
||||
assert_eq!(
|
||||
outcome.live_stage_runs, 0,
|
||||
"teardown must release mock stage run state"
|
||||
);
|
||||
}
|
||||
fn count_kind(outcome: &LocalMockOutcome, kind: obs::EventKind) -> usize {
|
||||
outcome
|
||||
.trace
|
||||
.iter()
|
||||
.filter(|event| event.kind() == kind)
|
||||
.count()
|
||||
}
|
||||
|
||||
fn first_position(outcome: &LocalMockOutcome, kind: obs::EventKind) -> usize {
|
||||
outcome
|
||||
.trace
|
||||
.iter()
|
||||
.position(|event| event.kind() == kind)
|
||||
.unwrap_or_else(|| panic!("missing event kind {kind:?}"))
|
||||
}
|
||||
|
||||
fn last_position(outcome: &LocalMockOutcome, kind: obs::EventKind) -> usize {
|
||||
outcome
|
||||
.trace
|
||||
.iter()
|
||||
.rposition(|event| event.kind() == kind)
|
||||
.unwrap_or_else(|| panic!("missing event kind {kind:?}"))
|
||||
}
|
||||
|
||||
fn assert_order(outcome: &LocalMockOutcome, before: obs::EventKind, after: obs::EventKind) {
|
||||
assert!(
|
||||
first_position(outcome, before) < first_position(outcome, after),
|
||||
"{before:?} must occur before {after:?}"
|
||||
);
|
||||
}
|
||||
|
||||
fn sequences_for_kind(outcome: &LocalMockOutcome, kind: obs::EventKind) -> Vec<u64> {
|
||||
outcome
|
||||
.trace
|
||||
.iter()
|
||||
.filter_map(|event| match event {
|
||||
obs::Event::ObjectScoped {
|
||||
kind: event_kind,
|
||||
sequence,
|
||||
..
|
||||
} if *event_kind == kind => Some(sequence.0),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn object_sequence_count(outcome: &LocalMockOutcome, kind: obs::EventKind, sequence: u64) -> usize {
|
||||
outcome
|
||||
.trace
|
||||
.iter()
|
||||
.filter(|event| match event {
|
||||
obs::Event::ObjectScoped {
|
||||
kind: event_kind,
|
||||
sequence: event_sequence,
|
||||
..
|
||||
} => *event_kind == kind && event_sequence.0 == sequence,
|
||||
_ => false,
|
||||
})
|
||||
.count()
|
||||
}
|
||||
|
||||
fn stage_position(outcome: &LocalMockOutcome, kind: obs::EventKind, stage_index: u32) -> usize {
|
||||
outcome
|
||||
.trace
|
||||
.iter()
|
||||
.position(|event| match event {
|
||||
obs::Event::StageScoped {
|
||||
kind: event_kind,
|
||||
stage_index: event_stage_index,
|
||||
..
|
||||
} => *event_kind == kind && event_stage_index.0 == stage_index,
|
||||
_ => false,
|
||||
})
|
||||
.unwrap_or_else(|| panic!("missing stage event {kind:?} for stage {stage_index}"))
|
||||
}
|
||||
|
||||
fn assert_only_teardown_after_terminal(outcome: &LocalMockOutcome, terminal: obs::EventKind) {
|
||||
let terminal_position = first_position(outcome, terminal);
|
||||
for event in &outcome.trace[terminal_position + 1..] {
|
||||
assert!(
|
||||
matches!(
|
||||
event.kind(),
|
||||
obs::EventKind::StopRunSent
|
||||
| obs::EventKind::StageStopped
|
||||
| obs::EventKind::RunTornDown
|
||||
),
|
||||
"non-teardown event after terminal outcome: {event:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,926 +0,0 @@
|
|||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
use crate::run_fsm as fsm;
|
||||
use crate::run_plan as plan;
|
||||
use crate::tests::harness::OrchestratorHarness;
|
||||
use data_plane::object_record::ObjectIdAllocator;
|
||||
use myelin::observability::lifecycle as obs;
|
||||
use myelin::orchestration::engine_builder as engine;
|
||||
use myelin::staging as stage;
|
||||
|
||||
use super::mock_node::MockNode;
|
||||
use super::mock_transport::{Delivery, MockObject, MockObjectKind, MockTransport};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct LocalMockConfig {
|
||||
pub stage_count: u32,
|
||||
pub max_tokens: u32,
|
||||
pub eos_after_sequence: u64,
|
||||
}
|
||||
|
||||
impl Default for LocalMockConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
stage_count: 2,
|
||||
max_tokens: 4,
|
||||
eos_after_sequence: 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct LocalMockCluster {
|
||||
run_id: plan::RunId,
|
||||
orchestrator_node_id: plan::NodeId,
|
||||
max_tokens: u32,
|
||||
engine_events: Vec<engine::EngineEvent>,
|
||||
plan: plan::RunPlan,
|
||||
nodes: BTreeMap<u32, MockNode>,
|
||||
orchestrator: Option<OrchestratorHarness>,
|
||||
orchestrator_command_cursor: usize,
|
||||
orchestrator_event_cursor: usize,
|
||||
trace: Vec<obs::Event>,
|
||||
transport: MockTransport,
|
||||
resources: ResourceTracker,
|
||||
observed_edges: BTreeSet<plan::EdgeId>,
|
||||
object_allocators: BTreeMap<plan::EdgeId, ObjectIdAllocator>,
|
||||
scenario: LocalMockScenario,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct LocalMockOutcome {
|
||||
pub trace: Vec<obs::Event>,
|
||||
pub engine_events: Vec<engine::EngineEvent>,
|
||||
pub injected_sequences: Vec<u64>,
|
||||
pub stage_count: usize,
|
||||
pub live_edges: usize,
|
||||
pub live_rings: usize,
|
||||
pub live_stage_runs: usize,
|
||||
pub transport_delivery_count: usize,
|
||||
pub transport_deliveries: Vec<Delivery>,
|
||||
pub edge_chain: Vec<plan::EdgeId>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
enum LocalMockScenario {
|
||||
#[default]
|
||||
Happy,
|
||||
UnauthorizedProvision {
|
||||
stage_index: u32,
|
||||
},
|
||||
WorkerCrashDuringExecution {
|
||||
stage_index: u32,
|
||||
sequence: u64,
|
||||
},
|
||||
SequenceViolation {
|
||||
stage_index: u32,
|
||||
sequence: u64,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ResourceTracker {
|
||||
live_edges: BTreeSet<u64>,
|
||||
live_rings: BTreeSet<u64>,
|
||||
live_stage_runs: BTreeSet<u32>,
|
||||
}
|
||||
|
||||
impl ResourceTracker {
|
||||
fn provision_edge(&mut self, edge_id: plan::EdgeId) {
|
||||
self.live_edges.insert(edge_id.0);
|
||||
self.live_rings.insert(20_000 + edge_id.0);
|
||||
}
|
||||
|
||||
fn provision_stage_run(&mut self, stage_index: u32) {
|
||||
self.live_stage_runs.insert(stage_index);
|
||||
}
|
||||
|
||||
fn release_stage_run(&mut self, stage_index: u32) {
|
||||
self.live_stage_runs.remove(&stage_index);
|
||||
}
|
||||
|
||||
fn release_all_edges(&mut self) {
|
||||
self.live_edges.clear();
|
||||
self.live_rings.clear();
|
||||
}
|
||||
}
|
||||
|
||||
fn mock_pool(orchestrator_node_id: plan::NodeId, stage_count: u32) -> engine::StaticPoolProvider {
|
||||
let mut nodes = Vec::with_capacity(stage_count as usize + 1);
|
||||
nodes.push(engine::NodeFacts {
|
||||
node_id: engine::NodeId(orchestrator_node_id.0),
|
||||
capabilities: vec![engine::NodeCapability::Coordinator],
|
||||
});
|
||||
for stage_index in 0..stage_count {
|
||||
nodes.push(engine::NodeFacts {
|
||||
node_id: engine::NodeId(11 + u64::from(stage_index)),
|
||||
capabilities: vec![engine::NodeCapability::Worker],
|
||||
});
|
||||
}
|
||||
engine::StaticPoolProvider::new(nodes)
|
||||
}
|
||||
|
||||
impl LocalMockCluster {
|
||||
pub fn two_stage() -> Self {
|
||||
Self::with_config(LocalMockConfig::default())
|
||||
}
|
||||
|
||||
pub fn with_config(config: LocalMockConfig) -> Self {
|
||||
assert!(
|
||||
config.stage_count > 0,
|
||||
"local mock needs at least one stage"
|
||||
);
|
||||
assert!(config.max_tokens > 0, "local mock needs at least one token");
|
||||
|
||||
let run_id = plan::RunId(77);
|
||||
let orchestrator_node_id = plan::NodeId(900);
|
||||
let engine_cluster = engine::ClusterBuilder::new(
|
||||
"local-mock",
|
||||
engine::ModelSpec::pipelined_causal_llm(
|
||||
"mock-gguf",
|
||||
plan::GgufSource::LocalPath("local-mock://mock-gguf".to_owned()),
|
||||
config.stage_count * 2,
|
||||
8,
|
||||
engine::DTypeFamily::BFloat,
|
||||
2,
|
||||
8,
|
||||
99,
|
||||
plan::TokenizerSource::EmbeddedGguf,
|
||||
),
|
||||
)
|
||||
.run_id(run_id.0)
|
||||
.pool_provider(mock_pool(orchestrator_node_id, config.stage_count))
|
||||
.planner(
|
||||
engine::FixedLinearPipelinePlanner::new(config.stage_count).runtime(
|
||||
plan::RuntimeConfig {
|
||||
max_tokens: config.max_tokens,
|
||||
sampling: plan::SamplingPolicy {
|
||||
temperature_millis: 0,
|
||||
top_k: 1,
|
||||
},
|
||||
},
|
||||
),
|
||||
)
|
||||
.launch()
|
||||
.expect("local mock engine builder must launch");
|
||||
let engine_events = engine_cluster.events().to_vec();
|
||||
let role_plan = engine_cluster.role_plan().clone();
|
||||
engine_cluster
|
||||
.shutdown()
|
||||
.expect("local mock engine builder must shutdown");
|
||||
let plan = role_plan.run_plan;
|
||||
let nodes = plan
|
||||
.stages
|
||||
.iter()
|
||||
.map(|stage| {
|
||||
(
|
||||
stage.stage_index,
|
||||
MockNode::new(
|
||||
stage.stage_index,
|
||||
stage.node_id,
|
||||
stage.stage_count,
|
||||
config.eos_after_sequence,
|
||||
),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
Self {
|
||||
run_id,
|
||||
orchestrator_node_id,
|
||||
engine_events,
|
||||
max_tokens: config.max_tokens,
|
||||
plan,
|
||||
nodes,
|
||||
orchestrator: None,
|
||||
orchestrator_command_cursor: 0,
|
||||
orchestrator_event_cursor: 0,
|
||||
trace: Vec::new(),
|
||||
transport: MockTransport::default(),
|
||||
resources: ResourceTracker::default(),
|
||||
observed_edges: BTreeSet::new(),
|
||||
object_allocators: BTreeMap::new(),
|
||||
scenario: LocalMockScenario::Happy,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn run_prompt(&mut self, prompt: &str) -> LocalMockOutcome {
|
||||
self.run_prompt_in_scenario(prompt, LocalMockScenario::Happy)
|
||||
}
|
||||
|
||||
pub fn run_prompt_with_delayed_stage_ready(
|
||||
&mut self,
|
||||
prompt: &str,
|
||||
delayed_stage_index: u32,
|
||||
) -> LocalMockOutcome {
|
||||
self.start_run(prompt, LocalMockScenario::Happy);
|
||||
self.process_orchestrator_commands_with_delayed_ready(Some(delayed_stage_index));
|
||||
assert_eq!(
|
||||
self.count_kind(obs::EventKind::PromptInjected),
|
||||
0,
|
||||
"prompt injection must be gated while one stage is not ready"
|
||||
);
|
||||
self.mark_stage_ready(delayed_stage_index);
|
||||
self.process_orchestrator_commands();
|
||||
self.finish_outcome()
|
||||
}
|
||||
|
||||
pub fn run_with_unauthorized_provision(
|
||||
&mut self,
|
||||
prompt: &str,
|
||||
stage_index: u32,
|
||||
) -> LocalMockOutcome {
|
||||
self.run_prompt_in_scenario(
|
||||
prompt,
|
||||
LocalMockScenario::UnauthorizedProvision { stage_index },
|
||||
)
|
||||
}
|
||||
|
||||
pub fn run_with_worker_crash_during_execution(
|
||||
&mut self,
|
||||
prompt: &str,
|
||||
stage_index: u32,
|
||||
sequence: u64,
|
||||
) -> LocalMockOutcome {
|
||||
self.run_prompt_in_scenario(
|
||||
prompt,
|
||||
LocalMockScenario::WorkerCrashDuringExecution {
|
||||
stage_index,
|
||||
sequence,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
pub fn run_with_sequence_violation(
|
||||
&mut self,
|
||||
prompt: &str,
|
||||
stage_index: u32,
|
||||
sequence: u64,
|
||||
) -> LocalMockOutcome {
|
||||
self.run_prompt_in_scenario(
|
||||
prompt,
|
||||
LocalMockScenario::SequenceViolation {
|
||||
stage_index,
|
||||
sequence,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
pub fn run_wrong_edge_object_then_prompt(
|
||||
&mut self,
|
||||
prompt: &str,
|
||||
stage_index: u32,
|
||||
) -> LocalMockOutcome {
|
||||
self.start_run(prompt, LocalMockScenario::Happy);
|
||||
self.process_orchestrator_commands_until(|command| {
|
||||
matches!(
|
||||
command,
|
||||
fsm::RunCommand::CreateTokenInEndpoint { .. }
|
||||
| fsm::RunCommand::CreateTokenOutEndpoint { .. }
|
||||
)
|
||||
});
|
||||
assert!(
|
||||
!self.inject_wrong_edge_object(stage_index),
|
||||
"wrong-edge object must not reach ExecuteStep"
|
||||
);
|
||||
self.process_orchestrator_commands();
|
||||
self.finish_outcome()
|
||||
}
|
||||
|
||||
fn run_prompt_in_scenario(
|
||||
&mut self,
|
||||
prompt: &str,
|
||||
scenario: LocalMockScenario,
|
||||
) -> LocalMockOutcome {
|
||||
self.start_run(prompt, scenario);
|
||||
self.process_orchestrator_commands();
|
||||
self.finish_outcome()
|
||||
}
|
||||
|
||||
fn start_run(&mut self, prompt: &str, scenario: LocalMockScenario) {
|
||||
self.trace.clear();
|
||||
self.transport = MockTransport::default();
|
||||
self.resources = ResourceTracker::default();
|
||||
self.observed_edges.clear();
|
||||
self.object_allocators.clear();
|
||||
self.orchestrator_command_cursor = 0;
|
||||
self.orchestrator_event_cursor = 0;
|
||||
self.scenario = scenario;
|
||||
self.orchestrator = Some(OrchestratorHarness::new(fsm::RunConfig {
|
||||
run_id: fsm::RunId(self.run_id.0),
|
||||
max_tokens: u64::from(self.max_tokens),
|
||||
prompt: tokenize(prompt),
|
||||
}));
|
||||
|
||||
let stage_node_ids: Vec<_> = self.plan.stages.iter().map(|stage| stage.node_id).collect();
|
||||
for node_id in &stage_node_ids {
|
||||
self.push_node(obs::EventKind::NodeStarted, *node_id);
|
||||
self.push_node(obs::EventKind::NodeAvailable, *node_id);
|
||||
}
|
||||
|
||||
self.push_run(obs::EventKind::PoolReady, obs::Component::Membership);
|
||||
self.orchestrator_mut().observe(fsm::RunEvent::PoolReady {
|
||||
nodes: stage_node_ids
|
||||
.iter()
|
||||
.map(|node_id| fsm::NodeId(node_id.0))
|
||||
.collect(),
|
||||
});
|
||||
|
||||
self.push_run(obs::EventKind::RunPlanned, obs::Component::Orchestrator);
|
||||
let fsm_plan = self.fsm_plan();
|
||||
self.orchestrator_mut()
|
||||
.observe(fsm::RunEvent::PlanAvailable(fsm_plan));
|
||||
}
|
||||
|
||||
fn process_orchestrator_commands(&mut self) {
|
||||
self.process_orchestrator_commands_with_delayed_ready(None);
|
||||
}
|
||||
|
||||
fn process_orchestrator_commands_with_delayed_ready(&mut self, delayed_stage: Option<u32>) {
|
||||
self.process_orchestrator_commands_until_with_delayed_ready(|_| false, delayed_stage);
|
||||
}
|
||||
|
||||
fn process_orchestrator_commands_until<F>(&mut self, should_pause: F)
|
||||
where
|
||||
F: FnMut(&fsm::RunCommand) -> bool,
|
||||
{
|
||||
self.process_orchestrator_commands_until_with_delayed_ready(should_pause, None);
|
||||
}
|
||||
|
||||
fn process_orchestrator_commands_until_with_delayed_ready<F>(
|
||||
&mut self,
|
||||
mut should_pause: F,
|
||||
delayed_stage: Option<u32>,
|
||||
) where
|
||||
F: FnMut(&fsm::RunCommand) -> bool,
|
||||
{
|
||||
loop {
|
||||
self.drain_orchestrator_lifecycle();
|
||||
let Some(command) = self.peek_orchestrator_command() else {
|
||||
break;
|
||||
};
|
||||
if should_pause(&command) {
|
||||
break;
|
||||
}
|
||||
let command = self
|
||||
.next_orchestrator_command()
|
||||
.expect("peeked command must still be present");
|
||||
self.process_orchestrator_command(command, delayed_stage);
|
||||
}
|
||||
self.drain_orchestrator_lifecycle();
|
||||
}
|
||||
|
||||
fn peek_orchestrator_command(&self) -> Option<fsm::RunCommand> {
|
||||
self.orchestrator
|
||||
.as_ref()?
|
||||
.commands()
|
||||
.get(self.orchestrator_command_cursor)
|
||||
.cloned()
|
||||
}
|
||||
|
||||
fn next_orchestrator_command(&mut self) -> Option<fsm::RunCommand> {
|
||||
let command = self.peek_orchestrator_command()?;
|
||||
self.orchestrator_command_cursor += 1;
|
||||
Some(command)
|
||||
}
|
||||
|
||||
fn process_orchestrator_command(
|
||||
&mut self,
|
||||
command: fsm::RunCommand,
|
||||
delayed_stage: Option<u32>,
|
||||
) {
|
||||
if self.terminal_observed()
|
||||
&& !matches!(
|
||||
command,
|
||||
fsm::RunCommand::StopRun { .. } | fsm::RunCommand::TearDownTokenEndpoints { .. }
|
||||
)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
match command {
|
||||
fsm::RunCommand::ProvisionStage { provision } => {
|
||||
self.provision_stage(provision.stage_index, delayed_stage);
|
||||
}
|
||||
fsm::RunCommand::CreateTokenInEndpoint { .. } => {
|
||||
self.orchestrator_mut()
|
||||
.observe(fsm::RunEvent::TokenInEndpointReady);
|
||||
}
|
||||
fsm::RunCommand::CreateTokenOutEndpoint { .. } => {
|
||||
self.orchestrator_mut()
|
||||
.observe(fsm::RunEvent::TokenOutEndpointReady);
|
||||
}
|
||||
fsm::RunCommand::InjectTokenObject { object, .. } => self.inject_token_object(object),
|
||||
fsm::RunCommand::StopRun { stage_index, .. } => self.stop_stage(stage_index),
|
||||
fsm::RunCommand::TearDownTokenEndpoints { .. } => {
|
||||
self.resources.release_all_edges();
|
||||
self.orchestrator_mut()
|
||||
.observe(fsm::RunEvent::TokenEndpointsStopped);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn provision_stage(&mut self, stage_index: u32, delayed_stage: Option<u32>) {
|
||||
let provision = plan::derive_stage_provision(&self.plan, stage_index)
|
||||
.expect("orchestrator must provision planned stages");
|
||||
self.resources.provision_stage_run(stage_index);
|
||||
self.push_stage(
|
||||
obs::EventKind::StageProvisionStarted,
|
||||
stage_index,
|
||||
obs::Component::Orchestrator,
|
||||
);
|
||||
self.push_stage(
|
||||
obs::EventKind::WeightsDownloadStarted,
|
||||
stage_index,
|
||||
obs::Component::WeightLifecycle,
|
||||
);
|
||||
self.push_stage(
|
||||
obs::EventKind::WeightsDownloaded,
|
||||
stage_index,
|
||||
obs::Component::WeightLifecycle,
|
||||
);
|
||||
self.push_stage(
|
||||
obs::EventKind::WeightsLoaded,
|
||||
stage_index,
|
||||
obs::Component::WeightLifecycle,
|
||||
);
|
||||
self.record_edge_ready(provision.inbound.edge_id);
|
||||
self.record_edge_ready(provision.outbound.edge_id);
|
||||
|
||||
let stage_provision = self.to_stage_provision(provision);
|
||||
let lifecycle_events = {
|
||||
let node = self
|
||||
.nodes
|
||||
.get_mut(&stage_index)
|
||||
.expect("mock node must exist for stage");
|
||||
if matches!(self.scenario, LocalMockScenario::UnauthorizedProvision { stage_index: target } if target == stage_index)
|
||||
{
|
||||
node.provision_from_wrong_orchestrator(stage_provision);
|
||||
} else {
|
||||
node.provision(stage::NodeId(self.orchestrator_node_id.0), stage_provision);
|
||||
if delayed_stage != Some(stage_index) {
|
||||
node.mark_ready();
|
||||
}
|
||||
}
|
||||
node.drain_lifecycle_events()
|
||||
};
|
||||
for event in lifecycle_events {
|
||||
self.handle_stage_lifecycle_event(event);
|
||||
}
|
||||
}
|
||||
|
||||
fn mark_stage_ready(&mut self, stage_index: u32) {
|
||||
let lifecycle_events = {
|
||||
let node = self
|
||||
.nodes
|
||||
.get_mut(&stage_index)
|
||||
.expect("mock node must exist for stage");
|
||||
node.mark_ready();
|
||||
node.drain_lifecycle_events()
|
||||
};
|
||||
for event in lifecycle_events {
|
||||
self.handle_stage_lifecycle_event(event);
|
||||
}
|
||||
}
|
||||
|
||||
fn record_edge_ready(&mut self, edge_id: plan::EdgeId) {
|
||||
if self.observed_edges.insert(edge_id) {
|
||||
self.resources.provision_edge(edge_id);
|
||||
self.push_edge(obs::EventKind::EdgeProvisionStarted, edge_id);
|
||||
self.push_edge(obs::EventKind::EdgeReady, edge_id);
|
||||
}
|
||||
}
|
||||
|
||||
fn inject_token_object(&mut self, object: fsm::TokenObjectInjection) {
|
||||
debug_assert!(matches!(
|
||||
object.payload,
|
||||
fsm::TokenObjectPayload::Prompt { .. } | fsm::TokenObjectPayload::Decode { .. }
|
||||
));
|
||||
if !self
|
||||
.trace
|
||||
.iter()
|
||||
.any(|event| event.kind() == obs::EventKind::ReadinessBarrierPassed)
|
||||
{
|
||||
self.push_run(
|
||||
obs::EventKind::ReadinessBarrierPassed,
|
||||
obs::Component::Orchestrator,
|
||||
);
|
||||
}
|
||||
let token_in_edge = self.edge_by_kind(plan::EdgeKind::TokenIn).edge_id;
|
||||
let object_id = self.allocate_object_id(token_in_edge);
|
||||
self.push_object(
|
||||
obs::EventKind::PromptInjected,
|
||||
object_id,
|
||||
object.sequence,
|
||||
obs::Component::TokenEndpoint,
|
||||
);
|
||||
let token_id = match object.payload {
|
||||
fsm::TokenObjectPayload::Prompt { .. } => None,
|
||||
fsm::TokenObjectPayload::Decode { token_id, .. } => Some(token_id),
|
||||
};
|
||||
let object = self.transport.deliver(MockObject {
|
||||
edge_id: token_in_edge,
|
||||
object_id,
|
||||
sequence: object.sequence,
|
||||
kind: MockObjectKind::Token,
|
||||
token_id,
|
||||
eos: false,
|
||||
});
|
||||
self.route_object_through_stages(object);
|
||||
}
|
||||
|
||||
fn route_object_through_stages(&mut self, object: MockObject) {
|
||||
let mut current = object;
|
||||
let stage_indices: Vec<_> = self
|
||||
.plan
|
||||
.stages
|
||||
.iter()
|
||||
.map(|stage| stage.stage_index)
|
||||
.collect();
|
||||
for stage_index in stage_indices {
|
||||
let mut stage_object = current;
|
||||
if self.scenario
|
||||
== (LocalMockScenario::SequenceViolation {
|
||||
stage_index,
|
||||
sequence: stage_object.sequence,
|
||||
})
|
||||
{
|
||||
stage_object.sequence += 1;
|
||||
}
|
||||
self.push_object(
|
||||
obs::EventKind::ObjectLoaded,
|
||||
stage_object.object_id,
|
||||
stage_object.sequence,
|
||||
obs::Component::GpuWorkerCtl,
|
||||
);
|
||||
|
||||
if self.scenario
|
||||
== (LocalMockScenario::WorkerCrashDuringExecution {
|
||||
stage_index,
|
||||
sequence: stage_object.sequence,
|
||||
})
|
||||
{
|
||||
let lifecycle_events = {
|
||||
let node = self
|
||||
.nodes
|
||||
.get_mut(&stage_index)
|
||||
.expect("mock node must exist for stage");
|
||||
node.crash_worker();
|
||||
node.drain_lifecycle_events()
|
||||
};
|
||||
for event in lifecycle_events {
|
||||
self.handle_stage_lifecycle_event(event);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let execution = {
|
||||
let node = self
|
||||
.nodes
|
||||
.get_mut(&stage_index)
|
||||
.expect("mock node must exist for stage");
|
||||
let execution = node.execute_loaded_object(stage_object);
|
||||
let lifecycle_events = node.drain_lifecycle_events();
|
||||
(execution, lifecycle_events)
|
||||
};
|
||||
for event in execution.1 {
|
||||
self.handle_stage_lifecycle_event(event);
|
||||
}
|
||||
let Some(execution) = execution.0 else {
|
||||
return;
|
||||
};
|
||||
self.push_step(obs::EventKind::ExecuteStepStarted, execution.step_id);
|
||||
self.push_object(
|
||||
obs::EventKind::ObjectProduced,
|
||||
execution.produced.object_id,
|
||||
execution.produced.sequence,
|
||||
obs::Component::GpuWorkerCtl,
|
||||
);
|
||||
self.push_step(obs::EventKind::StepCompleted, execution.step_id);
|
||||
let delivered = self.transport.deliver(execution.produced);
|
||||
let expected_edge = self
|
||||
.plan
|
||||
.stages
|
||||
.iter()
|
||||
.find(|stage| stage.stage_index == stage_index)
|
||||
.expect("mock stage must exist")
|
||||
.outbound_edge;
|
||||
debug_assert_eq!(delivered.edge_id, expected_edge);
|
||||
current = delivered;
|
||||
}
|
||||
self.push_object(
|
||||
obs::EventKind::TokenReceived,
|
||||
current.object_id,
|
||||
current.sequence,
|
||||
obs::Component::TokenEndpoint,
|
||||
);
|
||||
self.orchestrator_mut()
|
||||
.observe(fsm::RunEvent::TokenReceived {
|
||||
sequence: current.sequence,
|
||||
token_id: current.token_id.expect("last stage must produce token"),
|
||||
eos: current.eos,
|
||||
});
|
||||
self.drain_orchestrator_lifecycle();
|
||||
}
|
||||
|
||||
fn inject_wrong_edge_object(&mut self, stage_index: u32) -> bool {
|
||||
let stage = self
|
||||
.plan
|
||||
.stages
|
||||
.iter()
|
||||
.find(|stage| stage.stage_index == stage_index)
|
||||
.expect("mock stage must exist");
|
||||
let wrong_edge = stage.outbound_edge;
|
||||
let object_id = self.allocate_object_id(wrong_edge);
|
||||
let object = MockObject {
|
||||
edge_id: wrong_edge,
|
||||
object_id,
|
||||
sequence: 0,
|
||||
kind: MockObjectKind::Token,
|
||||
token_id: None,
|
||||
eos: false,
|
||||
};
|
||||
self.push_object(
|
||||
obs::EventKind::ObjectLoaded,
|
||||
object.object_id,
|
||||
object.sequence,
|
||||
obs::Component::GpuWorkerCtl,
|
||||
);
|
||||
let (execution, lifecycle_events) = {
|
||||
let node = self
|
||||
.nodes
|
||||
.get_mut(&stage_index)
|
||||
.expect("mock node must exist for stage");
|
||||
let execution = node.execute_loaded_object(object);
|
||||
let lifecycle_events = node.drain_lifecycle_events();
|
||||
(execution, lifecycle_events)
|
||||
};
|
||||
for event in lifecycle_events {
|
||||
self.handle_stage_lifecycle_event(event);
|
||||
}
|
||||
execution.is_some()
|
||||
}
|
||||
|
||||
fn stop_stage(&mut self, stage_index: u32) {
|
||||
self.push_stage(
|
||||
obs::EventKind::StopRunSent,
|
||||
stage_index,
|
||||
obs::Component::Orchestrator,
|
||||
);
|
||||
let lifecycle_events = {
|
||||
let node = self
|
||||
.nodes
|
||||
.get_mut(&stage_index)
|
||||
.expect("mock node must exist for stage");
|
||||
node.stop(self.run_id);
|
||||
node.drain_lifecycle_events()
|
||||
};
|
||||
for event in lifecycle_events {
|
||||
self.handle_stage_lifecycle_event(event);
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_stage_lifecycle_event(&mut self, event: stage::StageLifecycleEvent) {
|
||||
match event {
|
||||
stage::StageLifecycleEvent::StageReady {
|
||||
run_id,
|
||||
stage_index,
|
||||
} => {
|
||||
self.push_stage(
|
||||
obs::EventKind::StageReady,
|
||||
stage_index,
|
||||
obs::Component::StageController,
|
||||
);
|
||||
self.orchestrator_mut().observe(fsm::RunEvent::StageReady {
|
||||
run_id: fsm::RunId(run_id.0),
|
||||
stage_index,
|
||||
});
|
||||
}
|
||||
stage::StageLifecycleEvent::StageStopped {
|
||||
run_id,
|
||||
stage_index,
|
||||
} => {
|
||||
self.resources.release_stage_run(stage_index);
|
||||
self.push_stage(
|
||||
obs::EventKind::StageStopped,
|
||||
stage_index,
|
||||
obs::Component::StageController,
|
||||
);
|
||||
self.orchestrator_mut()
|
||||
.observe(fsm::RunEvent::StageStopped {
|
||||
run_id: fsm::RunId(run_id.0),
|
||||
stage_index,
|
||||
});
|
||||
}
|
||||
stage::StageLifecycleEvent::StageFault {
|
||||
run_id,
|
||||
stage_index,
|
||||
..
|
||||
} => {
|
||||
self.push_stage(
|
||||
obs::EventKind::StageFaulted,
|
||||
stage_index,
|
||||
obs::Component::StageController,
|
||||
);
|
||||
self.orchestrator_mut().observe(fsm::RunEvent::StageFault {
|
||||
run_id: fsm::RunId(run_id.0),
|
||||
stage_index,
|
||||
reason: fsm::StageFaultReason::WorkerCrashed,
|
||||
});
|
||||
}
|
||||
stage::StageLifecycleEvent::StepAccepted { .. } => {}
|
||||
}
|
||||
self.drain_orchestrator_lifecycle();
|
||||
}
|
||||
|
||||
fn drain_orchestrator_lifecycle(&mut self) {
|
||||
let Some(orchestrator) = &self.orchestrator else {
|
||||
return;
|
||||
};
|
||||
let events = orchestrator.events()[self.orchestrator_event_cursor..].to_vec();
|
||||
self.orchestrator_event_cursor = orchestrator.events().len();
|
||||
for event in events {
|
||||
match event {
|
||||
fsm::LifecycleEvent::RunCompleted { .. } => {
|
||||
self.push_run(obs::EventKind::RunCompleted, obs::Component::Orchestrator);
|
||||
}
|
||||
fsm::LifecycleEvent::RunFaulted { .. }
|
||||
| fsm::LifecycleEvent::RunOperatorStopped { .. } => {
|
||||
self.push_run(obs::EventKind::RunFaulted, obs::Component::Orchestrator);
|
||||
}
|
||||
fsm::LifecycleEvent::RunTornDown { .. } => {
|
||||
self.push_run(obs::EventKind::RunTornDown, obs::Component::Orchestrator);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn fsm_plan(&self) -> fsm::RunPlan {
|
||||
fsm::RunPlan::test_linear(
|
||||
fsm::RunId(self.run_id.0),
|
||||
self.plan
|
||||
.stages
|
||||
.iter()
|
||||
.map(|stage| fsm::StageRef {
|
||||
stage_index: stage.stage_index,
|
||||
node_id: fsm::NodeId(stage.node_id.0),
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
fn to_stage_provision(&self, provision: plan::ProvisionStage) -> stage::ProvisionStage {
|
||||
stage::ProvisionStage {
|
||||
run_id: stage::RunId(provision.run_id.0),
|
||||
authorized_orchestrator: stage::NodeId(self.orchestrator_node_id.0),
|
||||
node_id: stage::NodeId(provision.node_id.0),
|
||||
stage_index: provision.stage_index,
|
||||
stage_count: provision.stage_count,
|
||||
layer_range: stage::LayerRange {
|
||||
start: provision.layer_start,
|
||||
end_exclusive: provision.layer_end_exclusive,
|
||||
},
|
||||
inbound: stage::EdgeProvision::inbound(stage::EdgeId(provision.inbound.edge_id.0)),
|
||||
outbound: stage::EdgeProvision::outbound(stage::EdgeId(provision.outbound.edge_id.0)),
|
||||
weight_source: stage::WeightSource::new(
|
||||
provision.model.model_id,
|
||||
provision.gguf_source,
|
||||
provision.tokenizer,
|
||||
),
|
||||
shard_plan: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn edge_by_kind(&self, kind: plan::EdgeKind) -> &plan::EdgePlan {
|
||||
self.plan
|
||||
.edges
|
||||
.iter()
|
||||
.find(|edge| edge.kind == kind)
|
||||
.expect("mock plan must contain requested edge")
|
||||
}
|
||||
|
||||
fn edge_chain(&self) -> Vec<plan::EdgeId> {
|
||||
let mut stages = self.plan.stages.iter().collect::<Vec<_>>();
|
||||
stages.sort_by_key(|stage| stage.stage_index);
|
||||
let mut chain = Vec::with_capacity(stages.len() + 1);
|
||||
if let Some(first) = stages.first() {
|
||||
chain.push(first.inbound_edge);
|
||||
}
|
||||
chain.extend(stages.into_iter().map(|stage| stage.outbound_edge));
|
||||
chain
|
||||
}
|
||||
|
||||
fn allocate_object_id(&mut self, edge_id: plan::EdgeId) -> u64 {
|
||||
self.object_allocators
|
||||
.entry(edge_id)
|
||||
.or_insert_with(ObjectIdAllocator::new)
|
||||
.alloc()
|
||||
.0
|
||||
}
|
||||
|
||||
fn finish_outcome(&self) -> LocalMockOutcome {
|
||||
LocalMockOutcome {
|
||||
engine_events: self.engine_events.clone(),
|
||||
trace: self.trace.clone(),
|
||||
injected_sequences: self
|
||||
.orchestrator
|
||||
.as_ref()
|
||||
.map(|orchestrator| orchestrator.injected_sequences())
|
||||
.unwrap_or_default(),
|
||||
stage_count: self.plan.stages.len(),
|
||||
live_edges: self.resources.live_edges.len(),
|
||||
live_rings: self.resources.live_rings.len(),
|
||||
live_stage_runs: self.resources.live_stage_runs.len(),
|
||||
transport_delivery_count: self.transport.deliveries().len(),
|
||||
transport_deliveries: self.transport.deliveries().to_vec(),
|
||||
edge_chain: self.edge_chain(),
|
||||
}
|
||||
}
|
||||
|
||||
fn terminal_observed(&self) -> bool {
|
||||
self.trace.iter().any(|event| {
|
||||
matches!(
|
||||
event.kind(),
|
||||
obs::EventKind::RunCompleted | obs::EventKind::RunFaulted
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn count_kind(&self, kind: obs::EventKind) -> usize {
|
||||
self.trace
|
||||
.iter()
|
||||
.filter(|event| event.kind() == kind)
|
||||
.count()
|
||||
}
|
||||
|
||||
fn orchestrator_mut(&mut self) -> &mut OrchestratorHarness {
|
||||
self.orchestrator
|
||||
.as_mut()
|
||||
.expect("run_prompt must initialize orchestrator")
|
||||
}
|
||||
|
||||
fn push_run(&mut self, kind: obs::EventKind, component: obs::Component) {
|
||||
self.trace.push(obs::Event::RunScoped {
|
||||
kind,
|
||||
run_id: obs::RunId(self.run_id.0),
|
||||
reason: None,
|
||||
component,
|
||||
});
|
||||
}
|
||||
|
||||
fn push_node(&mut self, kind: obs::EventKind, node_id: plan::NodeId) {
|
||||
self.trace.push(obs::Event::NodeScoped {
|
||||
kind,
|
||||
node_id: obs::NodeId(node_id.0),
|
||||
component: obs::Component::NodeBoot,
|
||||
});
|
||||
}
|
||||
|
||||
fn push_stage(&mut self, kind: obs::EventKind, stage_index: u32, component: obs::Component) {
|
||||
self.trace.push(obs::Event::StageScoped {
|
||||
kind,
|
||||
run_id: obs::RunId(self.run_id.0),
|
||||
stage_index: obs::StageIndex(stage_index),
|
||||
reason: None,
|
||||
component,
|
||||
});
|
||||
}
|
||||
|
||||
fn push_edge(&mut self, kind: obs::EventKind, edge_id: plan::EdgeId) {
|
||||
self.trace.push(obs::Event::EdgeScoped {
|
||||
kind,
|
||||
edge_id: obs::EdgeId(edge_id.0),
|
||||
component: obs::Component::EdgeEstablisher,
|
||||
});
|
||||
}
|
||||
|
||||
fn push_object(
|
||||
&mut self,
|
||||
kind: obs::EventKind,
|
||||
object_id: u64,
|
||||
sequence: u64,
|
||||
component: obs::Component,
|
||||
) {
|
||||
self.trace.push(obs::Event::ObjectScoped {
|
||||
kind,
|
||||
object_id: obs::ObjectId(object_id),
|
||||
sequence: obs::Sequence(sequence),
|
||||
component,
|
||||
});
|
||||
}
|
||||
|
||||
fn push_step(&mut self, kind: obs::EventKind, step_id: u64) {
|
||||
self.trace.push(obs::Event::StepScoped {
|
||||
kind,
|
||||
step_id: obs::StepId(step_id),
|
||||
component: obs::Component::StageController,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
fn tokenize(prompt: &str) -> Vec<u32> {
|
||||
let tokens: Vec<u32> = prompt
|
||||
.split_whitespace()
|
||||
.enumerate()
|
||||
.map(|(index, _)| index as u32 + 1)
|
||||
.collect();
|
||||
if tokens.is_empty() { vec![0] } else { tokens }
|
||||
}
|
||||
|
|
@ -1,131 +0,0 @@
|
|||
use crate::run_plan as plan;
|
||||
use crate::tests::harness::StageControllerHarness;
|
||||
use data_plane::object_record::ObjectIdAllocator;
|
||||
use myelin::staging as stage;
|
||||
|
||||
use super::mock_transport::MockObject;
|
||||
use super::mock_worker::MockWorker;
|
||||
|
||||
pub struct MockExecution {
|
||||
pub step_id: u64,
|
||||
pub produced: MockObject,
|
||||
}
|
||||
|
||||
pub struct MockNode {
|
||||
pub stage_index: u32,
|
||||
stage_count: u32,
|
||||
inbound_edge: Option<plan::EdgeId>,
|
||||
outbound_edge: Option<plan::EdgeId>,
|
||||
outbound_object_allocator: Option<ObjectIdAllocator>,
|
||||
controller: StageControllerHarness,
|
||||
worker: MockWorker,
|
||||
event_cursor: usize,
|
||||
}
|
||||
|
||||
impl MockNode {
|
||||
pub fn new(
|
||||
stage_index: u32,
|
||||
node_id: plan::NodeId,
|
||||
stage_count: u32,
|
||||
eos_after_sequence: u64,
|
||||
) -> Self {
|
||||
Self {
|
||||
stage_index,
|
||||
stage_count,
|
||||
inbound_edge: None,
|
||||
outbound_edge: None,
|
||||
outbound_object_allocator: None,
|
||||
controller: StageControllerHarness::new(stage::NodeId(node_id.0)),
|
||||
worker: MockWorker::new(stage_index, eos_after_sequence),
|
||||
event_cursor: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn provision(&mut self, from: stage::NodeId, provision: stage::ProvisionStage) {
|
||||
self.inbound_edge = Some(plan::EdgeId(provision.inbound.edge_id.0));
|
||||
self.outbound_edge = Some(plan::EdgeId(provision.outbound.edge_id.0));
|
||||
self.outbound_object_allocator = Some(ObjectIdAllocator::new());
|
||||
self.controller
|
||||
.observe(stage::StageEvent::ProvisionStage { from, provision });
|
||||
}
|
||||
|
||||
pub fn provision_from_wrong_orchestrator(&mut self, provision: stage::ProvisionStage) {
|
||||
self.inbound_edge = Some(plan::EdgeId(provision.inbound.edge_id.0));
|
||||
self.outbound_edge = Some(plan::EdgeId(provision.outbound.edge_id.0));
|
||||
self.outbound_object_allocator = Some(ObjectIdAllocator::new());
|
||||
self.controller.observe(stage::StageEvent::ProvisionStage {
|
||||
from: stage::NodeId(provision.authorized_orchestrator.0 + 1),
|
||||
provision,
|
||||
});
|
||||
}
|
||||
|
||||
pub fn crash_worker(&mut self) {
|
||||
self.controller.observe(stage::StageEvent::WorkerCrashed);
|
||||
}
|
||||
|
||||
pub fn mark_ready(&mut self) {
|
||||
let inbound_edge = self.inbound_edge.expect("stage must be provisioned first");
|
||||
let outbound_edge = self.outbound_edge.expect("stage must be provisioned first");
|
||||
self.controller.observe(stage::StageEvent::WorkerReady);
|
||||
self.controller.observe(stage::StageEvent::WeightsReady);
|
||||
self.controller
|
||||
.observe(stage::StageEvent::InboundEdgeReady {
|
||||
edge_id: stage::EdgeId(inbound_edge.0),
|
||||
});
|
||||
self.controller
|
||||
.observe(stage::StageEvent::OutboundEdgeReady {
|
||||
edge_id: stage::EdgeId(outbound_edge.0),
|
||||
});
|
||||
}
|
||||
|
||||
pub fn execute_loaded_object(&mut self, object: MockObject) -> Option<MockExecution> {
|
||||
let outbound_edge = self.outbound_edge?;
|
||||
let command_start = self.controller.commands().len();
|
||||
self.controller.observe(stage::StageEvent::ObjectLoaded {
|
||||
edge_id: stage::EdgeId(object.edge_id.0),
|
||||
object_id: stage::ObjectId(object.object_id),
|
||||
sequence: object.sequence,
|
||||
handle: stage::DeviceHandle::new_current(object.object_id),
|
||||
});
|
||||
let step = self.controller.commands()[command_start..]
|
||||
.iter()
|
||||
.find_map(|command| match command {
|
||||
stage::StageCommand::ExecuteStep(step) => Some(step.clone()),
|
||||
_ => None,
|
||||
})?;
|
||||
let output_object_id = self.outbound_object_allocator.as_mut()?.alloc().0;
|
||||
let produced = self.worker.execute(
|
||||
&step,
|
||||
outbound_edge,
|
||||
output_object_id,
|
||||
self.stage_index + 1 == self.stage_count,
|
||||
);
|
||||
self.controller.observe(stage::StageEvent::StepCompleted {
|
||||
step_id: step.step_id,
|
||||
});
|
||||
Some(MockExecution {
|
||||
step_id: step.step_id.0,
|
||||
produced,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn stop(&mut self, run_id: plan::RunId) {
|
||||
let run_id = stage::RunId(run_id.0);
|
||||
self.controller
|
||||
.observe(stage::StageEvent::StopRun { run_id });
|
||||
self.controller
|
||||
.observe(stage::StageEvent::LocalEdgesStopped { run_id });
|
||||
self.controller
|
||||
.observe(stage::StageEvent::WorkerRingsQuiesced { run_id });
|
||||
self.controller
|
||||
.observe(stage::StageEvent::DeviceObjectsReleased { run_id });
|
||||
self.controller
|
||||
.observe(stage::StageEvent::WorkerRoleReset { run_id });
|
||||
}
|
||||
|
||||
pub fn drain_lifecycle_events(&mut self) -> Vec<stage::StageLifecycleEvent> {
|
||||
let events = self.controller.events()[self.event_cursor..].to_vec();
|
||||
self.event_cursor = self.controller.events().len();
|
||||
events
|
||||
}
|
||||
}
|
||||
|
|
@ -1,46 +0,0 @@
|
|||
use crate::run_plan as plan;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum MockObjectKind {
|
||||
Token,
|
||||
Activation,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct MockObject {
|
||||
pub edge_id: plan::EdgeId,
|
||||
pub object_id: u64,
|
||||
pub sequence: u64,
|
||||
pub kind: MockObjectKind,
|
||||
pub token_id: Option<u32>,
|
||||
pub eos: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Delivery {
|
||||
pub edge_id: plan::EdgeId,
|
||||
pub object_id: u64,
|
||||
pub sequence: u64,
|
||||
pub kind: MockObjectKind,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct MockTransport {
|
||||
deliveries: Vec<Delivery>,
|
||||
}
|
||||
|
||||
impl MockTransport {
|
||||
pub fn deliver(&mut self, object: MockObject) -> MockObject {
|
||||
self.deliveries.push(Delivery {
|
||||
edge_id: object.edge_id,
|
||||
object_id: object.object_id,
|
||||
sequence: object.sequence,
|
||||
kind: object.kind,
|
||||
});
|
||||
object
|
||||
}
|
||||
|
||||
pub fn deliveries(&self) -> &[Delivery] {
|
||||
&self.deliveries
|
||||
}
|
||||
}
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
use crate::run_plan as plan;
|
||||
use myelin::staging as stage;
|
||||
|
||||
use super::mock_transport::{MockObject, MockObjectKind};
|
||||
|
||||
pub struct MockWorker {
|
||||
eos_after_sequence: u64,
|
||||
}
|
||||
|
||||
impl MockWorker {
|
||||
pub fn new(_stage_index: u32, eos_after_sequence: u64) -> Self {
|
||||
Self { eos_after_sequence }
|
||||
}
|
||||
|
||||
pub fn execute(
|
||||
&self,
|
||||
step: &stage::ExecuteStep,
|
||||
outbound_edge: plan::EdgeId,
|
||||
object_id: u64,
|
||||
is_last_stage: bool,
|
||||
) -> MockObject {
|
||||
let sequence = step.input.sequence;
|
||||
if is_last_stage {
|
||||
MockObject {
|
||||
edge_id: outbound_edge,
|
||||
object_id,
|
||||
sequence,
|
||||
kind: MockObjectKind::Token,
|
||||
token_id: Some(if sequence >= self.eos_after_sequence {
|
||||
99
|
||||
} else {
|
||||
42
|
||||
}),
|
||||
eos: sequence >= self.eos_after_sequence,
|
||||
}
|
||||
} else {
|
||||
MockObject {
|
||||
edge_id: outbound_edge,
|
||||
object_id,
|
||||
sequence,
|
||||
kind: MockObjectKind::Activation,
|
||||
token_id: None,
|
||||
eos: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
mod assertions;
|
||||
mod environment;
|
||||
mod mock_node;
|
||||
mod mock_transport;
|
||||
mod mock_worker;
|
||||
|
||||
pub use assertions::{
|
||||
assert_happy_path_lifecycle, assert_terminal_fault, assert_terminal_success,
|
||||
assert_topology_surface,
|
||||
};
|
||||
pub use environment::{LocalMockCluster, LocalMockConfig, LocalMockOutcome};
|
||||
|
|
@ -1,8 +1,7 @@
|
|||
mod engine_composition;
|
||||
pub(crate) mod fuzz_support;
|
||||
mod harness;
|
||||
mod job_runner_iroh;
|
||||
mod local_e2e_guarantees;
|
||||
mod local_mock;
|
||||
mod node_guarantees;
|
||||
mod observability_guarantees;
|
||||
mod orchestration_guarantees;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
//! Behavior guarantees for the `node` module.
|
||||
//!
|
||||
//! These unit tests drive a manual `SingleThreadRuntime` host in isolation to verify actor
|
||||
//! message routing — they are not engine integration tests.
|
||||
//! These tests use the engine's deterministic stepping backend to verify actor
|
||||
//! message routing without constructing or driving a runtime directly.
|
||||
|
||||
use crate::node_actor::{NodeAgentActor, NodeAgentMsg, NodeAgentReport};
|
||||
use crate::orchestration::actor::OrchestratorMsg;
|
||||
|
|
@ -9,13 +9,15 @@ use iroh::{EndpointAddr, SecretKey};
|
|||
use myelin::staging as stage;
|
||||
use swactor::actor::ActorAddress;
|
||||
use swactor::config::RuntimeConfig;
|
||||
use swactor::runtime::{RuntimeParts, SingleThreadRuntime};
|
||||
use swactor::runtime::RuntimeParts;
|
||||
use swactor_engine::{Engine, SteppingBackend};
|
||||
|
||||
#[test]
|
||||
fn node_agent_runtime_loaded_reports_orchestrator() {
|
||||
let parts = RuntimeParts::new(RuntimeConfig::default());
|
||||
let runtime = parts.runtime().clone();
|
||||
let mut host = SingleThreadRuntime::new(parts);
|
||||
let backend = SteppingBackend::new();
|
||||
let _engine = Engine::new(parts, backend.clone()).expect("stepping engine");
|
||||
let orchestrator_inbox = runtime
|
||||
.new_inbox::<OrchestratorMsg>()
|
||||
.expect("orchestrator inbox");
|
||||
|
|
@ -39,7 +41,7 @@ fn node_agent_runtime_loaded_reports_orchestrator() {
|
|||
},
|
||||
)
|
||||
.expect("send runtime loaded");
|
||||
host.tick();
|
||||
backend.step();
|
||||
|
||||
assert_eq!(
|
||||
orchestrator_inbox.try_recv(),
|
||||
|
|
@ -58,7 +60,8 @@ fn node_agent_runtime_loaded_reports_orchestrator() {
|
|||
fn node_agent_runtime_ready_ack_reports_worker_loop() {
|
||||
let parts = RuntimeParts::new(RuntimeConfig::default());
|
||||
let runtime = parts.runtime().clone();
|
||||
let mut host = SingleThreadRuntime::new(parts);
|
||||
let backend = SteppingBackend::new();
|
||||
let _engine = Engine::new(parts, backend.clone()).expect("stepping engine");
|
||||
let orchestrator_inbox = runtime
|
||||
.new_inbox::<OrchestratorMsg>()
|
||||
.expect("orchestrator inbox");
|
||||
|
|
@ -84,7 +87,7 @@ fn node_agent_runtime_ready_ack_reports_worker_loop() {
|
|||
},
|
||||
)
|
||||
.expect("send runtime ready ack");
|
||||
host.tick();
|
||||
backend.step();
|
||||
|
||||
assert_eq!(
|
||||
reports.try_recv(),
|
||||
|
|
|
|||
|
|
@ -1,495 +1,5 @@
|
|||
//! Behavior guarantees for the `orchestration` module.
|
||||
|
||||
mod run_plan {
|
||||
//! Black-box contract tests for Myelin RunPlan formation.
|
||||
//!
|
||||
//! These tests intentionally know only the public planning surface:
|
||||
//!
|
||||
//! - `plan_run(input) -> Result<RunPlan, PlanRejection>`
|
||||
//! - `derive_stage_provision(&plan, stage_index) -> Result<ProvisionStage, ProjectionRejection>`
|
||||
//!
|
||||
//! They assert the guarantees in `specs/BEHAVIOR_GUARANTEES.md`.
|
||||
//! The planner implementation, placement heuristic, helper APIs, internal graph
|
||||
//! representation, and allocation strategy are not observable here.
|
||||
|
||||
use crate::run_plan as plan;
|
||||
|
||||
// Local aliases keep the test prose readable while importing only the public
|
||||
// planning module. The aliases do not grant access to planner internals.
|
||||
type DTypeFamily = plan::DTypeFamily;
|
||||
type EdgeEndpoint = plan::EdgeEndpoint;
|
||||
type EdgeKind = plan::EdgeKind;
|
||||
type GgufSource = plan::GgufSource;
|
||||
type ModelFacts = plan::ModelFacts;
|
||||
use plan::NodeId;
|
||||
type PlacementInput = plan::PlacementInput;
|
||||
type PlanRejectionKind = plan::PlanRejectionKind;
|
||||
type PlannerInput = plan::PlannerInput;
|
||||
type RingSpec = plan::RingSpec;
|
||||
type RuntimeConfig = plan::RuntimeConfig;
|
||||
type SamplingPolicy = plan::SamplingPolicy;
|
||||
type StagePlacement = plan::StagePlacement;
|
||||
type TokenizerSource = plan::TokenizerSource;
|
||||
|
||||
// Keep test node ids small and readable. The concrete identity mechanism is
|
||||
// outside this contract; these ids exist only so assertions can name topology
|
||||
// facts without depending on any address or discovery machinery.
|
||||
fn node(id: u64) -> NodeId {
|
||||
NodeId(id)
|
||||
}
|
||||
|
||||
// The candidate pool is deliberately larger than some test placements. That
|
||||
// lets the tests distinguish "known to the orchestrator" from "assigned to a
|
||||
// stage", which is one of the planner authority boundaries.
|
||||
fn valid_nodes() -> Vec<NodeId> {
|
||||
vec![node(10), node(11), node(12), node(13)]
|
||||
}
|
||||
|
||||
// Fixed linear placement is the smallest placement input that still exercises
|
||||
// the contract. It supplies stage-to-node intent, while the planner remains
|
||||
// responsible for validating it and minting the full RunPlan topology.
|
||||
fn linear_placement(stage_count: u32) -> PlacementInput {
|
||||
PlacementInput::FixedLinear(
|
||||
(0..stage_count)
|
||||
.map(|stage_index| StagePlacement {
|
||||
stage_index,
|
||||
node_id: node(10 + u64::from(stage_index)),
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
// This is the canonical valid fixture for RunPlan guarantees. Each test tweaks
|
||||
// only the fact it is trying to prove, so a failure points at the violated
|
||||
// contract instead of at accidental fixture drift.
|
||||
fn valid_input(stage_count: u32, num_layers: u32) -> PlannerInput {
|
||||
PlannerInput {
|
||||
run_id: 7.into(),
|
||||
orchestrator_node_id: node(99),
|
||||
model: ModelFacts {
|
||||
model_id: "test-gguf".into(),
|
||||
gguf_source: GgufSource::LocalPath("/models/test-gguf.gguf".into()),
|
||||
num_layers,
|
||||
hidden_dim: 4096,
|
||||
dtype_family: DTypeFamily::BFloat,
|
||||
dtype_width_bytes: 2,
|
||||
max_seq_len: 2048,
|
||||
eos_token_id: 2,
|
||||
tokenizer: TokenizerSource::LocalPath("/tokenizers/test-gguf.json".into()),
|
||||
},
|
||||
runtime: RuntimeConfig {
|
||||
max_tokens: 4,
|
||||
sampling: SamplingPolicy {
|
||||
temperature_millis: 125,
|
||||
top_k: 7,
|
||||
},
|
||||
},
|
||||
candidate_pool: valid_nodes(),
|
||||
stage_count,
|
||||
placement: linear_placement(stage_count),
|
||||
activation_ring: RingSpec {
|
||||
data_capacity: 1 << 20,
|
||||
alignment: 64,
|
||||
direction: plan::RingDirection::Egress,
|
||||
host_pinning: plan::HostPinning::Pageable,
|
||||
wake_coalescing: plan::WakeCoalescing::PendingBit,
|
||||
},
|
||||
token_ring: RingSpec {
|
||||
data_capacity: 4096,
|
||||
alignment: 8,
|
||||
direction: plan::RingDirection::Egress,
|
||||
host_pinning: plan::HostPinning::Pageable,
|
||||
wake_coalescing: plan::WakeCoalescing::PendingBit,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Edge endpoints can be orchestrator or stage endpoints. Tests use this helper
|
||||
// when they care only about stage adjacency and want orchestrator endpoints to
|
||||
// remain visibly outside the stage index space.
|
||||
fn edge_stage_index(endpoint: &EdgeEndpoint) -> Option<u32> {
|
||||
match endpoint {
|
||||
EdgeEndpoint::Orchestrator { .. } => None,
|
||||
EdgeEndpoint::Stage { stage_index, .. } => Some(*stage_index),
|
||||
}
|
||||
}
|
||||
|
||||
// This proves RunPlan formation is a total public boundary for valid input:
|
||||
// the caller observes one complete plan, not hidden follow-up topology work or
|
||||
// a partially initialized result.
|
||||
#[test]
|
||||
fn valid_input_emits_one_complete_plan() {
|
||||
// Build one ordinary valid planning request.
|
||||
let input = valid_input(3, 36);
|
||||
|
||||
// Planning valid input must produce a usable plan, not a deferred partial.
|
||||
let plan = plan::plan_run(input).expect("valid input must emit a plan");
|
||||
|
||||
// The plan-level identifiers and counts must be complete immediately.
|
||||
assert_eq!(plan.run_id, 7.into());
|
||||
assert_eq!(plan.stages.len(), 3);
|
||||
assert_eq!(plan.edges.len(), 4);
|
||||
assert_eq!(plan.max_tokens, 4);
|
||||
assert_eq!(plan.model.model_id, "test-gguf");
|
||||
assert_eq!(
|
||||
plan.model.gguf_source,
|
||||
GgufSource::LocalPath("/models/test-gguf.gguf".into())
|
||||
);
|
||||
assert_eq!(plan.model.num_layers, 36);
|
||||
assert_eq!(plan.model.hidden_dim, 4096);
|
||||
assert_eq!(plan.model.dtype_family, DTypeFamily::BFloat);
|
||||
assert_eq!(plan.model.dtype_width_bytes, 2);
|
||||
assert_eq!(plan.model.max_seq_len, 2048);
|
||||
assert_eq!(plan.model.eos_token_id, 2);
|
||||
assert_eq!(
|
||||
plan.model.tokenizer,
|
||||
TokenizerSource::LocalPath("/tokenizers/test-gguf.json".into())
|
||||
);
|
||||
assert_eq!(
|
||||
plan.sampling,
|
||||
SamplingPolicy {
|
||||
temperature_millis: 125,
|
||||
top_k: 7,
|
||||
}
|
||||
);
|
||||
|
||||
// Every stage must be bound to this run and know the run's stage count.
|
||||
for stage in &plan.stages {
|
||||
assert_eq!(stage.run_id, plan.run_id);
|
||||
assert_eq!(stage.stage_count, 3);
|
||||
assert_eq!(stage.gguf_source, plan.model.gguf_source);
|
||||
}
|
||||
|
||||
// Every edge must also be bound to this run; no edge can be a loose fact.
|
||||
for edge in &plan.edges {
|
||||
assert_eq!(edge.run_id, plan.run_id);
|
||||
}
|
||||
}
|
||||
// This proves layer assignment is a contiguous, non-overlapping partition of
|
||||
// the intended GGUF block range, with one non-empty range per stage.
|
||||
#[test]
|
||||
fn stage_ranges_partition_the_model_layers() {
|
||||
// Exercise several deterministic sizes so the check covers one-stage and
|
||||
// multi-stage partitioning without relying on random generation.
|
||||
for (stage_count, num_layers) in [(1, 12), (2, 24), (3, 36), (4, 40)] {
|
||||
// Produce the plan from public inputs.
|
||||
let plan = plan::plan_run(valid_input(stage_count, num_layers)).unwrap();
|
||||
|
||||
// Read only the public stage assignments and sort by stage index.
|
||||
let mut ranges = plan
|
||||
.stages
|
||||
.iter()
|
||||
.map(|stage| {
|
||||
(
|
||||
stage.stage_index,
|
||||
stage.layer_start,
|
||||
stage.layer_end_exclusive,
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
ranges.sort_by_key(|(stage_index, _, _)| *stage_index);
|
||||
|
||||
// Walk the sorted ranges as a proof of contiguity. The next start must
|
||||
// equal the previous end, and every range must consume at least one
|
||||
// layer inside the model range.
|
||||
let mut expected_start = 0;
|
||||
for (_, start, end) in ranges {
|
||||
assert_eq!(start, expected_start, "range gap or overlap");
|
||||
assert!(end > start, "stage range must be non-empty");
|
||||
assert!(end <= num_layers, "stage range exceeds model layer range");
|
||||
expected_start = end;
|
||||
}
|
||||
|
||||
// The final end must cover the whole intended block range.
|
||||
assert_eq!(expected_start, num_layers);
|
||||
}
|
||||
}
|
||||
// This proves every edge has one public producer and one public consumer, and
|
||||
// that the returned edge graph is exactly the Myelin linear pipeline.
|
||||
#[test]
|
||||
fn edge_graph_is_exactly_the_linear_pipeline() {
|
||||
// Use four stages so the activation chain has multiple interior edges.
|
||||
let stage_count = 4;
|
||||
let plan = plan::plan_run(valid_input(stage_count, 40)).unwrap();
|
||||
|
||||
// Token-in must be unique and must enter stage 0 from the orchestrator.
|
||||
let token_in = plan
|
||||
.edges
|
||||
.iter()
|
||||
.filter(|edge| edge.kind == EdgeKind::TokenIn)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(token_in.len(), 1);
|
||||
assert!(matches!(
|
||||
token_in[0].producer,
|
||||
EdgeEndpoint::Orchestrator { node_id } if node_id == node(99)
|
||||
));
|
||||
assert_eq!(
|
||||
token_in[0].consumer,
|
||||
EdgeEndpoint::Stage {
|
||||
node_id: node(10),
|
||||
stage_index: 0,
|
||||
}
|
||||
);
|
||||
|
||||
// Activation edges must be the only stage-to-stage edges, one per adjacent
|
||||
// stage pair.
|
||||
let activation_edges = plan
|
||||
.edges
|
||||
.iter()
|
||||
.filter(|edge| edge.kind == EdgeKind::Activation)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(activation_edges.len(), (stage_count - 1) as usize);
|
||||
|
||||
// Each activation edge produced by stage i must be consumed by stage i+1.
|
||||
for stage_index in 0..stage_count - 1 {
|
||||
let edge = activation_edges
|
||||
.iter()
|
||||
.find(|edge| edge_stage_index(&edge.producer) == Some(stage_index))
|
||||
.expect("activation edge produced by stage");
|
||||
|
||||
assert_eq!(
|
||||
edge.consumer,
|
||||
EdgeEndpoint::Stage {
|
||||
node_id: node(11 + u64::from(stage_index)),
|
||||
stage_index: stage_index + 1,
|
||||
}
|
||||
);
|
||||
assert_ne!(edge.producer, edge.consumer, "self-edge is forbidden");
|
||||
}
|
||||
|
||||
// Token-out must be unique and must leave the final stage for the
|
||||
// orchestrator.
|
||||
let token_out = plan
|
||||
.edges
|
||||
.iter()
|
||||
.filter(|edge| edge.kind == EdgeKind::TokenOut)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(token_out.len(), 1);
|
||||
assert_eq!(
|
||||
edge_stage_index(&token_out[0].producer),
|
||||
Some(stage_count - 1)
|
||||
);
|
||||
assert!(matches!(
|
||||
token_out[0].consumer,
|
||||
EdgeEndpoint::Orchestrator { node_id } if node_id == node(99)
|
||||
));
|
||||
}
|
||||
|
||||
// This proves edge ids are run-unique and that stage plans refer only to edge
|
||||
// ids present in the returned RunPlan, so stages receive assigned ids rather
|
||||
// than deriving data-flow identity themselves.
|
||||
#[test]
|
||||
fn edge_ids_are_unique_and_stage_references_resolve_to_plan_edges() {
|
||||
// Produce a plan with enough edges to make duplicate ids observable.
|
||||
let plan = plan::plan_run(valid_input(4, 40)).unwrap();
|
||||
|
||||
// Insert every public edge id into a set; a duplicate shrinks the set.
|
||||
let edge_ids = plan
|
||||
.edges
|
||||
.iter()
|
||||
.map(|edge| edge.edge_id)
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
|
||||
assert_eq!(edge_ids.len(), plan.edges.len(), "duplicate edge id");
|
||||
|
||||
// Stage plans may reference only ids that the RunPlan itself assigned.
|
||||
for stage in &plan.stages {
|
||||
assert!(
|
||||
edge_ids.contains(&stage.inbound_edge),
|
||||
"stage inbound edge id must come from RunPlan edges"
|
||||
);
|
||||
assert!(
|
||||
edge_ids.contains(&stage.outbound_edge),
|
||||
"stage outbound edge id must come from RunPlan edges"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// This proves deriving ProvisionStage is deterministic and stage-local:
|
||||
// repeated projection returns the same value, and the projected layer range is
|
||||
// exactly the range assigned to that stage in the RunPlan.
|
||||
#[test]
|
||||
fn provision_stage_projection_is_deterministic_and_stage_local() {
|
||||
// Start from one committed plan; projection is a pure public view of it.
|
||||
let plan = plan::plan_run(valid_input(3, 36)).unwrap();
|
||||
|
||||
// Check every stage projection, not only one representative stage.
|
||||
for stage_index in 0..3 {
|
||||
// Derive twice to prove projection does not depend on hidden mutable
|
||||
// state or call order.
|
||||
let first = plan::derive_stage_provision(&plan, stage_index).unwrap();
|
||||
let second = plan::derive_stage_provision(&plan, stage_index).unwrap();
|
||||
|
||||
// Find the corresponding public stage assignment in the plan.
|
||||
let stage = plan
|
||||
.stages
|
||||
.iter()
|
||||
.find(|stage| stage.stage_index == stage_index)
|
||||
.unwrap();
|
||||
|
||||
// The projected message must be stable and expose only that stage's
|
||||
// assigned run position and layer range.
|
||||
assert_eq!(first, second);
|
||||
assert_eq!(first.stage_index, stage_index);
|
||||
assert_eq!(first.stage_count, 3);
|
||||
assert_eq!(first.layer_start, stage.layer_start);
|
||||
assert_eq!(first.layer_end_exclusive, stage.layer_end_exclusive);
|
||||
assert_eq!(first.gguf_source, stage.gguf_source);
|
||||
assert_eq!(first.tokenizer, plan.model.tokenizer);
|
||||
assert_eq!(first.model.model_id, plan.model.model_id);
|
||||
assert_eq!(first.model.hidden_dim, plan.model.hidden_dim);
|
||||
assert_eq!(first.model.dtype_family, plan.model.dtype_family);
|
||||
assert_eq!(first.model.dtype_width_bytes, plan.model.dtype_width_bytes);
|
||||
assert_eq!(first.model.max_seq_len, plan.model.max_seq_len);
|
||||
assert_eq!(first.runtime.role_id, plan::RoleId(u64::from(stage_index)));
|
||||
assert_eq!(first.runtime.input_port, plan::PortId("input".into()));
|
||||
assert_eq!(first.runtime.output_port, plan::PortId("output".into()));
|
||||
let expected_sampling = if stage_index + 1 == stage.stage_count {
|
||||
Some(plan.sampling)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
assert_eq!(first.runtime.sampling, expected_sampling);
|
||||
}
|
||||
}
|
||||
|
||||
// This proves each stage receives exactly one inbound and one outbound edge
|
||||
// provision, and that the provisioned ids are the ids assigned to that stage by
|
||||
// the RunPlan.
|
||||
#[test]
|
||||
fn provision_stage_contains_exactly_the_assigned_inbound_and_outbound_edges() {
|
||||
// Build a valid plan and test projection for every stage in it.
|
||||
let plan = plan::plan_run(valid_input(3, 36)).unwrap();
|
||||
|
||||
for stage in &plan.stages {
|
||||
// Derive the public provisioning message for this stage.
|
||||
let provision = plan::derive_stage_provision(&plan, stage.stage_index).unwrap();
|
||||
|
||||
// The message exposes exactly the inbound and outbound ids assigned in
|
||||
// that stage's StagePlan.
|
||||
assert_eq!(provision.inbound.edge_id, stage.inbound_edge);
|
||||
assert_eq!(provision.outbound.edge_id, stage.outbound_edge);
|
||||
}
|
||||
}
|
||||
// This proves typed rejection is the public behavior for invalid authority and
|
||||
// topology inputs. No invalid case is allowed to emit a partial plan.
|
||||
#[test]
|
||||
fn invalid_authority_and_topology_inputs_reject_without_plan() {
|
||||
// Each case changes one authority/topology fact from the valid fixture and
|
||||
// names the typed rejection the planner must expose.
|
||||
let cases = [
|
||||
(
|
||||
invalid_unknown_node(),
|
||||
PlanRejectionKind::UnknownNode,
|
||||
"unknown node id",
|
||||
),
|
||||
(
|
||||
invalid_duplicate_stage_assignment(),
|
||||
PlanRejectionKind::DuplicateStageAssignment,
|
||||
"duplicate stage assignment",
|
||||
),
|
||||
(
|
||||
invalid_missing_stage_assignment(),
|
||||
PlanRejectionKind::MissingStage,
|
||||
"missing stage",
|
||||
),
|
||||
(
|
||||
invalid_zero_stage_count(),
|
||||
PlanRejectionKind::InvalidStageCount,
|
||||
"invalid stage count",
|
||||
),
|
||||
(
|
||||
invalid_model_stage_layout(),
|
||||
PlanRejectionKind::ModelStageLayoutMismatch,
|
||||
"model/stage layout mismatch",
|
||||
),
|
||||
];
|
||||
|
||||
// Invalid inputs must not produce a partial plan. The observable result is
|
||||
// a typed rejection kind.
|
||||
for (input, expected, label) in cases {
|
||||
let err = plan::plan_run(input).expect_err(label);
|
||||
assert_eq!(err.kind(), expected, "{label}");
|
||||
}
|
||||
}
|
||||
// Unknown-node rejection needs the placement to name a node outside the
|
||||
// orchestrator's candidate pool. The rest of the input stays valid so the
|
||||
// expected rejection is isolated to authority over node identity.
|
||||
fn invalid_unknown_node() -> PlannerInput {
|
||||
let mut input = valid_input(3, 36);
|
||||
input.placement = PlacementInput::FixedLinear(vec![
|
||||
StagePlacement {
|
||||
stage_index: 0,
|
||||
node_id: node(10),
|
||||
},
|
||||
StagePlacement {
|
||||
stage_index: 1,
|
||||
node_id: node(404),
|
||||
},
|
||||
StagePlacement {
|
||||
stage_index: 2,
|
||||
node_id: node(12),
|
||||
},
|
||||
]);
|
||||
input
|
||||
}
|
||||
|
||||
// Duplicate-stage rejection is observable when two placement entries claim the
|
||||
// same stage index. This checks that the planner does not silently pick one and
|
||||
// continue with ambiguous authority.
|
||||
fn invalid_duplicate_stage_assignment() -> PlannerInput {
|
||||
let mut input = valid_input(3, 36);
|
||||
input.placement = PlacementInput::FixedLinear(vec![
|
||||
StagePlacement {
|
||||
stage_index: 0,
|
||||
node_id: node(10),
|
||||
},
|
||||
StagePlacement {
|
||||
stage_index: 1,
|
||||
node_id: node(11),
|
||||
},
|
||||
StagePlacement {
|
||||
stage_index: 1,
|
||||
node_id: node(12),
|
||||
},
|
||||
]);
|
||||
input
|
||||
}
|
||||
|
||||
// Missing-stage rejection is observable when placement skips an index inside
|
||||
// `0..stage_count`. This checks that the planner does not invent hidden stage
|
||||
// ownership to patch an incomplete placement.
|
||||
fn invalid_missing_stage_assignment() -> PlannerInput {
|
||||
let mut input = valid_input(3, 36);
|
||||
input.placement = PlacementInput::FixedLinear(vec![
|
||||
StagePlacement {
|
||||
stage_index: 0,
|
||||
node_id: node(10),
|
||||
},
|
||||
StagePlacement {
|
||||
stage_index: 2,
|
||||
node_id: node(12),
|
||||
},
|
||||
]);
|
||||
input
|
||||
}
|
||||
|
||||
// Zero stages cannot form the Myelin pipeline. This fixture isolates the invalid
|
||||
// stage-count path without adding any other contradictory facts.
|
||||
fn invalid_zero_stage_count() -> PlannerInput {
|
||||
valid_input(0, 36)
|
||||
}
|
||||
|
||||
// The current contract requires one non-empty layer range per stage. Fewer
|
||||
// layers than stages forces an empty range unless explicitly allowed, so this
|
||||
// fixture should reject at the model/stage-layout boundary.
|
||||
fn invalid_model_stage_layout() -> PlannerInput {
|
||||
let mut input = valid_input(4, 3);
|
||||
input.placement = linear_placement(4);
|
||||
input
|
||||
}
|
||||
}
|
||||
|
||||
mod run_fsm {
|
||||
//! Black-box contract tests for the Myelin orchestrator run FSM.
|
||||
//!
|
||||
|
|
|
|||
|
|
@ -601,29 +601,16 @@ mod weight_lifecycle {
|
|||
harness.observe(weights::WeightEvent::Provisioned(valid_assignment()));
|
||||
|
||||
// The load command may use the physical source, but its graph-visible layer
|
||||
// range must be the assigned range.
|
||||
for command in harness.commands() {
|
||||
if let weights::WeightCommand::LoadOrBindRange { range, .. } = command {
|
||||
assert_eq!(
|
||||
*range,
|
||||
weights::LayerRange {
|
||||
start: 12,
|
||||
end_exclusive: 24,
|
||||
}
|
||||
);
|
||||
// range must be exactly the assigned range.
|
||||
assert_eq!(harness.commands().len(), 1);
|
||||
let weights::WeightCommand::LoadOrBindRange { range, .. } = &harness.commands()[0];
|
||||
assert_eq!(
|
||||
*range,
|
||||
weights::LayerRange {
|
||||
start: 12,
|
||||
end_exclusive: 24,
|
||||
}
|
||||
}
|
||||
|
||||
// There must be no command claiming ownership of neighboring layers.
|
||||
assert!(!harness.commands().iter().any(|command| {
|
||||
matches!(
|
||||
command,
|
||||
weights::WeightCommand::AdvertiseLoadedLayerRange {
|
||||
range,
|
||||
..
|
||||
} if range.start < 12 || range.end_exclusive > 24
|
||||
)
|
||||
}));
|
||||
);
|
||||
}
|
||||
|
||||
// This proves whole GGUF download, shard download, and cache use are physical
|
||||
|
|
|
|||
2319
apps/myelin/tests/stateful_vastai.rs
Normal file
2319
apps/myelin/tests/stateful_vastai.rs
Normal file
File diff suppressed because it is too large
Load diff
55
clippy.toml
55
clippy.toml
|
|
@ -1,52 +1,3 @@
|
|||
# Clippy enforcement policy for the swactor engine boundary
|
||||
# (ENGINE_SPEC.md §2 / §3.1).
|
||||
#
|
||||
# These direct runtime / scheduling / time / core-driving operations are
|
||||
# disallowed outside the engine's own substrate implementation. Integrations
|
||||
# (iroh-driver, myelin, ...) must go through `EngineHandle`. The
|
||||
# `swactor-engine` Tokio backend and the core driver carry narrow
|
||||
# `#[allow(clippy::disallowed_methods)]` exemptions because they ARE the
|
||||
# substrate implementor; the VastAI provider module carries a temporary
|
||||
# module-level exemption pending its separate redesign (out of scope per §2).
|
||||
#
|
||||
# In-scope work that backs actors, transport, RPC, sampling, or node/orchestrator
|
||||
# progression must schedule through `EngineHandle`. The only retained direct
|
||||
# uses are narrow exclusions (§2): provider adapters/lifecycle (VastAI),
|
||||
# provider-specific process supervision and log capture, and top-level OS-signal
|
||||
# / blocking user-stdin / synchronous process-control sequencing. Each retained
|
||||
# use carries a local `#[allow]` with its exclusion reason.
|
||||
#
|
||||
# Workspace-wide enforcement: `swactor-engine`, `iroh-driver`, and in-scope
|
||||
# `myelin` carry `#![deny(clippy::disallowed_methods)]` and pass clean.
|
||||
|
||||
disallowed-methods = [
|
||||
{ path = "tokio::runtime::Runtime::new", reason = "runtime ownership belongs to the engine; construct an engine-owned substrate instead" },
|
||||
{ path = "tokio::runtime::Builder::new_current_thread", reason = "runtime ownership belongs to the engine; use EngineHandle" },
|
||||
{ path = "tokio::runtime::Builder::new_multi_thread", reason = "runtime ownership belongs to the engine; use EngineHandle" },
|
||||
{ path = "tokio::runtime::Handle::current", reason = "ambient runtime detection is forbidden; construct an engine-owned substrate instead" },
|
||||
{ path = "tokio::runtime::Handle::try_current", reason = "ambient runtime detection is forbidden; construct an engine-owned substrate instead" },
|
||||
{ path = "tokio::runtime::Runtime::block_on", reason = "blocking on a runtime is forbidden; schedule through EngineHandle" },
|
||||
{ path = "tokio::runtime::Handle::block_on", reason = "blocking on a runtime is forbidden; schedule through EngineHandle" },
|
||||
|
||||
{ path = "tokio::spawn", reason = "direct scheduling is forbidden; use EngineHandle::spawn" },
|
||||
{ path = "tokio::task::spawn", reason = "direct scheduling is forbidden; use EngineHandle::spawn" },
|
||||
{ path = "tokio::task::spawn_blocking", reason = "use EngineHandle::spawn_blocking" },
|
||||
{ path = "tokio::runtime::Runtime::spawn", reason = "direct scheduling is forbidden; use EngineHandle::spawn" },
|
||||
{ path = "tokio::runtime::Handle::spawn", reason = "direct scheduling is forbidden; use EngineHandle::spawn" },
|
||||
{ path = "tokio::runtime::Runtime::spawn_blocking", reason = "use EngineHandle::spawn_blocking" },
|
||||
{ path = "tokio::runtime::Handle::spawn_blocking", reason = "use EngineHandle::spawn_blocking" },
|
||||
|
||||
{ path = "tokio::time::sleep", reason = "use EngineHandle::timer" },
|
||||
{ path = "tokio::time::sleep_until", reason = "use EngineHandle::timer" },
|
||||
{ path = "tokio::time::interval", reason = "use EngineHandle::interval" },
|
||||
{ path = "tokio::time::interval_at", reason = "use EngineHandle::interval" },
|
||||
{ path = "tokio::time::timeout", reason = "use an engine-derived timeout" },
|
||||
{ path = "tokio::time::timeout_at", reason = "use an engine-derived timeout" },
|
||||
|
||||
{ path = "std::thread::spawn", reason = "direct thread scheduling is forbidden; schedule through EngineHandle" },
|
||||
{ path = "std::thread::sleep", reason = "use EngineHandle::timer; retained only for narrow process-control exclusions (ENGINE_SPEC.md §2)" },
|
||||
|
||||
{ path = "swactor::runtime::Runtime::tick", reason = "manual core driving is forbidden; the engine owns core progression" },
|
||||
{ path = "swactor::runtime::Runtime::try_tick", reason = "manual core driving is forbidden; the engine owns core progression" },
|
||||
{ path = "swactor::runtime::Runtime::has_work", reason = "manual core driving is forbidden; the engine owns core progression" },
|
||||
]
|
||||
# Actor control-flow policy is enforced by the repository rustc workspace
|
||||
# wrapper configured in `.cargo/config.toml`. Clippy is intentionally not a
|
||||
# second architecture-policy mechanism.
|
||||
|
|
|
|||
|
|
@ -10,4 +10,5 @@ crate-type = ["cdylib"]
|
|||
|
||||
[dependencies]
|
||||
swactor = { path = "../../.." }
|
||||
swactor-engine = { path = "../../engine", default-features = false }
|
||||
pyo3 = { version = "0.23", features = ["extension-module"] }
|
||||
|
|
|
|||
|
|
@ -8,9 +8,8 @@ use ::swactor::actor::{
|
|||
Actor, ActorAddress, ActorInterface, AnyActor, Ctx, Environment, SpawnRequest,
|
||||
};
|
||||
use ::swactor::config::RuntimeConfig;
|
||||
use ::swactor::runtime::{
|
||||
Inbox, Runtime, RuntimeParts, SingleThreadRuntime as SingleThreadRuntimeHost,
|
||||
};
|
||||
use ::swactor::runtime::{Inbox, Runtime, RuntimeParts};
|
||||
use swactor_engine::{Engine, SteppingBackend};
|
||||
|
||||
// ─── PyMsg newtype ───────────────────────────────────────────────────────────
|
||||
|
||||
|
|
@ -279,7 +278,8 @@ impl From<PyRuntimeConfig> for RuntimeConfig {
|
|||
#[pyclass(name = "Runtime", unsendable)]
|
||||
pub struct PyRuntime {
|
||||
runtime: Runtime,
|
||||
host: SingleThreadRuntimeHost,
|
||||
_engine: Engine,
|
||||
backend: SteppingBackend,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
|
|
@ -293,8 +293,13 @@ impl PyRuntime {
|
|||
};
|
||||
let parts = RuntimeParts::new(config);
|
||||
let runtime = parts.runtime().clone();
|
||||
let host = SingleThreadRuntimeHost::new(parts);
|
||||
Self { runtime, host }
|
||||
let backend = SteppingBackend::new();
|
||||
let engine = Engine::new(parts, backend.clone()).expect("create Python actor engine");
|
||||
Self {
|
||||
runtime,
|
||||
_engine: engine,
|
||||
backend,
|
||||
}
|
||||
}
|
||||
|
||||
fn spawn(&self, handler: PyObject) -> PyResult<PyActorAddress> {
|
||||
|
|
@ -316,7 +321,7 @@ impl PyRuntime {
|
|||
}
|
||||
|
||||
fn tick(&mut self) -> PyResult<()> {
|
||||
self.host.tick();
|
||||
self.backend.step();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -326,7 +331,6 @@ impl PyRuntime {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
// ─── ActorInfo / RuntimeStats ────────────────────────────────────────────────
|
||||
|
||||
#[pyclass(name = "ActorInfo")]
|
||||
|
|
|
|||
|
|
@ -9,4 +9,5 @@ crate-type = ["cdylib"]
|
|||
|
||||
[dependencies]
|
||||
swactor = { path = "../../..", default-features = false, features = ["wasm", "std"] }
|
||||
swactor-engine = { path = "../../engine", default-features = false }
|
||||
wasm-bindgen = "0.2"
|
||||
|
|
|
|||
|
|
@ -3,11 +3,9 @@ use std::sync::Arc;
|
|||
use wasm_bindgen::prelude::*;
|
||||
|
||||
use swactor::actor::{ActorAddress, ActorExited, ActorInterface};
|
||||
use swactor::runtime::{
|
||||
Ctx, Inbox, Runtime, RuntimeConfig, RuntimeParts,
|
||||
SingleThreadRuntime as SingleThreadRuntimeHost,
|
||||
};
|
||||
use swactor::runtime::{Ctx, Inbox, Runtime, RuntimeConfig, RuntimeParts};
|
||||
use swactor::std::{CtxGroups, CtxWatching, RuntimeGroups, RuntimeNaming, StdExtension};
|
||||
use swactor_engine::{Engine, SteppingBackend};
|
||||
|
||||
// ─── Core JS-facing types ───────────────────────────────────────────────────
|
||||
|
||||
|
|
@ -94,14 +92,15 @@ impl WasmInboxString {
|
|||
|
||||
/// The browser-facing swactor runtime.
|
||||
///
|
||||
/// Owns a cloneable `swactor::Runtime` handle plus the single-threaded host that
|
||||
/// drives its workers, with StdExtension installed (naming, monitoring, groups).
|
||||
/// Owns a cloneable `swactor::Runtime` handle plus an engine-backed stepping
|
||||
/// substrate, with StdExtension installed (naming, monitoring, groups).
|
||||
/// Actors are spawned via dedicated spawn functions (one per actor type). The
|
||||
/// runtime is driven by calling `tick()`.
|
||||
#[wasm_bindgen]
|
||||
pub struct WasmRuntime {
|
||||
rt: Runtime,
|
||||
host: SingleThreadRuntimeHost,
|
||||
_engine: Engine,
|
||||
backend: SteppingBackend,
|
||||
}
|
||||
|
||||
#[wasm_bindgen]
|
||||
|
|
@ -111,13 +110,18 @@ impl WasmRuntime {
|
|||
let parts = RuntimeParts::new(RuntimeConfig::default())
|
||||
.with_extension(Arc::new(StdExtension::new()));
|
||||
let rt = parts.runtime().clone();
|
||||
let host = SingleThreadRuntimeHost::new(parts);
|
||||
Self { rt, host }
|
||||
let backend = SteppingBackend::new();
|
||||
let engine = Engine::new(parts, backend.clone()).expect("create wasm actor engine");
|
||||
Self {
|
||||
rt,
|
||||
_engine: engine,
|
||||
backend,
|
||||
}
|
||||
}
|
||||
|
||||
/// Drive one tick of the runtime.
|
||||
pub fn tick(&mut self) {
|
||||
self.host.tick();
|
||||
self.backend.step();
|
||||
}
|
||||
|
||||
/// Number of actors currently alive.
|
||||
|
|
|
|||
|
|
@ -8,4 +8,4 @@ Keep this crate read-only with respect to observed programs.
|
|||
- It must not send control signals to observed runtimes.
|
||||
- It must not require changes outside `crates/dashboard` for dashboard-only work.
|
||||
|
||||
Main built-in view: the fused control plane at `/` and `/view/fleet` (node cards with machine + actor rollup, per-node roster, per-actor dossier via `/api/view/fleet/detail`), backed by `host.*`, `proc.<label>.lifecycle`, `runtime.stats`, and `runtime.actors` frames when present. It is a pure frame consumer, tolerant of publisher shape. Message history is folded view-side from `messages_processed` deltas — no producer changes. The unified navbar is injected server-side from the view registry; pages opt in with a `<!--swactor:nav-->` placeholder.
|
||||
Main built-in view: the fused control plane at `/` and `/view/fleet` (node cards with machine + actor rollup, per-node roster, per-actor dossier via `/api/view/fleet/detail`), backed by `host.*`, `proc.<label>.lifecycle`, `runtime.stats`, and `runtime.actors` frames when present. It is a pure frame consumer, tolerant of publisher shape. Message history is folded view-side from `messages_processed` deltas - no producer changes. The unified navbar is injected server-side from the view and application plugin-page registries; pages opt in with a `<!--swactor:nav-->` placeholder. Plugin routers remain application-owned composition inputs and must not add dashboard-owned control state.
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ license = "AGPL-3.0-only"
|
|||
axum = "0.8"
|
||||
telemetry = { path = "../telemetry" }
|
||||
swactor = { path = "../..", features = ["serde"] }
|
||||
swactor-engine = { path = "../engine" }
|
||||
parking_lot = "0.12"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
|
|
@ -18,3 +19,7 @@ tokio-stream = "0.1"
|
|||
# Live control actions (kill processes, provision nodes) for the
|
||||
# provisioning-reconciler demo. Never enabled in shipping builds.
|
||||
demo-control = []
|
||||
|
||||
[dev-dependencies]
|
||||
proptest = "1"
|
||||
tower = { version = "0.5", features = ["util"] }
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
Read-only HTML/SSE dashboard over incoming telemetry frames.
|
||||
|
||||
The crate owns the Axum server, bounded raw frame window, and view registry. Component crates can keep their own view implementations beside their code and register them through `DashboardHandle::register_view`. The built-in control-plane view is hosted here because worker/actor/message processing is universal to swactor programs.
|
||||
The crate owns the Axum server, bounded raw frame window, view registry, and application plugin-page registry. Component crates can keep telemetry views beside their code and register them through `DashboardHandle::register_view`. An embedding application can pass `DashboardPlugin` values to `DashboardHandle::with_plugins`; each plugin contributes an application-owned router and optional `PluginPage` metadata/HTML. The built-in control-plane view is hosted here because worker/actor/message processing is universal to swactor programs.
|
||||
|
||||
The control-plane page fuses machine stats and actor stats per node stream: node cards (CPU/GPU/net + actor rollup) → per-node actor roster → per-actor dossier (identity, message diet, sampled message history). The Rust type name is the actor's display name; the address is the unique key. Stale streams (silent beyond the liveness window) render in a separate collapsed pool, superseded `life` generations are evicted immediately, and the stale pool is hard-capped.
|
||||
|
||||
|
|
@ -18,6 +18,6 @@ The control-plane page fuses machine stats and actor stats per node stream: node
|
|||
- `GET /api/view/fleet` — live/stale pools with per-node machine and roster snapshot
|
||||
- `GET /api/view/fleet/detail?stream=<node#life>&actor=<addr>` — bounded per-actor dossier detail (diet, history, sampled receipts)
|
||||
|
||||
Every page carries the unified top navbar, built from the view registry at serve time — pages include a `<!--swactor:nav-->` placeholder and the server substitutes the links, so app-registered views appear automatically.
|
||||
Every page carries the unified top navbar, built from the view and plugin-page registries at serve time. Pages include a `<!--swactor:nav-->` placeholder and the server substitutes the links, so registered telemetry views and application plugin pages appear automatically.
|
||||
|
||||
All state is derived from observed frames. The dashboard sends no control signals back to producers.
|
||||
Dashboard-owned state is derived only from observed frames, and the dashboard crate sends no control signals back to producers. A plugin router remains owned by the embedding application; composing that router does not grant dashboard views mutation capabilities.
|
||||
|
|
|
|||
7
crates/dashboard/proptest-regressions/control.txt
Normal file
7
crates/dashboard/proptest-regressions/control.txt
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
# Seeds for failure cases proptest has generated in the past. It is
|
||||
# automatically read and these particular cases re-run before any
|
||||
# novel cases are generated.
|
||||
#
|
||||
# It is recommended to check this file in to source control so that
|
||||
# everyone who runs the test benefits from these saved cases.
|
||||
cc 6dbd7f7ab6013cac23e3e815f076193eac00fc906957d486983845194cf20103 # shrinks to inputs = [(0, 0)], split = 0, concurrent = false
|
||||
|
|
@ -10,8 +10,8 @@
|
|||
//! - `POST /control/kill` body `{"Kill":{"command_id":"...","node":"..."}}`
|
||||
//! - `POST /control/provision` body `{"Provision":{"command_id":"...","count":1}}`
|
||||
|
||||
use std::sync::mpsc::Sender;
|
||||
use std::sync::OnceLock;
|
||||
use std::sync::mpsc::Sender;
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
|
|
@ -50,8 +50,283 @@ pub fn set_control_sender(sender: Sender<ControlCommand>) {
|
|||
let _ = CONTROL_SENDER.set(sender);
|
||||
}
|
||||
|
||||
/// Install an actor destination for dashboard-issued control commands.
|
||||
///
|
||||
/// The dashboard owns the blocking channel reader; each command becomes one
|
||||
/// typed actor observation.
|
||||
pub fn install_actor_sink(
|
||||
sender: swactor::runtime::ExternalSender,
|
||||
actor: swactor::actor::ActorAddress,
|
||||
) {
|
||||
let (control_sender, receiver) = std::sync::mpsc::channel();
|
||||
set_control_sender(control_sender);
|
||||
drop(spawn_actor_sink_forwarder(receiver, sender, actor));
|
||||
}
|
||||
|
||||
fn spawn_actor_sink_forwarder(
|
||||
receiver: std::sync::mpsc::Receiver<ControlCommand>,
|
||||
sender: swactor::runtime::ExternalSender,
|
||||
actor: swactor::actor::ActorAddress,
|
||||
) -> std::thread::JoinHandle<()> {
|
||||
std::thread::spawn(move || {
|
||||
while let Ok(command) = receiver.recv() {
|
||||
if sender.send_to(actor, command).is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn dispatch(command: ControlCommand) -> bool {
|
||||
CONTROL_SENDER
|
||||
.get()
|
||||
.is_some_and(|sender| sender.send(command).is_ok())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod properties {
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use parking_lot::Mutex;
|
||||
use proptest::prelude::*;
|
||||
use swactor::actor::{ActorInterface, Ctx};
|
||||
use swactor::config::RuntimeConfig;
|
||||
use swactor::runtime::{Runtime, RuntimeParts};
|
||||
use swactor_engine::{Engine, SteppingBackend};
|
||||
|
||||
use super::*;
|
||||
|
||||
const STEP_BUDGET: usize = 64;
|
||||
const FORWARDER_BUDGET: Duration = Duration::from_secs(1);
|
||||
|
||||
struct CommandProbe {
|
||||
observed: Arc<Mutex<Vec<String>>>,
|
||||
}
|
||||
|
||||
impl ActorInterface for CommandProbe {
|
||||
type Incoming = ControlCommand;
|
||||
type Response = ();
|
||||
|
||||
fn handle(&mut self, _ctx: &Ctx, command: Self::Incoming) {
|
||||
self.observed.lock().push(format!("{command:?}"));
|
||||
}
|
||||
}
|
||||
|
||||
fn command(kind: u8, value: u8) -> ControlCommand {
|
||||
let command_id = format!("repeated-{}", value % 4);
|
||||
match kind % 4 {
|
||||
0 => ControlCommand::Kill {
|
||||
command_id,
|
||||
node: format!("node-{value}"),
|
||||
},
|
||||
1 => ControlCommand::Provision {
|
||||
command_id,
|
||||
count: u32::from(value),
|
||||
},
|
||||
2 => ControlCommand::Remove {
|
||||
command_id,
|
||||
count: u32::from(value),
|
||||
},
|
||||
_ => ControlCommand::EstablishEdge {
|
||||
command_id,
|
||||
node: format!("node-{value}"),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn bridge_invariant_failure(
|
||||
expected: &[String],
|
||||
observed: &[String],
|
||||
runtime: &Runtime,
|
||||
expected_actor_count: usize,
|
||||
) -> Option<String> {
|
||||
let stats = runtime.stats();
|
||||
let panics = stats
|
||||
.workers
|
||||
.iter()
|
||||
.map(|worker| worker.panics)
|
||||
.sum::<u64>();
|
||||
let mailbox_depth = stats
|
||||
.workers
|
||||
.iter()
|
||||
.map(|worker| worker.mailbox_depth)
|
||||
.sum::<usize>()
|
||||
+ stats
|
||||
.actor_details
|
||||
.iter()
|
||||
.map(|actor| actor.mailbox_depth)
|
||||
.sum::<usize>();
|
||||
if observed != expected
|
||||
|| stats.actors.len() != expected_actor_count
|
||||
|| stats.actor_details.iter().any(|actor| actor.poisoned)
|
||||
|| panics != 0
|
||||
|| mailbox_depth != 0
|
||||
{
|
||||
Some(format!(
|
||||
"expected={expected:?}, observed={observed:?}, \
|
||||
expected_actor_count={expected_actor_count}, mailbox_depth={mailbox_depth}, \
|
||||
actor_census={stats:?}"
|
||||
))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#![proptest_config(ProptestConfig {
|
||||
cases: 128,
|
||||
max_shrink_iters: 2_000,
|
||||
..ProptestConfig::default()
|
||||
})]
|
||||
|
||||
#[test]
|
||||
fn generated_concurrent_bridge_commands_forward_once_and_shutdown(
|
||||
inputs in prop::collection::vec((any::<u8>(), any::<u8>()), 0..=32),
|
||||
split in 0_usize..=32,
|
||||
concurrent in any::<bool>(),
|
||||
destination_disappears in any::<bool>(),
|
||||
) {
|
||||
let mut config = RuntimeConfig::default();
|
||||
config.worker_count = 1;
|
||||
let parts = RuntimeParts::new(config);
|
||||
let runtime = parts.runtime().clone();
|
||||
let backend = SteppingBackend::new();
|
||||
let _engine =
|
||||
Engine::new(parts, backend.clone()).expect("dashboard bridge stepping engine");
|
||||
let observed = Arc::new(Mutex::new(Vec::new()));
|
||||
let probe = runtime
|
||||
.spawn(CommandProbe {
|
||||
observed: Arc::clone(&observed),
|
||||
})
|
||||
.expect("spawn dashboard control probe");
|
||||
if destination_disappears {
|
||||
runtime
|
||||
.stop_actor(probe)
|
||||
.expect("stop dashboard destination before forwarding");
|
||||
for _ in 0..STEP_BUDGET {
|
||||
backend.step();
|
||||
}
|
||||
}
|
||||
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
let forwarder =
|
||||
spawn_actor_sink_forwarder(rx, runtime.create_sender(), probe);
|
||||
let commands = inputs
|
||||
.iter()
|
||||
.map(|(kind, value)| command(*kind, *value))
|
||||
.collect::<Vec<_>>();
|
||||
let command_log = commands
|
||||
.iter()
|
||||
.map(|command| format!("{command:?}"))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
if concurrent {
|
||||
let split = split.min(commands.len());
|
||||
let left = commands[..split].to_vec();
|
||||
let right = commands[split..].to_vec();
|
||||
let left_tx = tx.clone();
|
||||
let left_sender = std::thread::spawn(move || {
|
||||
left.into_iter()
|
||||
.map(|command| left_tx.send(command).is_ok())
|
||||
.collect::<Vec<_>>()
|
||||
});
|
||||
let right_tx = tx.clone();
|
||||
let right_sender = std::thread::spawn(move || {
|
||||
right
|
||||
.into_iter()
|
||||
.map(|command| right_tx.send(command).is_ok())
|
||||
.collect::<Vec<_>>()
|
||||
});
|
||||
let _ = left_sender.join().expect("join left dashboard sender");
|
||||
let _ = right_sender.join().expect("join right dashboard sender");
|
||||
} else {
|
||||
for command in commands {
|
||||
if !destination_disappears {
|
||||
tx.send(command).expect("send dashboard command");
|
||||
} else {
|
||||
let _ = tx.send(command);
|
||||
}
|
||||
}
|
||||
}
|
||||
drop(tx);
|
||||
|
||||
let (done_tx, done_rx) = std::sync::mpsc::channel();
|
||||
std::thread::spawn(move || {
|
||||
let _ = done_tx.send(forwarder.join());
|
||||
});
|
||||
let forwarder_result = done_rx.recv_timeout(FORWARDER_BUDGET).unwrap_or_else(|error| {
|
||||
panic!(
|
||||
"dashboard bridge did not terminate after disconnect: {error}; \
|
||||
actions={command_log:?}; actor_census={:?}",
|
||||
runtime.stats(),
|
||||
)
|
||||
});
|
||||
forwarder_result.expect("dashboard bridge forwarder panicked");
|
||||
for _ in 0..STEP_BUDGET {
|
||||
backend.step();
|
||||
}
|
||||
|
||||
let mut actual = observed.lock().clone();
|
||||
actual.sort();
|
||||
let mut expected = if destination_disappears {
|
||||
Vec::new()
|
||||
} else {
|
||||
command_log.clone()
|
||||
};
|
||||
expected.sort();
|
||||
let expected_actor_count = usize::from(!destination_disappears);
|
||||
prop_assert!(
|
||||
bridge_invariant_failure(
|
||||
&expected,
|
||||
&actual,
|
||||
&runtime,
|
||||
expected_actor_count,
|
||||
)
|
||||
.is_none(),
|
||||
"dashboard bridge invariant failed; actions={:?}; disconnect={}; failure={}",
|
||||
command_log,
|
||||
destination_disappears,
|
||||
bridge_invariant_failure(
|
||||
&expected,
|
||||
&actual,
|
||||
&runtime,
|
||||
expected_actor_count,
|
||||
)
|
||||
.unwrap_or_default(),
|
||||
);
|
||||
|
||||
if !destination_disappears {
|
||||
runtime
|
||||
.stop_actor(probe)
|
||||
.expect("stop dashboard control probe");
|
||||
for _ in 0..STEP_BUDGET {
|
||||
backend.step();
|
||||
}
|
||||
}
|
||||
prop_assert!(
|
||||
bridge_invariant_failure(&expected, &actual, &runtime, 0).is_none(),
|
||||
"dashboard bridge teardown leaked forwarding actors; actions={:?}; failure={}",
|
||||
command_log,
|
||||
bridge_invariant_failure(&expected, &actual, &runtime, 0)
|
||||
.unwrap_or_default(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bridge_invariant_rejects_a_controlled_duplicate_delivery() {
|
||||
let parts = RuntimeParts::new(RuntimeConfig::default());
|
||||
let runtime = parts.runtime().clone();
|
||||
let expected = vec!["Provision repeated-0".to_owned()];
|
||||
let duplicated = vec![
|
||||
"Provision repeated-0".to_owned(),
|
||||
"Provision repeated-0".to_owned(),
|
||||
];
|
||||
|
||||
assert!(
|
||||
bridge_invariant_failure(&expected, &duplicated, &runtime, 0).is_some(),
|
||||
"bridge invariant accepted a controlled duplicate delivery"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,10 +16,10 @@
|
|||
use std::collections::BTreeMap;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use telemetry::frame::{Frame, StreamId};
|
||||
use parking_lot::RwLock;
|
||||
use serde::Serialize;
|
||||
use serde_json::{Value, json};
|
||||
use telemetry::frame::{Frame, StreamId};
|
||||
|
||||
use crate::hardware_view::{
|
||||
CpuSnapshot, GpuSnapshot, HardwareHistorySnapshot, NodeHardwareState, duration_ms,
|
||||
|
|
@ -34,6 +34,9 @@ const CONTROL_PLANE_HTML: &str = include_str!("control_plane_page.html");
|
|||
const LIVE_TTL: Duration = Duration::from_secs(8);
|
||||
/// Hard cap on retained stale streams; oldest are evicted.
|
||||
const STALE_POOL_CAP: usize = 50;
|
||||
/// Stream origin of the process hosting this dashboard. Its card can never
|
||||
/// go meaningfully stale: if that publisher were silent, no page would render.
|
||||
const ORIGIN_ORCHESTRATOR: &str = "orchestrator";
|
||||
|
||||
/// Fused control-plane view serving `/` and `/view/fleet`.
|
||||
#[derive(Default)]
|
||||
|
|
@ -55,6 +58,12 @@ struct FusedNode {
|
|||
label: Option<String>,
|
||||
}
|
||||
|
||||
impl FusedNode {
|
||||
fn is_orchestrator(&self) -> bool {
|
||||
self.origin.as_deref() == Some(ORIGIN_ORCHESTRATOR)
|
||||
}
|
||||
}
|
||||
|
||||
impl DashboardView for ControlPlaneView {
|
||||
fn id(&self) -> &'static str {
|
||||
"fleet"
|
||||
|
|
@ -113,16 +122,38 @@ impl DashboardView for ControlPlaneView {
|
|||
}
|
||||
}
|
||||
live.sort_by(|left, right| {
|
||||
left.stream
|
||||
.node
|
||||
.cmp(&right.stream.node)
|
||||
.then_with(|| left.stream.life.cmp(&right.stream.life))
|
||||
// The orchestrator card leads the live pool: it is the control
|
||||
// plane every other node hangs off of.
|
||||
right
|
||||
.stream
|
||||
.origin
|
||||
.as_deref()
|
||||
.map(|origin| origin == ORIGIN_ORCHESTRATOR)
|
||||
.unwrap_or(false)
|
||||
.cmp(
|
||||
&left
|
||||
.stream
|
||||
.origin
|
||||
.as_deref()
|
||||
.map(|origin| origin == ORIGIN_ORCHESTRATOR)
|
||||
.unwrap_or(false),
|
||||
)
|
||||
.then_with(|| {
|
||||
left.stream
|
||||
.node
|
||||
.cmp(&right.stream.node)
|
||||
.then_with(|| left.stream.life.cmp(&right.stream.life))
|
||||
})
|
||||
});
|
||||
// Stale pool: most recently seen first, bounded by the physical cap.
|
||||
stale.sort_by_key(|right| std::cmp::Reverse(right.last_seen_ms_ago));
|
||||
stale.truncate(STALE_POOL_CAP);
|
||||
let totals = fused_totals(live.len(), stale.len());
|
||||
let snapshot = FusedSnapshot { totals, live, stale };
|
||||
let snapshot = FusedSnapshot {
|
||||
totals,
|
||||
live,
|
||||
stale,
|
||||
};
|
||||
serde_json::to_value(snapshot).unwrap_or_else(|_| {
|
||||
json!({
|
||||
"totals": FusedTotals::default(),
|
||||
|
|
@ -160,9 +191,7 @@ fn prune(streams: &mut BTreeMap<String, FusedNode>, fresh: &StreamEvent, now: In
|
|||
// process is gone by construction once its successor publishes.
|
||||
let superseded: Vec<String> = streams
|
||||
.iter()
|
||||
.filter(|(_, node)| {
|
||||
node.stream.node == fresh.node && node.stream.life < fresh.life
|
||||
})
|
||||
.filter(|(_, node)| node.stream.node == fresh.node && node.stream.life < fresh.life)
|
||||
.map(|(key, _)| key.clone())
|
||||
.collect();
|
||||
for key in superseded {
|
||||
|
|
@ -171,7 +200,9 @@ fn prune(streams: &mut BTreeMap<String, FusedNode>, fresh: &StreamEvent, now: In
|
|||
|
||||
let mut stale: Vec<(String, Instant)> = streams
|
||||
.iter()
|
||||
.filter(|(_, node)| now.duration_since(node.last_seen) > LIVE_TTL)
|
||||
.filter(|(_, node)| {
|
||||
!node.is_orchestrator() && now.duration_since(node.last_seen) > LIVE_TTL
|
||||
})
|
||||
.map(|(key, node)| (key.clone(), node.last_seen))
|
||||
.collect();
|
||||
if stale.len() > STALE_POOL_CAP {
|
||||
|
|
@ -298,12 +329,7 @@ fn stream_key(stream: &StreamEvent) -> String {
|
|||
fn node_card(node: &FusedNode, now: Instant) -> NodeCard {
|
||||
let summary = node.hardware.summary();
|
||||
let totals = node.actors.totals();
|
||||
let mut roster: Vec<RosterRow> = node
|
||||
.actors
|
||||
.actors
|
||||
.values()
|
||||
.map(roster_row)
|
||||
.collect();
|
||||
let mut roster: Vec<RosterRow> = node.actors.actors.values().map(roster_row).collect();
|
||||
// Busiest actors first; ties fall back to address for stable rendering.
|
||||
roster.sort_by(|left, right| {
|
||||
right
|
||||
|
|
@ -320,7 +346,7 @@ fn node_card(node: &FusedNode, now: Instant) -> NodeCard {
|
|||
origin: node.origin.clone(),
|
||||
label: node.label.clone(),
|
||||
},
|
||||
live: now.duration_since(node.last_seen) <= LIVE_TTL,
|
||||
live: node.is_orchestrator() || now.duration_since(node.last_seen) <= LIVE_TTL,
|
||||
last_seen_ms_ago: duration_ms(now.duration_since(node.last_seen)),
|
||||
last_sample_unix_ms: summary.sample_unix_ms,
|
||||
errors: node.hardware.errors(),
|
||||
|
|
@ -361,7 +387,11 @@ fn roster_row(actor: &ActorState) -> RosterRow {
|
|||
name: actor.name.clone(),
|
||||
actor_type: actor.actor_type.clone(),
|
||||
message_type: actor.message_type.clone(),
|
||||
state: if actor.poisoned { "poisoned" } else { "running" },
|
||||
state: if actor.poisoned {
|
||||
"poisoned"
|
||||
} else {
|
||||
"running"
|
||||
},
|
||||
mailbox_depth: actor.mailbox_depth,
|
||||
messages_processed: actor.messages_processed,
|
||||
msg_per_sec: actor.msg_per_sec,
|
||||
|
|
@ -392,7 +422,11 @@ fn actor_detail(
|
|||
actor_type: actor.actor_type.clone(),
|
||||
message_type: actor.message_type.clone(),
|
||||
worker_id: actor.worker_id,
|
||||
state: if actor.poisoned { "poisoned" } else { "running" },
|
||||
state: if actor.poisoned {
|
||||
"poisoned"
|
||||
} else {
|
||||
"running"
|
||||
},
|
||||
poisoned: actor.poisoned,
|
||||
mailbox_depth: actor.mailbox_depth,
|
||||
mailbox_growth: actor.mailbox_growth,
|
||||
|
|
@ -574,13 +608,84 @@ mod tests {
|
|||
assert_eq!(snapshot["totals"]["stale_nodes"].as_u64(), Some(1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn orchestrator_stream_stays_live_beyond_ttl_and_survives_pool_eviction() {
|
||||
let view = ControlPlaneView::default();
|
||||
let orch = StreamId::new(NodeId::new("orch"), Lifetime(1));
|
||||
let worker = StreamId::new(NodeId::new("worker"), Lifetime(1));
|
||||
let frame = Frame::new(ChannelId(1), Position(0), actors_payload(0, json!([])));
|
||||
for (stream, origin) in [(&orch, Some("orchestrator")), (&worker, None)] {
|
||||
let event = FrameEvent {
|
||||
stream: crate::StreamEvent {
|
||||
node: stream.node.as_str().to_string(),
|
||||
life: stream.life.0,
|
||||
origin: origin.map(str::to_owned),
|
||||
label: None,
|
||||
},
|
||||
channel: "runtime.actors".to_owned(),
|
||||
position: 0,
|
||||
payload: frame.payload.clone(),
|
||||
};
|
||||
view.ingest(stream, &frame, &event);
|
||||
}
|
||||
{
|
||||
let mut state = view.state.write();
|
||||
for node in state.streams.values_mut() {
|
||||
node.last_seen -= LIVE_TTL + Duration::from_secs(1);
|
||||
}
|
||||
}
|
||||
|
||||
let snapshot = view.snapshot_json();
|
||||
let live = snapshot["live"].as_array().expect("live array");
|
||||
assert_eq!(live.len(), 1, "only the orchestrator stays live");
|
||||
assert_eq!(live[0]["stream"]["node"], json!("orch"));
|
||||
assert_eq!(live[0]["live"], json!(true));
|
||||
assert_eq!(snapshot["stale"].as_array().map(Vec::len), Some(1));
|
||||
|
||||
// The stale-pool eviction sweep must not remove the orchestrator
|
||||
// stream even when it is the oldest entry.
|
||||
let fresh = StreamId::new(NodeId::new("fresh"), Lifetime(1));
|
||||
ingest_json(
|
||||
&view,
|
||||
&fresh,
|
||||
0,
|
||||
"runtime.actors",
|
||||
actors_payload(0, json!([])),
|
||||
);
|
||||
let snapshot = view.snapshot_json();
|
||||
let still_live: Vec<&str> = snapshot["live"]
|
||||
.as_array()
|
||||
.expect("live array")
|
||||
.iter()
|
||||
.map(|card| card["stream"]["node"].as_str().expect("node"))
|
||||
.collect();
|
||||
assert!(
|
||||
still_live.contains(&"orch"),
|
||||
"orchestrator evicted: {still_live:?}"
|
||||
);
|
||||
// Orchestrator leads the live pool regardless of node name order.
|
||||
assert_eq!(still_live.first(), Some(&"orch"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn newer_life_generation_evicts_superseded_stream() {
|
||||
let view = ControlPlaneView::default();
|
||||
let old = StreamId::new(NodeId::new("node"), Lifetime(1));
|
||||
let new = StreamId::new(NodeId::new("node"), Lifetime(2));
|
||||
ingest_json(&view, &old, 0, "runtime.actors", actors_payload(0, json!([])));
|
||||
ingest_json(&view, &new, 0, "runtime.actors", actors_payload(0, json!([])));
|
||||
ingest_json(
|
||||
&view,
|
||||
&old,
|
||||
0,
|
||||
"runtime.actors",
|
||||
actors_payload(0, json!([])),
|
||||
);
|
||||
ingest_json(
|
||||
&view,
|
||||
&new,
|
||||
0,
|
||||
"runtime.actors",
|
||||
actors_payload(0, json!([])),
|
||||
);
|
||||
|
||||
let snapshot = view.snapshot_json();
|
||||
let nodes: Vec<&str> = snapshot["live"]
|
||||
|
|
@ -597,7 +702,13 @@ mod tests {
|
|||
let view = ControlPlaneView::default();
|
||||
for index in 0..(STALE_POOL_CAP as u64 + 5) {
|
||||
let stream = StreamId::new(NodeId::new(&format!("old-{index}")), Lifetime(1));
|
||||
ingest_json(&view, &stream, 0, "runtime.actors", actors_payload(0, json!([])));
|
||||
ingest_json(
|
||||
&view,
|
||||
&stream,
|
||||
0,
|
||||
"runtime.actors",
|
||||
actors_payload(0, json!([])),
|
||||
);
|
||||
}
|
||||
{
|
||||
let mut state = view.state.write();
|
||||
|
|
@ -607,7 +718,13 @@ mod tests {
|
|||
}
|
||||
// Physical cap applies on the next ingest…
|
||||
let fresh = StreamId::new(NodeId::new("fresh"), Lifetime(1));
|
||||
ingest_json(&view, &fresh, 0, "runtime.actors", actors_payload(0, json!([])));
|
||||
ingest_json(
|
||||
&view,
|
||||
&fresh,
|
||||
0,
|
||||
"runtime.actors",
|
||||
actors_payload(0, json!([])),
|
||||
);
|
||||
|
||||
let state = view.state.read();
|
||||
assert!(state.streams.len() <= STALE_POOL_CAP + 1);
|
||||
|
|
@ -643,7 +760,10 @@ mod tests {
|
|||
assert_eq!(detail["address"], json!("ff00"));
|
||||
assert_eq!(detail["actor_type"], json!("OrchestratorActor"));
|
||||
assert_eq!(detail["worker_id"], json!(7));
|
||||
assert_eq!(detail["message_type_counts"][0]["message_type"], json!("Ping"));
|
||||
assert_eq!(
|
||||
detail["message_type_counts"][0]["message_type"],
|
||||
json!("Ping")
|
||||
);
|
||||
assert_eq!(detail["receipts"].as_array().map(Vec::len), Some(1));
|
||||
assert_eq!(detail["receipts"][0]["ty"], json!("Ping"));
|
||||
}
|
||||
|
|
@ -676,7 +796,9 @@ mod tests {
|
|||
),
|
||||
);
|
||||
|
||||
let detail = view.detail_json("stream=node%231&actor=aa").expect("detail");
|
||||
let detail = view
|
||||
.detail_json("stream=node%231&actor=aa")
|
||||
.expect("detail");
|
||||
let receipts = detail["receipts"].as_array().expect("receipts");
|
||||
assert_eq!(receipts.len(), 1);
|
||||
assert_eq!(detail["sampled_out"].as_u64(), Some(4));
|
||||
|
|
@ -691,7 +813,10 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn percent_decoding_handles_hash_and_plus() {
|
||||
assert_eq!(query_param("stream=node%234&actor=ab", "stream").as_deref(), Some("node#4"));
|
||||
assert_eq!(
|
||||
query_param("stream=node%234&actor=ab", "stream").as_deref(),
|
||||
Some("node#4")
|
||||
);
|
||||
assert_eq!(query_param("a=1&actor=cd", "actor").as_deref(), Some("cd"));
|
||||
assert_eq!(query_param("stream=x", "actor"), None);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@
|
|||
.mono { font-family: var(--mono); font-size: 12px; }
|
||||
#status { font-family: var(--mono); font-size: 13px; color: var(--amber); }
|
||||
a.node-card { color: inherit; text-decoration: none; display: block; }
|
||||
.node-card.orchestrator { grid-column: 1 / -1; border-color: var(--amber); background: var(--card-hover); }
|
||||
.node-card.orchestrator { grid-column: 1 / -1; border-color: var(--amber); background: var(--ghost); box-shadow: inset 3px 0 0 var(--amber); padding-left: 17px; }
|
||||
.node-card.orchestrator .role { margin-bottom: 6px; color: var(--muted); font: 600 9px/1.2 var(--mono); text-transform: uppercase; letter-spacing: .12em; }
|
||||
:is(a, button, input, summary):focus-visible, tr[data-addr]:focus-visible { outline: 2px solid var(--cyan); outline-offset: 2px; }
|
||||
.grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(300px, 1fr)); gap: 12px; }
|
||||
|
|
@ -141,6 +141,7 @@ let detailTimer = null;
|
|||
// restarted animations every second even in a fully converged steady state.
|
||||
let lastPageHtml = null;
|
||||
let lastRosterHtml = null;
|
||||
let lastMachineHtml = null;
|
||||
let lastDossierHtml = null;
|
||||
|
||||
// Receipt ages (dossier) render as empty spans carrying an absolute epoch —
|
||||
|
|
@ -292,23 +293,20 @@ function updateCardTexts(nodes) {
|
|||
|
||||
function renderNode(page, node, live, stale) {
|
||||
const summary = node.actor_summary || {};
|
||||
const machine = machineDetail(node);
|
||||
// The machine panel churns with live metrics, so swaps here are real; keep
|
||||
// the roster filter usable by restoring focus and caret across them.
|
||||
const active = document.activeElement;
|
||||
const refocus = active && active.id === 'roster-filter' && page.contains(active)
|
||||
? { start: active.selectionStart, end: active.selectionEnd, dir: active.selectionDirection }
|
||||
: null;
|
||||
const html = `
|
||||
<a class="focus-back" href="/">← Fleet (${fmt(live.length)} live, ${fmt(stale.length)} stale)</a>
|
||||
<a class="focus-back" href="/">← Fleet (<span data-live-count></span> live, <span data-stale-count></span> stale)</a>
|
||||
<div class="layout ${selectedActor ? 'with-dossier' : ''}">
|
||||
<div class="node-view">
|
||||
<div class="node-view" data-node="${esc(node.stream.node)}">
|
||||
<section class="panel">
|
||||
<h2 style="margin:0 0 8px">${esc(node.stream.node)}${node.stream.label ? ` <span class="muted" style="font-size:13px">${esc(node.stream.label)}</span>` : ''} <span class="muted" style="font-size:13px">life ${fmt(node.stream.life)} · <span data-seen></span></span></h2>
|
||||
${machine}
|
||||
<div id="machine-detail"></div>
|
||||
</section>
|
||||
<section class="panel">
|
||||
<h2 style="margin:0 0 8px">Actors (${fmt(summary.actors)})</h2>
|
||||
<h2 style="margin:0 0 8px">Actors (<span data-actor-count></span>)</h2>
|
||||
<input type="search" id="roster-filter" aria-label="Filter actors" placeholder="filter by type, address, worker…" value="${esc(rosterFilter)}">
|
||||
<div id="roster-wrap" style="margin-top:10px"></div>
|
||||
</section>
|
||||
|
|
@ -318,6 +316,8 @@ function renderNode(page, node, live, stale) {
|
|||
if (html !== lastPageHtml) {
|
||||
page.innerHTML = html;
|
||||
lastPageHtml = html;
|
||||
lastMachineHtml = null;
|
||||
lastRosterHtml = null;
|
||||
const filter = document.getElementById('roster-filter');
|
||||
filter.addEventListener('input', e => { rosterFilter = e.target.value; renderRoster(node); });
|
||||
if (refocus) {
|
||||
|
|
@ -325,10 +325,26 @@ function renderNode(page, node, live, stale) {
|
|||
try { filter.setSelectionRange(refocus.start, refocus.end, refocus.dir); } catch { /* caret shape changed */ }
|
||||
}
|
||||
}
|
||||
const seen = document.querySelector('#page h2 [data-seen]');
|
||||
const liveCount = page.querySelector('[data-live-count]');
|
||||
if (liveCount) liveCount.textContent = fmt(live.length);
|
||||
const staleCount = page.querySelector('[data-stale-count]');
|
||||
if (staleCount) staleCount.textContent = fmt(stale.length);
|
||||
const actorCount = page.querySelector('[data-actor-count]');
|
||||
if (actorCount) actorCount.textContent = fmt(summary.actors);
|
||||
const seen = page.querySelector('h2 [data-seen]');
|
||||
if (seen) seen.textContent = node.live ? 'live' : 'stale ' + ago(node.last_seen_ms_ago) + ' ago';
|
||||
const machine = machineDetail(node);
|
||||
const machineSlot = document.getElementById('machine-detail');
|
||||
if (machineSlot && machine !== lastMachineHtml) {
|
||||
machineSlot.innerHTML = machine;
|
||||
lastMachineHtml = machine;
|
||||
}
|
||||
renderRoster(node);
|
||||
if (selectedActor) startDetailPolling();
|
||||
if (selectedActor) {
|
||||
if (!detailTimer) startDetailPolling();
|
||||
} else {
|
||||
stopDetailPolling();
|
||||
}
|
||||
}
|
||||
|
||||
function machineDetail(node) {
|
||||
|
|
@ -406,17 +422,36 @@ function renderRoster(node) {
|
|||
<td class="mono">${esc(shortAddr(a.address))}${a.name ? `<br><span class="muted">${esc(a.name)}</span>` : ''}</td>
|
||||
<td><span class="muted" title="${esc(a.actor_type || '')}">${typeShort(a.actor_type)}</span></td>
|
||||
<td>${a.poisoned ? '<span class="pill poisoned">poisoned</span>' : '<span class="muted">running</span>'}</td>
|
||||
<td>${fmt(a.mailbox_depth)}</td>
|
||||
<td>${fmtRate(a.msg_per_sec)}</td>
|
||||
<td>${fmt(a.messages_processed)}</td>
|
||||
<td data-mailbox></td>
|
||||
<td data-rate></td>
|
||||
<td data-processed></td>
|
||||
<td>${a.worker_id == null ? '—' : fmt(a.worker_id)}</td>
|
||||
<td><span class="muted" title="${esc(a.last_msg_type || '')}">${typeShort(a.last_msg_type)}</span></td>
|
||||
<td><span class="muted" data-last-message></span></td>
|
||||
</tr>`).join('')}</tbody>
|
||||
</table>`;
|
||||
}
|
||||
if (html === lastRosterHtml) return;
|
||||
wrap.innerHTML = html;
|
||||
lastRosterHtml = html;
|
||||
if (html !== lastRosterHtml) {
|
||||
wrap.innerHTML = html;
|
||||
lastRosterHtml = html;
|
||||
}
|
||||
updateRosterTexts(wrap, capped);
|
||||
}
|
||||
|
||||
function updateRosterTexts(wrap, actors) {
|
||||
const byAddress = new Map(actors.map(actor => [actor.address, actor]));
|
||||
for (const row of wrap.querySelectorAll('tr[data-addr]')) {
|
||||
const actor = byAddress.get(row.getAttribute('data-addr'));
|
||||
if (!actor) continue;
|
||||
row.querySelector('[data-mailbox]').textContent = fmt(actor.mailbox_depth);
|
||||
row.querySelector('[data-rate]').textContent = fmtRate(actor.msg_per_sec);
|
||||
row.querySelector('[data-processed]').textContent = fmt(actor.messages_processed);
|
||||
const last = row.querySelector('[data-last-message]');
|
||||
const messageType = actor.last_msg_type || '';
|
||||
last.title = messageType;
|
||||
const parts = String(messageType).split('::');
|
||||
last.textContent = messageType ? parts[parts.length - 1] || messageType : '—';
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function shortAddr(address) {
|
||||
|
|
|
|||
|
|
@ -15,8 +15,8 @@ use serde::Serialize;
|
|||
use serde_json::Value;
|
||||
use telemetry::frame::{Frame, StreamId};
|
||||
|
||||
use crate::view::DashboardView;
|
||||
use crate::FrameEvent;
|
||||
use crate::view::DashboardView;
|
||||
|
||||
#[derive(Clone, Serialize)]
|
||||
struct ProcessEntry {
|
||||
|
|
|
|||
|
|
@ -147,7 +147,7 @@ RISK: terminal cosplay if decoration creeps past data; held by the palette law.
|
|||
<section class="panel">
|
||||
<table>
|
||||
<thead>
|
||||
<tr><th>node</th><th>stage</th><th>state</th><th>pid</th><th>seen</th><th>edge</th><th></th></tr>
|
||||
<tr><th>node</th><th>stage</th><th>state</th><th>pid</th><th>seen</th><th data-edge hidden>edge</th><th></th></tr>
|
||||
</thead>
|
||||
<tbody id="rows"></tbody>
|
||||
</table>
|
||||
|
|
@ -229,7 +229,7 @@ RISK: terminal cosplay if decoration creeps past data; held by the palette law.
|
|||
// One row per logical node: reconciler state (stage, attempt, ready) fused
|
||||
// with the observed process (state, pid, seen). Reconciler order leads;
|
||||
// processes the reconciler no longer tracks still render.
|
||||
function renderRow(entry) {
|
||||
function renderRow(entry, edgeCapable) {
|
||||
const reconciler = entry.reconciler;
|
||||
const process = entry.process;
|
||||
const node = reconciler ? reconciler.id : process.node;
|
||||
|
|
@ -245,9 +245,15 @@ RISK: terminal cosplay if decoration creeps past data; held by the palette law.
|
|||
? `<div class="row-error">${escapeHtml(reconciler.failure)}</div>`
|
||||
: '';
|
||||
const killable = process && process.state === 'running';
|
||||
const edge = entry.edge
|
||||
? `<span class="badge ${EDGE_CLASS[entry.edge.state] || 'b-progress'}">e${escapeHtml(entry.edge.edge)} ${escapeHtml(entry.edge.state)}</span>`
|
||||
: '<span class="muted">—</span>';
|
||||
const edgeCell = edgeCapable
|
||||
? `<td>${entry.edge
|
||||
? `<span class="badge ${EDGE_CLASS[entry.edge.state] || 'b-progress'}">e${escapeHtml(entry.edge.edge)} ${escapeHtml(entry.edge.state)}</span>`
|
||||
: '<span class="muted">—</span>'}</td>`
|
||||
: '';
|
||||
const edgeButton = edgeCapable
|
||||
? ` <button class="edge" data-action="edge" data-node="${escapeHtml(node)}" ${killable ? '' : 'disabled'}>edge</button>`
|
||||
: '';
|
||||
const colspan = edgeCapable ? 7 : 6;
|
||||
// "seen" is deliberately absent from this HTML: it is written per-poll
|
||||
// as textContent by updateSeen() so a ticking relative age never
|
||||
// invalidates the cached row markup.
|
||||
|
|
@ -257,12 +263,9 @@ RISK: terminal cosplay if decoration creeps past data; held by the palette law.
|
|||
<td>${state}</td>
|
||||
<td>${pid == null ? '—' : escapeHtml(pid)}</td>
|
||||
<td class="muted" data-seen>—</td>
|
||||
<td>${edge}</td>
|
||||
<td>
|
||||
<button class="edge" data-action="edge" data-node="${escapeHtml(node)}" ${killable ? '' : 'disabled'}>edge</button>
|
||||
<button class="danger" data-action="kill" data-node="${escapeHtml(node)}" ${killable ? '' : 'disabled'}>kill</button>
|
||||
</td>
|
||||
</tr>${failure ? `<tr><td colspan="7" style="padding-top:0">${failure}</td></tr>` : ''}`;
|
||||
${edgeCell}
|
||||
<td>${edgeButton}<button class="danger" data-action="kill" data-node="${escapeHtml(node)}" ${killable ? '' : 'disabled'}>kill</button></td>
|
||||
</tr>${failure ? `<tr><td colspan="${colspan}" style="padding-top:0">${failure}</td></tr>` : ''}`;
|
||||
}
|
||||
|
||||
function fuse(processes, reconcilerNodes, edges) {
|
||||
|
|
@ -292,9 +295,13 @@ RISK: terminal cosplay if decoration creeps past data; held by the palette law.
|
|||
const processes = (processSnapshot && processSnapshot.nodes) || [];
|
||||
const reconcilerNodes = (reconcilerSnapshot && reconcilerSnapshot.nodes) || [];
|
||||
const edges = (reconcilerSnapshot && reconcilerSnapshot.edges) || [];
|
||||
// The edge column exists only when a reconciler view supplies edge
|
||||
// records; without it the edge button can never succeed.
|
||||
const edgeCapable = !!reconcilerSnapshot && Array.isArray(reconcilerSnapshot.edges);
|
||||
document.querySelector('th[data-edge]').hidden = !edgeCapable;
|
||||
const fused = fuse(processes, reconcilerNodes, edges);
|
||||
empty.hidden = fused.length > 0;
|
||||
const rowsHtml = fused.map(renderRow).join('');
|
||||
const rowsHtml = fused.map((entry) => renderRow(entry, edgeCapable)).join('');
|
||||
if (rowsHtml !== lastRowsHtml) {
|
||||
rows.innerHTML = rowsHtml;
|
||||
lastRowsHtml = rowsHtml;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
use std::collections::{BTreeMap, VecDeque};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
|
||||
use telemetry::Record;
|
||||
use telemetry::hardware::cpu::{
|
||||
CpuCoreSample, CpuHostSample, CpuProcessSample, HOST_CPU_CHANNEL, HostCpuSample,
|
||||
};
|
||||
|
|
@ -9,12 +9,9 @@ use telemetry::hardware::gpu::{
|
|||
GpuDeviceSample, GpuProcessSample, HOST_GPU_CHANNEL, HostGpuSample,
|
||||
};
|
||||
use telemetry::hardware::net::{HOST_NET_CHANNEL, HostNetSample, NetInterfaceSample};
|
||||
use telemetry::Record;
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
|
||||
|
||||
const HISTORY_CAP: usize = 300;
|
||||
const HISTORY_MIN_INTERVAL: Duration = Duration::from_millis(900);
|
||||
|
||||
|
|
@ -112,8 +109,8 @@ impl NodeHardwareState {
|
|||
gpu_max_percent =
|
||||
Some(gpu_max_percent.map_or(percent, |current: u64| current.max(percent)));
|
||||
}
|
||||
gpu_memory_used_mib = gpu_memory_used_mib
|
||||
.saturating_add(device.memory_used_mib.unwrap_or_default());
|
||||
gpu_memory_used_mib =
|
||||
gpu_memory_used_mib.saturating_add(device.memory_used_mib.unwrap_or_default());
|
||||
gpu_memory_total_mib = gpu_memory_total_mib
|
||||
.saturating_add(device.memory_total_mib.unwrap_or_default());
|
||||
}
|
||||
|
|
@ -333,7 +330,6 @@ pub(crate) struct HardwareHistoryState {
|
|||
pub(crate) net_tx_bps: f64,
|
||||
}
|
||||
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(crate) struct CpuSnapshot {
|
||||
pub(crate) seq: u64,
|
||||
|
|
@ -394,7 +390,6 @@ pub(crate) struct HardwareHistorySnapshot {
|
|||
pub(crate) net_tx_bps: f64,
|
||||
}
|
||||
|
||||
|
||||
pub(crate) fn duration_ms(duration: Duration) -> u64 {
|
||||
u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
|
||||
}
|
||||
|
|
@ -402,4 +397,3 @@ pub(crate) fn duration_ms(duration: Duration) -> u64 {
|
|||
pub(crate) fn saturating_u32(value: usize) -> u32 {
|
||||
u32::try_from(value).unwrap_or(u32::MAX)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
#[cfg(feature = "demo-control")]
|
||||
pub mod control;
|
||||
mod control_plane;
|
||||
#[cfg(feature = "demo-control")]
|
||||
mod demo_control;
|
||||
mod control_plane;
|
||||
mod hardware_view;
|
||||
mod live_explorer;
|
||||
mod server;
|
||||
|
|
@ -21,6 +21,51 @@ use tokio::sync::broadcast;
|
|||
use crate::store::DashboardStore;
|
||||
use crate::view::{DashboardView, ViewRegistry};
|
||||
|
||||
/// An application-owned page rendered inside the dashboard shell.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct PluginPage {
|
||||
pub id: &'static str,
|
||||
pub title: &'static str,
|
||||
pub path: &'static str,
|
||||
pub html: &'static str,
|
||||
}
|
||||
|
||||
impl PluginPage {
|
||||
pub const fn new(
|
||||
id: &'static str,
|
||||
title: &'static str,
|
||||
path: &'static str,
|
||||
html: &'static str,
|
||||
) -> Self {
|
||||
Self {
|
||||
id,
|
||||
title,
|
||||
path,
|
||||
html,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Application routes and their dashboard-visible pages.
|
||||
pub struct DashboardPlugin {
|
||||
pub pages: Vec<PluginPage>,
|
||||
pub routes: axum::Router,
|
||||
}
|
||||
|
||||
impl DashboardPlugin {
|
||||
pub fn new(routes: axum::Router) -> Self {
|
||||
Self {
|
||||
pages: Vec::new(),
|
||||
routes,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_page(mut self, page: PluginPage) -> Self {
|
||||
self.pages.push(page);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// Configuration for the telemetry dashboard server.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DashboardConfig {
|
||||
|
|
@ -29,6 +74,9 @@ pub struct DashboardConfig {
|
|||
pub frame_buffer: usize,
|
||||
/// Number of recent raw frames retained for `/api/frames`.
|
||||
pub raw_frame_history: usize,
|
||||
/// Embedding-owned scripts appended to dashboard pages. The dashboard does
|
||||
/// not define their behavior or gain mutation capabilities from them.
|
||||
pub page_script_urls: Vec<String>,
|
||||
}
|
||||
|
||||
impl Default for DashboardConfig {
|
||||
|
|
@ -37,6 +85,7 @@ impl Default for DashboardConfig {
|
|||
port: 9090,
|
||||
frame_buffer: 1024,
|
||||
raw_frame_history: 1024,
|
||||
page_script_urls: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -106,6 +155,7 @@ pub struct DashboardHandle {
|
|||
frames: broadcast::Sender<FrameEvent>,
|
||||
store: Arc<DashboardStore>,
|
||||
views: Arc<ViewRegistry>,
|
||||
page_script_urls: Arc<Vec<String>>,
|
||||
shutdown_notify: Arc<tokio::sync::Notify>,
|
||||
}
|
||||
|
||||
|
|
@ -129,6 +179,7 @@ impl DashboardHandle {
|
|||
port: config.port,
|
||||
frames,
|
||||
store,
|
||||
page_script_urls: Arc::new(config.page_script_urls),
|
||||
views,
|
||||
shutdown_notify: Arc::new(tokio::sync::Notify::new()),
|
||||
}
|
||||
|
|
@ -163,6 +214,8 @@ impl DashboardHandle {
|
|||
frames: self.frames.clone(),
|
||||
store: Arc::clone(&self.store),
|
||||
views: Arc::clone(&self.views),
|
||||
plugin_pages: Arc::new(Vec::new()),
|
||||
page_script_urls: Arc::clone(&self.page_script_urls),
|
||||
shutdown_notify: Arc::clone(&self.shutdown_notify),
|
||||
};
|
||||
let port = self.port;
|
||||
|
|
@ -171,6 +224,65 @@ impl DashboardHandle {
|
|||
}
|
||||
}
|
||||
|
||||
/// Build the dashboard server with application-owned routes on the same
|
||||
/// origin. The dashboard router remains read-only; mutation handlers stay
|
||||
/// in the embedding application.
|
||||
pub fn http_server_with_routes(
|
||||
&self,
|
||||
routes: axum::Router,
|
||||
) -> impl Future<Output = ()> + Send + 'static {
|
||||
let state = server::AppState {
|
||||
frames: self.frames.clone(),
|
||||
store: Arc::clone(&self.store),
|
||||
views: Arc::clone(&self.views),
|
||||
plugin_pages: Arc::new(Vec::new()),
|
||||
page_script_urls: Arc::clone(&self.page_script_urls),
|
||||
shutdown_notify: Arc::clone(&self.shutdown_notify),
|
||||
};
|
||||
let port = self.port;
|
||||
async move {
|
||||
server::run_server_with_routes(state, port, routes).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the dashboard with application plugins. Plugin pages are served
|
||||
/// by the dashboard and automatically participate in shared navigation.
|
||||
pub fn http_server_with_plugins(
|
||||
&self,
|
||||
plugins: Vec<DashboardPlugin>,
|
||||
) -> impl Future<Output = ()> + Send + 'static {
|
||||
let mut routes = axum::Router::new();
|
||||
let mut plugin_pages = Vec::new();
|
||||
for plugin in plugins {
|
||||
routes = routes.merge(plugin.routes);
|
||||
plugin_pages.extend(plugin.pages);
|
||||
}
|
||||
let state = server::AppState {
|
||||
frames: self.frames.clone(),
|
||||
store: Arc::clone(&self.store),
|
||||
views: Arc::clone(&self.views),
|
||||
plugin_pages: Arc::new(plugin_pages),
|
||||
page_script_urls: Arc::clone(&self.page_script_urls),
|
||||
shutdown_notify: Arc::clone(&self.shutdown_notify),
|
||||
};
|
||||
let port = self.port;
|
||||
async move {
|
||||
server::run_server_with_routes(state, port, routes).await;
|
||||
}
|
||||
}
|
||||
/// Schedule the dashboard HTTP server on its approved execution owner.
|
||||
pub fn spawn(&self, engine: &swactor_engine::EngineHandle) {
|
||||
engine.spawn(self.http_server());
|
||||
}
|
||||
|
||||
/// Schedule the dashboard HTTP server on its approved execution owner.
|
||||
pub fn spawn_with_plugins(
|
||||
&self,
|
||||
engine: &swactor_engine::EngineHandle,
|
||||
plugins: Vec<DashboardPlugin>,
|
||||
) {
|
||||
engine.spawn(self.http_server_with_plugins(plugins));
|
||||
}
|
||||
}
|
||||
|
||||
/// Create the telemetry dashboard state.
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
use std::collections::{BTreeMap, VecDeque};
|
||||
|
||||
use telemetry::frame::{Frame, StreamId};
|
||||
use parking_lot::RwLock;
|
||||
use serde_json::{Value, json};
|
||||
use telemetry::frame::{Frame, StreamId};
|
||||
|
||||
use crate::FrameEvent;
|
||||
use crate::view::DashboardView;
|
||||
|
|
|
|||
|
|
@ -11,37 +11,52 @@ use tokio::sync::{broadcast, mpsc};
|
|||
use tokio_stream::StreamExt;
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
|
||||
use crate::FrameEvent;
|
||||
use crate::store::DashboardStore;
|
||||
use crate::view::ViewRegistry;
|
||||
use crate::{FrameEvent, PluginPage};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct AppState {
|
||||
pub frames: broadcast::Sender<FrameEvent>,
|
||||
pub store: Arc<DashboardStore>,
|
||||
pub views: Arc<ViewRegistry>,
|
||||
pub plugin_pages: Arc<Vec<PluginPage>>,
|
||||
pub page_script_urls: Arc<Vec<String>>,
|
||||
pub shutdown_notify: Arc<tokio::sync::Notify>,
|
||||
}
|
||||
|
||||
pub(crate) async fn run_server(state: AppState, port: u16) {
|
||||
run_server_with_routes(state, port, Router::new()).await;
|
||||
}
|
||||
|
||||
pub(crate) async fn run_server_with_routes(state: AppState, port: u16, extra: Router) {
|
||||
let listener = tokio::net::TcpListener::bind(format!("0.0.0.0:{port}"))
|
||||
.await
|
||||
.expect("failed to bind HTTP server");
|
||||
let shutdown = Arc::clone(&state.shutdown_notify);
|
||||
axum::serve(listener, router(state))
|
||||
axum::serve(listener, router(state).merge(extra))
|
||||
.with_graceful_shutdown(async move { shutdown.notified().await })
|
||||
.await
|
||||
.expect("HTTP server error");
|
||||
}
|
||||
|
||||
fn router(state: AppState) -> Router {
|
||||
let router = Router::new()
|
||||
let mut router = Router::new()
|
||||
.route("/", get(root_page))
|
||||
.route("/events", get(frame_stream))
|
||||
.route("/api/frames", get(recent_frames))
|
||||
.route("/api/views", get(views_json))
|
||||
.route("/api/view/{*path}", get(view_snapshot))
|
||||
.route("/view/{*path}", get(view_page));
|
||||
for page in state.plugin_pages.iter().cloned() {
|
||||
router = router.route(
|
||||
page.path,
|
||||
get(move |State(state): State<AppState>| {
|
||||
let page = page.clone();
|
||||
async move { Html(inject_nav(page.html, page.id, &state)) }
|
||||
}),
|
||||
);
|
||||
}
|
||||
#[cfg(feature = "demo-control")]
|
||||
let router = router
|
||||
.route("/control/kill", axum::routing::post(control_kill))
|
||||
|
|
@ -52,9 +67,7 @@ fn router(state: AppState) -> Router {
|
|||
}
|
||||
|
||||
#[cfg(feature = "demo-control")]
|
||||
async fn control_kill(
|
||||
Json(command): Json<crate::control::ControlCommand>,
|
||||
) -> impl IntoResponse {
|
||||
async fn control_kill(Json(command): Json<crate::control::ControlCommand>) -> impl IntoResponse {
|
||||
match command {
|
||||
crate::control::ControlCommand::Kill { .. } => {
|
||||
if crate::control::dispatch(command) {
|
||||
|
|
@ -84,9 +97,7 @@ async fn control_provision(
|
|||
}
|
||||
|
||||
#[cfg(feature = "demo-control")]
|
||||
async fn control_remove(
|
||||
Json(command): Json<crate::control::ControlCommand>,
|
||||
) -> impl IntoResponse {
|
||||
async fn control_remove(Json(command): Json<crate::control::ControlCommand>) -> impl IntoResponse {
|
||||
match command {
|
||||
crate::control::ControlCommand::Remove { .. } => {
|
||||
if crate::control::dispatch(command) {
|
||||
|
|
@ -100,9 +111,7 @@ async fn control_remove(
|
|||
}
|
||||
|
||||
#[cfg(feature = "demo-control")]
|
||||
async fn control_edge(
|
||||
Json(command): Json<crate::control::ControlCommand>,
|
||||
) -> impl IntoResponse {
|
||||
async fn control_edge(Json(command): Json<crate::control::ControlCommand>) -> impl IntoResponse {
|
||||
match command {
|
||||
crate::control::ControlCommand::EstablishEdge { .. } => {
|
||||
if crate::control::dispatch(command) {
|
||||
|
|
@ -133,9 +142,6 @@ const NAV_PLACEHOLDER: &str = "<!--swactor:nav-->";
|
|||
/// Build the unified top navbar from the view registry (registration order,
|
||||
/// active by served path). Fleet links to `/` — it is the home page.
|
||||
fn inject_nav(html: &str, active_path: &str, state: &AppState) -> String {
|
||||
if !html.contains(NAV_PLACEHOLDER) {
|
||||
return html.to_owned();
|
||||
}
|
||||
let mut links = Vec::new();
|
||||
for view in state.views.descriptors() {
|
||||
if !view.show_in_nav {
|
||||
|
|
@ -150,10 +156,26 @@ fn inject_nav(html: &str, active_path: &str, state: &AppState) -> String {
|
|||
links.push(format!(
|
||||
r#"<a href="{}"{}>{}</a>"#,
|
||||
escape_html(&href),
|
||||
if active { r#" aria-current="page" data-active="true""# } else { "" },
|
||||
if active {
|
||||
r#" aria-current="page" data-active="true""#
|
||||
} else {
|
||||
""
|
||||
},
|
||||
escape_html(view.title),
|
||||
));
|
||||
}
|
||||
for page in state.plugin_pages.iter() {
|
||||
links.push(format!(
|
||||
r#"<a href="{}"{}>{}</a>"#,
|
||||
escape_html(page.path),
|
||||
if page.id == active_path {
|
||||
r#" aria-current="page" data-active="true""#
|
||||
} else {
|
||||
""
|
||||
},
|
||||
escape_html(page.title),
|
||||
));
|
||||
}
|
||||
let nav = format!(
|
||||
concat!(
|
||||
r#"<script>try{{var t=localStorage.getItem('swactor-theme');document.documentElement.dataset.theme=t||(matchMedia('(prefers-color-scheme: light)').matches?'light':'dark')}}catch(e){{}}</script>"#,
|
||||
|
|
@ -182,7 +204,16 @@ fn inject_nav(html: &str, active_path: &str, state: &AppState) -> String {
|
|||
),
|
||||
links.join("")
|
||||
);
|
||||
html.replace(NAV_PLACEHOLDER, &nav)
|
||||
let rendered = html.replace(NAV_PLACEHOLDER, &nav);
|
||||
if state.page_script_urls.is_empty() {
|
||||
return rendered;
|
||||
}
|
||||
let scripts = state
|
||||
.page_script_urls
|
||||
.iter()
|
||||
.map(|url| format!(r#"<script src="{}"></script>"#, escape_html(url)))
|
||||
.collect::<String>();
|
||||
rendered.replacen("</body>", &format!("{scripts}</body>"), 1)
|
||||
}
|
||||
|
||||
fn escape_html(value: &str) -> String {
|
||||
|
|
@ -198,7 +229,21 @@ async fn recent_frames(State(state): State<AppState>) -> Json<Vec<FrameEvent>> {
|
|||
}
|
||||
|
||||
async fn views_json(State(state): State<AppState>) -> Json<serde_json::Value> {
|
||||
Json(serde_json::json!({ "views": state.views.descriptors() }))
|
||||
let pages = state
|
||||
.plugin_pages
|
||||
.iter()
|
||||
.map(|page| {
|
||||
serde_json::json!({
|
||||
"id": page.id,
|
||||
"title": page.title,
|
||||
"path": page.path,
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
Json(serde_json::json!({
|
||||
"views": state.views.descriptors(),
|
||||
"plugin_pages": pages,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn view_snapshot(
|
||||
|
|
@ -265,3 +310,351 @@ async fn frame_stream(
|
|||
|
||||
Sse::new(ReceiverStream::new(out).map(Ok)).keep_alive(KeepAlive::default())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[cfg(feature = "demo-control")]
|
||||
use std::sync::{LazyLock, Mutex};
|
||||
#[cfg(feature = "demo-control")]
|
||||
use std::time::Duration;
|
||||
|
||||
use super::*;
|
||||
#[cfg(feature = "demo-control")]
|
||||
use axum::body::Body;
|
||||
#[cfg(feature = "demo-control")]
|
||||
use axum::http::Request;
|
||||
#[cfg(feature = "demo-control")]
|
||||
use proptest::prelude::*;
|
||||
#[cfg(feature = "demo-control")]
|
||||
use tower::util::ServiceExt;
|
||||
|
||||
fn state_with_plugin(page: PluginPage) -> AppState {
|
||||
let views = Arc::new(ViewRegistry::new());
|
||||
let store = Arc::new(DashboardStore::new(1, Arc::clone(&views)));
|
||||
let (frames, _) = broadcast::channel(1);
|
||||
AppState {
|
||||
frames,
|
||||
store,
|
||||
views,
|
||||
plugin_pages: Arc::new(vec![page]),
|
||||
page_script_urls: Arc::new(Vec::new()),
|
||||
shutdown_notify: Arc::new(tokio::sync::Notify::new()),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registered_plugin_page_appears_in_shared_navigation() {
|
||||
let state = state_with_plugin(PluginPage::new(
|
||||
"provision",
|
||||
"Provision",
|
||||
"/provision",
|
||||
"<!--swactor:nav-->",
|
||||
));
|
||||
let rendered = inject_nav("<!--swactor:nav-->", "provision", &state);
|
||||
assert!(rendered.contains(r#"<a href="/provision""#));
|
||||
assert!(rendered.contains(r#"aria-current="page" data-active="true">Provision</a>"#));
|
||||
}
|
||||
|
||||
#[cfg(feature = "demo-control")]
|
||||
const HTTP_RESPONSE_BUDGET: Duration = Duration::from_secs(1);
|
||||
|
||||
#[cfg(feature = "demo-control")]
|
||||
static TEST_CONTROL_RECEIVER: LazyLock<
|
||||
Mutex<std::sync::mpsc::Receiver<crate::control::ControlCommand>>,
|
||||
> = LazyLock::new(|| {
|
||||
let (sender, receiver) = std::sync::mpsc::channel();
|
||||
crate::control::set_control_sender(sender);
|
||||
Mutex::new(receiver)
|
||||
});
|
||||
|
||||
#[cfg(feature = "demo-control")]
|
||||
fn test_control_receiver()
|
||||
-> &'static Mutex<std::sync::mpsc::Receiver<crate::control::ControlCommand>> {
|
||||
&TEST_CONTROL_RECEIVER
|
||||
}
|
||||
|
||||
#[cfg(feature = "demo-control")]
|
||||
#[derive(Clone, Debug)]
|
||||
struct HttpAction {
|
||||
route: u8,
|
||||
payload: u8,
|
||||
value: u8,
|
||||
}
|
||||
|
||||
#[cfg(feature = "demo-control")]
|
||||
impl HttpAction {
|
||||
fn uri(&self) -> &'static str {
|
||||
match self.route % 4 {
|
||||
0 => "/control/kill",
|
||||
1 => "/control/provision",
|
||||
2 => "/control/remove",
|
||||
_ => "/control/edge",
|
||||
}
|
||||
}
|
||||
|
||||
fn command_json(&self, route: u8) -> String {
|
||||
let command_id = format!("repeated-{}", self.value % 4);
|
||||
match route % 4 {
|
||||
0 => serde_json::json!({
|
||||
"Kill": {
|
||||
"command_id": command_id,
|
||||
"node": format!("node-{}", self.value),
|
||||
}
|
||||
})
|
||||
.to_string(),
|
||||
1 => serde_json::json!({
|
||||
"Provision": {
|
||||
"command_id": command_id,
|
||||
"count": self.value,
|
||||
}
|
||||
})
|
||||
.to_string(),
|
||||
2 => serde_json::json!({
|
||||
"Remove": {
|
||||
"command_id": command_id,
|
||||
"count": self.value,
|
||||
}
|
||||
})
|
||||
.to_string(),
|
||||
_ => serde_json::json!({
|
||||
"EstablishEdge": {
|
||||
"command_id": command_id,
|
||||
"node": format!("node-{}", self.value),
|
||||
}
|
||||
})
|
||||
.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn body(&self) -> String {
|
||||
match self.payload % 5 {
|
||||
0 => self.command_json(self.route),
|
||||
1 => match self.value % 6 {
|
||||
0 => String::new(),
|
||||
1 => "{".to_owned(),
|
||||
2 => "[".to_owned(),
|
||||
3 => "{\"".to_owned(),
|
||||
4 => "{\"command_id\":".to_owned(),
|
||||
_ => "not-json".to_owned(),
|
||||
},
|
||||
2 => {
|
||||
let command_id = format!("repeated-{}", self.value % 4);
|
||||
match self.route % 4 {
|
||||
0 => serde_json::json!({
|
||||
"Kill": {"command_id": command_id}
|
||||
})
|
||||
.to_string(),
|
||||
1 => serde_json::json!({
|
||||
"Provision": {"count": self.value}
|
||||
})
|
||||
.to_string(),
|
||||
2 => serde_json::json!({
|
||||
"Remove": {"command_id": command_id}
|
||||
})
|
||||
.to_string(),
|
||||
_ => serde_json::json!({
|
||||
"EstablishEdge": {"node": format!("node-{}", self.value)}
|
||||
})
|
||||
.to_string(),
|
||||
}
|
||||
}
|
||||
3 => self.command_json(self.route.wrapping_add(1)),
|
||||
_ => match self.route % 4 {
|
||||
0 => r#"{"Kill":{"command_id":7,"node":[]}}"#.to_owned(),
|
||||
1 => r#"{"Provision":{"command_id":7,"count":"one"}}"#.to_owned(),
|
||||
2 => r#"{"Remove":{"command_id":[],"count":-1}}"#.to_owned(),
|
||||
_ => r#"{"EstablishEdge":{"command_id":false,"node":7}}"#.to_owned(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn is_valid_for_route(&self) -> bool {
|
||||
self.payload % 5 == 0
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "demo-control")]
|
||||
#[derive(Clone, Debug)]
|
||||
struct HttpObservation {
|
||||
index: usize,
|
||||
uri: &'static str,
|
||||
body: String,
|
||||
expected_valid: bool,
|
||||
status: StatusCode,
|
||||
}
|
||||
|
||||
#[cfg(feature = "demo-control")]
|
||||
async fn send_control_request(
|
||||
app: Router,
|
||||
index: usize,
|
||||
action: HttpAction,
|
||||
) -> Result<HttpObservation, String> {
|
||||
let uri = action.uri();
|
||||
let body = action.body();
|
||||
let expected_valid = action.is_valid_for_route();
|
||||
let request = Request::post(uri)
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(body.clone()))
|
||||
.map_err(|error| format!("build dashboard request: {error}"))?;
|
||||
let response = tokio::time::timeout(HTTP_RESPONSE_BUDGET, app.oneshot(request))
|
||||
.await
|
||||
.map_err(|_| format!("request {index} {uri} exceeded {HTTP_RESPONSE_BUDGET:?}"))?
|
||||
.map_err(|error| format!("route dashboard request: {error}"))?;
|
||||
Ok(HttpObservation {
|
||||
index,
|
||||
uri,
|
||||
body,
|
||||
expected_valid,
|
||||
status: response.status(),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(feature = "demo-control")]
|
||||
fn http_invariant_failure(
|
||||
actions: &[HttpAction],
|
||||
responses: &[HttpObservation],
|
||||
forwarded: usize,
|
||||
) -> Option<String> {
|
||||
let expected_forwarded = actions
|
||||
.iter()
|
||||
.filter(|action| action.is_valid_for_route())
|
||||
.count();
|
||||
let terminal = responses.len() == actions.len();
|
||||
let response_log = responses
|
||||
.iter()
|
||||
.map(|response| {
|
||||
format!(
|
||||
"#{} {} body={:?} -> {}",
|
||||
response.index, response.uri, response.body, response.status,
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let statuses_valid = responses.iter().all(|response| {
|
||||
!response.status.is_server_error()
|
||||
&& if response.expected_valid {
|
||||
response.status == StatusCode::ACCEPTED
|
||||
} else {
|
||||
response.status.is_client_error()
|
||||
}
|
||||
});
|
||||
if terminal && statuses_valid && forwarded == expected_forwarded {
|
||||
None
|
||||
} else {
|
||||
Some(format!(
|
||||
"terminal={terminal}, expected_forwarded={expected_forwarded}, \
|
||||
forwarded={forwarded}, responses={response_log:?}, \
|
||||
actor_census=dashboard HTTP routes own no actors"
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "demo-control")]
|
||||
proptest! {
|
||||
#![proptest_config(ProptestConfig {
|
||||
cases: 128,
|
||||
max_shrink_iters: 2_000,
|
||||
..ProptestConfig::default()
|
||||
})]
|
||||
|
||||
#[test]
|
||||
fn generated_control_http_sequences_are_bounded_and_typed(
|
||||
raw_actions in prop::collection::vec(
|
||||
(any::<u8>(), any::<u8>(), any::<u8>()),
|
||||
0..=32,
|
||||
),
|
||||
concurrent in any::<bool>(),
|
||||
) {
|
||||
let actions = raw_actions
|
||||
.into_iter()
|
||||
.map(|(route, payload, value)| HttpAction {
|
||||
route,
|
||||
payload,
|
||||
value,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let receiver = test_control_receiver();
|
||||
while receiver
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.try_recv()
|
||||
.is_ok()
|
||||
{}
|
||||
let state = state_with_plugin(PluginPage::new(
|
||||
"control-test",
|
||||
"Control test",
|
||||
"/control-test",
|
||||
"",
|
||||
));
|
||||
let current_thread = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("build current-thread dashboard HTTP runtime");
|
||||
let outcome = current_thread.block_on(async {
|
||||
let mut responses = Vec::with_capacity(actions.len());
|
||||
if concurrent {
|
||||
let mut requests = tokio::task::JoinSet::new();
|
||||
for (index, action) in actions.iter().cloned().enumerate() {
|
||||
requests.spawn(send_control_request(router(state.clone()), index, action));
|
||||
}
|
||||
while let Some(result) = requests.join_next().await {
|
||||
responses.push(
|
||||
result
|
||||
.map_err(|error| format!("dashboard request task failed: {error}"))??,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
for (index, action) in actions.iter().cloned().enumerate() {
|
||||
responses.push(
|
||||
send_control_request(router(state.clone()), index, action).await?,
|
||||
);
|
||||
}
|
||||
}
|
||||
responses.sort_by_key(|response| response.index);
|
||||
Ok::<_, String>(responses)
|
||||
});
|
||||
prop_assert!(
|
||||
outcome.is_ok(),
|
||||
"dashboard HTTP request did not terminate; actions={:?}; error={:?}; \
|
||||
responses=[]; actor_census=dashboard HTTP routes own no actors",
|
||||
actions,
|
||||
outcome.as_ref().err(),
|
||||
);
|
||||
let responses = outcome.expect("outcome checked above");
|
||||
let forwarded = {
|
||||
let receiver = receiver
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
receiver.try_iter().count()
|
||||
};
|
||||
let failure = http_invariant_failure(&actions, &responses, forwarded);
|
||||
prop_assert!(
|
||||
failure.is_none(),
|
||||
"dashboard HTTP invariant failed; actions={:?}; responses={:?}; failure={}",
|
||||
actions,
|
||||
responses,
|
||||
failure.unwrap_or_default(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "demo-control")]
|
||||
#[test]
|
||||
fn control_http_invariant_rejects_a_controlled_server_error() {
|
||||
let actions = vec![HttpAction {
|
||||
route: 0,
|
||||
payload: 1,
|
||||
value: 0,
|
||||
}];
|
||||
let responses = vec![HttpObservation {
|
||||
index: 0,
|
||||
uri: actions[0].uri(),
|
||||
body: actions[0].body(),
|
||||
expected_valid: false,
|
||||
status: StatusCode::INTERNAL_SERVER_ERROR,
|
||||
}];
|
||||
assert!(
|
||||
http_invariant_failure(&actions, &responses, 0).is_some(),
|
||||
"HTTP invariant accepted a controlled 5xx response for invalid JSON"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
use std::collections::VecDeque;
|
||||
use std::sync::Arc;
|
||||
|
||||
use telemetry::frame::{Frame, StreamId};
|
||||
use parking_lot::Mutex;
|
||||
use telemetry::frame::{Frame, StreamId};
|
||||
|
||||
use crate::FrameEvent;
|
||||
use crate::view::ViewRegistry;
|
||||
|
|
|
|||
|
|
@ -52,10 +52,9 @@ pub(crate) struct RuntimeState {
|
|||
pub(crate) uptime_ms: Option<u64>,
|
||||
pub(crate) actors: BTreeMap<String, ActorState>,
|
||||
pub(crate) history: VecDeque<HistorySample>,
|
||||
actor_snapshot_generation: u64,
|
||||
}
|
||||
|
||||
|
||||
|
||||
impl RuntimeState {
|
||||
pub(crate) fn new(now: Instant) -> Self {
|
||||
Self {
|
||||
|
|
@ -64,6 +63,7 @@ impl RuntimeState {
|
|||
uptime_ms: None,
|
||||
actors: BTreeMap::new(),
|
||||
history: VecDeque::with_capacity(HISTORY_CAP),
|
||||
actor_snapshot_generation: 0,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -81,28 +81,45 @@ impl RuntimeState {
|
|||
}
|
||||
|
||||
fn apply_actors(&mut self, value: &Value, now: Instant) {
|
||||
// Per-worker `worker_id` wrapper (TelemetryStatsHook shape) is the
|
||||
// default placement for actors that do not carry one inline.
|
||||
// A wrapped actor list is a complete snapshot for one worker. A list
|
||||
// without a worker is a complete merged-runtime snapshot. Bare actor
|
||||
// frames remain incremental.
|
||||
let wrapper_worker = u32_field(value, &["worker_id", "worker"]);
|
||||
if let Some(actors) = value.get("actors").and_then(Value::as_array) {
|
||||
for actor in actors {
|
||||
self.apply_actor(actor, now, wrapper_worker);
|
||||
}
|
||||
return;
|
||||
}
|
||||
// A bare actor object per frame (no envelope).
|
||||
self.apply_actor(value, now, wrapper_worker);
|
||||
}
|
||||
|
||||
fn apply_actor(&mut self, value: &Value, now: Instant, default_worker: Option<u32>) {
|
||||
let Some(address) = string_field(value, &["address", "addr", "actor_addr"]) else {
|
||||
let Some(actors) = value.get("actors").and_then(Value::as_array) else {
|
||||
let _ = self.apply_actor(value, now, wrapper_worker);
|
||||
return;
|
||||
};
|
||||
|
||||
self.actor_snapshot_generation = self.actor_snapshot_generation.wrapping_add(1);
|
||||
let generation = self.actor_snapshot_generation;
|
||||
for actor in actors {
|
||||
if let Some(actor) = self.apply_actor(actor, now, wrapper_worker) {
|
||||
actor.snapshot_generation = generation;
|
||||
}
|
||||
}
|
||||
match wrapper_worker {
|
||||
Some(worker_id) => self.actors.retain(|_, actor| {
|
||||
actor.worker_id != Some(worker_id) || actor.snapshot_generation == generation
|
||||
}),
|
||||
None => self
|
||||
.actors
|
||||
.retain(|_, actor| actor.snapshot_generation == generation),
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_actor(
|
||||
&mut self,
|
||||
value: &Value,
|
||||
now: Instant,
|
||||
default_worker: Option<u32>,
|
||||
) -> Option<&mut ActorState> {
|
||||
let address = string_field(value, &["address", "addr", "actor_addr"])?;
|
||||
let actor = self
|
||||
.actors
|
||||
.entry(address.clone())
|
||||
.or_insert_with(|| ActorState::new(address));
|
||||
actor.apply_json(value, now, default_worker);
|
||||
Some(actor)
|
||||
}
|
||||
|
||||
fn apply_stats(&mut self, value: &Value, now: Instant) {
|
||||
|
|
@ -130,7 +147,7 @@ impl RuntimeState {
|
|||
// Some publishers carry full per-actor detail under `actor_details`.
|
||||
if let Some(details) = value.get("actor_details").and_then(Value::as_array) {
|
||||
for actor in details {
|
||||
self.apply_actor(actor, now, None);
|
||||
let _ = self.apply_actor(actor, now, None);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -187,6 +204,7 @@ pub(crate) struct ActorState {
|
|||
/// Messages folded away by the receipt sampling interval.
|
||||
pub(crate) sampled_out: u64,
|
||||
pub(crate) last_update: Option<Instant>,
|
||||
snapshot_generation: u64,
|
||||
}
|
||||
|
||||
impl ActorState {
|
||||
|
|
@ -208,6 +226,7 @@ impl ActorState {
|
|||
receipts: VecDeque::with_capacity(RECEIPT_CAP),
|
||||
sampled_out: 0,
|
||||
last_update: None,
|
||||
snapshot_generation: 0,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -222,7 +241,8 @@ impl ActorState {
|
|||
if let Some(actor_type) = string_field(value, &["actor_type"]).filter(|t| !t.is_empty()) {
|
||||
self.actor_type = Some(actor_type);
|
||||
}
|
||||
if let Some(message_type) = string_field(value, &["message_type"]).filter(|t| !t.is_empty()) {
|
||||
if let Some(message_type) = string_field(value, &["message_type"]).filter(|t| !t.is_empty())
|
||||
{
|
||||
self.message_type = Some(message_type);
|
||||
}
|
||||
if let Some(worker_id) = u32_field(value, &["worker_id", "worker"]) {
|
||||
|
|
@ -254,7 +274,10 @@ impl ActorState {
|
|||
self.message_type_counts = counts;
|
||||
}
|
||||
self.last_update = Some(now);
|
||||
self.fold_receipt(now, self.messages_processed.saturating_sub(processed_before));
|
||||
self.fold_receipt(
|
||||
now,
|
||||
self.messages_processed.saturating_sub(processed_before),
|
||||
);
|
||||
self.push_history(now);
|
||||
self.recompute_growth();
|
||||
}
|
||||
|
|
@ -276,10 +299,7 @@ impl ActorState {
|
|||
if self.receipts.len() == RECEIPT_CAP {
|
||||
self.receipts.pop_front();
|
||||
}
|
||||
let ty = self
|
||||
.last_msg_type
|
||||
.clone()
|
||||
.unwrap_or_else(|| "?".to_owned());
|
||||
let ty = self.last_msg_type.clone().unwrap_or_else(|| "?".to_owned());
|
||||
self.receipts.push_back(ActorReceipt {
|
||||
at: now,
|
||||
ty,
|
||||
|
|
@ -346,8 +366,6 @@ pub(crate) struct Totals {
|
|||
pub(crate) poisoned: u32,
|
||||
}
|
||||
|
||||
|
||||
|
||||
// --- tolerant JSON helpers (publisher-shape-agnostic readers) ---------------
|
||||
|
||||
fn assign_u32(slot: &mut u32, value: &Value, names: &[&str]) {
|
||||
|
|
@ -437,3 +455,64 @@ fn parse_message_type_counts(value: Option<&Value>) -> Option<Vec<(String, u64)>
|
|||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn per_worker_snapshots_remove_stopped_actors_without_touching_other_workers() {
|
||||
let now = Instant::now();
|
||||
let mut runtime = RuntimeState::new(now);
|
||||
|
||||
runtime.apply_actors(
|
||||
&json!({
|
||||
"worker_id": 0,
|
||||
"actors": [
|
||||
{"address": "stable", "actor_type": "StableActor"},
|
||||
{"address": "observer", "actor_type": "ControlReplyObserver"},
|
||||
],
|
||||
}),
|
||||
now,
|
||||
);
|
||||
runtime.apply_actors(
|
||||
&json!({
|
||||
"worker_id": 1,
|
||||
"actors": [
|
||||
{"address": "other-worker", "actor_type": "OtherActor"},
|
||||
],
|
||||
}),
|
||||
now,
|
||||
);
|
||||
assert_eq!(runtime.actors.len(), 3);
|
||||
|
||||
runtime.apply_actors(
|
||||
&json!({
|
||||
"worker_id": 0,
|
||||
"actors": [
|
||||
{"address": "stable", "actor_type": "StableActor"},
|
||||
],
|
||||
}),
|
||||
now,
|
||||
);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.actors
|
||||
.keys()
|
||||
.map(String::as_str)
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["other-worker", "stable"]
|
||||
);
|
||||
|
||||
runtime.apply_actors(&json!({"worker_id": 0, "actors": []}), now);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.actors
|
||||
.keys()
|
||||
.map(String::as_str)
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["other-worker"]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
use telemetry::frame::{Frame, StreamId};
|
||||
use parking_lot::RwLock;
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
use telemetry::frame::{Frame, StreamId};
|
||||
|
||||
use crate::FrameEvent;
|
||||
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@
|
|||
use std::collections::{BTreeMap, VecDeque};
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use telemetry::Record;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use telemetry::Record;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ArenaSnapshot {
|
||||
|
|
|
|||
|
|
@ -27,8 +27,7 @@ use crate::arena::{
|
|||
};
|
||||
use crate::edge_lifecycle::{
|
||||
EdgeCommand, EdgeEstablisher, EdgeEvent, EdgeFaultReason, EdgeLifecycleEvent, NodeId,
|
||||
ObjectSpec, ProvisionRx, ProvisionTx, RingDirection, RingLeaseRejection,
|
||||
StreamFaultReason,
|
||||
ObjectSpec, ProvisionRx, ProvisionTx, RingDirection, RingLeaseRejection, StreamFaultReason,
|
||||
};
|
||||
use crate::edge_wire::EdgeTransport;
|
||||
use crate::ids::{EdgeId, LeaseRequestId, RingId, StreamId};
|
||||
|
|
@ -231,9 +230,14 @@ impl<T: EdgeTransport> EdgeRuntime<T> {
|
|||
match event {
|
||||
crate::edge_wire::WireEvent::StreamArrived { edge_id, stream_id } => {
|
||||
self.incoming_stream(edge_id, stream_id);
|
||||
self.observations.push(Observation::StreamArrived { edge_id, stream_id });
|
||||
self.observations
|
||||
.push(Observation::StreamArrived { edge_id, stream_id });
|
||||
}
|
||||
crate::edge_wire::WireEvent::BytesRead { edge_id, stream_id, bytes } => {
|
||||
crate::edge_wire::WireEvent::BytesRead {
|
||||
edge_id,
|
||||
stream_id,
|
||||
bytes,
|
||||
} => {
|
||||
self.observations.push(Observation::BytesRead {
|
||||
edge_id,
|
||||
stream_id,
|
||||
|
|
@ -242,10 +246,8 @@ impl<T: EdgeTransport> EdgeRuntime<T> {
|
|||
self.ingress_bytes(arena, worker, edge_id, stream_id, bytes)?;
|
||||
}
|
||||
crate::edge_wire::WireEvent::StreamEnded { edge_id, stream_id } => {
|
||||
if let Some(inbound) = self
|
||||
.inbound
|
||||
.as_mut()
|
||||
.filter(|edge| edge.edge_id == edge_id)
|
||||
if let Some(inbound) =
|
||||
self.inbound.as_mut().filter(|edge| edge.edge_id == edge_id)
|
||||
{
|
||||
inbound.buffers.remove(&stream_id);
|
||||
}
|
||||
|
|
@ -407,8 +409,8 @@ impl<T: EdgeTransport> EdgeRuntime<T> {
|
|||
worker: &mut dyn WorkerPort,
|
||||
) -> Result<(), String> {
|
||||
loop {
|
||||
let progressed = self.execute_commands(transport, arena, worker)?
|
||||
|| self.drain_lifecycle()?;
|
||||
let progressed =
|
||||
self.execute_commands(transport, arena, worker)? || self.drain_lifecycle()?;
|
||||
if !progressed {
|
||||
break;
|
||||
}
|
||||
|
|
@ -473,11 +475,10 @@ impl<T: EdgeTransport> EdgeRuntime<T> {
|
|||
RingLeaseRejection::ArenaShuttingDown
|
||||
}
|
||||
};
|
||||
self.establisher
|
||||
.observe(EdgeEvent::RingLeaseRejected {
|
||||
request_id: LeaseRequestId(request_id.0),
|
||||
reason,
|
||||
});
|
||||
self.establisher.observe(EdgeEvent::RingLeaseRejected {
|
||||
request_id: LeaseRequestId(request_id.0),
|
||||
reason,
|
||||
});
|
||||
}
|
||||
ArenaEvent::RingLeaseQueued { .. }
|
||||
| ArenaEvent::RingReleased { .. }
|
||||
|
|
@ -500,10 +501,8 @@ impl<T: EdgeTransport> EdgeRuntime<T> {
|
|||
worker.install_ring(edge_id, ring_id, direction, &layout, &object_spec)?;
|
||||
match direction {
|
||||
RingDirection::Ingress => {
|
||||
if let Some(inbound) = self
|
||||
.inbound
|
||||
.as_mut()
|
||||
.filter(|edge| edge.edge_id == edge_id)
|
||||
if let Some(inbound) =
|
||||
self.inbound.as_mut().filter(|edge| edge.edge_id == edge_id)
|
||||
{
|
||||
inbound.ring_id = Some(ring_id);
|
||||
}
|
||||
|
|
@ -530,7 +529,10 @@ impl<T: EdgeTransport> EdgeRuntime<T> {
|
|||
.map(|outbound| outbound.peer.clone())
|
||||
.ok_or_else(|| "outbound edge missing".to_owned())?;
|
||||
let writer = transport.open_writer(edge_id, &peer)?;
|
||||
if let Some(outbound) = self.outbound.as_mut().filter(|edge| edge.edge_id == edge_id)
|
||||
if let Some(outbound) = self
|
||||
.outbound
|
||||
.as_mut()
|
||||
.filter(|edge| edge.edge_id == edge_id)
|
||||
{
|
||||
outbound.writer = Some(writer);
|
||||
}
|
||||
|
|
@ -600,8 +602,7 @@ impl<T: EdgeTransport> EdgeRuntime<T> {
|
|||
return Err(format!("edge {} faulted: {reason:?}", edge_id.0));
|
||||
}
|
||||
EdgeLifecycleEvent::EdgeStopped { edge_id } => {
|
||||
self.observations
|
||||
.push(Observation::EdgeStopped { edge_id });
|
||||
self.observations.push(Observation::EdgeStopped { edge_id });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -694,10 +695,5 @@ fn edge_layout(layout: &ArenaRingLayout) -> crate::edge_lifecycle::RingLayout {
|
|||
}
|
||||
|
||||
fn elapsed_ms(started: Instant) -> u64 {
|
||||
started
|
||||
.elapsed()
|
||||
.as_millis()
|
||||
.try_into()
|
||||
.unwrap_or(u64::MAX)
|
||||
started.elapsed().as_millis().try_into().unwrap_or(u64::MAX)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,10 @@ use crate::ids::{EdgeId, StreamId};
|
|||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum WireEvent {
|
||||
/// A new inbound stream arrived, tagged for `edge_id`.
|
||||
StreamArrived { edge_id: EdgeId, stream_id: StreamId },
|
||||
StreamArrived {
|
||||
edge_id: EdgeId,
|
||||
stream_id: StreamId,
|
||||
},
|
||||
/// Bytes were read from an inbound stream.
|
||||
BytesRead {
|
||||
edge_id: EdgeId,
|
||||
|
|
@ -24,7 +27,10 @@ pub enum WireEvent {
|
|||
bytes: Vec<u8>,
|
||||
},
|
||||
/// An inbound stream ended cleanly.
|
||||
StreamEnded { edge_id: EdgeId, stream_id: StreamId },
|
||||
StreamEnded {
|
||||
edge_id: EdgeId,
|
||||
stream_id: StreamId,
|
||||
},
|
||||
/// A stream or connection-level fault. `None` ids mean the fault could
|
||||
/// not be attributed to a specific edge or stream.
|
||||
StreamFault {
|
||||
|
|
@ -56,7 +62,11 @@ pub trait EdgeTransport {
|
|||
type PeerAddr: Clone;
|
||||
|
||||
/// Open (or continue) the writer pumping bytes to `edge_id`'s peer.
|
||||
fn open_writer(&mut self, edge_id: EdgeId, peer: &Self::PeerAddr) -> Result<Self::Writer, String>;
|
||||
fn open_writer(
|
||||
&mut self,
|
||||
edge_id: EdgeId,
|
||||
peer: &Self::PeerAddr,
|
||||
) -> Result<Self::Writer, String>;
|
||||
|
||||
/// Drain all transport events observed since the last call.
|
||||
fn drain_events(&mut self) -> Vec<WireEvent>;
|
||||
|
|
|
|||
|
|
@ -22,8 +22,7 @@ use data_plane::edge_runtime::{EdgeRuntime, LoadedObject, Observation, WorkerPor
|
|||
use data_plane::edge_wire::{EdgeTransport, EdgeWriter, WireEvent};
|
||||
use data_plane::ids::{EdgeId, RingId, StreamId};
|
||||
use data_plane::object_record::{
|
||||
ObjectFlags, ObjectId, ObjectLayout, ObjectRecord, ObjectRecordBuilder,
|
||||
ObjectSpec as ParseSpec,
|
||||
ObjectFlags, ObjectId, ObjectLayout, ObjectRecord, ObjectRecordBuilder, ObjectSpec as ParseSpec,
|
||||
};
|
||||
|
||||
// ─── Mock transport ─────────────────────────────────────────────────────────
|
||||
|
|
@ -171,7 +170,9 @@ fn record_bytes(object_id: u64, sequence: u64) -> Vec<u8> {
|
|||
.encode()
|
||||
}
|
||||
|
||||
fn arena_runtime(events: Vec<WireEvent>) -> (
|
||||
fn arena_runtime(
|
||||
events: Vec<WireEvent>,
|
||||
) -> (
|
||||
EdgeRuntime<MockTransport>,
|
||||
MockTransport,
|
||||
ArenaManager,
|
||||
|
|
@ -213,7 +214,9 @@ fn inbound_edge_reaches_ready_and_delivers_objects() {
|
|||
]);
|
||||
runtime.establish_inbound(provision_rx(7001), parse_spec());
|
||||
|
||||
runtime.poll(&mut transport, &mut arena, &mut worker).expect("poll");
|
||||
runtime
|
||||
.poll(&mut transport, &mut arena, &mut worker)
|
||||
.expect("poll");
|
||||
let observations = runtime.take_observations();
|
||||
|
||||
assert_eq!(worker.installed.len(), 1, "worker ring must be installed");
|
||||
|
|
@ -223,7 +226,10 @@ fn inbound_edge_reaches_ready_and_delivers_objects() {
|
|||
.expect("edge ready observation") else {
|
||||
unreachable!()
|
||||
};
|
||||
assert_eq!(*direction, data_plane::edge_lifecycle::RingDirection::Ingress);
|
||||
assert_eq!(
|
||||
*direction,
|
||||
data_plane::edge_lifecycle::RingDirection::Ingress
|
||||
);
|
||||
|
||||
let Observation::ObjectLoaded { object, .. } = find_observation(&observations, |obs| {
|
||||
matches!(obs, Observation::ObjectLoaded { .. })
|
||||
|
|
@ -256,7 +262,9 @@ fn outbound_edge_opens_writer_and_allocates_object_ids() {
|
|||
let (mut runtime, mut transport, mut arena, mut worker) = arena_runtime(Vec::new());
|
||||
runtime.establish_outbound(provision_tx(7002), ());
|
||||
|
||||
runtime.poll(&mut transport, &mut arena, &mut worker).expect("poll");
|
||||
runtime
|
||||
.poll(&mut transport, &mut arena, &mut worker)
|
||||
.expect("poll");
|
||||
let observations = runtime.take_observations();
|
||||
|
||||
assert!(matches!(
|
||||
|
|
@ -277,13 +285,16 @@ fn outbound_edge_opens_writer_and_allocates_object_ids() {
|
|||
// still becomes Ready once establishment completes.
|
||||
#[test]
|
||||
fn early_stream_waits_for_recv_establishment() {
|
||||
let (mut runtime, mut transport, mut arena, mut worker) = arena_runtime(vec![WireEvent::StreamArrived {
|
||||
edge_id: EdgeId(7003),
|
||||
stream_id: StreamId(9),
|
||||
}]);
|
||||
let (mut runtime, mut transport, mut arena, mut worker) =
|
||||
arena_runtime(vec![WireEvent::StreamArrived {
|
||||
edge_id: EdgeId(7003),
|
||||
stream_id: StreamId(9),
|
||||
}]);
|
||||
runtime.establish_inbound(provision_rx(7003), parse_spec());
|
||||
|
||||
runtime.poll(&mut transport, &mut arena, &mut worker).expect("poll");
|
||||
runtime
|
||||
.poll(&mut transport, &mut arena, &mut worker)
|
||||
.expect("poll");
|
||||
let observations = runtime.take_observations();
|
||||
assert!(matches!(
|
||||
find_observation(&observations, |obs| matches!(
|
||||
|
|
@ -299,11 +310,12 @@ fn early_stream_waits_for_recv_establishment() {
|
|||
#[test]
|
||||
fn malformed_ingress_record_is_fatal_and_reported() {
|
||||
let garbage = vec![0xDE; 64];
|
||||
let (mut runtime, mut transport, mut arena, mut worker) = arena_runtime(vec![WireEvent::BytesRead {
|
||||
edge_id: EdgeId(7004),
|
||||
stream_id: StreamId(1),
|
||||
bytes: garbage,
|
||||
}]);
|
||||
let (mut runtime, mut transport, mut arena, mut worker) =
|
||||
arena_runtime(vec![WireEvent::BytesRead {
|
||||
edge_id: EdgeId(7004),
|
||||
stream_id: StreamId(1),
|
||||
bytes: garbage,
|
||||
}]);
|
||||
runtime.establish_inbound(provision_rx(7004), parse_spec());
|
||||
|
||||
let result = runtime.poll(&mut transport, &mut arena, &mut worker);
|
||||
|
|
@ -313,7 +325,10 @@ fn malformed_ingress_record_is_fatal_and_reported() {
|
|||
find_observation(&observations, |obs| {
|
||||
matches!(
|
||||
obs,
|
||||
Observation::ObjectFailed { object_id: None, .. }
|
||||
Observation::ObjectFailed {
|
||||
object_id: None,
|
||||
..
|
||||
}
|
||||
)
|
||||
}),
|
||||
Some(_)
|
||||
|
|
@ -324,11 +339,12 @@ fn malformed_ingress_record_is_fatal_and_reported() {
|
|||
// remain fatal.
|
||||
#[test]
|
||||
fn worker_load_failure_reports_object_and_is_fatal() {
|
||||
let (mut runtime, mut transport, mut arena, mut worker) = arena_runtime(vec![WireEvent::BytesRead {
|
||||
edge_id: EdgeId(7005),
|
||||
stream_id: StreamId(1),
|
||||
bytes: record_bytes(9002, 1),
|
||||
}]);
|
||||
let (mut runtime, mut transport, mut arena, mut worker) =
|
||||
arena_runtime(vec![WireEvent::BytesRead {
|
||||
edge_id: EdgeId(7005),
|
||||
stream_id: StreamId(1),
|
||||
bytes: record_bytes(9002, 1),
|
||||
}]);
|
||||
worker.fail_load = true;
|
||||
runtime.establish_inbound(provision_rx(7005), parse_spec());
|
||||
|
||||
|
|
|
|||
|
|
@ -21,3 +21,4 @@ libc = "0.2"
|
|||
[dev-dependencies]
|
||||
serde_json = "1"
|
||||
proptest = "1"
|
||||
swactor-engine = { path = "../engine", default-features = false }
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
//! Distribution-owned telemetry records.
|
||||
|
||||
use telemetry::Record;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use telemetry::Record;
|
||||
|
||||
/// Transport internals — connectivity to peers and relay.
|
||||
pub const TRANSPORT_INTERNALS: &str = "transport.internals";
|
||||
|
|
|
|||
|
|
@ -228,14 +228,14 @@ mod standalone_gossip_transport {
|
|||
//! Actorized registry/metadata/directory gossip over one codec and transport, proving
|
||||
//! standalone frames converge without piggybacking on SWIM.
|
||||
|
||||
use std::cell::RefCell;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, RwLock};
|
||||
|
||||
use swactor::Error;
|
||||
use swactor::actor::ActorAddress;
|
||||
use swactor::runtime::{Inbox, Runtime, RuntimeConfig, RuntimeParts, SingleThreadRuntime};
|
||||
use swactor::runtime::{Inbox, Runtime, RuntimeConfig, RuntimeParts};
|
||||
use swactor::std::StdExtension;
|
||||
use swactor_engine::{Engine, SteppingBackend};
|
||||
use swactor_transport::{CodecRegistry, Transport, TransportRouter, WireEnvelope};
|
||||
|
||||
use distribution::crypto::{Keypair, KeypairExt};
|
||||
|
|
@ -272,7 +272,8 @@ mod standalone_gossip_transport {
|
|||
/// plus the shared state needed to wire it into a mesh.
|
||||
struct Node {
|
||||
rt: Runtime,
|
||||
host: RefCell<SingleThreadRuntime>,
|
||||
_engine: Engine,
|
||||
backend: SteppingBackend,
|
||||
registry: ActorAddress,
|
||||
metadata: ActorAddress,
|
||||
directory: ActorAddress,
|
||||
|
|
@ -305,7 +306,9 @@ mod standalone_gossip_transport {
|
|||
codec.clone(),
|
||||
router.clone(),
|
||||
)));
|
||||
let host = RefCell::new(SingleThreadRuntime::new(parts));
|
||||
let backend = SteppingBackend::new();
|
||||
let engine =
|
||||
Engine::new(parts, backend.clone()).expect("create stepping actor engine");
|
||||
|
||||
let dir = SharedPeerDirectory::new();
|
||||
let relay_mirror: RelayMirror = Arc::new(RwLock::new(HashMap::new()));
|
||||
|
|
@ -336,7 +339,8 @@ mod standalone_gossip_transport {
|
|||
|
||||
nodes.push(Node {
|
||||
rt,
|
||||
host,
|
||||
_engine: engine,
|
||||
backend,
|
||||
registry,
|
||||
metadata,
|
||||
directory,
|
||||
|
|
@ -407,7 +411,7 @@ mod standalone_gossip_transport {
|
|||
fn pump(&self, k: usize) {
|
||||
for _ in 0..k {
|
||||
for node in &self.nodes {
|
||||
node.host.borrow_mut().tick();
|
||||
node.backend.step();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -448,7 +452,7 @@ mod standalone_gossip_transport {
|
|||
},
|
||||
)
|
||||
.unwrap();
|
||||
self.nodes[observer].host.borrow_mut().tick();
|
||||
self.nodes[observer].backend.step();
|
||||
inbox.try_recv().and_then(|r| r.binding)
|
||||
}
|
||||
|
||||
|
|
@ -465,7 +469,7 @@ mod standalone_gossip_transport {
|
|||
},
|
||||
)
|
||||
.unwrap();
|
||||
self.nodes[observer].host.borrow_mut().tick();
|
||||
self.nodes[observer].backend.step();
|
||||
inbox.try_recv().and_then(|r| r.relay_url)
|
||||
}
|
||||
|
||||
|
|
@ -482,7 +486,7 @@ mod standalone_gossip_transport {
|
|||
},
|
||||
)
|
||||
.unwrap();
|
||||
self.nodes[observer].host.borrow_mut().tick();
|
||||
self.nodes[observer].backend.step();
|
||||
inbox.try_recv().and_then(|located| located.host)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,15 +16,15 @@ mod directory_actor {
|
|||
//! DirectoryActor convergence and safety: signed claims, supersede, deterministic conflict
|
||||
//! resolution, dead-host hiding, catch-up, quieting, and retained recovery claims.
|
||||
|
||||
use std::cell::RefCell;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, RwLock};
|
||||
|
||||
use swactor::Error;
|
||||
use swactor::actor::ActorAddress;
|
||||
use swactor::runtime::{Inbox, Runtime, RuntimeConfig, RuntimeParts, SingleThreadRuntime};
|
||||
use swactor::runtime::{Inbox, Runtime, RuntimeConfig, RuntimeParts};
|
||||
use swactor::std::StdExtension;
|
||||
use swactor_engine::{Engine, SteppingBackend};
|
||||
use swactor_transport::{CodecRegistry, Transport, TransportRouter, WireEnvelope};
|
||||
|
||||
use distribution::crypto::{Keypair, KeypairExt};
|
||||
|
|
@ -63,7 +63,8 @@ mod directory_actor {
|
|||
/// wire it into a mesh and observe it.
|
||||
struct Node {
|
||||
rt: Runtime,
|
||||
host: RefCell<SingleThreadRuntime>,
|
||||
_engine: Engine,
|
||||
backend: SteppingBackend,
|
||||
directory: ActorAddress,
|
||||
dir: SharedPeerDirectory,
|
||||
router: Arc<TransportRouter>,
|
||||
|
|
@ -94,7 +95,9 @@ mod directory_actor {
|
|||
codec.clone(),
|
||||
router.clone(),
|
||||
)));
|
||||
let host = RefCell::new(SingleThreadRuntime::new(parts));
|
||||
let backend = SteppingBackend::new();
|
||||
let engine =
|
||||
Engine::new(parts, backend.clone()).expect("create stepping actor engine");
|
||||
|
||||
let dir = SharedPeerDirectory::new();
|
||||
let route_view: RouteView = Arc::new(RwLock::new(HashMap::new()));
|
||||
|
|
@ -109,7 +112,8 @@ mod directory_actor {
|
|||
|
||||
nodes.push(Node {
|
||||
rt,
|
||||
host,
|
||||
_engine: engine,
|
||||
backend,
|
||||
directory,
|
||||
dir,
|
||||
router,
|
||||
|
|
@ -181,7 +185,7 @@ mod directory_actor {
|
|||
fn pump(&self, k: usize) {
|
||||
for _ in 0..k {
|
||||
for node in &self.nodes {
|
||||
node.host.borrow_mut().tick();
|
||||
node.backend.step();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -246,7 +250,7 @@ mod directory_actor {
|
|||
},
|
||||
)
|
||||
.unwrap();
|
||||
self.nodes[observer].host.borrow_mut().tick();
|
||||
self.nodes[observer].backend.step();
|
||||
inbox.try_recv().and_then(|located| located.host)
|
||||
}
|
||||
|
||||
|
|
@ -585,15 +589,15 @@ mod directory_route_path {
|
|||
//! Application delivery through the directory route view: address-only sends, supersede, and
|
||||
//! best-effort drops.
|
||||
|
||||
use std::cell::RefCell;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex, RwLock};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use swactor::Error;
|
||||
use swactor::actor::{ActorAddress, ActorInterface};
|
||||
use swactor::runtime::{Ctx, Runtime, RuntimeConfig, RuntimeParts, SingleThreadRuntime};
|
||||
use swactor::runtime::{Ctx, Runtime, RuntimeConfig, RuntimeParts};
|
||||
use swactor::std::StdExtension;
|
||||
use swactor_engine::{Engine, SteppingBackend};
|
||||
use swactor_transport::{CodecRegistry, NetworkMessage, TransportRouter};
|
||||
|
||||
use distribution::crypto::{Keypair, KeypairExt};
|
||||
|
|
@ -659,7 +663,8 @@ mod directory_route_path {
|
|||
|
||||
struct RouteNode {
|
||||
rt: Runtime,
|
||||
host: RefCell<SingleThreadRuntime>,
|
||||
_engine: Engine,
|
||||
backend: SteppingBackend,
|
||||
outbox: Outbox,
|
||||
route_view: RouteView,
|
||||
directory: ActorAddress,
|
||||
|
|
@ -702,7 +707,9 @@ mod directory_route_path {
|
|||
codec.clone(),
|
||||
router.clone(),
|
||||
)));
|
||||
let host = RefCell::new(SingleThreadRuntime::new(parts));
|
||||
let backend = SteppingBackend::new();
|
||||
let engine =
|
||||
Engine::new(parts, backend.clone()).expect("create stepping actor engine");
|
||||
|
||||
let outbox: Outbox = Arc::new(Mutex::new(Vec::new()));
|
||||
let route_view: RouteView = Arc::new(RwLock::new(HashMap::new()));
|
||||
|
|
@ -731,7 +738,8 @@ mod directory_route_path {
|
|||
|
||||
nodes.push(RouteNode {
|
||||
rt,
|
||||
host,
|
||||
_engine: engine,
|
||||
backend,
|
||||
outbox,
|
||||
route_view,
|
||||
directory,
|
||||
|
|
@ -801,7 +809,7 @@ mod directory_route_path {
|
|||
fn settle(&self, k: usize) {
|
||||
for _ in 0..k {
|
||||
for node in &self.nodes {
|
||||
node.host.borrow_mut().tick();
|
||||
node.backend.step();
|
||||
}
|
||||
self.deliver_wire();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,13 +16,13 @@ mod single_runtime_actor {
|
|||
//! SwimActor behavior in one runtime: subscription stream convergence and genuine unreachable-
|
||||
//! peer death detection.
|
||||
|
||||
use std::cell::RefCell;
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use swactor::runtime::{Runtime, RuntimeConfig, RuntimeParts, SingleThreadRuntime};
|
||||
use swactor::runtime::{Runtime, RuntimeConfig, RuntimeParts};
|
||||
use swactor::std::StdExtension;
|
||||
use swactor_engine::{Engine, SteppingBackend};
|
||||
|
||||
use distribution::swim::actor::{
|
||||
MembershipChanged, PeerDirectory, SharedPeerDirectory, SwimActor, SwimIn,
|
||||
|
|
@ -55,7 +55,8 @@ mod single_runtime_actor {
|
|||
/// `MembershipChanged` subscriber inbox, and a shared Binding.
|
||||
struct ActorCluster {
|
||||
rt: Runtime,
|
||||
host: RefCell<SingleThreadRuntime>,
|
||||
_engine: Engine,
|
||||
backend: SteppingBackend,
|
||||
ids: Vec<NodeId>,
|
||||
addrs: Vec<swactor::actor::ActorAddress>,
|
||||
inboxes: Vec<swactor::runtime::Inbox<MembershipChanged>>,
|
||||
|
|
@ -72,7 +73,8 @@ mod single_runtime_actor {
|
|||
let parts = RuntimeParts::new(RuntimeConfig::default())
|
||||
.with_extension(Arc::new(StdExtension::new()));
|
||||
let rt = parts.runtime().clone();
|
||||
let host = RefCell::new(SingleThreadRuntime::new(parts));
|
||||
let backend = SteppingBackend::new();
|
||||
let engine = Engine::new(parts, backend.clone()).expect("create stepping actor engine");
|
||||
let dir = SharedPeerDirectory::new();
|
||||
let now = Instant::now();
|
||||
let ids: Vec<NodeId> = (0..n).map(|i| id(i as u8)).collect();
|
||||
|
|
@ -103,7 +105,8 @@ mod single_runtime_actor {
|
|||
}
|
||||
let mut c = ActorCluster {
|
||||
rt,
|
||||
host,
|
||||
_engine: engine,
|
||||
backend,
|
||||
ids,
|
||||
addrs,
|
||||
inboxes,
|
||||
|
|
@ -128,7 +131,7 @@ mod single_runtime_actor {
|
|||
|
||||
fn pump(&self, n: usize) {
|
||||
for _ in 0..n {
|
||||
self.host.borrow_mut().tick();
|
||||
self.backend.step();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -242,15 +245,15 @@ mod transport_runtime_actor {
|
|||
//! SwimActor behavior across separate runtimes through codec, TransportRouter, deliver_raw, and
|
||||
//! transport send failure.
|
||||
|
||||
use std::cell::RefCell;
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use swactor::Error;
|
||||
use swactor::actor::ActorAddress;
|
||||
use swactor::runtime::{Inbox, Runtime, RuntimeConfig, RuntimeParts, SingleThreadRuntime};
|
||||
use swactor::runtime::{Inbox, Runtime, RuntimeConfig, RuntimeParts};
|
||||
use swactor::std::StdExtension;
|
||||
use swactor_engine::{Engine, SteppingBackend};
|
||||
use swactor_transport::{CodecRegistry, Transport, TransportRouter, WireEnvelope};
|
||||
|
||||
use distribution::messages::actor_codec_registry;
|
||||
|
|
@ -321,7 +324,8 @@ mod transport_runtime_actor {
|
|||
/// `Link` transports — the multi-runtime analog of `swim_actor.rs::ActorCluster`.
|
||||
struct TransportCluster {
|
||||
rts: Vec<Runtime>,
|
||||
hosts: Vec<RefCell<SingleThreadRuntime>>,
|
||||
_engines: Vec<Engine>,
|
||||
backends: Vec<SteppingBackend>,
|
||||
swims: Vec<ActorAddress>,
|
||||
inboxes: Vec<Inbox<MembershipChanged>>,
|
||||
streams: Vec<Vec<MembershipChanged>>,
|
||||
|
|
@ -338,7 +342,8 @@ mod transport_runtime_actor {
|
|||
let partition = Arc::new(Mutex::new(HashSet::new()));
|
||||
|
||||
let mut rts = Vec::new();
|
||||
let mut hosts = Vec::new();
|
||||
let mut engines = Vec::new();
|
||||
let mut backends = Vec::new();
|
||||
let mut swims = Vec::new();
|
||||
let mut dirs = Vec::new();
|
||||
let mut routers = Vec::new();
|
||||
|
|
@ -355,7 +360,9 @@ mod transport_runtime_actor {
|
|||
codec.clone(),
|
||||
router.clone(),
|
||||
)));
|
||||
let host = RefCell::new(SingleThreadRuntime::new(parts));
|
||||
let backend = SteppingBackend::new();
|
||||
let engine =
|
||||
Engine::new(parts, backend.clone()).expect("create stepping actor engine");
|
||||
|
||||
let dir = SharedPeerDirectory::new();
|
||||
let swim = rt
|
||||
|
|
@ -378,7 +385,8 @@ mod transport_runtime_actor {
|
|||
.unwrap();
|
||||
|
||||
rts.push(rt);
|
||||
hosts.push(host);
|
||||
engines.push(engine);
|
||||
backends.push(backend);
|
||||
swims.push(swim);
|
||||
dirs.push(dir);
|
||||
routers.push(router);
|
||||
|
|
@ -412,7 +420,8 @@ mod transport_runtime_actor {
|
|||
|
||||
let mut c = TransportCluster {
|
||||
rts,
|
||||
hosts,
|
||||
_engines: engines,
|
||||
backends,
|
||||
swims,
|
||||
inboxes,
|
||||
streams: vec![Vec::new(); n],
|
||||
|
|
@ -443,8 +452,8 @@ mod transport_runtime_actor {
|
|||
/// iterations; `k` is sized so a probe + its notification settle per round.
|
||||
fn pump(&self, k: usize) {
|
||||
for _ in 0..k {
|
||||
for host in &self.hosts {
|
||||
host.borrow_mut().tick();
|
||||
for backend in &self.backends {
|
||||
backend.step();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -547,13 +556,13 @@ mod actor_membership_safety_edges {
|
|||
//! Actor-observable safety edges: resurrection after silence, multi-hop death dissemination,
|
||||
//! and bounded stale-refute behavior.
|
||||
|
||||
use std::cell::RefCell;
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use swactor::runtime::{Runtime, RuntimeConfig, RuntimeParts, SingleThreadRuntime};
|
||||
use swactor::runtime::{Runtime, RuntimeConfig, RuntimeParts};
|
||||
use swactor::std::StdExtension;
|
||||
use swactor_engine::{Engine, SteppingBackend};
|
||||
|
||||
use distribution::swim::actor::{
|
||||
MembershipChanged, PeerDirectory, SharedPeerDirectory, SwimActor, SwimIn,
|
||||
|
|
@ -589,7 +598,8 @@ mod actor_membership_safety_edges {
|
|||
/// but lets the test choose the config and rebind a dropped node.
|
||||
struct Cluster {
|
||||
rt: Runtime,
|
||||
host: RefCell<SingleThreadRuntime>,
|
||||
_engine: Engine,
|
||||
backend: SteppingBackend,
|
||||
ids: Vec<NodeId>,
|
||||
addrs: Vec<swactor::actor::ActorAddress>,
|
||||
inboxes: Vec<swactor::runtime::Inbox<MembershipChanged>>,
|
||||
|
|
@ -603,7 +613,8 @@ mod actor_membership_safety_edges {
|
|||
let parts = RuntimeParts::new(RuntimeConfig::default())
|
||||
.with_extension(Arc::new(StdExtension::new()));
|
||||
let rt = parts.runtime().clone();
|
||||
let host = RefCell::new(SingleThreadRuntime::new(parts));
|
||||
let backend = SteppingBackend::new();
|
||||
let engine = Engine::new(parts, backend.clone()).expect("create stepping actor engine");
|
||||
let dir = SharedPeerDirectory::new();
|
||||
let now = Instant::now();
|
||||
let ids: Vec<NodeId> = (0..n).map(|i| id(i as u8)).collect();
|
||||
|
|
@ -633,7 +644,8 @@ mod actor_membership_safety_edges {
|
|||
}
|
||||
let mut c = Cluster {
|
||||
rt,
|
||||
host,
|
||||
_engine: engine,
|
||||
backend,
|
||||
ids,
|
||||
addrs,
|
||||
inboxes,
|
||||
|
|
@ -655,7 +667,7 @@ mod actor_membership_safety_edges {
|
|||
|
||||
fn pump(&self, n: usize) {
|
||||
for _ in 0..n {
|
||||
self.host.borrow_mut().tick();
|
||||
self.backend.step();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -13,12 +13,12 @@ mod telemetry_records {
|
|||
//! Distribution-owned telemetry records keep their channels and JSON payloads stable through
|
||||
//! the mux.
|
||||
|
||||
use telemetry::frame::{Lifetime, NodeId, StreamId};
|
||||
use telemetry::{ChannelId, Mux, Position, Record};
|
||||
use distribution::telemetry::{
|
||||
CacheEntryRec, DIST_STATE, DistributionState, MEMBERSHIP, MembershipTransition,
|
||||
RegistryEntryRec, SWIM_PROBES, SwimProbeEvent, TRANSPORT_INTERNALS, TransportInternals,
|
||||
};
|
||||
use telemetry::frame::{Lifetime, NodeId, StreamId};
|
||||
use telemetry::{ChannelId, Mux, Position, Record};
|
||||
|
||||
#[test]
|
||||
fn distribution_state_record_round_trips_from_owner_crate() {
|
||||
|
|
|
|||
|
|
@ -12,3 +12,6 @@ tokio = ["dep:tokio"]
|
|||
swactor = { path = "../.." }
|
||||
parking_lot = "0.12"
|
||||
tokio = { workspace = true, optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
proptest = "1"
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@ Constructing a swactor engine consumes configured `RuntimeParts` and the selecte
|
|||
| operation | meaning |
|
||||
|---|---|
|
||||
| `spawn(task)` | Schedule an async unit of work on the substrate. |
|
||||
| `spawn_blocking(work)` | Schedule blocking CPU / syscall work off the async path. |
|
||||
| `blocking_work_sender().submit(work)` | Route blocking I/O work to the substrate's dedicated blocking pool. |
|
||||
| `timer(delay)` / `interval(period)` | Schedule future or recurring work. |
|
||||
| `now()` | The engine's monotonic clock. |
|
||||
|
||||
|
|
@ -88,7 +88,7 @@ The primitives an engine may provide. Capabilities are **per-implementation and
|
|||
- **Tasks** — `spawn` of an async unit of work; the substrate's unit of concurrency.
|
||||
- **Timers** — one-shot delay and recurring interval.
|
||||
- **I/O** — streams, sockets, files, and protocol endpoints used by engine-hosted work. An engine may implement I/O through asynchronous operations, blocking operations on managed threads, callbacks, or host-native facilities. Integrations declare the I/O capabilities they require, and binding fails at construction when the selected engine cannot provide them.
|
||||
- **Blocking** — `spawn_blocking` for CPU-bound or syscall work that must not stall the executor.
|
||||
- **Blocking** — a `BlockingWorkSender` routes syscall work off actor and async workers.
|
||||
- **Time** — `now()`. In a test engine this is virtual, advanced by the test; this is what makes deterministic testing possible.
|
||||
|
||||
An engine that provides only tasks + time is still valid. Blocking and I/O are additional capabilities declared by integrations that require them.
|
||||
|
|
|
|||
|
|
@ -57,7 +57,6 @@ struct CoreDriver {
|
|||
|
||||
// Core drivers are the engine's sole core-progression path; this is the one
|
||||
// place permitted to call `Worker::try_tick` (ENGINE_SPEC.md §2).
|
||||
#[allow(clippy::disallowed_methods)]
|
||||
impl Future for CoreDriver {
|
||||
type Output = ();
|
||||
|
||||
|
|
|
|||
|
|
@ -1,11 +1,14 @@
|
|||
//! Composite engine and cloneable scheduler handle.
|
||||
|
||||
use parking_lot::{Condvar, Mutex};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Weak};
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::backend::{Capabilities, EngineError, ExecutionBackend};
|
||||
use crate::time::{EngineInstant, Interval, Timeout, Timer};
|
||||
use swactor::runtime::{Runtime, RuntimeParts};
|
||||
use swactor::actor::{ActorAddress, Message};
|
||||
use swactor::runtime::{ExternalSender, Runtime, RuntimeParts};
|
||||
|
||||
/// The composite engine: retains a configured core runtime handle and its
|
||||
/// execution backend, and owns one core-driving loop per worker.
|
||||
|
|
@ -67,6 +70,116 @@ impl Engine {
|
|||
pub struct EngineHandle {
|
||||
backend: Weak<dyn ExecutionBackend>,
|
||||
}
|
||||
/// A clonable substrate route for one-shot blocking I/O work.
|
||||
///
|
||||
/// Domain actors decide which effect to execute; this handle only moves its
|
||||
/// mechanics onto the engine backend's dedicated blocking pool.
|
||||
#[derive(Clone)]
|
||||
pub struct BlockingWorkSender {
|
||||
backend: Weak<dyn ExecutionBackend>,
|
||||
}
|
||||
|
||||
impl BlockingWorkSender {
|
||||
/// Submit one blocking I/O operation without occupying an actor worker.
|
||||
///
|
||||
/// Returns the operation unchanged if the owning engine has stopped.
|
||||
pub fn submit(&self, work: crate::BoxWork) -> Result<(), crate::BoxWork> {
|
||||
let Some(backend) = self.backend.upgrade() else {
|
||||
return Err(work);
|
||||
};
|
||||
backend.spawn_blocking(work);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Cancellation handle for an engine-owned actor message timer.
|
||||
///
|
||||
/// Cancellation is idempotent. A timer may already have fired when cancellation
|
||||
/// races its deadline, so actor messages should still carry an operation or
|
||||
/// generation identity that lets the receiver reject stale work.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ActorTimer {
|
||||
cancelled: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl ActorTimer {
|
||||
/// Prevent this timer from delivering future messages.
|
||||
pub fn cancel(&self) {
|
||||
self.cancelled.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Report whether cancellation has been requested.
|
||||
pub fn is_cancelled(&self) -> bool {
|
||||
self.cancelled.load(Ordering::Acquire)
|
||||
}
|
||||
}
|
||||
|
||||
enum CompletionState<T> {
|
||||
Pending,
|
||||
Ready(T),
|
||||
Consumed,
|
||||
}
|
||||
|
||||
/// One actor-owned terminal observation for a synchronous process entrypoint.
|
||||
///
|
||||
/// The waiting thread cannot poll, set a deadline, or advance domain state.
|
||||
/// An actor decides when the operation is complete and publishes the value.
|
||||
pub struct ActorCompletion<T> {
|
||||
inner: Arc<(Mutex<CompletionState<T>>, Condvar)>,
|
||||
}
|
||||
|
||||
impl<T> Clone for ActorCompletion<T> {
|
||||
fn clone(&self) -> Self {
|
||||
Self {
|
||||
inner: Arc::clone(&self.inner),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> Default for ActorCompletion<T> {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> ActorCompletion<T> {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
inner: Arc::new((Mutex::new(CompletionState::Pending), Condvar::new())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Publish the terminal observation once.
|
||||
pub fn complete(&self, value: T) -> Result<(), T> {
|
||||
let (state, ready) = &*self.inner;
|
||||
let mut state = state.lock();
|
||||
if !matches!(*state, CompletionState::Pending) {
|
||||
return Err(value);
|
||||
}
|
||||
*state = CompletionState::Ready(value);
|
||||
ready.notify_all();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Block the process entrypoint until the owning actor completes.
|
||||
pub fn wait(&self) -> T {
|
||||
let (state, ready) = &*self.inner;
|
||||
let mut state = state.lock();
|
||||
loop {
|
||||
if matches!(*state, CompletionState::Pending) {
|
||||
ready.wait(&mut state);
|
||||
continue;
|
||||
}
|
||||
match std::mem::replace(&mut *state, CompletionState::Consumed) {
|
||||
CompletionState::Ready(value) => return value,
|
||||
CompletionState::Consumed => {
|
||||
panic!("ActorCompletion::wait called after value was consumed")
|
||||
}
|
||||
CompletionState::Pending => unreachable!("pending state handled above"),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl EngineHandle {
|
||||
/// Upgrade to the live backend, or `None` if the owning engine is gone.
|
||||
|
|
@ -86,16 +199,10 @@ impl EngineHandle {
|
|||
backend.spawn(Box::pin(task));
|
||||
}
|
||||
}
|
||||
|
||||
/// Schedule `work` on a dedicated blocking thread.
|
||||
///
|
||||
/// A no-op once the owning engine has been dropped.
|
||||
pub fn spawn_blocking<F>(&self, work: F)
|
||||
where
|
||||
F: FnOnce() + Send + 'static,
|
||||
{
|
||||
if let Some(backend) = self.backend() {
|
||||
backend.spawn_blocking(Box::new(work));
|
||||
/// Create a route to the backend's dedicated blocking-I/O pool.
|
||||
pub fn blocking_work_sender(&self) -> BlockingWorkSender {
|
||||
BlockingWorkSender {
|
||||
backend: self.backend.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -120,6 +227,76 @@ impl EngineHandle {
|
|||
current: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Schedule one typed message for delivery after `delay`.
|
||||
///
|
||||
/// The engine owns the timer task; domain code receives no future or
|
||||
/// scheduling callback. The actor receiving `message` owns the deadline
|
||||
/// decision and should reject stale operation identities.
|
||||
pub fn send_after<M>(
|
||||
&self,
|
||||
delay: Duration,
|
||||
sender: ExternalSender,
|
||||
actor: ActorAddress,
|
||||
message: M,
|
||||
) -> ActorTimer
|
||||
where
|
||||
M: Message,
|
||||
{
|
||||
let actor_timer = ActorTimer {
|
||||
cancelled: Arc::new(AtomicBool::new(false)),
|
||||
};
|
||||
let cancelled = Arc::clone(&actor_timer.cancelled);
|
||||
let timer = self.timer(delay);
|
||||
self.spawn(async move {
|
||||
timer.await;
|
||||
if !cancelled.load(Ordering::Acquire) {
|
||||
let _ = sender.send_to(actor, message);
|
||||
}
|
||||
});
|
||||
actor_timer
|
||||
}
|
||||
|
||||
/// Schedule a cloned typed message after every `period`.
|
||||
///
|
||||
/// Delivery stops after cancellation or when the actor address no longer
|
||||
/// accepts messages.
|
||||
pub fn send_every<M>(
|
||||
&self,
|
||||
period: Duration,
|
||||
sender: ExternalSender,
|
||||
actor: ActorAddress,
|
||||
message: M,
|
||||
) -> ActorTimer
|
||||
where
|
||||
M: Message,
|
||||
{
|
||||
let actor_timer = ActorTimer {
|
||||
cancelled: Arc::new(AtomicBool::new(false)),
|
||||
};
|
||||
let cancelled = Arc::clone(&actor_timer.cancelled);
|
||||
let handle = self.clone();
|
||||
let mut interval = Box::pin(handle.interval(period));
|
||||
self.spawn(std::future::poll_fn(move |cx| {
|
||||
if cancelled.load(Ordering::Acquire) {
|
||||
return std::task::Poll::Ready(());
|
||||
}
|
||||
match std::future::Future::poll(interval.as_mut(), cx) {
|
||||
std::task::Poll::Ready(()) => {
|
||||
if cancelled.load(Ordering::Acquire)
|
||||
|| sender.send_to(actor, message.clone()).is_err()
|
||||
{
|
||||
std::task::Poll::Ready(())
|
||||
} else {
|
||||
cx.waker().wake_by_ref();
|
||||
std::task::Poll::Pending
|
||||
}
|
||||
}
|
||||
std::task::Poll::Pending => std::task::Poll::Pending,
|
||||
}
|
||||
}));
|
||||
actor_timer
|
||||
}
|
||||
/// Race `future` against an engine timer.
|
||||
///
|
||||
/// Resolves to `Ok` with the future's output if it completes within
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ mod time;
|
|||
mod tokio;
|
||||
|
||||
pub use backend::{BoxTask, BoxTimer, BoxWork, Capabilities, EngineError, ExecutionBackend};
|
||||
pub use engine::{Engine, EngineHandle};
|
||||
pub use engine::{ActorCompletion, ActorTimer, BlockingWorkSender, Engine, EngineHandle};
|
||||
pub use stepping::SteppingBackend;
|
||||
pub use time::{Elapsed, EngineInstant, Interval, Timeout, Timer};
|
||||
|
||||
|
|
|
|||
|
|
@ -181,11 +181,22 @@ impl SteppingBackend {
|
|||
pub fn pending_task_count(&self) -> usize {
|
||||
self.inner.tasks.lock().len()
|
||||
}
|
||||
|
||||
/// Join every blocking operation submitted so far.
|
||||
///
|
||||
/// Deterministic tests use this as a barrier before stepping actor replies.
|
||||
/// Returns the first worker panic instead of silently discarding it.
|
||||
pub fn join_blocking(&self) -> std::thread::Result<()> {
|
||||
let handles: Vec<_> = self.inner.blocking.lock().drain(..).collect();
|
||||
for handle in handles {
|
||||
handle.join()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// This impl is the substrate implementor for the deterministic stepping
|
||||
/// backend; blocking work runs on a std thread (ENGINE_SPEC.md §2).
|
||||
#[allow(clippy::disallowed_methods)]
|
||||
impl ExecutionBackend for SteppingBackend {
|
||||
fn spawn(&self, task: BoxTask) {
|
||||
self.inner.tasks.lock().push(task);
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue