Compare commits
3 commits
0c74ee2824
...
0411c20b94
| Author | SHA1 | Date | |
|---|---|---|---|
| 0411c20b94 | |||
| df72b4ff60 | |||
| 4854d85906 |
102 changed files with 7176 additions and 3462 deletions
|
|
@ -3,4 +3,4 @@ rustc-workspace-wrapper = "tools/actor-control-flow-lint/rustc-wrapper.py"
|
||||||
|
|
||||||
[alias]
|
[alias]
|
||||||
xtask = "run --package xtask --"
|
xtask = "run --package xtask --"
|
||||||
myelin-chat = "run --package xtask -- myelin-chat"
|
lint = "clippy --workspace --all-targets --all-features"
|
||||||
|
|
|
||||||
2
.config/nextest.toml
Normal file
2
.config/nextest.toml
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
[profile.default]
|
||||||
|
slow-timeout = { period = "60s", terminate-after = 1 }
|
||||||
119
.github/workflows/myelin-properties.yml
vendored
119
.github/workflows/myelin-properties.yml
vendored
|
|
@ -1,119 +0,0 @@
|
||||||
name: Myelin properties
|
|
||||||
|
|
||||||
on:
|
|
||||||
pull_request:
|
|
||||||
schedule:
|
|
||||||
- cron: "17 3 * * *"
|
|
||||||
workflow_dispatch:
|
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
component-properties:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
- name: Install pinned Rust toolchain
|
|
||||||
run: rustup toolchain install nightly-2026-02-07 --profile minimal --component rustfmt --component rustc-dev --component rust-src --component llvm-tools-preview
|
|
||||||
- name: Check formatting
|
|
||||||
run: cargo fmt --all -- --check
|
|
||||||
- name: Check default workspace members
|
|
||||||
run: cargo check
|
|
||||||
- name: Check compiler-policy contracts
|
|
||||||
run: cargo test -p actor-control-flow-lint-tests --test contracts compiler_policy_contracts -- --exact
|
|
||||||
- name: Engine scheduling properties
|
|
||||||
run: |
|
|
||||||
cargo test -p swactor-engine --test engine_unit generated_actor_timers_and_completion_are_bounded -- --exact
|
|
||||||
cargo test -p swactor-engine --test engine_unit lifecycle_invariant_detects_injected_duplicate_completion -- --exact
|
|
||||||
cargo test -p swactor-engine --test engine_unit lifecycle_invariant_detects_injected_uncancelled_periodic_timer -- --exact
|
|
||||||
- name: Process adapter properties
|
|
||||||
run: |
|
|
||||||
cargo test -p swactor-process --lib operations::properties::generated_stream_observations_close_once_and_stay_closed -- --exact
|
|
||||||
cargo test -p swactor-process --lib operations::properties::generated_lifecycle_actions_make_stop_idempotent_and_exit_terminal -- --exact
|
|
||||||
cargo test -p swactor-process --lib operations::properties::generated_stop_notifications_are_delivered_at_most_once -- --exact
|
|
||||||
cargo test -p swactor-process --lib operations::properties::generated_stdin_commands_and_eof_notify_once -- --exact
|
|
||||||
cargo test -p swactor-process --lib operations::properties::property_invariants_reject_controlled_defects -- --exact
|
|
||||||
cargo test -p swactor-process --lib operations::properties::trivial_real_child_exit_has_a_hard_timeout -- --exact
|
|
||||||
- name: Myelin component properties (exclude job and reconciler tests)
|
|
||||||
run: |
|
|
||||||
cargo test -p myelin --lib orchestration::manual_control::tests::aggressive_random_event_stream_preserves_control_invariants -- --exact
|
|
||||||
cargo test -p myelin --lib orchestration::manual_control::tests::rental_free_end_to_end_sequences_converge -- --exact
|
|
||||||
cargo test -p myelin --lib orchestration::manual_control::tests::manual_actor_generated_public_actions_and_callbacks_are_bounded -- --exact
|
|
||||||
cargo test -p myelin --lib orchestration::manual_control::tests::fixed_helper_cardinality_invariant_detects_controlled_extra_spawn -- --exact
|
|
||||||
cargo test -p myelin --lib orchestration::manual_control::tests::callback_panic_reports_typed_failure_without_poisoning_work_actor -- --exact
|
|
||||||
cargo test -p myelin --lib orchestration::manual_control::tests::callback_panic_invariant_detects_controlled_unguarded_panic -- --exact
|
|
||||||
cargo test -p myelin --lib provisioning::tests::mock_vastai_handle_state_survives_random_create_and_stop_sequences -- --exact
|
|
||||||
cargo test -p myelin --lib provisioning::tests::docker_generated_attempt_lifecycles_are_idempotent_and_bounded -- --exact
|
|
||||||
cargo test -p myelin --lib provisioning::tests::docker_duplicate_resource_detector_rejects_controlled_fault -- --exact
|
|
||||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::offer_status_classes_are_offers_or_typed_rejections -- --exact
|
|
||||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::malformed_offer_bodies_are_typed_rejections -- --exact
|
|
||||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::wrong_or_missing_offer_fields_are_typed_rejections -- --exact
|
|
||||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::duplicate_offer_records_remain_explicit_values -- --exact
|
|
||||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::provider_monitor_preserves_contract_identity_and_cardinality -- --exact
|
|
||||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::provider_monitor_terminal_polling_stops_after_one_typed_outcome -- --exact
|
|
||||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::provider_monitor_poll_stop_orderings_cease_polling -- --exact
|
|
||||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::duplicate_terminal_detector_rejects_controlled_fault -- --exact
|
|
||||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_output_lines_preserve_stream_and_protocol -- --exact
|
|
||||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_malformed_protocol_is_data_not_poison -- --exact
|
|
||||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_eof_orderings_stop_relay_and_actor -- --exact
|
|
||||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_child_failures_have_typed_attempt_outcomes -- --exact
|
|
||||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_timeout_is_typed_and_stops_polling -- --exact
|
|
||||||
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_stop_orderings_emit_one_terminal_and_stop_all_actors -- --exact
|
|
||||||
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::runtime_actors_generated_transitions_complete_once_on_one_worker -- --exact
|
|
||||||
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::runtime_invariant_checker_rejects_duplicate_readiness_publication -- --exact
|
|
||||||
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::helper_wait_generated_terminal_sequences_complete_once_on_one_worker -- --exact
|
|
||||||
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::helper_invariant_checker_rejects_expected_output_after_terminal_error -- --exact
|
|
||||||
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::stage_fetch_generated_observations_complete_once_on_one_worker -- --exact
|
|
||||||
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::stage_invariant_checker_rejects_wrong_terminal_classification -- --exact
|
|
||||||
cargo test -p myelin --lib orchestration::app::serve_cluster_properties::serve_cluster_production_transitions_converge_once_without_growth -- --exact
|
|
||||||
cargo test -p myelin --lib orchestration::app::serve_cluster_properties::serve_cluster_lifecycle_invariants_reject_injected_duplicate_and_growth -- --exact
|
|
||||||
cargo test -p myelin --lib orchestration::control::properties::generated_http_bridge_sequences_terminate_without_control_actor_growth -- --exact
|
|
||||||
cargo test -p myelin --lib orchestration::control::properties::generated_duplicate_control_replies_deliver_first_once_and_remove_observer -- --exact
|
|
||||||
cargo test -p myelin --lib orchestration::control::properties::reply_observer_disappearance_returns_a_bounded_terminal_http_response -- --exact
|
|
||||||
cargo test -p myelin --lib orchestration::control::properties::http_bridge_invariant_rejects_a_controlled_duplicate_forward -- --exact
|
|
||||||
- name: Controlled E2E oracle contract
|
|
||||||
run: cargo test -p myelin --features test-support --test stateful_vastai e2e_oracle_rejects_controlled_lifecycle_faults -- --exact
|
|
||||||
- name: Dashboard bridge properties
|
|
||||||
run: |
|
|
||||||
cargo test -p dashboard --features demo-control --lib control::properties::generated_concurrent_bridge_commands_forward_once_and_shutdown -- --exact
|
|
||||||
cargo test -p dashboard --features demo-control --lib control::properties::bridge_invariant_rejects_a_controlled_duplicate_delivery -- --exact
|
|
||||||
cargo test -p dashboard --features demo-control --lib server::tests::generated_control_http_sequences_are_bounded_and_typed -- --exact
|
|
||||||
cargo test -p dashboard --features demo-control --lib server::tests::control_http_invariant_rejects_a_controlled_server_error -- --exact
|
|
||||||
- name: Demo actor properties
|
|
||||||
run: |
|
|
||||||
cargo test -p xtask --bin xtask demo::control::properties::generated_control_commands_forward_only_after_supervisor_registration -- --exact
|
|
||||||
cargo test -p xtask --bin xtask demo::control::properties::control_transition_oracle_rejects_duplicate_forwarding -- --exact
|
|
||||||
cargo test -p xtask --bin xtask demo::feed::properties::generated_supervisor_transitions_are_once_only_nonblocking_and_clean -- --exact
|
|
||||||
cargo test -p xtask --bin xtask demo::feed::properties::supervisor_transition_oracle_rejects_duplicate_identity_resources -- --exact
|
|
||||||
cargo test -p xtask --bin xtask demo::node::properties::generated_node_runtime_transitions_emit_heartbeats_and_stop_once -- --exact
|
|
||||||
cargo test -p xtask --bin xtask demo::node::properties::node_transition_oracle_rejects_duplicate_resources -- --exact
|
|
||||||
cargo test -p xtask --bin xtask demo::provider::properties::generated_process_reports_complete_exit_watchers_once_and_preserve_last_state -- --exact
|
|
||||||
cargo test -p xtask --bin xtask demo::provider::properties::process_relay_oracle_rejects_lost_exit -- --exact
|
|
||||||
cargo test -p xtask --bin xtask demo::properties::direct_binary_signal_smoke_has_a_hard_timeout -- --exact
|
|
||||||
|
|
||||||
process-e2e:
|
|
||||||
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
|
||||||
needs: component-properties
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
timeout-minutes: 30
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
- name: Install pinned Rust toolchain
|
|
||||||
run: rustup toolchain install nightly-2026-02-07 --profile minimal --component rustc-dev --component rust-src --component llvm-tools-preview
|
|
||||||
- name: Stateful VastAI process E2E
|
|
||||||
env:
|
|
||||||
PROPTEST_CASES: "4"
|
|
||||||
run: |
|
|
||||||
set -o pipefail
|
|
||||||
mkdir -p artifacts
|
|
||||||
cargo test -p myelin --features test-support --test stateful_vastai stateful_vastai_dashboard_control_survives_restarts -- --ignored --exact --nocapture 2>&1 | tee artifacts/stateful-vastai.log
|
|
||||||
- name: Preserve E2E failure artifacts
|
|
||||||
if: failure()
|
|
||||||
uses: actions/upload-artifact@v4
|
|
||||||
with:
|
|
||||||
name: stateful-vastai-failure-${{ github.run_id }}
|
|
||||||
path: |
|
|
||||||
artifacts/stateful-vastai.log
|
|
||||||
apps/myelin/proptest-regressions/tests/e2e_vastai.txt
|
|
||||||
if-no-files-found: warn
|
|
||||||
6
Cargo.lock
generated
6
Cargo.lock
generated
|
|
@ -962,7 +962,9 @@ dependencies = [
|
||||||
"futures-lite",
|
"futures-lite",
|
||||||
"libc",
|
"libc",
|
||||||
"parking_lot",
|
"parking_lot",
|
||||||
|
"proptest",
|
||||||
"serde",
|
"serde",
|
||||||
|
"serde_json",
|
||||||
"sha2 0.10.9",
|
"sha2 0.10.9",
|
||||||
"swactor",
|
"swactor",
|
||||||
"swactor-engine",
|
"swactor-engine",
|
||||||
|
|
@ -4335,9 +4337,11 @@ dependencies = [
|
||||||
"getrandom 0.2.17",
|
"getrandom 0.2.17",
|
||||||
"loom",
|
"loom",
|
||||||
"parking_lot",
|
"parking_lot",
|
||||||
|
"proc-macro2",
|
||||||
"proptest",
|
"proptest",
|
||||||
"proptest-state-machine",
|
"proptest-state-machine",
|
||||||
"serde",
|
"serde",
|
||||||
|
"syn",
|
||||||
"tracing",
|
"tracing",
|
||||||
"web-time 0.2.4",
|
"web-time 0.2.4",
|
||||||
]
|
]
|
||||||
|
|
@ -4489,11 +4493,13 @@ name = "telemetry"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"crossbeam-channel",
|
"crossbeam-channel",
|
||||||
|
"futures-channel",
|
||||||
"iroh",
|
"iroh",
|
||||||
"libc",
|
"libc",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"swactor",
|
"swactor",
|
||||||
|
"swactor-engine",
|
||||||
"swactor-transport",
|
"swactor-transport",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
|
||||||
11
Cargo.toml
11
Cargo.toml
|
|
@ -75,4 +75,15 @@ criterion = { version = "0.5", features = ["html_reports"] }
|
||||||
proptest = "1"
|
proptest = "1"
|
||||||
proptest-state-machine = "0.3"
|
proptest-state-machine = "0.3"
|
||||||
loom = "0.7"
|
loom = "0.7"
|
||||||
|
proc-macro2 = "1"
|
||||||
|
syn = { version = "2", features = ["full", "visit"] }
|
||||||
|
|
||||||
|
|
||||||
|
[workspace.lints.rust]
|
||||||
|
warnings = "deny"
|
||||||
|
|
||||||
|
[workspace.lints.clippy]
|
||||||
|
allow_attributes = "forbid"
|
||||||
|
|
||||||
|
[lints]
|
||||||
|
workspace = true
|
||||||
|
|
|
||||||
|
|
@ -1,54 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
BASE_IMAGE=${BASE_IMAGE:-myelin-node-base:cuda12.6}
|
|
||||||
IMAGE=${IMAGE:-myelin-node:latest}
|
|
||||||
CONTAINER=${CONTAINER:-myelin-node-e2e-$$}
|
|
||||||
GPUS=${MYELIN_CUDA_GPUS:-all}
|
|
||||||
PROMPT=${MYELIN_NODE_SELF_TEST_PROMPT:-ping}
|
|
||||||
TIMEOUT_SECS=${MYELIN_NODE_E2E_TIMEOUT_SECS:-1800}
|
|
||||||
FRAME_LOG=${MYELIN_TELEMETRY_FRAME_LOG:-/var/log/myelin-telemetry.ndjson}
|
|
||||||
|
|
||||||
cleanup() {
|
|
||||||
docker rm -f "$CONTAINER" >/dev/null 2>&1 || true
|
|
||||||
}
|
|
||||||
trap cleanup EXIT
|
|
||||||
|
|
||||||
cargo build --release -p myelin --bin myelin-worker
|
|
||||||
docker build -f apps/myelin/node-image/Dockerfile.base -t "$BASE_IMAGE" .
|
|
||||||
docker build -f apps/myelin/node-image/Dockerfile --build-arg BASE_IMAGE="$BASE_IMAGE" -t "$IMAGE" .
|
|
||||||
|
|
||||||
docker run -d \
|
|
||||||
--name "$CONTAINER" \
|
|
||||||
--gpus "$GPUS" \
|
|
||||||
-e MYELIN_NODE_SELF_TEST_PROMPT="$PROMPT" \
|
|
||||||
-e MYELIN_NODE_MAX_RUNTIME_SECS=1 \
|
|
||||||
-e MYELIN_SELF_TEST_MAX_TOKENS="${MYELIN_SELF_TEST_MAX_TOKENS:-1}" \
|
|
||||||
-e MYELIN_MODEL_CACHE_DIR=/var/cache/myelin-models \
|
|
||||||
-e MYELIN_TELEMETRY_FRAME_LOG="$FRAME_LOG" \
|
|
||||||
${HF_TOKEN:+-e HF_TOKEN="$HF_TOKEN"} \
|
|
||||||
"$IMAGE" >/dev/null
|
|
||||||
|
|
||||||
deadline=$((SECONDS + TIMEOUT_SECS))
|
|
||||||
while (( SECONDS < deadline )); do
|
|
||||||
logs=$(docker logs "$CONTAINER" 2>&1 || true)
|
|
||||||
if grep -q '"type":"ready"' <<<"$logs" && grep -q '"type":"self_test_completed"' <<<"$logs"; then
|
|
||||||
frames=$(docker exec "$CONTAINER" cat "$FRAME_LOG" 2>/dev/null || true)
|
|
||||||
if grep -q '"channel":"myelin.node.ready"' <<<"$frames" &&
|
|
||||||
grep -q '"channel":"myelin.worker.weights"' <<<"$frames" &&
|
|
||||||
grep -q '"channel":"myelin.worker.prompt"' <<<"$frames"; then
|
|
||||||
printf '%s\n' "$logs"
|
|
||||||
printf '%s\n' "$frames"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
if grep -q 'WorkerFatal\|myelin-node: .*failed\|ModelLoadFailed\|GgufDownloadFailed' <<<"$logs"; then
|
|
||||||
printf '%s\n' "$logs" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
sleep 5
|
|
||||||
done
|
|
||||||
|
|
||||||
docker logs "$CONTAINER" 2>&1 || true
|
|
||||||
echo "myelin-node Docker E2E timed out after ${TIMEOUT_SECS}s" >&2
|
|
||||||
exit 1
|
|
||||||
|
|
@ -1,5 +0,0 @@
|
||||||
# Seeds for stateful VastAI E2E failures. Proptest replays these before generated cases.
|
|
||||||
cc 26c3cd5944cb25de70a984389373e7380a0c6e6ba1c8f3e6eff5a0f78dbef168 # current-strategy replay; E2eCase is derived from e2e_case()
|
|
||||||
cc 5efb0a6c5c348475c859a2c4267d5177aaecd5adefcc5e8b7fc2e6d728deecd1 # shrinks to case = E2eCase { seed: 7099259626237328177, node_seed: 245, kill_mask: 24, offer_offset: 3, actions: [ConcurrentQueries, Search { count: 2 }, Restart { mode: FlushSafeAbrupt }, Query, Kill { node_slot: 218, command_slot: 5 }, Query, Provision { command_slot: 5, use_searched_offers: true }, EndpointProbe { node_slot: 217 }, Restart { mode: Graceful }, Flush, Kill { node_slot: 217, command_slot: 5 }] }
|
|
||||||
cc 6d005687858520ed65af20ec5b2de056efcda15771b6420d7f67e3cb6c246d5b # shrinks to case = E2eCase { seed: 17649392557414661864, node_seed: 249, kill_mask: 159, offer_offset: 5, actions: [Kill { node_slot: 88, command_slot: 242 }, Flush, ConcurrentQueries, EndpointProbe { node_slot: 88 }, Query, Search { count: 2 }, Provision { command_slot: 242, use_searched_offers: true }, Restart { mode: FlushSafeAbrupt }, Kill { node_slot: 89, command_slot: 242 }, Query, Restart { mode: Graceful }] }
|
|
||||||
cc 2e36f51e7cb347b571ac838f81638748f8414a32ec91cb9f4b2730363d72be61 # shrinks to case = E2eCase { seed: 13301618846512983428, node_seed: 99, kill_mask: 30, offer_offset: 2, actions: [Query, Restart { mode: FlushSafeAbrupt }, Kill { node_slot: 136, command_slot: 187 }, ConcurrentQueries, Restart { mode: Graceful }, Query, EndpointProbe { node_slot: 136 }, Kill { node_slot: 137, command_slot: 187 }, Provision { command_slot: 187, use_searched_offers: false }, Flush, Search { count: 0 }] }
|
|
||||||
|
|
@ -1,9 +1,11 @@
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
|
use std::os::fd::{FromRawFd, OwnedFd};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use data_plane::arena::{ArenaConfig, ArenaManager, NodeId as ArenaNodeId};
|
use data_plane::arena::{ArenaConfig, ArenaManager, NodeId as ArenaNodeId};
|
||||||
use data_plane::blob_transfer::{BlobTransferReceiver, BlobTransferSender};
|
use data_plane::blob_transfer::{BlobTransferReceiver, BlobTransferSender};
|
||||||
use data_plane::bootstrap::{self, BootstrapSpec, ENV_DATA_PLANE_ENDPOINT, JobHandoff};
|
use data_plane::bootstrap::{self, BootstrapSpec, ENV_DATA_PLANE_ENDPOINT, JobHandoff};
|
||||||
|
use data_plane::data_plane::{DataPlane, DataPlaneBootstrap};
|
||||||
use data_plane::host::{
|
use data_plane::host::{
|
||||||
HostDataPlaneConfig, HostDataPlaneSessionActor, HostRouteRegistrar, install_session_env,
|
HostDataPlaneConfig, HostDataPlaneSessionActor, HostRouteRegistrar, install_session_env,
|
||||||
};
|
};
|
||||||
|
|
@ -11,6 +13,7 @@ use data_plane::namespace::NamespaceClient;
|
||||||
use data_plane::path::JobContext;
|
use data_plane::path::JobContext;
|
||||||
use data_plane::protocol::JobCapability;
|
use data_plane::protocol::JobCapability;
|
||||||
use data_plane::source::BlobSourcePublisher;
|
use data_plane::source::BlobSourcePublisher;
|
||||||
|
use data_plane::stream_transport::StreamTransport;
|
||||||
use distribution::transport_bridge::{OutboxRouteBinder, RouteBinder, RouteView};
|
use distribution::transport_bridge::{OutboxRouteBinder, RouteBinder, RouteView};
|
||||||
use distribution::types::NodeId;
|
use distribution::types::NodeId;
|
||||||
use swactor::actor::ActorAddress;
|
use swactor::actor::ActorAddress;
|
||||||
|
|
@ -76,6 +79,7 @@ const ARENA_ALIGNMENT: u64 = 64;
|
||||||
pub(crate) struct ActorJobDataPlane {
|
pub(crate) struct ActorJobDataPlane {
|
||||||
handoff: JobHandoff,
|
handoff: JobHandoff,
|
||||||
host_session: ActorAddress,
|
host_session: ActorAddress,
|
||||||
|
capability: JobCapability,
|
||||||
runtime: Runtime,
|
runtime: Runtime,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -90,6 +94,7 @@ pub(crate) struct ActorJobDataPlaneConfig {
|
||||||
pub(crate) source_sender: Option<Arc<dyn BlobTransferSender>>,
|
pub(crate) source_sender: Option<Arc<dyn BlobTransferSender>>,
|
||||||
pub(crate) source_publisher: Option<Arc<dyn BlobSourcePublisher>>,
|
pub(crate) source_publisher: Option<Arc<dyn BlobSourcePublisher>>,
|
||||||
pub(crate) route_registrar: Option<Arc<dyn HostRouteRegistrar>>,
|
pub(crate) route_registrar: Option<Arc<dyn HostRouteRegistrar>>,
|
||||||
|
pub(crate) stream_transport: Option<Arc<dyn StreamTransport>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ActorJobDataPlane {
|
impl ActorJobDataPlane {
|
||||||
|
|
@ -105,6 +110,7 @@ impl ActorJobDataPlane {
|
||||||
source_sender,
|
source_sender,
|
||||||
source_publisher,
|
source_publisher,
|
||||||
route_registrar,
|
route_registrar,
|
||||||
|
stream_transport,
|
||||||
} = config;
|
} = config;
|
||||||
let mut arena = ArenaManager::boot(ArenaConfig {
|
let mut arena = ArenaManager::boot(ArenaConfig {
|
||||||
node_id: ArenaNodeId(1),
|
node_id: ArenaNodeId(1),
|
||||||
|
|
@ -134,6 +140,7 @@ impl ActorJobDataPlane {
|
||||||
source_sender,
|
source_sender,
|
||||||
source_publisher,
|
source_publisher,
|
||||||
route_registrar,
|
route_registrar,
|
||||||
|
stream_transport,
|
||||||
})
|
})
|
||||||
.map_err(|error| format!("configure host data-plane session: {error}"))?,
|
.map_err(|error| format!("configure host data-plane session: {error}"))?,
|
||||||
)
|
)
|
||||||
|
|
@ -141,11 +148,31 @@ impl ActorJobDataPlane {
|
||||||
install_session_env(&mut handoff, host_session, capability);
|
install_session_env(&mut handoff, host_session, capability);
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
handoff,
|
handoff,
|
||||||
runtime: runtime.clone(),
|
|
||||||
host_session,
|
host_session,
|
||||||
|
capability,
|
||||||
|
runtime: runtime.clone(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn attach_local(&self) -> Result<DataPlane, String> {
|
||||||
|
let fd = unsafe { libc::dup(std::os::fd::AsRawFd::as_raw_fd(&self.handoff.arena_fd)) };
|
||||||
|
if fd < 0 {
|
||||||
|
return Err(format!(
|
||||||
|
"duplicate local data-plane arena: {}",
|
||||||
|
std::io::Error::last_os_error()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let owned = unsafe { OwnedFd::from_raw_fd(fd) };
|
||||||
|
futures_lite::future::block_on(DataPlaneBootstrap::attach(
|
||||||
|
owned,
|
||||||
|
self.runtime.clone(),
|
||||||
|
self.host_session,
|
||||||
|
self.capability,
|
||||||
|
))
|
||||||
|
.map(|bootstrap| bootstrap.data_plane)
|
||||||
|
.map_err(|error| format!("attach local data-plane client: {error}"))
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn configure_run(&self, run_id: String) -> Result<(), String> {
|
pub(crate) fn configure_run(&self, run_id: String) -> Result<(), String> {
|
||||||
futures_lite::future::block_on(async {
|
futures_lite::future::block_on(async {
|
||||||
self.runtime
|
self.runtime
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@
|
||||||
//! in the directory, and exchanges its `EndpointAddr` + actor address
|
//! in the directory, and exchanges its `EndpointAddr` + actor address
|
||||||
//! out-of-band so each side can route to the other over the iroh actor plane.
|
//! out-of-band so each side can route to the other over the iroh actor plane.
|
||||||
|
|
||||||
|
use crate::data_namespace::{DataNamespaceAuthority, install_namespace_client};
|
||||||
use crate::job_data_plane::{
|
use crate::job_data_plane::{
|
||||||
ActorJobDataPlane, ActorJobDataPlaneConfig, MyelinChildRouteRegistrar,
|
ActorJobDataPlane, ActorJobDataPlaneConfig, MyelinChildRouteRegistrar,
|
||||||
};
|
};
|
||||||
|
|
@ -11,7 +12,7 @@ use parking_lot::Mutex;
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
use std::env;
|
use std::env;
|
||||||
use std::io::{BufRead, BufReader};
|
use std::io::{BufRead, BufReader};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::PathBuf;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::sync::atomic::{AtomicBool, Ordering};
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
@ -21,25 +22,25 @@ use swactor::actor::{ActorAddress, ActorInterface};
|
||||||
use swactor::runtime::{Ctx, ExternalSender};
|
use swactor::runtime::{Ctx, ExternalSender};
|
||||||
use swactor_engine::{ActorCompletion, Engine, EngineHandle, TokioBackend, TokioConfig};
|
use swactor_engine::{ActorCompletion, Engine, EngineHandle, TokioBackend, TokioConfig};
|
||||||
use swactor_job_runner::{
|
use swactor_job_runner::{
|
||||||
INFERENCE_RESULTS_EDGE_ID, Job, JobDataPlanePort, JobDone, NodeJobActor, OUTPUTS_EDGE_ID,
|
Job, JobDataPlanePort, JobDone, NodeJobActor, OUTPUTS_EDGE_ID, OrchestratorJobActor,
|
||||||
OrchestratorJobActor, OrchestratorJobMsg, WORKSPACE_EDGE_ID, register_job_codecs,
|
OrchestratorJobMsg, WORKSPACE_EDGE_ID, register_job_codecs,
|
||||||
};
|
};
|
||||||
use swactor_transport::hex_encode;
|
use swactor_transport::hex_encode;
|
||||||
|
|
||||||
use data_plane::blob_transfer::{BlobTransferReceiver, BlobTransferSender};
|
use data_plane::blob_transfer::{BlobTransferReceiver, BlobTransferSender};
|
||||||
|
use data_plane::data_plane::StreamConsumer;
|
||||||
use data_plane::edge_wire::WireEvent;
|
use data_plane::edge_wire::WireEvent;
|
||||||
use data_plane::namespace::NamespaceClient;
|
use data_plane::namespace::NamespaceClient;
|
||||||
use data_plane::path::{DataPath, JobContext};
|
use data_plane::path::{DataPath, JobContext};
|
||||||
|
use data_plane::protocol::DataPlaneError;
|
||||||
use data_plane::protocol::{JobCapability, register_data_plane_codecs};
|
use data_plane::protocol::{JobCapability, register_data_plane_codecs};
|
||||||
use data_plane::source::BlobSourcePublisher;
|
use data_plane::source::BlobSourcePublisher;
|
||||||
use distribution::node::DistributedNodeConfig;
|
use distribution::node::DistributedNodeConfig;
|
||||||
use iroh::{EndpointAddr, RelayMode};
|
use iroh::{EndpointAddr, RelayMode};
|
||||||
use iroh_driver::{
|
use iroh_driver::{
|
||||||
EDGE_ALPN, EdgeConnector, EdgeSendHandle, EndpointAddrMask, IrohDriver, IrohDriverConfig,
|
EDGE_ALPN, EndpointAddrMask, IrohDriver, IrohDriverConfig, MVP_IROH_ENDPOINT_ADDR_MASK_ENV,
|
||||||
MVP_IROH_ENDPOINT_ADDR_MASK_ENV, advertised_endpoint,
|
advertised_endpoint,
|
||||||
};
|
};
|
||||||
use tokio::io::AsyncReadExt;
|
|
||||||
use tokio::sync::Notify;
|
|
||||||
|
|
||||||
use crate::orchestration::distribution_stack::DistributionRuntimeStack;
|
use crate::orchestration::distribution_stack::DistributionRuntimeStack;
|
||||||
|
|
||||||
|
|
@ -50,57 +51,37 @@ const RELAY_WAIT_DEADLINE: Duration = Duration::from_secs(30);
|
||||||
const MYELIN_IROH_RELAY_MODE_ENV: &str = "MYELIN_IROH_RELAY_MODE";
|
const MYELIN_IROH_RELAY_MODE_ENV: &str = "MYELIN_IROH_RELAY_MODE";
|
||||||
const MYELIN_IROH_RELAY_URL_ENV: &str = "MYELIN_IROH_RELAY_URL";
|
const MYELIN_IROH_RELAY_URL_ENV: &str = "MYELIN_IROH_RELAY_URL";
|
||||||
const SWACTOR_IROH_RELAY_URL_ENV: &str = "SWACTOR_IROH_RELAY_URL";
|
const SWACTOR_IROH_RELAY_URL_ENV: &str = "SWACTOR_IROH_RELAY_URL";
|
||||||
const JOB_OUTPUT_SOCKET: &str = "inference-results.sock";
|
|
||||||
const DATA_PLANE_CONNECT_DEADLINE: Duration = Duration::from_secs(30);
|
|
||||||
const JOB_ARENA_BYTES: u64 = 1 << 20;
|
const JOB_ARENA_BYTES: u64 = 1 << 20;
|
||||||
|
|
||||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
/// Actor-driven finite-blob ingress and namespace-addressed result streams.
|
||||||
struct EmbeddedDataPlaneAssignment {
|
|
||||||
result_endpoint: EndpointAddr,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Actor-driven finite-blob ingress plus the retained temporary Unix output
|
|
||||||
/// stream bridge. Remote bytes remain on `EDGE_ALPN`.
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub(crate) struct EmbeddedJobDataPlane {
|
pub(crate) struct EmbeddedJobDataPlane {
|
||||||
result_sink: Arc<Mutex<Option<EdgeSendHandle>>>,
|
|
||||||
result_ready: Arc<Notify>,
|
|
||||||
connector: EdgeConnector,
|
|
||||||
actor_plane: Arc<ActorJobDataPlane>,
|
actor_plane: Arc<ActorJobDataPlane>,
|
||||||
host_endpoint_json: String,
|
host_endpoint_json: String,
|
||||||
output_path: PathBuf,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) struct EmbeddedJobDataPlaneConfig<'a> {
|
pub(crate) struct EmbeddedJobDataPlaneConfig {
|
||||||
pub(crate) engine: EngineHandle,
|
|
||||||
pub(crate) connector: EdgeConnector,
|
|
||||||
pub(crate) root: &'a Path,
|
|
||||||
pub(crate) host_endpoint: EndpointAddr,
|
pub(crate) host_endpoint: EndpointAddr,
|
||||||
pub(crate) namespace: NamespaceClient,
|
pub(crate) namespace: NamespaceClient,
|
||||||
pub(crate) transfer_receiver: Arc<dyn BlobTransferReceiver>,
|
pub(crate) transfer_receiver: Arc<dyn BlobTransferReceiver>,
|
||||||
pub(crate) source_sender: Arc<dyn BlobTransferSender>,
|
pub(crate) source_sender: Arc<dyn BlobTransferSender>,
|
||||||
pub(crate) source_publisher: Arc<dyn BlobSourcePublisher>,
|
pub(crate) source_publisher: Arc<dyn BlobSourcePublisher>,
|
||||||
|
pub(crate) stream_transport: Arc<dyn data_plane::stream_transport::StreamTransport>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl EmbeddedJobDataPlane {
|
impl EmbeddedJobDataPlane {
|
||||||
pub(crate) fn start(
|
pub(crate) fn start(
|
||||||
stack: &DistributionRuntimeStack,
|
stack: &DistributionRuntimeStack,
|
||||||
config: EmbeddedJobDataPlaneConfig<'_>,
|
config: EmbeddedJobDataPlaneConfig,
|
||||||
) -> Result<Self, String> {
|
) -> Result<Self, String> {
|
||||||
let EmbeddedJobDataPlaneConfig {
|
let EmbeddedJobDataPlaneConfig {
|
||||||
engine,
|
|
||||||
connector,
|
|
||||||
root,
|
|
||||||
host_endpoint,
|
host_endpoint,
|
||||||
namespace,
|
namespace,
|
||||||
transfer_receiver,
|
transfer_receiver,
|
||||||
source_sender,
|
source_sender,
|
||||||
source_publisher,
|
source_publisher,
|
||||||
|
stream_transport,
|
||||||
} = config;
|
} = config;
|
||||||
std::fs::create_dir_all(root)
|
|
||||||
.map_err(|error| format!("create job data-plane root {}: {error}", root.display()))?;
|
|
||||||
let output_path = root.join(JOB_OUTPUT_SOCKET);
|
|
||||||
remove_stale_socket(&output_path)?;
|
|
||||||
let capability = JobCapability::new(ActorAddress::new_random().0);
|
let capability = JobCapability::new(ActorAddress::new_random().0);
|
||||||
let route_registrar = Arc::new(MyelinChildRouteRegistrar::new(
|
let route_registrar = Arc::new(MyelinChildRouteRegistrar::new(
|
||||||
stack.route_view.clone(),
|
stack.route_view.clone(),
|
||||||
|
|
@ -124,49 +105,13 @@ impl EmbeddedJobDataPlane {
|
||||||
source_sender: Some(source_sender),
|
source_sender: Some(source_sender),
|
||||||
source_publisher: Some(source_publisher),
|
source_publisher: Some(source_publisher),
|
||||||
route_registrar: Some(route_registrar),
|
route_registrar: Some(route_registrar),
|
||||||
|
stream_transport: Some(stream_transport),
|
||||||
},
|
},
|
||||||
)?);
|
)?);
|
||||||
let host_endpoint_json = serde_json::to_string(&host_endpoint)
|
let host_endpoint_json = serde_json::to_string(&host_endpoint)
|
||||||
.map_err(|error| format!("serialize host data-plane endpoint: {error}"))?;
|
.map_err(|error| format!("serialize host data-plane endpoint: {error}"))?;
|
||||||
|
|
||||||
let result_sink: Arc<Mutex<Option<EdgeSendHandle>>> = Arc::new(Mutex::new(None));
|
|
||||||
let result_ready = Arc::new(Notify::new());
|
|
||||||
|
|
||||||
let output_slot = Arc::clone(&result_sink);
|
|
||||||
let output_ready = Arc::clone(&result_ready);
|
|
||||||
swactor_process::spawn_unix_stream_listener(engine, &output_path, move |mut stream| {
|
|
||||||
let output_slot = Arc::clone(&output_slot);
|
|
||||||
let output_ready = Arc::clone(&output_ready);
|
|
||||||
async move {
|
|
||||||
let sink = loop {
|
|
||||||
let notified = output_ready.notified();
|
|
||||||
if let Some(sink) = output_slot.lock().take() {
|
|
||||||
break sink;
|
|
||||||
}
|
|
||||||
notified.await;
|
|
||||||
};
|
|
||||||
let mut bytes = vec![0_u8; 64 * 1024];
|
|
||||||
loop {
|
|
||||||
match stream.read(&mut bytes).await {
|
|
||||||
Ok(0) => break,
|
|
||||||
Ok(count) => {
|
|
||||||
if sink.send(bytes[..count].to_vec()).is_err() {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(_) => break,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
drop(sink);
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.map_err(|error| format!("bind job data-plane output: {error}"))?;
|
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
result_sink,
|
|
||||||
result_ready,
|
|
||||||
connector,
|
|
||||||
output_path,
|
|
||||||
actor_plane,
|
actor_plane,
|
||||||
host_endpoint_json,
|
host_endpoint_json,
|
||||||
})
|
})
|
||||||
|
|
@ -177,43 +122,17 @@ impl JobDataPlanePort for EmbeddedJobDataPlane {
|
||||||
fn configure(
|
fn configure(
|
||||||
&self,
|
&self,
|
||||||
job_id: u64,
|
job_id: u64,
|
||||||
result_peer: &str,
|
_result_peer: &str,
|
||||||
) -> Result<BTreeMap<String, String>, String> {
|
) -> Result<BTreeMap<String, String>, String> {
|
||||||
let assignment = serde_json::from_str::<EmbeddedDataPlaneAssignment>(result_peer)
|
|
||||||
.map_err(|error| format!("parse job data-plane assignment: {error}"))?;
|
|
||||||
let sink = self.connector.connect(
|
|
||||||
assignment.result_endpoint,
|
|
||||||
INFERENCE_RESULTS_EDGE_ID,
|
|
||||||
DATA_PLANE_CONNECT_DEADLINE,
|
|
||||||
)?;
|
|
||||||
*self.result_sink.lock() = Some(sink);
|
|
||||||
self.result_ready.notify_one();
|
|
||||||
self.actor_plane.configure_run(job_id.to_string())?;
|
self.actor_plane.configure_run(job_id.to_string())?;
|
||||||
let mut env = self.actor_plane.handoff_env(&self.host_endpoint_json);
|
Ok(self.actor_plane.handoff_env(&self.host_endpoint_json))
|
||||||
env.insert(
|
|
||||||
"SWACTOR_DATA_PLANE_OUTPUT".to_owned(),
|
|
||||||
self.output_path.to_string_lossy().into_owned(),
|
|
||||||
);
|
|
||||||
Ok(env)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn session_ended(&self, _job_id: u64) {
|
fn session_ended(&self, _job_id: u64) {
|
||||||
self.result_sink.lock().take();
|
|
||||||
self.actor_plane.close();
|
self.actor_plane.close();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn remove_stale_socket(path: &Path) -> Result<(), String> {
|
|
||||||
match std::fs::remove_file(path) {
|
|
||||||
Ok(()) => Ok(()),
|
|
||||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
|
|
||||||
Err(error) => Err(format!(
|
|
||||||
"remove stale job data-plane socket {}: {error}",
|
|
||||||
path.display()
|
|
||||||
)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Out-of-band identity one side publishes so the other can route to it.
|
/// Out-of-band identity one side publishes so the other can route to it.
|
||||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||||
pub struct NodeIdentity {
|
pub struct NodeIdentity {
|
||||||
|
|
@ -221,6 +140,17 @@ pub struct NodeIdentity {
|
||||||
pub actor_hex: String,
|
pub actor_hex: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct OrchestratorResultSink {
|
||||||
|
bytes: Arc<Mutex<Vec<u8>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StreamConsumer for OrchestratorResultSink {
|
||||||
|
fn consume(&self, bytes: &[u8]) -> Result<(), String> {
|
||||||
|
self.bytes.lock().extend_from_slice(bytes);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) struct JobOrchestratorSession {
|
pub(crate) struct JobOrchestratorSession {
|
||||||
_engine: Engine,
|
_engine: Engine,
|
||||||
driver: IrohDriver,
|
driver: IrohDriver,
|
||||||
|
|
@ -229,6 +159,10 @@ pub(crate) struct JobOrchestratorSession {
|
||||||
orch: ActorAddress,
|
orch: ActorAddress,
|
||||||
identity: NodeIdentity,
|
identity: NodeIdentity,
|
||||||
landing: PathBuf,
|
landing: PathBuf,
|
||||||
|
_namespace_authority: DataNamespaceAuthority,
|
||||||
|
_result_plane: Arc<ActorJobDataPlane>,
|
||||||
|
_result_completion: ActorCompletion<Result<(), DataPlaneError>>,
|
||||||
|
_result_bytes: Arc<Mutex<Vec<u8>>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
type JobComposition = (Engine, IrohDriver, DistributionRuntimeStack);
|
type JobComposition = (Engine, IrohDriver, DistributionRuntimeStack);
|
||||||
|
|
@ -611,6 +545,47 @@ fn start_orchestrator_mode(
|
||||||
Some(relay_mode) => build_composition_with_relay(relay_mode)?,
|
Some(relay_mode) => build_composition_with_relay(relay_mode)?,
|
||||||
None => build_composition()?,
|
None => build_composition()?,
|
||||||
};
|
};
|
||||||
|
let namespace_authority = DataNamespaceAuthority::start(
|
||||||
|
&stack,
|
||||||
|
&driver,
|
||||||
|
landing.join(".swactor-data-namespace.json"),
|
||||||
|
)?;
|
||||||
|
let namespace = install_namespace_client(&stack, &driver)?;
|
||||||
|
let result_capability = JobCapability::new(ActorAddress::new_random().0);
|
||||||
|
let result_transport: Arc<dyn data_plane::stream_transport::StreamTransport> =
|
||||||
|
driver.stream_transport();
|
||||||
|
let result_plane = Arc::new(ActorJobDataPlane::new(
|
||||||
|
&stack.runtime,
|
||||||
|
ActorJobDataPlaneConfig {
|
||||||
|
arena_bytes: JOB_ARENA_BYTES,
|
||||||
|
arena_generation: 2,
|
||||||
|
session_generation: 2,
|
||||||
|
capability: result_capability,
|
||||||
|
job_context: JobContext {
|
||||||
|
run_id: "unconfigured".to_owned(),
|
||||||
|
read_prefixes: vec![DataPath::parse("/runs").expect("static run prefix")],
|
||||||
|
write_prefixes: Vec::new(),
|
||||||
|
},
|
||||||
|
namespace: Some(namespace.client),
|
||||||
|
transfer_receiver: None,
|
||||||
|
source_sender: None,
|
||||||
|
source_publisher: Some(namespace.source_publisher),
|
||||||
|
route_registrar: None,
|
||||||
|
stream_transport: Some(result_transport),
|
||||||
|
},
|
||||||
|
)?);
|
||||||
|
result_plane.configure_run("0".to_owned())?;
|
||||||
|
let result_client = result_plane.attach_local()?;
|
||||||
|
let result_bytes = Arc::new(Mutex::new(Vec::new()));
|
||||||
|
let result_consumer: Arc<dyn StreamConsumer> = Arc::new(OrchestratorResultSink {
|
||||||
|
bytes: Arc::clone(&result_bytes),
|
||||||
|
});
|
||||||
|
let result_completion = result_client
|
||||||
|
.collect_stream(
|
||||||
|
DataPath::parse("/runs/0/results/inference").expect("static result path"),
|
||||||
|
result_consumer,
|
||||||
|
)
|
||||||
|
.map_err(|error| format!("register inference result sink: {error}"))?;
|
||||||
let done = stack
|
let done = stack
|
||||||
.runtime
|
.runtime
|
||||||
.new_inbox::<JobDone>()
|
.new_inbox::<JobDone>()
|
||||||
|
|
@ -630,6 +605,10 @@ fn start_orchestrator_mode(
|
||||||
orch,
|
orch,
|
||||||
identity,
|
identity,
|
||||||
landing,
|
landing,
|
||||||
|
_namespace_authority: namespace_authority,
|
||||||
|
_result_plane: result_plane,
|
||||||
|
_result_completion: result_completion,
|
||||||
|
_result_bytes: result_bytes,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1983,14 +1983,12 @@ fn run() -> Result<(), String> {
|
||||||
let data_plane = EmbeddedJobDataPlane::start(
|
let data_plane = EmbeddedJobDataPlane::start(
|
||||||
&stack,
|
&stack,
|
||||||
crate::job_deploy::EmbeddedJobDataPlaneConfig {
|
crate::job_deploy::EmbeddedJobDataPlaneConfig {
|
||||||
engine: engine.handle(),
|
|
||||||
connector: driver.edge_connector(),
|
|
||||||
root: &workdir,
|
|
||||||
host_endpoint: driver.endpoint_addr(),
|
host_endpoint: driver.endpoint_addr(),
|
||||||
namespace: namespace.client,
|
namespace: namespace.client,
|
||||||
transfer_receiver,
|
transfer_receiver,
|
||||||
source_sender,
|
source_sender,
|
||||||
source_publisher: namespace.source_publisher,
|
source_publisher: namespace.source_publisher,
|
||||||
|
stream_transport: driver.stream_transport(),
|
||||||
},
|
},
|
||||||
)?;
|
)?;
|
||||||
let job_route_registrar = Arc::new(MyelinChildRouteRegistrar::new(
|
let job_route_registrar = Arc::new(MyelinChildRouteRegistrar::new(
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,7 @@
|
||||||
//! only through these closures.
|
//! only through these closures.
|
||||||
|
|
||||||
use iroh::EndpointAddr;
|
use iroh::EndpointAddr;
|
||||||
|
use iroh_driver::telemetry_transport::PullCollectorConfig;
|
||||||
use iroh_driver::{IrohDriver, PullCollectorHandle, TelemetryQuicHeader, spawn_pull_collector};
|
use iroh_driver::{IrohDriver, PullCollectorHandle, TelemetryQuicHeader, spawn_pull_collector};
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
use std::collections::{BTreeMap, BTreeSet};
|
use std::collections::{BTreeMap, BTreeSet};
|
||||||
|
|
@ -82,12 +83,14 @@ impl FrameCollector {
|
||||||
flow_id[8..].copy_from_slice(&node_id.to_le_bytes());
|
flow_id[8..].copy_from_slice(&node_id.to_le_bytes());
|
||||||
let collector = spawn_pull_collector(
|
let collector = spawn_pull_collector(
|
||||||
engine,
|
engine,
|
||||||
endpoint,
|
PullCollectorConfig {
|
||||||
peer,
|
endpoint,
|
||||||
flow_id,
|
peer,
|
||||||
Vec::new(),
|
flow_id,
|
||||||
SubscriptionRequest::all(),
|
token: Vec::new(),
|
||||||
Arc::clone(&self.pull_fanout),
|
request: SubscriptionRequest::all(),
|
||||||
|
fanout: Arc::clone(&self.pull_fanout),
|
||||||
|
},
|
||||||
self.pull_header_tx.clone(),
|
self.pull_header_tx.clone(),
|
||||||
);
|
);
|
||||||
if let Some(previous) = self
|
if let Some(previous) = self
|
||||||
|
|
|
||||||
|
|
@ -43,6 +43,15 @@ pub(crate) struct OrchTelemetry {
|
||||||
descriptor: StreamDescriptor,
|
descriptor: StreamDescriptor,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) struct BootstrapEmission<'a> {
|
||||||
|
pub(crate) dashboard: Option<&'a DashboardSupport>,
|
||||||
|
pub(crate) channel: &'a str,
|
||||||
|
pub(crate) run_id: u64,
|
||||||
|
pub(crate) node_id: u64,
|
||||||
|
pub(crate) phase: &'a str,
|
||||||
|
pub(crate) status: &'a str,
|
||||||
|
pub(crate) detail: Value,
|
||||||
|
}
|
||||||
impl OrchTelemetry {
|
impl OrchTelemetry {
|
||||||
pub(crate) fn new(run_id: u64, frame_log: Option<&Path>) -> Result<Self, String> {
|
pub(crate) fn new(run_id: u64, frame_log: Option<&Path>) -> Result<Self, String> {
|
||||||
let stream = StreamId::new(NodeId::new("myelin-orchestrator"), Lifetime(run_id));
|
let stream = StreamId::new(NodeId::new("myelin-orchestrator"), Lifetime(run_id));
|
||||||
|
|
@ -110,6 +119,10 @@ impl OrchTelemetry {
|
||||||
id
|
id
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn producer(&self) -> TelemetryProducer {
|
||||||
|
self.producer.clone()
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn emit_event(
|
pub(crate) fn emit_event(
|
||||||
&mut self,
|
&mut self,
|
||||||
dashboard: Option<&DashboardSupport>,
|
dashboard: Option<&DashboardSupport>,
|
||||||
|
|
@ -140,27 +153,27 @@ impl OrchTelemetry {
|
||||||
status: &str,
|
status: &str,
|
||||||
detail: Value,
|
detail: Value,
|
||||||
) {
|
) {
|
||||||
self.emit_bootstrap_to_channel(
|
self.emit_bootstrap_to_channel(BootstrapEmission {
|
||||||
dashboard,
|
dashboard,
|
||||||
MYELIN_ORCH_BOOTSTRAP,
|
channel: MYELIN_ORCH_BOOTSTRAP,
|
||||||
run_id,
|
run_id,
|
||||||
node_id,
|
node_id,
|
||||||
phase,
|
phase,
|
||||||
status,
|
status,
|
||||||
detail,
|
detail,
|
||||||
);
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn emit_bootstrap_to_channel(
|
pub(crate) fn emit_bootstrap_to_channel(&mut self, emission: BootstrapEmission<'_>) {
|
||||||
&mut self,
|
let BootstrapEmission {
|
||||||
dashboard: Option<&DashboardSupport>,
|
dashboard,
|
||||||
channel: &str,
|
channel,
|
||||||
run_id: u64,
|
run_id,
|
||||||
node_id: u64,
|
node_id,
|
||||||
phase: &str,
|
phase,
|
||||||
status: &str,
|
status,
|
||||||
detail: Value,
|
detail,
|
||||||
) {
|
} = emission;
|
||||||
let benchmark = benchmark::stamp("myelin-orchestrator");
|
let benchmark = benchmark::stamp("myelin-orchestrator");
|
||||||
let payload = serde_json::to_vec(&json!({
|
let payload = serde_json::to_vec(&json!({
|
||||||
"schema_version": benchmark["schema_version"].clone(),
|
"schema_version": benchmark["schema_version"].clone(),
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@ use crate::provisioning::{
|
||||||
};
|
};
|
||||||
|
|
||||||
pub(crate) fn node_stream_id(run_id: u64, node_id: u64) -> StreamId {
|
pub(crate) fn node_stream_id(run_id: u64, node_id: u64) -> StreamId {
|
||||||
StreamId::new(NodeId::new(&node_id.to_string()), Lifetime(run_id))
|
StreamId::new(NodeId::new(node_id.to_string()), Lifetime(run_id))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
|
|
|
||||||
|
|
@ -538,7 +538,7 @@ async fn submit_node_job(
|
||||||
ManualControlMsg::Query { reply_to }
|
ManualControlMsg::Query { reply_to }
|
||||||
}) {
|
}) {
|
||||||
Ok(response_rx) => response_rx,
|
Ok(response_rx) => response_rx,
|
||||||
Err(response) => return response,
|
Err(response) => return *response,
|
||||||
};
|
};
|
||||||
let model = match response_rx.await {
|
let model = match response_rx.await {
|
||||||
Ok(ManualControlReply::Status(model)) => model,
|
Ok(ManualControlReply::Status(model)) => model,
|
||||||
|
|
@ -689,7 +689,7 @@ async fn request_reply(
|
||||||
) -> Response {
|
) -> Response {
|
||||||
let response_rx = match begin_request_reply(state, timeout, build) {
|
let response_rx = match begin_request_reply(state, timeout, build) {
|
||||||
Ok(response_rx) => response_rx,
|
Ok(response_rx) => response_rx,
|
||||||
Err(response) => return response,
|
Err(response) => return *response,
|
||||||
};
|
};
|
||||||
|
|
||||||
match response_rx.await {
|
match response_rx.await {
|
||||||
|
|
@ -718,7 +718,7 @@ fn begin_request_reply(
|
||||||
state: &ControlHttpState,
|
state: &ControlHttpState,
|
||||||
timeout: Duration,
|
timeout: Duration,
|
||||||
build: impl FnOnce(ActorAddress) -> ManualControlMsg,
|
build: impl FnOnce(ActorAddress) -> ManualControlMsg,
|
||||||
) -> Result<tokio::sync::oneshot::Receiver<ManualControlReply>, Response> {
|
) -> Result<tokio::sync::oneshot::Receiver<ManualControlReply>, Box<Response>> {
|
||||||
let (response_tx, response_rx) = tokio::sync::oneshot::channel();
|
let (response_tx, response_rx) = tokio::sync::oneshot::channel();
|
||||||
let response_tx = Arc::new(Mutex::new(Some(response_tx)));
|
let response_tx = Arc::new(Mutex::new(Some(response_tx)));
|
||||||
let reply_to = state
|
let reply_to = state
|
||||||
|
|
@ -730,26 +730,30 @@ fn begin_request_reply(
|
||||||
timeout,
|
timeout,
|
||||||
})
|
})
|
||||||
.map_err(|error| {
|
.map_err(|error| {
|
||||||
(
|
Box::new(
|
||||||
StatusCode::SERVICE_UNAVAILABLE,
|
(
|
||||||
Json(ErrorResponse {
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
error: format!("create control reply observer: {error}"),
|
Json(ErrorResponse {
|
||||||
}),
|
error: format!("create control reply observer: {error}"),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
)
|
)
|
||||||
.into_response()
|
|
||||||
})?;
|
})?;
|
||||||
if let Err(error) = state
|
if let Err(error) = state
|
||||||
.runtime
|
.runtime
|
||||||
.send_to(state.orchestrator, OrchestratorMsg::Manual(build(reply_to)))
|
.send_to(state.orchestrator, OrchestratorMsg::Manual(build(reply_to)))
|
||||||
{
|
{
|
||||||
let _ = state.runtime.stop_actor(reply_to);
|
let _ = state.runtime.stop_actor(reply_to);
|
||||||
return Err((
|
return Err(Box::new(
|
||||||
StatusCode::SERVICE_UNAVAILABLE,
|
(
|
||||||
Json(ErrorResponse {
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
error: format!("orchestrator control actor unavailable: {error}"),
|
Json(ErrorResponse {
|
||||||
}),
|
error: format!("orchestrator control actor unavailable: {error}"),
|
||||||
)
|
}),
|
||||||
.into_response());
|
)
|
||||||
|
.into_response(),
|
||||||
|
));
|
||||||
}
|
}
|
||||||
Ok(response_rx)
|
Ok(response_rx)
|
||||||
}
|
}
|
||||||
|
|
@ -998,7 +1002,7 @@ mod properties {
|
||||||
status,
|
status,
|
||||||
})
|
})
|
||||||
};
|
};
|
||||||
let reply = |result: Result<_, Response>| match result {
|
let reply = |result: Result<_, Box<Response>>| match result {
|
||||||
Ok(receiver) => PendingHttpObservation::Reply {
|
Ok(receiver) => PendingHttpObservation::Reply {
|
||||||
index,
|
index,
|
||||||
action: action.clone(),
|
action: action.clone(),
|
||||||
|
|
@ -1191,9 +1195,10 @@ mod properties {
|
||||||
HttpAction::from_raw(kind, command_slot, value)
|
HttpAction::from_raw(kind, command_slot, value)
|
||||||
})
|
})
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
let mut config = RuntimeConfig::default();
|
let parts = RuntimeParts::new(RuntimeConfig {
|
||||||
config.worker_count = 1;
|
worker_count: 1,
|
||||||
let parts = RuntimeParts::new(config);
|
..RuntimeConfig::default()
|
||||||
|
});
|
||||||
let runtime = parts.runtime().clone();
|
let runtime = parts.runtime().clone();
|
||||||
let backend = SteppingBackend::new();
|
let backend = SteppingBackend::new();
|
||||||
let engine =
|
let engine =
|
||||||
|
|
@ -1294,9 +1299,10 @@ mod properties {
|
||||||
fn generated_duplicate_control_replies_deliver_first_once_and_remove_observer(
|
fn generated_duplicate_control_replies_deliver_first_once_and_remove_observer(
|
||||||
replies in prop::collection::vec(any::<u8>(), 0..=16)
|
replies in prop::collection::vec(any::<u8>(), 0..=16)
|
||||||
) {
|
) {
|
||||||
let mut config = RuntimeConfig::default();
|
let parts = RuntimeParts::new(RuntimeConfig {
|
||||||
config.worker_count = 1;
|
worker_count: 1,
|
||||||
let parts = RuntimeParts::new(config);
|
..RuntimeConfig::default()
|
||||||
|
});
|
||||||
let runtime = parts.runtime().clone();
|
let runtime = parts.runtime().clone();
|
||||||
let backend = SteppingBackend::new();
|
let backend = SteppingBackend::new();
|
||||||
let engine =
|
let engine =
|
||||||
|
|
@ -1380,9 +1386,10 @@ mod properties {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn reply_observer_disappearance_returns_a_bounded_terminal_http_response() {
|
fn reply_observer_disappearance_returns_a_bounded_terminal_http_response() {
|
||||||
let mut config = RuntimeConfig::default();
|
let parts = RuntimeParts::new(RuntimeConfig {
|
||||||
config.worker_count = 1;
|
worker_count: 1,
|
||||||
let parts = RuntimeParts::new(config);
|
..RuntimeConfig::default()
|
||||||
|
});
|
||||||
let runtime = parts.runtime().clone();
|
let runtime = parts.runtime().clone();
|
||||||
let backend = SteppingBackend::new();
|
let backend = SteppingBackend::new();
|
||||||
let engine =
|
let engine =
|
||||||
|
|
|
||||||
|
|
@ -1,14 +1,50 @@
|
||||||
(() => {
|
(() => {
|
||||||
const CONTROL_ID = 'myelin-fleet-control';
|
const CONTROL_ID = 'myelin-fleet-control';
|
||||||
|
const CONFIRM_ID = 'myelin-confirm-dialog';
|
||||||
|
const CONFIRM_STYLE_ID = 'myelin-confirm-dialog-style';
|
||||||
const selectedJobs = new Map();
|
const selectedJobs = new Map();
|
||||||
|
|
||||||
async function syncControl() {
|
function confirmKill(logicalNodeId) {
|
||||||
const nodeView = document.querySelector('.node-view[data-node]');
|
let dialog = document.getElementById(CONFIRM_ID);
|
||||||
if (!nodeView) return;
|
if (!dialog) {
|
||||||
const rawNodeId = nodeView.getAttribute('data-node') || '';
|
const style = document.createElement('style');
|
||||||
if (!/^\d+$/.test(rawNodeId)) return;
|
style.id = CONFIRM_STYLE_ID;
|
||||||
const logicalNodeId = Number(rawNodeId);
|
style.textContent = `
|
||||||
|
.myelin-confirm { width:min(440px,calc(100vw - 32px));padding:0;color:var(--text);background:var(--panel);border:1px solid var(--bad);border-radius:var(--r);box-shadow:0 18px 60px rgba(0,0,0,.55) }
|
||||||
|
.myelin-confirm::backdrop { background:rgba(0,6,12,.78) }
|
||||||
|
.myelin-confirm form { display:grid;gap:14px;padding:18px }
|
||||||
|
.myelin-confirm h2,.myelin-confirm p { margin:0 }
|
||||||
|
.myelin-confirm h2 { color:var(--bad) }
|
||||||
|
.myelin-confirm-actions { display:flex;justify-content:flex-end;gap:8px }
|
||||||
|
.myelin-confirm button { padding:6px 12px;background:transparent;color:var(--text);border:1px solid var(--border);border-radius:var(--r);cursor:pointer;font:600 13px var(--mono) }
|
||||||
|
.myelin-confirm button[value="confirm"] { color:var(--danger-ink);background:var(--danger-fill);border-color:var(--danger-border) }
|
||||||
|
`;
|
||||||
|
document.head.append(style);
|
||||||
|
dialog = document.createElement('dialog');
|
||||||
|
dialog.id = CONFIRM_ID;
|
||||||
|
dialog.className = 'myelin-confirm';
|
||||||
|
dialog.setAttribute('aria-labelledby', 'myelin-confirm-title');
|
||||||
|
dialog.setAttribute('aria-describedby', 'myelin-confirm-message');
|
||||||
|
dialog.innerHTML = `<form method="dialog">
|
||||||
|
<h2 id="myelin-confirm-title">Terminate managed node?</h2>
|
||||||
|
<p id="myelin-confirm-message"></p>
|
||||||
|
<div class="myelin-confirm-actions">
|
||||||
|
<button value="cancel" autofocus>Cancel</button>
|
||||||
|
<button value="confirm">Terminate node</button>
|
||||||
|
</div>
|
||||||
|
</form>`;
|
||||||
|
document.body.append(dialog);
|
||||||
|
}
|
||||||
|
dialog.querySelector('#myelin-confirm-message').textContent =
|
||||||
|
`Terminate managed node ${logicalNodeId}? Vast.ai contracts are destroyed and billing stops.`;
|
||||||
|
dialog.returnValue = 'cancel';
|
||||||
|
return new Promise(resolve => {
|
||||||
|
dialog.addEventListener('close', () => resolve(dialog.returnValue === 'confirm'), { once: true });
|
||||||
|
dialog.showModal();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function syncControl() {
|
||||||
let model;
|
let model;
|
||||||
try {
|
try {
|
||||||
const response = await fetch('/api/control/status', { cache: 'no-store' });
|
const response = await fetch('/api/control/status', { cache: 'no-store' });
|
||||||
|
|
@ -18,6 +54,17 @@
|
||||||
} catch (_) {
|
} catch (_) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
window.dispatchEvent(new CustomEvent('dashboard-hardware-source', {
|
||||||
|
detail: {
|
||||||
|
source: model?.provider?.provisioning_mode === 'mock' ? 'orchestrator' : 'node',
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
const nodeView = document.querySelector('.node-view[data-node]');
|
||||||
|
if (!nodeView) return;
|
||||||
|
const rawNodeId = nodeView.getAttribute('data-node') || '';
|
||||||
|
if (!/^\d+$/.test(rawNodeId)) return;
|
||||||
|
const logicalNodeId = Number(rawNodeId);
|
||||||
const node = model?.nodes?.find(candidate => candidate.logical_node_id === logicalNodeId);
|
const node = model?.nodes?.find(candidate => candidate.logical_node_id === logicalNodeId);
|
||||||
if (!node) return;
|
if (!node) return;
|
||||||
|
|
||||||
|
|
@ -130,7 +177,7 @@
|
||||||
};
|
};
|
||||||
|
|
||||||
killButton.onclick = async () => {
|
killButton.onclick = async () => {
|
||||||
if (!window.confirm(`Kill managed node ${logicalNodeId}? Vast.ai contracts are destroyed and billing stops.`)) return;
|
if (!await confirmKill(logicalNodeId)) return;
|
||||||
killButton.disabled = true;
|
killButton.disabled = true;
|
||||||
message.textContent = 'Submitting kill…';
|
message.textContent = 'Submitting kill…';
|
||||||
const commandId = `fleet-kill-${globalThis.crypto?.randomUUID?.() || Date.now()}`;
|
const commandId = `fleet-kill-${globalThis.crypto?.randomUUID?.() || Date.now()}`;
|
||||||
|
|
|
||||||
|
|
@ -3441,8 +3441,8 @@ mod tests {
|
||||||
|
|
||||||
proptest! {
|
proptest! {
|
||||||
#![proptest_config(ProptestConfig {
|
#![proptest_config(ProptestConfig {
|
||||||
cases: 128,
|
cases: 16,
|
||||||
max_shrink_iters: 2_000,
|
max_shrink_iters: 256,
|
||||||
..ProptestConfig::default()
|
..ProptestConfig::default()
|
||||||
})]
|
})]
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -92,6 +92,19 @@
|
||||||
.sort-button[data-direction="asc"]::before { top: 3px; border-bottom: 5px solid currentColor; }
|
.sort-button[data-direction="asc"]::before { top: 3px; border-bottom: 5px solid currentColor; }
|
||||||
.sort-button[data-direction="desc"]::before { bottom: 3px; border-top: 5px solid currentColor; }
|
.sort-button[data-direction="desc"]::before { bottom: 3px; border-top: 5px solid currentColor; }
|
||||||
.sort-button:hover:not(:disabled), .sort-button[aria-pressed="true"] { color: var(--cyan); border-color: var(--cyan); background: var(--panel-hover); }
|
.sort-button:hover:not(:disabled), .sort-button[aria-pressed="true"] { color: var(--cyan); border-color: var(--cyan); background: var(--panel-hover); }
|
||||||
|
.offers-heading { display: flex; align-items: center; justify-content: space-between; gap: 12px; flex-wrap: wrap; margin-bottom: 12px; }
|
||||||
|
.offers-heading h2 { margin: 0; }
|
||||||
|
.offer-actions { display: flex; align-items: center; gap: 10px; flex-wrap: wrap; }
|
||||||
|
.offer-selection { margin: 0; font: 12px var(--mono); }
|
||||||
|
dialog.confirm-dialog {
|
||||||
|
width: min(440px, calc(100vw - 32px)); padding: 0; color: var(--text);
|
||||||
|
background: var(--panel); border: 1px solid var(--amber); border-radius: var(--r);
|
||||||
|
box-shadow: 0 18px 60px rgba(0, 0, 0, .55);
|
||||||
|
}
|
||||||
|
dialog.confirm-dialog::backdrop { background: rgba(0, 6, 12, .78); }
|
||||||
|
.confirm-form { display: grid; gap: 14px; padding: 18px; }
|
||||||
|
.confirm-form h2, .confirm-form p { margin: 0; }
|
||||||
|
.confirm-actions { display: flex; justify-content: flex-end; gap: 8px; }
|
||||||
@media (max-width: 680px) { #search-form { grid-template-columns: 1fr; } }
|
@media (max-width: 680px) { #search-form { grid-template-columns: 1fr; } }
|
||||||
@media (prefers-reduced-motion: reduce) { * { transition-duration: .01ms !important; } }
|
@media (prefers-reduced-motion: reduce) { * { transition-duration: .01ms !important; } }
|
||||||
</style>
|
</style>
|
||||||
|
|
@ -144,12 +157,27 @@
|
||||||
</section>
|
</section>
|
||||||
</div>
|
</div>
|
||||||
<section id="vast-offers-card" class="card" style="margin-top:1rem">
|
<section id="vast-offers-card" class="card" style="margin-top:1rem">
|
||||||
<h2>Offers</h2><p id="offer-status" class="muted"></p><p id="offer-error" class="error"></p>
|
<div class="offers-heading">
|
||||||
|
<h2>Offers</h2>
|
||||||
|
<div class="offer-actions">
|
||||||
|
<p class="offer-selection">Selected: <code id="selection">none (0/8)</code></p>
|
||||||
|
<button id="provision-button" class="primary" disabled>Provision selected offers</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<p id="offer-status" class="muted"></p><p id="offer-error" class="error"></p>
|
||||||
<div class="offers-scroll"><table id="offers-table"><thead><tr><th>Select</th><th>Offer<span class="sort-controls"><button type="button" class="sort-button" data-sort="offer_id" data-direction="asc" title="Sort offer ascending" aria-label="Sort offer ascending">↑</button><button type="button" class="sort-button" data-sort="offer_id" data-direction="desc" title="Sort offer descending" aria-label="Sort offer descending">↓</button></span></th><th>Host<span class="sort-controls"><button type="button" class="sort-button" data-sort="host_id" data-direction="asc" title="Sort host ascending" aria-label="Sort host ascending">↑</button><button type="button" class="sort-button" data-sort="host_id" data-direction="desc" title="Sort host descending" aria-label="Sort host descending">↓</button></span></th><th>GPU<span class="sort-controls"><button type="button" class="sort-button" data-sort="gpu_model" data-direction="asc" title="Sort GPU ascending" aria-label="Sort GPU ascending">↑</button><button type="button" class="sort-button" data-sort="gpu_model" data-direction="desc" title="Sort GPU descending" aria-label="Sort GPU descending">↓</button></span></th><th>RAM<span class="sort-controls"><button type="button" class="sort-button" data-sort="gpu_ram_mb" data-direction="asc" title="Sort RAM ascending" aria-label="Sort RAM ascending">↑</button><button type="button" class="sort-button" data-sort="gpu_ram_mb" data-direction="desc" title="Sort RAM descending" aria-label="Sort RAM descending">↓</button></span></th><th>CC<span class="sort-controls"><button type="button" class="sort-button" data-sort="compute_cap" data-direction="asc" title="Sort compute capability ascending" aria-label="Sort compute capability ascending">↑</button><button type="button" class="sort-button" data-sort="compute_cap" data-direction="desc" title="Sort compute capability descending" aria-label="Sort compute capability descending">↓</button></span></th><th>Verified<span class="sort-controls"><button type="button" class="sort-button" data-sort="verification" data-direction="asc" title="Sort verification ascending" aria-label="Sort verification ascending">↑</button><button type="button" class="sort-button" data-sort="verification" data-direction="desc" title="Sort verification descending" aria-label="Sort verification descending">↓</button></span></th><th>Reliability<span class="sort-controls"><button type="button" class="sort-button" data-sort="reliability" data-direction="asc" title="Sort reliability ascending" aria-label="Sort reliability ascending">↑</button><button type="button" class="sort-button" data-sort="reliability" data-direction="desc" title="Sort reliability descending" aria-label="Sort reliability descending">↓</button></span></th><th>Down<span class="sort-controls"><button type="button" class="sort-button" data-sort="download_mbps" data-direction="asc" title="Sort download speed ascending" aria-label="Sort download speed ascending">↑</button><button type="button" class="sort-button" data-sort="download_mbps" data-direction="desc" title="Sort download speed descending" aria-label="Sort download speed descending">↓</button></span></th><th>Up<span class="sort-controls"><button type="button" class="sort-button" data-sort="upload_mbps" data-direction="asc" title="Sort upload speed ascending" aria-label="Sort upload speed ascending">↑</button><button type="button" class="sort-button" data-sort="upload_mbps" data-direction="desc" title="Sort upload speed descending" aria-label="Sort upload speed descending">↓</button></span></th><th>Location<span class="sort-controls"><button type="button" class="sort-button" data-sort="location" data-direction="asc" title="Sort location ascending" aria-label="Sort location ascending">↑</button><button type="button" class="sort-button" data-sort="location" data-direction="desc" title="Sort location descending" aria-label="Sort location descending">↓</button></span></th><th>$/hr<span class="sort-controls"><button type="button" class="sort-button" data-sort="hourly_price" data-direction="asc" title="Sort price ascending" aria-label="Sort price ascending">↑</button><button type="button" class="sort-button" data-sort="hourly_price" data-direction="desc" title="Sort price descending" aria-label="Sort price descending">↓</button></span></th></tr></thead><tbody id="offers"></tbody></table></div>
|
<div class="offers-scroll"><table id="offers-table"><thead><tr><th>Select</th><th>Offer<span class="sort-controls"><button type="button" class="sort-button" data-sort="offer_id" data-direction="asc" title="Sort offer ascending" aria-label="Sort offer ascending">↑</button><button type="button" class="sort-button" data-sort="offer_id" data-direction="desc" title="Sort offer descending" aria-label="Sort offer descending">↓</button></span></th><th>Host<span class="sort-controls"><button type="button" class="sort-button" data-sort="host_id" data-direction="asc" title="Sort host ascending" aria-label="Sort host ascending">↑</button><button type="button" class="sort-button" data-sort="host_id" data-direction="desc" title="Sort host descending" aria-label="Sort host descending">↓</button></span></th><th>GPU<span class="sort-controls"><button type="button" class="sort-button" data-sort="gpu_model" data-direction="asc" title="Sort GPU ascending" aria-label="Sort GPU ascending">↑</button><button type="button" class="sort-button" data-sort="gpu_model" data-direction="desc" title="Sort GPU descending" aria-label="Sort GPU descending">↓</button></span></th><th>RAM<span class="sort-controls"><button type="button" class="sort-button" data-sort="gpu_ram_mb" data-direction="asc" title="Sort RAM ascending" aria-label="Sort RAM ascending">↑</button><button type="button" class="sort-button" data-sort="gpu_ram_mb" data-direction="desc" title="Sort RAM descending" aria-label="Sort RAM descending">↓</button></span></th><th>CC<span class="sort-controls"><button type="button" class="sort-button" data-sort="compute_cap" data-direction="asc" title="Sort compute capability ascending" aria-label="Sort compute capability ascending">↑</button><button type="button" class="sort-button" data-sort="compute_cap" data-direction="desc" title="Sort compute capability descending" aria-label="Sort compute capability descending">↓</button></span></th><th>Verified<span class="sort-controls"><button type="button" class="sort-button" data-sort="verification" data-direction="asc" title="Sort verification ascending" aria-label="Sort verification ascending">↑</button><button type="button" class="sort-button" data-sort="verification" data-direction="desc" title="Sort verification descending" aria-label="Sort verification descending">↓</button></span></th><th>Reliability<span class="sort-controls"><button type="button" class="sort-button" data-sort="reliability" data-direction="asc" title="Sort reliability ascending" aria-label="Sort reliability ascending">↑</button><button type="button" class="sort-button" data-sort="reliability" data-direction="desc" title="Sort reliability descending" aria-label="Sort reliability descending">↓</button></span></th><th>Down<span class="sort-controls"><button type="button" class="sort-button" data-sort="download_mbps" data-direction="asc" title="Sort download speed ascending" aria-label="Sort download speed ascending">↑</button><button type="button" class="sort-button" data-sort="download_mbps" data-direction="desc" title="Sort download speed descending" aria-label="Sort download speed descending">↓</button></span></th><th>Up<span class="sort-controls"><button type="button" class="sort-button" data-sort="upload_mbps" data-direction="asc" title="Sort upload speed ascending" aria-label="Sort upload speed ascending">↑</button><button type="button" class="sort-button" data-sort="upload_mbps" data-direction="desc" title="Sort upload speed descending" aria-label="Sort upload speed descending">↓</button></span></th><th>Location<span class="sort-controls"><button type="button" class="sort-button" data-sort="location" data-direction="asc" title="Sort location ascending" aria-label="Sort location ascending">↑</button><button type="button" class="sort-button" data-sort="location" data-direction="desc" title="Sort location descending" aria-label="Sort location descending">↓</button></span></th><th>$/hr<span class="sort-controls"><button type="button" class="sort-button" data-sort="hourly_price" data-direction="asc" title="Sort price ascending" aria-label="Sort price ascending">↑</button><button type="button" class="sort-button" data-sort="hourly_price" data-direction="desc" title="Sort price descending" aria-label="Sort price descending">↓</button></span></th></tr></thead><tbody id="offers"></tbody></table></div>
|
||||||
<p>Selected exact offer IDs: <code id="selection">none (0/8)</code></p>
|
|
||||||
<button id="provision-button" class="primary" disabled>Provision selected offers</button>
|
|
||||||
</section>
|
</section>
|
||||||
</main>
|
</main>
|
||||||
|
<dialog id="confirm-dialog" class="confirm-dialog" aria-labelledby="confirm-title" aria-describedby="confirm-message">
|
||||||
|
<form method="dialog" class="confirm-form">
|
||||||
|
<h2 id="confirm-title">Confirm action</h2>
|
||||||
|
<p id="confirm-message"></p>
|
||||||
|
<div class="confirm-actions">
|
||||||
|
<button value="cancel" autofocus>Cancel</button>
|
||||||
|
<button id="confirm-submit" class="primary" value="confirm">Confirm</button>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</dialog>
|
||||||
<script>
|
<script>
|
||||||
'use strict';
|
'use strict';
|
||||||
const $=id=>document.getElementById(id); const MAX_SELECTED_OFFERS=8; const PAGE_STATE_KEY='myelin.provision.page.v1'; const SEARCH_FIELDS=['gpu_model','min_gpu_ram_mb','min_compute_cap','min_reliability','min_download_mbps','min_upload_mbps','max_hourly_price','blacklist_hosts','require_verified']; let selected=new Set(); let provisioningMode='real'; let currentOffers=[]; let offerSorts=[]; let sortDoubleClickStartedActive=null;
|
const $=id=>document.getElementById(id); const MAX_SELECTED_OFFERS=8; const PAGE_STATE_KEY='myelin.provision.page.v1'; const SEARCH_FIELDS=['gpu_model','min_gpu_ram_mb','min_compute_cap','min_reliability','min_download_mbps','min_upload_mbps','max_hourly_price','blacklist_hosts','require_verified']; let selected=new Set(); let provisioningMode='real'; let currentOffers=[]; let offerSorts=[]; let sortDoubleClickStartedActive=null;
|
||||||
|
|
@ -158,6 +186,7 @@ const number=(form,name)=>{const raw=value(form,name);return raw===null?null:Num
|
||||||
const escapeHtml=raw=>String(raw??'').replace(/[&<>"']/g,char=>({'&':'&','<':'<','>':'>','"':'"',"'":'''}[char]));
|
const escapeHtml=raw=>String(raw??'').replace(/[&<>"']/g,char=>({'&':'&','<':'<','>':'>','"':'"',"'":'''}[char]));
|
||||||
async function jsonFetch(url,options={}){const response=await fetch(url,{...options,headers:{'content-type':'application/json',...(options.headers||{})}});const text=await response.text();let body=null;try{body=text?JSON.parse(text):null}catch{body={error:text}}if(!response.ok)throw new Error(body?.error||`${response.status} ${response.statusText}`);return body}
|
async function jsonFetch(url,options={}){const response=await fetch(url,{...options,headers:{'content-type':'application/json',...(options.headers||{})}});const text=await response.text();let body=null;try{body=text?JSON.parse(text):null}catch{body={error:text}}if(!response.ok)throw new Error(body?.error||`${response.status} ${response.statusText}`);return body}
|
||||||
function commandId(prefix){return `${prefix}-${crypto.randomUUID()}`}
|
function commandId(prefix){return `${prefix}-${crypto.randomUUID()}`}
|
||||||
|
function confirmAction(title,message,confirmLabel){const dialog=$('confirm-dialog');$('confirm-title').textContent=title;$('confirm-message').textContent=message;$('confirm-submit').textContent=confirmLabel;dialog.returnValue='cancel';return new Promise(resolve=>{dialog.addEventListener('close',()=>resolve(dialog.returnValue==='confirm'),{once:true});dialog.showModal()})}
|
||||||
function formSnapshot(form,names){return Object.fromEntries(names.map(name=>{const field=form.elements[name];return [name,field.type==='checkbox'?field.checked:field.value]}))}
|
function formSnapshot(form,names){return Object.fromEntries(names.map(name=>{const field=form.elements[name];return [name,field.type==='checkbox'?field.checked:field.value]}))}
|
||||||
function restoreForm(form,state){if(!state)return;Object.entries(state).forEach(([name,saved])=>{const field=form.elements[name];if(!field)return;if(field.type==='checkbox')field.checked=Boolean(saved);else field.value=String(saved??'')})}
|
function restoreForm(form,state){if(!state)return;Object.entries(state).forEach(([name,saved])=>{const field=form.elements[name];if(!field)return;if(field.type==='checkbox')field.checked=Boolean(saved);else field.value=String(saved??'')})}
|
||||||
function persistPageState(){try{sessionStorage.setItem(PAGE_STATE_KEY,JSON.stringify({config:formSnapshot($('config-form'),['ssh_identity','bootstrap_command']),search:formSnapshot($('search-form'),SEARCH_FIELDS),local:formSnapshot($('local-provision-form'),['count']),offers:currentOffers,selected:[...selected],sorts:offerSorts}))}catch{}}
|
function persistPageState(){try{sessionStorage.setItem(PAGE_STATE_KEY,JSON.stringify({config:formSnapshot($('config-form'),['ssh_identity','bootstrap_command']),search:formSnapshot($('search-form'),SEARCH_FIELDS),local:formSnapshot($('local-provision-form'),['count']),offers:currentOffers,selected:[...selected],sorts:offerSorts}))}catch{}}
|
||||||
|
|
@ -165,13 +194,13 @@ function restorePageState(){try{const saved=JSON.parse(sessionStorage.getItem(PA
|
||||||
function updateSelectionUi(){const count=selected.size;$('selection').textContent=count?`${[...selected].join(', ')} (${count}/${MAX_SELECTED_OFFERS})`:`none (0/${MAX_SELECTED_OFFERS})`;$('provision-button').disabled=count===0;$('provision-button').textContent=count===0?(provisioningMode==='mock'?'Simulate selected offers':'Provision selected offers'):`${provisioningMode==='mock'?'Simulate':'Provision'} ${count} selected offer${count===1?'':'s'}`}
|
function updateSelectionUi(){const count=selected.size;$('selection').textContent=count?`${[...selected].join(', ')} (${count}/${MAX_SELECTED_OFFERS})`:`none (0/${MAX_SELECTED_OFFERS})`;$('provision-button').disabled=count===0;$('provision-button').textContent=count===0?(provisioningMode==='mock'?'Simulate selected offers':'Provision selected offers'):`${provisioningMode==='mock'?'Simulate':'Provision'} ${count} selected offer${count===1?'':'s'}`}
|
||||||
async function refresh(){try{const reply=await jsonFetch('/api/control/status');const model=reply.Status;const provider=model.provider.name;provisioningMode=model.provider.provisioning_mode||'real';const vast=provider==='vastai';const simulated=vast&&provisioningMode==='mock';const local=provider==='process'||provider==='docker';$('provider-config-card').hidden=!vast;$('vast-search-card').hidden=!vast;$('vast-offers-card').hidden=!vast;$('local-provision-card').hidden=!local;$('simulation-banner').hidden=!simulated;document.querySelectorAll('[data-real-vastai]').forEach(element=>element.hidden=simulated);$('local-provider-name').textContent=provider;$('readiness').textContent=`${provider} · ${provisioningMode} · ${model.provider.kind}`;$('readiness').className=model.provider.kind==='ready'?'status ok':'status';$('readiness-error').textContent=model.provider.error||'';$('search-button').disabled=!vast||model.provider.kind!=='ready';$('local-provision-button').disabled=!local||model.provider.kind!=='ready';updateSelectionUi()}catch(error){$('readiness-error').textContent=error.message}}
|
async function refresh(){try{const reply=await jsonFetch('/api/control/status');const model=reply.Status;const provider=model.provider.name;provisioningMode=model.provider.provisioning_mode||'real';const vast=provider==='vastai';const simulated=vast&&provisioningMode==='mock';const local=provider==='process'||provider==='docker';$('provider-config-card').hidden=!vast;$('vast-search-card').hidden=!vast;$('vast-offers-card').hidden=!vast;$('local-provision-card').hidden=!local;$('simulation-banner').hidden=!simulated;document.querySelectorAll('[data-real-vastai]').forEach(element=>element.hidden=simulated);$('local-provider-name').textContent=provider;$('readiness').textContent=`${provider} · ${provisioningMode} · ${model.provider.kind}`;$('readiness').className=model.provider.kind==='ready'?'status ok':'status';$('readiness-error').textContent=model.provider.error||'';$('search-button').disabled=!vast||model.provider.kind!=='ready';$('local-provision-button').disabled=!local||model.provider.kind!=='ready';updateSelectionUi()}catch(error){$('readiness-error').textContent=error.message}}
|
||||||
$('config-form').addEventListener('submit',async event=>{event.preventDefault();const form=event.currentTarget;try{await jsonFetch('/api/control/provider',{method:'POST',body:JSON.stringify({api_key:value(form,'api_key'),ssh_identity:value(form,'ssh_identity'),bootstrap_command:value(form,'bootstrap_command')})});form.elements.api_key.value='';persistPageState();await refresh()}catch(error){$('readiness-error').textContent=error.message}});
|
$('config-form').addEventListener('submit',async event=>{event.preventDefault();const form=event.currentTarget;try{await jsonFetch('/api/control/provider',{method:'POST',body:JSON.stringify({api_key:value(form,'api_key'),ssh_identity:value(form,'ssh_identity'),bootstrap_command:value(form,'bootstrap_command')})});form.elements.api_key.value='';persistPageState();await refresh()}catch(error){$('readiness-error').textContent=error.message}});
|
||||||
$('local-provision-form').addEventListener('submit',async event=>{event.preventDefault();const form=event.currentTarget;const count=number(form,'count')||1;const provider=$('local-provider-name').textContent;if(!confirm(`Provision ${count} ${provider} node${count===1?'':'s'}?`))return;try{await jsonFetch('/api/control/provision',{method:'POST',body:JSON.stringify({command_id:commandId('provision'),count,selected_offer_ids:[],image:$('runtime-image').value.trim()})});$('local-provision-error').textContent='';await refresh()}catch(error){$('local-provision-error').textContent=error.message}});
|
$('local-provision-form').addEventListener('submit',async event=>{event.preventDefault();const form=event.currentTarget;const count=number(form,'count')||1;const provider=$('local-provider-name').textContent;if(!await confirmAction('Provision nodes?',`Provision ${count} ${provider} node${count===1?'':'s'}?`,'Provision'))return;try{await jsonFetch('/api/control/provision',{method:'POST',body:JSON.stringify({command_id:commandId('provision'),count,selected_offer_ids:[],image:$('runtime-image').value.trim()})});$('local-provision-error').textContent='';await refresh()}catch(error){$('local-provision-error').textContent=error.message}});
|
||||||
$('search-form').addEventListener('submit',async event=>{event.preventDefault();const form=event.currentTarget;const blacklist=(value(form,'blacklist_hosts')||'').split(',').map(v=>Number(v.trim())).filter(Number.isFinite);$('offer-status').textContent='Searching…';$('offer-error').textContent='';try{const reply=await jsonFetch('/api/control/offers',{method:'POST',body:JSON.stringify({gpu_model:value(form,'gpu_model'),min_gpu_ram_mb:number(form,'min_gpu_ram_mb'),min_compute_cap:number(form,'min_compute_cap'),min_reliability:number(form,'min_reliability'),require_verified:form.elements.require_verified.checked,min_download_mbps:number(form,'min_download_mbps'),min_upload_mbps:number(form,'min_upload_mbps'),max_hourly_price:number(form,'max_hourly_price'),blacklist_hosts:blacklist,count:null})});currentOffers=reply.Offers;selected.clear();renderOffers();updateSelectionUi();$('offer-status').textContent=currentOffers.length===0?'No offers match this search.':`${currentOffers.length} matching offer${currentOffers.length===1?'':'s'}`;persistPageState()}catch(error){$('offer-status').textContent='';$('offer-error').textContent=error.message}});
|
$('search-form').addEventListener('submit',async event=>{event.preventDefault();const form=event.currentTarget;const blacklist=(value(form,'blacklist_hosts')||'').split(',').map(v=>Number(v.trim())).filter(Number.isFinite);$('offer-status').textContent='Searching…';$('offer-error').textContent='';try{const reply=await jsonFetch('/api/control/offers',{method:'POST',body:JSON.stringify({gpu_model:value(form,'gpu_model'),min_gpu_ram_mb:number(form,'min_gpu_ram_mb'),min_compute_cap:number(form,'min_compute_cap'),min_reliability:number(form,'min_reliability'),require_verified:form.elements.require_verified.checked,min_download_mbps:number(form,'min_download_mbps'),min_upload_mbps:number(form,'min_upload_mbps'),max_hourly_price:number(form,'max_hourly_price'),blacklist_hosts:blacklist,count:null})});currentOffers=reply.Offers;selected.clear();renderOffers();updateSelectionUi();$('offer-status').textContent=currentOffers.length===0?'No offers match this search.':`${currentOffers.length} matching offer${currentOffers.length===1?'':'s'}`;persistPageState()}catch(error){$('offer-status').textContent='';$('offer-error').textContent=error.message}});
|
||||||
function compareOfferValues(left,right){const leftMissing=left===null||left===undefined||left==='';const rightMissing=right===null||right===undefined||right==='';if(leftMissing||rightMissing)return leftMissing===rightMissing?0:leftMissing?1:-1;if(typeof left==='number'&&typeof right==='number')return left-right;return String(left).localeCompare(String(right),undefined,{numeric:true,sensitivity:'base'})}
|
function compareOfferValues(left,right){const leftMissing=left===null||left===undefined||left==='';const rightMissing=right===null||right===undefined||right==='';if(leftMissing||rightMissing)return leftMissing===rightMissing?0:leftMissing?1:-1;if(typeof left==='number'&&typeof right==='number')return left-right;return String(left).localeCompare(String(right),undefined,{numeric:true,sensitivity:'base'})}
|
||||||
function renderOffers(){const offers=[...currentOffers];if(offerSorts.length)offers.sort((left,right)=>{for(const sort of offerSorts){const compared=compareOfferValues(left[sort.key],right[sort.key]);if(compared!==0)return compared*(sort.direction==='asc'?1:-1)}return Number(left.offer_id)-Number(right.offer_id)});$('offers').replaceChildren(...offers.map(offer=>{const tr=document.createElement('tr');tr.innerHTML=`<td><input type="checkbox" data-offer="${Number(offer.offer_id)}" style="width:auto"></td><td>${escapeHtml(offer.offer_id)}</td><td>${escapeHtml(offer.host_id)}</td><td>${escapeHtml(offer.gpu_model)}</td><td>${escapeHtml(offer.gpu_ram_mb)}</td><td>${escapeHtml(offer.compute_cap)}</td><td>${escapeHtml(offer.verification)}</td><td>${escapeHtml(offer.reliability)}</td><td>${escapeHtml(offer.download_mbps)}</td><td>${escapeHtml(offer.upload_mbps)}</td><td>${escapeHtml(offer.location)}</td><td>${escapeHtml(Number(offer.hourly_price).toFixed(3))}</td>`;tr.querySelector('[data-offer]').checked=selected.has(Number(offer.offer_id));return tr}));document.querySelectorAll('.sort-button').forEach(button=>button.setAttribute('aria-pressed',String(offerSorts.some(sort=>sort.key===button.dataset.sort&&sort.direction===button.dataset.direction))))}
|
function renderOffers(){const offers=[...currentOffers];if(offerSorts.length)offers.sort((left,right)=>{for(const sort of offerSorts){const compared=compareOfferValues(left[sort.key],right[sort.key]);if(compared!==0)return compared*(sort.direction==='asc'?1:-1)}return Number(left.offer_id)-Number(right.offer_id)});$('offers').replaceChildren(...offers.map(offer=>{const tr=document.createElement('tr');tr.innerHTML=`<td><input type="checkbox" data-offer="${Number(offer.offer_id)}" style="width:auto"></td><td>${escapeHtml(offer.offer_id)}</td><td>${escapeHtml(offer.host_id)}</td><td>${escapeHtml(offer.gpu_model)}</td><td>${escapeHtml(offer.gpu_ram_mb)}</td><td>${escapeHtml(offer.compute_cap)}</td><td>${escapeHtml(offer.verification)}</td><td>${escapeHtml(offer.reliability)}</td><td>${escapeHtml(offer.download_mbps)}</td><td>${escapeHtml(offer.upload_mbps)}</td><td>${escapeHtml(offer.location)}</td><td>${escapeHtml(Number(offer.hourly_price).toFixed(3))}</td>`;tr.querySelector('[data-offer]').checked=selected.has(Number(offer.offer_id));return tr}));document.querySelectorAll('.sort-button').forEach(button=>button.setAttribute('aria-pressed',String(offerSorts.some(sort=>sort.key===button.dataset.sort&&sort.direction===button.dataset.direction))))}
|
||||||
$('offers-table').addEventListener('click',event=>{const button=event.target.closest('.sort-button');if(!button)return;const id=`${button.dataset.sort}:${button.dataset.direction}`;const activeIndex=offerSorts.findIndex(sort=>sort.key===button.dataset.sort&&sort.direction===button.dataset.direction);if(event.detail===1)sortDoubleClickStartedActive=activeIndex>=0?id:null;if(event.detail===2&&sortDoubleClickStartedActive===id){offerSorts.splice(activeIndex,1);sortDoubleClickStartedActive=null}else if(activeIndex<0){const columnIndex=offerSorts.findIndex(sort=>sort.key===button.dataset.sort);const next={key:button.dataset.sort,direction:button.dataset.direction};if(columnIndex>=0)offerSorts.splice(columnIndex,1,next);else offerSorts.push(next)}renderOffers();persistPageState()});
|
$('offers-table').addEventListener('click',event=>{const button=event.target.closest('.sort-button');if(!button)return;const id=`${button.dataset.sort}:${button.dataset.direction}`;const activeIndex=offerSorts.findIndex(sort=>sort.key===button.dataset.sort&&sort.direction===button.dataset.direction);if(event.detail===1)sortDoubleClickStartedActive=activeIndex>=0?id:null;if(event.detail===2&&sortDoubleClickStartedActive===id){offerSorts.splice(activeIndex,1);sortDoubleClickStartedActive=null}else if(activeIndex<0){const columnIndex=offerSorts.findIndex(sort=>sort.key===button.dataset.sort);const next={key:button.dataset.sort,direction:button.dataset.direction};if(columnIndex>=0)offerSorts.splice(columnIndex,1,next);else offerSorts.push(next)}renderOffers();persistPageState()});
|
||||||
$('offers').addEventListener('change',event=>{const id=Number(event.target.dataset.offer);if(!Number.isFinite(id))return;if(event.target.checked){if(selected.size>=MAX_SELECTED_OFFERS){event.target.checked=false;$('offer-error').textContent=`Select at most ${MAX_SELECTED_OFFERS} offers.`;return}selected.add(id)}else selected.delete(id);$('offer-error').textContent='';updateSelectionUi();persistPageState()});
|
$('offers').addEventListener('change',event=>{const id=Number(event.target.dataset.offer);if(!Number.isFinite(id))return;if(event.target.checked){if(selected.size>=MAX_SELECTED_OFFERS){event.target.checked=false;$('offer-error').textContent=`Select at most ${MAX_SELECTED_OFFERS} offers.`;return}selected.add(id)}else selected.delete(id);$('offer-error').textContent='';updateSelectionUi();persistPageState()});
|
||||||
$('provision-button').addEventListener('click',async()=>{try{const ids=[...selected];if(ids.length===0)return;const question=provisioningMode==='mock'?`Simulate provisioning exact Vast.ai offer IDs ${ids.join(', ')}? No rental will be created.`:`Provision exact Vast.ai offer IDs ${ids.join(', ')}? This can incur charges.`;if(!confirm(question))return;await jsonFetch('/api/control/provision',{method:'POST',body:JSON.stringify({command_id:commandId('provision'),count:ids.length,selected_offer_ids:ids,image:$('runtime-image').value.trim()})});const provisioned=new Set(ids);currentOffers=currentOffers.filter(offer=>!provisioned.has(Number(offer.offer_id)));selected.clear();renderOffers();updateSelectionUi();persistPageState();await refresh()}catch(error){$('offer-error').textContent=error.message}});
|
$('provision-button').addEventListener('click',async()=>{try{const ids=[...selected];if(ids.length===0)return;const simulated=provisioningMode==='mock';const question=simulated?`Simulate provisioning exact Vast.ai offer IDs ${ids.join(', ')}? No rental will be created.`:`Provision exact Vast.ai offer IDs ${ids.join(', ')}? This can incur charges.`;if(!await confirmAction(simulated?'Simulate provisioning?':'Provision selected offers?',question,simulated?'Simulate':'Provision'))return;await jsonFetch('/api/control/provision',{method:'POST',body:JSON.stringify({command_id:commandId('provision'),count:ids.length,selected_offer_ids:ids,image:$('runtime-image').value.trim()})});const provisioned=new Set(ids);currentOffers=currentOffers.filter(offer=>!provisioned.has(Number(offer.offer_id)));selected.clear();renderOffers();updateSelectionUi();persistPageState();await refresh()}catch(error){$('offer-error').textContent=error.message}});
|
||||||
$('config-form').addEventListener('input',persistPageState);$('search-form').addEventListener('input',persistPageState);$('local-provision-form').addEventListener('input',persistPageState);restorePageState();refresh();setInterval(refresh,1000);
|
$('config-form').addEventListener('input',persistPageState);$('search-form').addEventListener('input',persistPageState);$('local-provision-form').addEventListener('input',persistPageState);restorePageState();refresh();setInterval(refresh,1000);
|
||||||
</script>
|
</script>
|
||||||
</body></html>
|
</body></html>
|
||||||
|
|
|
||||||
|
|
@ -99,6 +99,7 @@ impl DataNode {
|
||||||
source_sender: Some(Arc::clone(&self.sender)),
|
source_sender: Some(Arc::clone(&self.sender)),
|
||||||
source_publisher: Some(Arc::clone(&self.namespace.source_publisher)),
|
source_publisher: Some(Arc::clone(&self.namespace.source_publisher)),
|
||||||
route_registrar: None,
|
route_registrar: None,
|
||||||
|
stream_transport: None,
|
||||||
})
|
})
|
||||||
.expect("host session"),
|
.expect("host session"),
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -1,14 +1,26 @@
|
||||||
#![cfg(target_os = "linux")]
|
#![cfg(target_os = "linux")]
|
||||||
|
|
||||||
use std::collections::BTreeSet;
|
use std::collections::BTreeSet;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
use data_plane::bootstrap::{
|
use data_plane::bootstrap::{
|
||||||
ENV_ARENA_FD, ENV_DATA_PLANE_ACTOR, ENV_DATA_PLANE_ENDPOINT, ENV_JOB_CAPABILITY,
|
ENV_ARENA_FD, ENV_DATA_PLANE_ACTOR, ENV_DATA_PLANE_ENDPOINT, ENV_JOB_CAPABILITY,
|
||||||
};
|
};
|
||||||
|
use data_plane::data_plane::StreamConsumer;
|
||||||
|
use data_plane::namespace::{
|
||||||
|
DataDirectoryActor, NamespaceClient, NamespaceClientActor, NamespaceDiscovery,
|
||||||
|
};
|
||||||
use data_plane::path::{DataPath, JobContext};
|
use data_plane::path::{DataPath, JobContext};
|
||||||
use data_plane::protocol::JobCapability;
|
use data_plane::protocol::JobCapability;
|
||||||
|
use data_plane::source::BlobSourcePublisher;
|
||||||
|
use data_plane::stream_transport::{LocalStreamTransport, StreamTransport};
|
||||||
|
use futures_lite::future;
|
||||||
|
use parking_lot::Mutex;
|
||||||
|
use swactor::actor::ActorAddress;
|
||||||
use swactor::config::RuntimeConfig;
|
use swactor::config::RuntimeConfig;
|
||||||
use swactor::runtime::RuntimeParts;
|
use swactor::runtime::RuntimeParts;
|
||||||
|
use swactor_engine::{Engine, TokioBackend, TokioConfig};
|
||||||
|
|
||||||
use crate::job_data_plane::ActorJobDataPlane;
|
use crate::job_data_plane::ActorJobDataPlane;
|
||||||
|
|
||||||
|
|
@ -39,6 +51,7 @@ fn plane() -> ActorJobDataPlane {
|
||||||
source_sender: None,
|
source_sender: None,
|
||||||
source_publisher: None,
|
source_publisher: None,
|
||||||
route_registrar: None,
|
route_registrar: None,
|
||||||
|
stream_transport: None,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.expect("actor data-plane")
|
.expect("actor data-plane")
|
||||||
|
|
@ -69,3 +82,121 @@ fn handoff_contains_one_descriptor_and_private_actor_metadata() {
|
||||||
assert!(flags >= 0);
|
assert!(flags >= 0);
|
||||||
assert_eq!(flags & libc::FD_CLOEXEC, 0);
|
assert_eq!(flags & libc::FD_CLOEXEC, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct StaticDiscovery(ActorAddress);
|
||||||
|
|
||||||
|
impl NamespaceDiscovery for StaticDiscovery {
|
||||||
|
fn current_directory(&self) -> Option<ActorAddress> {
|
||||||
|
Some(self.0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct LocalPublisher;
|
||||||
|
|
||||||
|
impl BlobSourcePublisher for LocalPublisher {
|
||||||
|
fn publish_source(&self, _source: ActorAddress) -> Result<(), String> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct BytesConsumer(Arc<Mutex<Vec<u8>>>);
|
||||||
|
|
||||||
|
impl StreamConsumer for BytesConsumer {
|
||||||
|
fn consume(&self, bytes: &[u8]) -> Result<(), String> {
|
||||||
|
self.0.lock().extend_from_slice(bytes);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn canonical_inference_result_uses_native_stream_end_to_end() {
|
||||||
|
let state_root = std::env::temp_dir().join(format!(
|
||||||
|
"myelin-stream-test-{}",
|
||||||
|
ActorAddress::new_random().to_full_hex()
|
||||||
|
));
|
||||||
|
let parts = RuntimeParts::new(RuntimeConfig::default());
|
||||||
|
let runtime = parts.runtime().clone();
|
||||||
|
let engine = Engine::new(
|
||||||
|
parts,
|
||||||
|
TokioBackend::new(TokioConfig::default()).expect("tokio backend"),
|
||||||
|
)
|
||||||
|
.expect("engine");
|
||||||
|
let directory = runtime
|
||||||
|
.spawn(
|
||||||
|
DataDirectoryActor::recover(state_root.join("namespace.json"), |_recovery, _length| {
|
||||||
|
Err(data_plane::namespace::NamespaceError::SourceRecovery(
|
||||||
|
"no recovered blob sources".to_owned(),
|
||||||
|
))
|
||||||
|
})
|
||||||
|
.expect("directory"),
|
||||||
|
)
|
||||||
|
.expect("spawn directory");
|
||||||
|
let proxy = runtime
|
||||||
|
.spawn(NamespaceClientActor::new(
|
||||||
|
engine.handle(),
|
||||||
|
runtime.create_sender(),
|
||||||
|
Arc::new(StaticDiscovery(directory)),
|
||||||
|
Duration::from_millis(5),
|
||||||
|
))
|
||||||
|
.expect("namespace proxy");
|
||||||
|
let namespace = NamespaceClient::new(runtime.clone(), proxy);
|
||||||
|
let transport: Arc<dyn StreamTransport> = Arc::new(LocalStreamTransport::new());
|
||||||
|
let publisher: Arc<dyn BlobSourcePublisher> = Arc::new(LocalPublisher);
|
||||||
|
|
||||||
|
let make_plane = |read: bool| {
|
||||||
|
ActorJobDataPlane::new(
|
||||||
|
&runtime,
|
||||||
|
crate::job_data_plane::ActorJobDataPlaneConfig {
|
||||||
|
arena_bytes: 1 << 20,
|
||||||
|
arena_generation: if read { 21 } else { 22 },
|
||||||
|
session_generation: if read { 31 } else { 32 },
|
||||||
|
capability: CAPABILITY,
|
||||||
|
job_context: JobContext {
|
||||||
|
run_id: "0".to_owned(),
|
||||||
|
read_prefixes: if read {
|
||||||
|
vec![path("/runs/0/results")]
|
||||||
|
} else {
|
||||||
|
Vec::new()
|
||||||
|
},
|
||||||
|
write_prefixes: if read {
|
||||||
|
Vec::new()
|
||||||
|
} else {
|
||||||
|
vec![path("/runs/0/results")]
|
||||||
|
},
|
||||||
|
},
|
||||||
|
namespace: Some(namespace.clone()),
|
||||||
|
transfer_receiver: None,
|
||||||
|
source_sender: None,
|
||||||
|
source_publisher: Some(Arc::clone(&publisher)),
|
||||||
|
route_registrar: None,
|
||||||
|
stream_transport: Some(Arc::clone(&transport)),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("actor plane")
|
||||||
|
};
|
||||||
|
let reader_plane = make_plane(true);
|
||||||
|
let writer_plane = make_plane(false);
|
||||||
|
let reader = reader_plane.attach_local().expect("reader attachment");
|
||||||
|
let writer = writer_plane.attach_local().expect("writer attachment");
|
||||||
|
let logical = path("/runs/0/results/inference");
|
||||||
|
let observed = Arc::new(Mutex::new(Vec::new()));
|
||||||
|
let consumer: Arc<dyn StreamConsumer> = Arc::new(BytesConsumer(Arc::clone(&observed)));
|
||||||
|
let completed = reader
|
||||||
|
.collect_stream(logical.clone(), consumer)
|
||||||
|
.expect("register result sink");
|
||||||
|
|
||||||
|
future::block_on(async {
|
||||||
|
let mut writer = writer.write_stream(&logical).await.expect("open writer");
|
||||||
|
writer
|
||||||
|
.write(br#"{"device":"CUDA:0","output":[2.75,-8.75]}"#)
|
||||||
|
.await
|
||||||
|
.expect("write result");
|
||||||
|
writer.close().await.expect("close writer");
|
||||||
|
});
|
||||||
|
completed.wait().expect("result sink completed");
|
||||||
|
assert_eq!(
|
||||||
|
&*observed.lock(),
|
||||||
|
br#"{"device":"CUDA:0","output":[2.75,-8.75]}"#
|
||||||
|
);
|
||||||
|
let _ = std::fs::remove_dir_all(state_root);
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,305 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
"""Tinygrad-backed device bridge probe for Rust Myelin bridge tests.
|
|
||||||
|
|
||||||
Line-delimited JSON control only. Payload bytes live in the arena file whose
|
|
||||||
path Rust passes during initialize.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import struct
|
|
||||||
import sys
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
try:
|
|
||||||
from tinygrad import Tensor # type: ignore
|
|
||||||
except Exception as exc: # pragma: no cover - exercised from Rust process tests
|
|
||||||
print(
|
|
||||||
json.dumps(
|
|
||||||
{
|
|
||||||
"type": "worker_fatal",
|
|
||||||
"reason": "tinygrad_unavailable",
|
|
||||||
"message": str(exc),
|
|
||||||
}
|
|
||||||
),
|
|
||||||
flush=True,
|
|
||||||
)
|
|
||||||
raise SystemExit(2)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class DeviceObject:
|
|
||||||
dtype: str
|
|
||||||
shape: str
|
|
||||||
extent: int
|
|
||||||
values: list[int]
|
|
||||||
tensor: Any
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class PendingCopy:
|
|
||||||
kind: str
|
|
||||||
copy_id: int
|
|
||||||
handle_id: int
|
|
||||||
host_offset: int
|
|
||||||
device_offset: int
|
|
||||||
length: int
|
|
||||||
|
|
||||||
|
|
||||||
arena_path: str | None = None
|
|
||||||
arena_bytes = 0
|
|
||||||
generation = 0
|
|
||||||
objects: dict[int, DeviceObject] = {}
|
|
||||||
pending_copies: dict[int, PendingCopy] = {}
|
|
||||||
fail_next: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
def emit(obj: dict[str, Any]) -> None:
|
|
||||||
print(json.dumps(obj, separators=(",", ":")), flush=True)
|
|
||||||
|
|
||||||
|
|
||||||
def backend_error(reason: str) -> None:
|
|
||||||
emit({"type": "backend_error", "reason": reason})
|
|
||||||
|
|
||||||
|
|
||||||
def fatal(reason: str, message: str) -> None:
|
|
||||||
emit({"type": "worker_fatal", "reason": reason, "message": message})
|
|
||||||
|
|
||||||
|
|
||||||
def require_arena() -> str:
|
|
||||||
if arena_path is None:
|
|
||||||
raise RuntimeError("arena is not initialized")
|
|
||||||
return arena_path
|
|
||||||
|
|
||||||
|
|
||||||
def consume_failure(reason: str) -> bool:
|
|
||||||
global fail_next
|
|
||||||
if fail_next == reason:
|
|
||||||
fail_next = None
|
|
||||||
backend_error(reason)
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def check_u32_range(offset: int, length: int) -> None:
|
|
||||||
if offset < 0 or length < 0 or offset % 4 != 0 or length % 4 != 0:
|
|
||||||
raise ValueError("u32 ranges must be non-negative and 4-byte aligned")
|
|
||||||
|
|
||||||
|
|
||||||
def tensor_from_values(values: list[int]) -> Any:
|
|
||||||
try:
|
|
||||||
return Tensor(values, dtype="uint32").realize()
|
|
||||||
except Exception:
|
|
||||||
return Tensor(values, dtype="int32").realize()
|
|
||||||
|
|
||||||
|
|
||||||
def allocate_tensor(dtype: str, extent: int) -> tuple[list[int], Any]:
|
|
||||||
if dtype == "u32":
|
|
||||||
if extent % 4 != 0:
|
|
||||||
raise ValueError("u32 extent must be 4-byte aligned")
|
|
||||||
values = [0] * (extent // 4)
|
|
||||||
return values, tensor_from_values(values)
|
|
||||||
if dtype == "f16":
|
|
||||||
values = [0] * (extent // 2)
|
|
||||||
return values, Tensor(values, dtype="float16").realize()
|
|
||||||
raise ValueError(f"unsupported dtype {dtype!r}")
|
|
||||||
|
|
||||||
|
|
||||||
def update_tensor(obj: DeviceObject) -> None:
|
|
||||||
if obj.dtype == "u32":
|
|
||||||
obj.tensor = tensor_from_values(obj.values)
|
|
||||||
elif obj.dtype == "f16":
|
|
||||||
obj.tensor = Tensor(obj.values, dtype="float16").realize()
|
|
||||||
else:
|
|
||||||
raise ValueError(f"unsupported dtype {obj.dtype!r}")
|
|
||||||
|
|
||||||
|
|
||||||
def materialized_values(obj: DeviceObject) -> list[int]:
|
|
||||||
return [int(v) for v in obj.tensor.tolist()]
|
|
||||||
|
|
||||||
|
|
||||||
def read_arena(offset: int, length: int) -> bytes:
|
|
||||||
path = require_arena()
|
|
||||||
with open(path, "rb", buffering=0) as f:
|
|
||||||
f.seek(offset)
|
|
||||||
data = f.read(length)
|
|
||||||
if len(data) != length:
|
|
||||||
raise EOFError("short arena read")
|
|
||||||
return data
|
|
||||||
|
|
||||||
|
|
||||||
def write_arena(offset: int, data: bytes) -> None:
|
|
||||||
path = require_arena()
|
|
||||||
with open(path, "r+b", buffering=0) as f:
|
|
||||||
f.seek(offset)
|
|
||||||
f.write(data)
|
|
||||||
f.flush()
|
|
||||||
|
|
||||||
|
|
||||||
def perform_host_to_device(copy: PendingCopy) -> None:
|
|
||||||
obj = objects[copy.handle_id]
|
|
||||||
if obj.dtype != "u32":
|
|
||||||
raise ValueError("payload copy is implemented for u32 test objects only")
|
|
||||||
check_u32_range(copy.device_offset, copy.length)
|
|
||||||
payload = read_arena(copy.host_offset, copy.length)
|
|
||||||
words = list(struct.unpack("<" + "I" * (copy.length // 4), payload))
|
|
||||||
start = copy.device_offset // 4
|
|
||||||
end = start + len(words)
|
|
||||||
if end > len(obj.values):
|
|
||||||
raise ValueError("device range out of bounds")
|
|
||||||
obj.values[start:end] = words
|
|
||||||
update_tensor(obj)
|
|
||||||
|
|
||||||
|
|
||||||
def perform_device_to_host(copy: PendingCopy) -> None:
|
|
||||||
obj = objects[copy.handle_id]
|
|
||||||
if obj.dtype != "u32":
|
|
||||||
raise ValueError("payload copy is implemented for u32 test objects only")
|
|
||||||
check_u32_range(copy.device_offset, copy.length)
|
|
||||||
values = materialized_values(obj)
|
|
||||||
start = copy.device_offset // 4
|
|
||||||
end = start + (copy.length // 4)
|
|
||||||
if end > len(values):
|
|
||||||
raise ValueError("device range out of bounds")
|
|
||||||
payload = struct.pack("<" + "I" * (end - start), *values[start:end])
|
|
||||||
write_arena(copy.host_offset, payload)
|
|
||||||
|
|
||||||
|
|
||||||
def perform_copy(copy: PendingCopy) -> None:
|
|
||||||
if copy.kind == "host_to_device":
|
|
||||||
perform_host_to_device(copy)
|
|
||||||
elif copy.kind == "device_to_host":
|
|
||||||
perform_device_to_host(copy)
|
|
||||||
else:
|
|
||||||
raise ValueError(f"unknown copy kind {copy.kind!r}")
|
|
||||||
|
|
||||||
|
|
||||||
def handle(req: dict[str, Any]) -> bool:
|
|
||||||
global arena_path, arena_bytes, generation, fail_next
|
|
||||||
|
|
||||||
typ = req.get("type")
|
|
||||||
if typ == "initialize":
|
|
||||||
arena_path = str(req["arena_path"])
|
|
||||||
arena_bytes = int(req["arena_bytes"])
|
|
||||||
generation = int(req["generation"])
|
|
||||||
with open(arena_path, "r+b", buffering=0) as f:
|
|
||||||
f.truncate(arena_bytes)
|
|
||||||
emit({"type": "worker_ready", "generation": generation})
|
|
||||||
return True
|
|
||||||
|
|
||||||
if typ == "alloc":
|
|
||||||
if consume_failure("allocation_failed"):
|
|
||||||
return True
|
|
||||||
handle_id = int(req["handle_id"])
|
|
||||||
dtype = str(req["dtype"])
|
|
||||||
shape = str(req["shape"])
|
|
||||||
extent = int(req["extent"])
|
|
||||||
values, tensor = allocate_tensor(dtype, extent)
|
|
||||||
objects[handle_id] = DeviceObject(dtype=dtype, shape=shape, extent=extent, values=values, tensor=tensor)
|
|
||||||
emit({"type": "allocated", "handle_id": handle_id})
|
|
||||||
return True
|
|
||||||
|
|
||||||
if typ in ("host_to_device", "device_to_host"):
|
|
||||||
if consume_failure("copy_failed"):
|
|
||||||
return True
|
|
||||||
copy = PendingCopy(
|
|
||||||
kind=typ,
|
|
||||||
copy_id=int(req["copy_id"]),
|
|
||||||
handle_id=int(req["handle_id"]),
|
|
||||||
host_offset=int(req["host_offset"]),
|
|
||||||
device_offset=int(req["device_offset"]),
|
|
||||||
length=int(req["len"]),
|
|
||||||
)
|
|
||||||
if copy.handle_id not in objects:
|
|
||||||
backend_error("copy_failed")
|
|
||||||
return True
|
|
||||||
if bool(req.get("defer", False)):
|
|
||||||
pending_copies[copy.copy_id] = copy
|
|
||||||
emit({"type": "copy_started", "copy_id": copy.copy_id})
|
|
||||||
else:
|
|
||||||
perform_copy(copy)
|
|
||||||
emit({"type": "copy_completed", "copy_id": copy.copy_id})
|
|
||||||
return True
|
|
||||||
|
|
||||||
if typ == "complete_copy":
|
|
||||||
copy_id = int(req["copy_id"])
|
|
||||||
copy = pending_copies.pop(copy_id, None)
|
|
||||||
if copy is None:
|
|
||||||
backend_error("copy_failed")
|
|
||||||
return True
|
|
||||||
perform_copy(copy)
|
|
||||||
emit({"type": "copy_completed", "copy_id": copy_id})
|
|
||||||
return True
|
|
||||||
|
|
||||||
if typ == "wrap_for_tinygrad":
|
|
||||||
if consume_failure("invalid_view"):
|
|
||||||
return True
|
|
||||||
handle_id = int(req["handle_id"])
|
|
||||||
obj = objects.get(handle_id)
|
|
||||||
if obj is None:
|
|
||||||
backend_error("invalid_view")
|
|
||||||
return True
|
|
||||||
dtype = str(req["dtype"])
|
|
||||||
shape = str(req["shape"])
|
|
||||||
if obj.dtype != dtype or obj.shape != shape:
|
|
||||||
backend_error("invalid_view")
|
|
||||||
return True
|
|
||||||
# Force materialization at view time so success depends on live tensor state.
|
|
||||||
_ = obj.tensor.tolist()
|
|
||||||
emit({"type": "view", "handle_id": handle_id, "dtype": dtype, "shape": shape})
|
|
||||||
return True
|
|
||||||
|
|
||||||
if typ == "free":
|
|
||||||
handle_id = int(req["handle_id"])
|
|
||||||
if handle_id not in objects:
|
|
||||||
backend_error("invalid_view")
|
|
||||||
return True
|
|
||||||
del objects[handle_id]
|
|
||||||
emit({"type": "freed", "handle_id": handle_id})
|
|
||||||
return True
|
|
||||||
|
|
||||||
if typ == "restart":
|
|
||||||
generation = int(req["generation"])
|
|
||||||
objects.clear()
|
|
||||||
pending_copies.clear()
|
|
||||||
fail_next = None
|
|
||||||
emit({"type": "worker_ready", "generation": generation})
|
|
||||||
return True
|
|
||||||
|
|
||||||
if typ == "fail_next":
|
|
||||||
fail_next = str(req["failure"])
|
|
||||||
emit({"type": "ok"})
|
|
||||||
return True
|
|
||||||
|
|
||||||
if typ == "shutdown":
|
|
||||||
emit({"type": "worker_stopped"})
|
|
||||||
return False
|
|
||||||
|
|
||||||
fatal("protocol_error", f"unknown command type {typ!r}")
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
for line in sys.stdin:
|
|
||||||
line = line.strip()
|
|
||||||
if not line:
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
req = json.loads(line)
|
|
||||||
if not isinstance(req, dict):
|
|
||||||
raise ValueError("request must be an object")
|
|
||||||
if not handle(req):
|
|
||||||
return 0
|
|
||||||
except SystemExit:
|
|
||||||
raise
|
|
||||||
except Exception as exc:
|
|
||||||
fatal("backend_exception", str(exc))
|
|
||||||
return 2
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -8,7 +8,7 @@ version = "0.1.0"
|
||||||
requires-python = ">=3.9"
|
requires-python = ">=3.9"
|
||||||
|
|
||||||
[dependency-groups]
|
[dependency-groups]
|
||||||
dev = ["jupyter", "ipykernel", "pytest"]
|
dev = ["jupyter", "ipykernel", "maturin>=1.7,<2", "pytest", "pytest-timeout"]
|
||||||
|
|
||||||
[tool.uv]
|
[tool.uv]
|
||||||
cache-keys = [
|
cache-keys = [
|
||||||
|
|
|
||||||
|
|
@ -2,12 +2,15 @@ use std::collections::HashMap;
|
||||||
use std::ffi::{CString, c_int, c_void};
|
use std::ffi::{CString, c_int, c_void};
|
||||||
use std::os::fd::{FromRawFd, OwnedFd, RawFd};
|
use std::os::fd::{FromRawFd, OwnedFd, RawFd};
|
||||||
use std::ptr;
|
use std::ptr;
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
use std::sync::{Arc, Mutex, RwLock};
|
use std::sync::{Arc, Mutex, RwLock};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use data_plane::blob::{Blob, BlobView, ContentDigest, WritableArenaView};
|
use data_plane::blob::{Blob, BlobView, ContentDigest, WritableArenaView};
|
||||||
use data_plane::bootstrap as dp_bootstrap;
|
use data_plane::bootstrap as dp_bootstrap;
|
||||||
use data_plane::data_plane::{BlobWriter, DataPlane, DataPlaneBootstrap, parse_actor_address};
|
use data_plane::data_plane::{
|
||||||
|
BlobWriter, DataPlane, DataPlaneBootstrap, StreamReader, StreamWriter, parse_actor_address,
|
||||||
|
};
|
||||||
use data_plane::path::DataPath;
|
use data_plane::path::DataPath;
|
||||||
use data_plane::protocol::{
|
use data_plane::protocol::{
|
||||||
BlobFailure, DataPlaneError, JobCapability, register_data_plane_codecs,
|
BlobFailure, DataPlaneError, JobCapability, register_data_plane_codecs,
|
||||||
|
|
@ -23,24 +26,22 @@ use parking_lot::Mutex as ParkingMutex;
|
||||||
use pyo3::exceptions::{PyBufferError, PyPermissionError, PyRuntimeError};
|
use pyo3::exceptions::{PyBufferError, PyPermissionError, PyRuntimeError};
|
||||||
use pyo3::ffi;
|
use pyo3::ffi;
|
||||||
use pyo3::prelude::*;
|
use pyo3::prelude::*;
|
||||||
use pyo3::types::{PyAny, PyModule};
|
use pyo3::types::{PyAny, PyBytes, PyModule};
|
||||||
use swactor::actor::{ActorAddress, ActorInterface, Ctx};
|
use swactor::actor::{ActorAddress, ActorInterface, Ctx};
|
||||||
use swactor::config::RuntimeConfig;
|
use swactor::config::RuntimeConfig;
|
||||||
use swactor::runtime::{Runtime, RuntimeParts};
|
use swactor::runtime::{Runtime, RuntimeParts};
|
||||||
use swactor_engine::{ActorCompletion, Engine, EngineHandle, TokioBackend, TokioConfig};
|
use swactor_engine::{ActorCompletion, Engine, EngineHandle, TokioBackend, TokioConfig};
|
||||||
use swactor_transport::{CodecRegistry, CodecRemoteSink, TransportRouter};
|
use swactor_transport::{CodecRegistry, CodecRemoteSink, TransportRouter};
|
||||||
use tokio::io::{AsyncWriteExt, BufWriter};
|
|
||||||
use tokio::net::UnixStream;
|
|
||||||
|
|
||||||
const ROUTE_POLL: Duration = Duration::from_millis(5);
|
const ROUTE_POLL: Duration = Duration::from_millis(5);
|
||||||
const ROUTE_DEADLINE: Duration = Duration::from_secs(5);
|
const ROUTE_DEADLINE: Duration = Duration::from_secs(5);
|
||||||
const LEGACY_OUTPUT_ENV: &str = "SWACTOR_DATA_PLANE_OUTPUT";
|
|
||||||
|
|
||||||
pyo3::create_exception!(swactor, SwactorError, pyo3::exceptions::PyException);
|
pyo3::create_exception!(swactor, SwactorError, pyo3::exceptions::PyException);
|
||||||
pyo3::create_exception!(swactor, BootstrapError, SwactorError);
|
pyo3::create_exception!(swactor, BootstrapError, SwactorError);
|
||||||
pyo3::create_exception!(swactor, DataPathError, SwactorError);
|
pyo3::create_exception!(swactor, DataPathError, SwactorError);
|
||||||
pyo3::create_exception!(swactor, BlobError, SwactorError);
|
pyo3::create_exception!(swactor, BlobError, SwactorError);
|
||||||
pyo3::create_exception!(swactor, SessionError, SwactorError);
|
pyo3::create_exception!(swactor, SessionError, SwactorError);
|
||||||
|
pyo3::create_exception!(swactor, StreamError, SwactorError);
|
||||||
|
|
||||||
fn bootstrap_error(message: impl Into<String>) -> PyErr {
|
fn bootstrap_error(message: impl Into<String>) -> PyErr {
|
||||||
PyErr::new::<BootstrapError, _>(message.into())
|
PyErr::new::<BootstrapError, _>(message.into())
|
||||||
|
|
@ -56,6 +57,11 @@ fn data_plane_error(error: DataPlaneError) -> PyErr {
|
||||||
PyErr::new::<DataPathError, _>(format!("data path not found: {path}"))
|
PyErr::new::<DataPathError, _>(format!("data path not found: {path}"))
|
||||||
}
|
}
|
||||||
DataPlaneError::Blob(reason) => PyErr::new::<BlobError, _>(format!("{reason:?}")),
|
DataPlaneError::Blob(reason) => PyErr::new::<BlobError, _>(format!("{reason:?}")),
|
||||||
|
DataPlaneError::WrongEntryType { .. }
|
||||||
|
| DataPlaneError::PathReplaced(_)
|
||||||
|
| DataPlaneError::PeerLost
|
||||||
|
| DataPlaneError::StreamFault(_)
|
||||||
|
| DataPlaneError::StreamClosed => PyErr::new::<StreamError, _>(error.to_string()),
|
||||||
DataPlaneError::Attachment(reason) => {
|
DataPlaneError::Attachment(reason) => {
|
||||||
PyErr::new::<SessionError, _>(format!("attachment failed: {reason:?}"))
|
PyErr::new::<SessionError, _>(format!("attachment failed: {reason:?}"))
|
||||||
}
|
}
|
||||||
|
|
@ -229,7 +235,6 @@ fn build_child_routing(
|
||||||
pub struct PyDataPlane {
|
pub struct PyDataPlane {
|
||||||
inner: Arc<DataPlane>,
|
inner: Arc<DataPlane>,
|
||||||
_routing: Arc<JobRouting>,
|
_routing: Arc<JobRouting>,
|
||||||
legacy_output: Option<String>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[pymethods]
|
#[pymethods]
|
||||||
|
|
@ -257,20 +262,34 @@ impl PyDataPlane {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn read_stream(&self, _path: String) -> PyResult<()> {
|
fn read_stream<'py>(&self, py: Python<'py>, path: String) -> PyResult<Bound<'py, PyAny>> {
|
||||||
Err(PyRuntimeError::new_err(
|
let path = DataPath::parse(path)
|
||||||
"actor-driven stream reads are not installed",
|
.map_err(|error| PyErr::new::<DataPathError, _>(error.to_string()))?;
|
||||||
))
|
let data_plane = self.inner.clone();
|
||||||
|
pyo3_async_runtimes::tokio::future_into_py(py, async move {
|
||||||
|
let reader = data_plane
|
||||||
|
.read_stream(&path)
|
||||||
|
.await
|
||||||
|
.map_err(data_plane_error)?;
|
||||||
|
Python::with_gil(|py| {
|
||||||
|
Py::new(
|
||||||
|
py,
|
||||||
|
PyStreamReader {
|
||||||
|
reader: Arc::new(tokio::sync::Mutex::new(reader)),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
})
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn write_stream(&self, path: String) -> PyResult<PyStreamContext> {
|
fn write_stream(&self, path: String) -> PyResult<PyStreamContext> {
|
||||||
DataPath::parse(path).map_err(|error| PyErr::new::<DataPathError, _>(error.to_string()))?;
|
let path = DataPath::parse(path)
|
||||||
let socket = self.legacy_output.clone().ok_or_else(|| {
|
.map_err(|error| PyErr::new::<DataPathError, _>(error.to_string()))?;
|
||||||
PyRuntimeError::new_err("temporary deployment stream bridge is not configured")
|
|
||||||
})?;
|
|
||||||
Ok(PyStreamContext {
|
Ok(PyStreamContext {
|
||||||
socket,
|
data_plane: self.inner.clone(),
|
||||||
|
path,
|
||||||
stream: Arc::new(tokio::sync::Mutex::new(None)),
|
stream: Arc::new(tokio::sync::Mutex::new(None)),
|
||||||
|
entered: Arc::new(AtomicBool::new(false)),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -618,9 +637,25 @@ unsafe fn release_buffer_format(view: *mut ffi::Py_buffer) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[pyclass(name = "StreamReader")]
|
||||||
|
pub struct PyStreamReader {
|
||||||
|
reader: Arc<tokio::sync::Mutex<StreamReader>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[pymethods]
|
||||||
|
impl PyStreamReader {
|
||||||
|
fn read<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
|
||||||
|
let reader = self.reader.clone();
|
||||||
|
pyo3_async_runtimes::tokio::future_into_py(py, async move {
|
||||||
|
let bytes = reader.lock().await.read().await.map_err(data_plane_error)?;
|
||||||
|
Python::with_gil(|py| Ok(bytes.map(|bytes| PyBytes::new(py, &bytes).unbind())))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[pyclass(name = "StreamWriter")]
|
#[pyclass(name = "StreamWriter")]
|
||||||
pub struct PyStreamWriter {
|
pub struct PyStreamWriter {
|
||||||
stream: Arc<tokio::sync::Mutex<Option<BufWriter<UnixStream>>>>,
|
stream: Arc<tokio::sync::Mutex<Option<StreamWriter>>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[pymethods]
|
#[pymethods]
|
||||||
|
|
@ -629,56 +664,67 @@ impl PyStreamWriter {
|
||||||
let stream = self.stream.clone();
|
let stream = self.stream.clone();
|
||||||
pyo3_async_runtimes::tokio::future_into_py(py, async move {
|
pyo3_async_runtimes::tokio::future_into_py(py, async move {
|
||||||
let mut stream = stream.lock().await;
|
let mut stream = stream.lock().await;
|
||||||
let stream = stream
|
let writer = stream
|
||||||
.as_mut()
|
.as_mut()
|
||||||
.ok_or_else(|| PyRuntimeError::new_err("stream writer is closed"))?;
|
.ok_or_else(|| PyRuntimeError::new_err("stream writer is closed"))?;
|
||||||
stream
|
writer.write(&bytes).await.map_err(data_plane_error)
|
||||||
.write_all(&bytes)
|
|
||||||
.await
|
|
||||||
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
|
|
||||||
Ok(())
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[pyclass(name = "_StreamWriteContext")]
|
#[pyclass(name = "_StreamWriteContext")]
|
||||||
pub struct PyStreamContext {
|
pub struct PyStreamContext {
|
||||||
socket: String,
|
data_plane: Arc<DataPlane>,
|
||||||
stream: Arc<tokio::sync::Mutex<Option<BufWriter<UnixStream>>>>,
|
path: DataPath,
|
||||||
|
stream: Arc<tokio::sync::Mutex<Option<StreamWriter>>>,
|
||||||
|
entered: Arc<AtomicBool>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[pymethods]
|
#[pymethods]
|
||||||
impl PyStreamContext {
|
impl PyStreamContext {
|
||||||
fn __aenter__<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
|
fn __aenter__<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
|
||||||
let socket = self.socket.clone();
|
if self.entered.swap(true, Ordering::AcqRel) {
|
||||||
|
return Err(PyRuntimeError::new_err(
|
||||||
|
"stream write context cannot be entered twice",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let data_plane = self.data_plane.clone();
|
||||||
|
let path = self.path.clone();
|
||||||
let state = self.stream.clone();
|
let state = self.stream.clone();
|
||||||
|
let entered = self.entered.clone();
|
||||||
pyo3_async_runtimes::tokio::future_into_py(py, async move {
|
pyo3_async_runtimes::tokio::future_into_py(py, async move {
|
||||||
let stream = UnixStream::connect(&socket).await.map_err(|error| {
|
match data_plane.write_stream(&path).await {
|
||||||
PyRuntimeError::new_err(format!("connect output stream: {error}"))
|
Ok(writer) => {
|
||||||
})?;
|
*state.lock().await = Some(writer);
|
||||||
*state.lock().await = Some(BufWriter::new(stream));
|
Python::with_gil(|py| Py::new(py, PyStreamWriter { stream: state }))
|
||||||
Python::with_gil(|py| Py::new(py, PyStreamWriter { stream: state }))
|
}
|
||||||
|
Err(error) => {
|
||||||
|
entered.store(false, Ordering::Release);
|
||||||
|
Err(data_plane_error(error))
|
||||||
|
}
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn __aexit__<'py>(
|
fn __aexit__<'py>(
|
||||||
&self,
|
&self,
|
||||||
py: Python<'py>,
|
py: Python<'py>,
|
||||||
_exception_type: &Bound<'_, PyAny>,
|
exception_type: &Bound<'_, PyAny>,
|
||||||
_exception: &Bound<'_, PyAny>,
|
_exception: &Bound<'_, PyAny>,
|
||||||
_traceback: &Bound<'_, PyAny>,
|
_traceback: &Bound<'_, PyAny>,
|
||||||
) -> PyResult<Bound<'py, PyAny>> {
|
) -> PyResult<Bound<'py, PyAny>> {
|
||||||
|
let clean = exception_type.is_none();
|
||||||
let state = self.stream.clone();
|
let state = self.stream.clone();
|
||||||
pyo3_async_runtimes::tokio::future_into_py(py, async move {
|
pyo3_async_runtimes::tokio::future_into_py(py, async move {
|
||||||
if let Some(mut stream) = state.lock().await.take() {
|
let mut writer = state
|
||||||
stream
|
.lock()
|
||||||
.flush()
|
.await
|
||||||
.await
|
.take()
|
||||||
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
|
.ok_or_else(|| PyRuntimeError::new_err("stream write context is not active"))?;
|
||||||
stream
|
if clean {
|
||||||
.shutdown()
|
writer.close().await.map_err(data_plane_error)?;
|
||||||
.await
|
} else {
|
||||||
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
|
writer.abort().map_err(data_plane_error)?;
|
||||||
}
|
}
|
||||||
Ok(false)
|
Ok(false)
|
||||||
})
|
})
|
||||||
|
|
@ -736,7 +782,6 @@ fn run(py: Python<'_>, main: Bound<'_, PyAny>) -> PyResult<()> {
|
||||||
PyDataPlane {
|
PyDataPlane {
|
||||||
inner: Arc::new(bootstrap.data_plane),
|
inner: Arc::new(bootstrap.data_plane),
|
||||||
_routing: Arc::new(routing),
|
_routing: Arc::new(routing),
|
||||||
legacy_output: std::env::var(LEGACY_OUTPUT_ENV).ok(),
|
|
||||||
},
|
},
|
||||||
)?;
|
)?;
|
||||||
let context = Py::new(py, PyContext { data })?;
|
let context = Py::new(py, PyContext { data })?;
|
||||||
|
|
@ -848,6 +893,19 @@ impl data_plane::source::BlobSourcePublisher for DebugSourceRegistrar {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(all(debug_assertions, target_os = "linux"))]
|
||||||
|
struct DebugStreamCaptureConsumer {
|
||||||
|
bytes: Arc<ParkingMutex<Vec<u8>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(all(debug_assertions, target_os = "linux"))]
|
||||||
|
impl data_plane::data_plane::StreamConsumer for DebugStreamCaptureConsumer {
|
||||||
|
fn consume(&self, bytes: &[u8]) -> Result<(), String> {
|
||||||
|
self.bytes.lock().extend_from_slice(bytes);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(all(debug_assertions, target_os = "linux"))]
|
#[cfg(all(debug_assertions, target_os = "linux"))]
|
||||||
#[pyclass(name = "_TestDataPlaneHost")]
|
#[pyclass(name = "_TestDataPlaneHost")]
|
||||||
struct PyTestDataPlaneHost {
|
struct PyTestDataPlaneHost {
|
||||||
|
|
@ -855,6 +913,7 @@ struct PyTestDataPlaneHost {
|
||||||
_engine: Engine,
|
_engine: Engine,
|
||||||
handoff: data_plane::bootstrap::JobHandoff,
|
handoff: data_plane::bootstrap::JobHandoff,
|
||||||
namespace_root: std::path::PathBuf,
|
namespace_root: std::path::PathBuf,
|
||||||
|
stream_data_plane: Arc<DataPlane>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(all(debug_assertions, target_os = "linux"))]
|
#[cfg(all(debug_assertions, target_os = "linux"))]
|
||||||
|
|
@ -879,6 +938,57 @@ impl PyTestDataPlaneHost {
|
||||||
fn arena_fd(&self) -> RawFd {
|
fn arena_fd(&self) -> RawFd {
|
||||||
std::os::fd::AsRawFd::as_raw_fd(&self.handoff.arena_fd)
|
std::os::fd::AsRawFd::as_raw_fd(&self.handoff.arena_fd)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn read_stream<'py>(&self, py: Python<'py>, path: String) -> PyResult<Bound<'py, PyAny>> {
|
||||||
|
let path = DataPath::parse(path)
|
||||||
|
.map_err(|error| PyErr::new::<DataPathError, _>(error.to_string()))?;
|
||||||
|
let data_plane = self.stream_data_plane.clone();
|
||||||
|
pyo3_async_runtimes::tokio::future_into_py(py, async move {
|
||||||
|
let reader = data_plane
|
||||||
|
.read_stream(&path)
|
||||||
|
.await
|
||||||
|
.map_err(data_plane_error)?;
|
||||||
|
Python::with_gil(|py| {
|
||||||
|
Py::new(
|
||||||
|
py,
|
||||||
|
PyStreamReader {
|
||||||
|
reader: Arc::new(tokio::sync::Mutex::new(reader)),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn capture_stream(&self, path: String) -> PyResult<PyTestStreamCapture> {
|
||||||
|
let path = DataPath::parse(path)
|
||||||
|
.map_err(|error| PyErr::new::<DataPathError, _>(error.to_string()))?;
|
||||||
|
let bytes = Arc::new(ParkingMutex::new(Vec::new()));
|
||||||
|
let consumer: Arc<dyn data_plane::data_plane::StreamConsumer> =
|
||||||
|
Arc::new(DebugStreamCaptureConsumer {
|
||||||
|
bytes: Arc::clone(&bytes),
|
||||||
|
});
|
||||||
|
let completion = self
|
||||||
|
.stream_data_plane
|
||||||
|
.collect_stream(path, consumer)
|
||||||
|
.map_err(data_plane_error)?;
|
||||||
|
Ok(PyTestStreamCapture { completion, bytes })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(all(debug_assertions, target_os = "linux"))]
|
||||||
|
#[pyclass(name = "_TestStreamCapture")]
|
||||||
|
struct PyTestStreamCapture {
|
||||||
|
completion: ActorCompletion<Result<(), DataPlaneError>>,
|
||||||
|
bytes: Arc<ParkingMutex<Vec<u8>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(all(debug_assertions, target_os = "linux"))]
|
||||||
|
#[pymethods]
|
||||||
|
impl PyTestStreamCapture {
|
||||||
|
fn result<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyBytes>> {
|
||||||
|
self.completion.wait().map_err(data_plane_error)?;
|
||||||
|
Ok(PyBytes::new(py, &self.bytes.lock()))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(all(debug_assertions, target_os = "linux"))]
|
#[cfg(all(debug_assertions, target_os = "linux"))]
|
||||||
|
|
@ -976,6 +1086,8 @@ fn _test_data_plane_host() -> PyResult<PyTestDataPlaneHost> {
|
||||||
))
|
))
|
||||||
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
|
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
|
||||||
let namespace = data_plane::namespace::NamespaceClient::new(runtime.clone(), namespace_proxy);
|
let namespace = data_plane::namespace::NamespaceClient::new(runtime.clone(), namespace_proxy);
|
||||||
|
let stream_transport: Arc<dyn data_plane::stream_transport::StreamTransport> =
|
||||||
|
Arc::new(data_plane::stream_transport::LocalStreamTransport::new());
|
||||||
let host_session = runtime
|
let host_session = runtime
|
||||||
.spawn(
|
.spawn(
|
||||||
data_plane::host::HostDataPlaneSessionActor::new(
|
data_plane::host::HostDataPlaneSessionActor::new(
|
||||||
|
|
@ -998,11 +1110,12 @@ fn _test_data_plane_host() -> PyResult<PyTestDataPlaneHost> {
|
||||||
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?,
|
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?,
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
namespace: Some(namespace),
|
namespace: Some(namespace.clone()),
|
||||||
transfer_receiver: Some(Arc::new(DebugBlobReceiver)),
|
transfer_receiver: Some(Arc::new(DebugBlobReceiver)),
|
||||||
source_sender: Some(source_sender),
|
source_sender: Some(Arc::clone(&source_sender)),
|
||||||
source_publisher: Some(source_publisher),
|
source_publisher: Some(Arc::clone(&source_publisher)),
|
||||||
route_registrar: Some(registrar),
|
route_registrar: Some(registrar),
|
||||||
|
stream_transport: Some(Arc::clone(&stream_transport)),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?,
|
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?,
|
||||||
|
|
@ -1010,6 +1123,59 @@ fn _test_data_plane_host() -> PyResult<PyTestDataPlaneHost> {
|
||||||
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
|
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
|
||||||
data_plane::host::install_session_env(&mut handoff, host_session, capability);
|
data_plane::host::install_session_env(&mut handoff, host_session, capability);
|
||||||
|
|
||||||
|
let mut sink_arena = data_plane::arena::ArenaManager::boot(data_plane::arena::ArenaConfig {
|
||||||
|
node_id: data_plane::arena::NodeId(2),
|
||||||
|
reservation_ceiling: 1 << 20,
|
||||||
|
base_alignment: 64,
|
||||||
|
})
|
||||||
|
.map_err(|error| PyRuntimeError::new_err(format!("debug sink arena: {error:?}")))?;
|
||||||
|
let sink_handoff = data_plane::bootstrap::write_bootstrap(
|
||||||
|
&mut sink_arena,
|
||||||
|
data_plane::bootstrap::BootstrapSpec {
|
||||||
|
arena_generation: 2,
|
||||||
|
alignment: 64,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
|
||||||
|
let sink_session = runtime
|
||||||
|
.spawn(
|
||||||
|
data_plane::host::HostDataPlaneSessionActor::new(
|
||||||
|
data_plane::host::HostDataPlaneConfig {
|
||||||
|
runtime: runtime.clone(),
|
||||||
|
arena: sink_arena,
|
||||||
|
arena_generation: 2,
|
||||||
|
session_generation: 2,
|
||||||
|
capability,
|
||||||
|
job_context: data_plane::path::JobContext {
|
||||||
|
run_id: "test-run".to_owned(),
|
||||||
|
read_prefixes: vec![
|
||||||
|
DataPath::parse("/runs/test-run/results")
|
||||||
|
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?,
|
||||||
|
],
|
||||||
|
write_prefixes: Vec::new(),
|
||||||
|
},
|
||||||
|
namespace: Some(namespace),
|
||||||
|
transfer_receiver: None,
|
||||||
|
source_sender: None,
|
||||||
|
source_publisher: None,
|
||||||
|
route_registrar: None,
|
||||||
|
stream_transport: Some(stream_transport),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?,
|
||||||
|
)
|
||||||
|
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
|
||||||
|
let stream_data_plane = Arc::new(
|
||||||
|
future::block_on(DataPlaneBootstrap::attach(
|
||||||
|
sink_handoff.arena_fd,
|
||||||
|
runtime.clone(),
|
||||||
|
sink_session,
|
||||||
|
capability,
|
||||||
|
))
|
||||||
|
.map_err(data_plane_error)?
|
||||||
|
.data_plane,
|
||||||
|
);
|
||||||
|
|
||||||
driver.enable_actor_bridge(iroh_driver::ActorBridgeConfig {
|
driver.enable_actor_bridge(iroh_driver::ActorBridgeConfig {
|
||||||
runtime: runtime.clone(),
|
runtime: runtime.clone(),
|
||||||
codec: codecs,
|
codec: codecs,
|
||||||
|
|
@ -1031,6 +1197,7 @@ fn _test_data_plane_host() -> PyResult<PyTestDataPlaneHost> {
|
||||||
_engine: engine,
|
_engine: engine,
|
||||||
handoff,
|
handoff,
|
||||||
namespace_root,
|
namespace_root,
|
||||||
|
stream_data_plane,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1040,14 +1207,17 @@ pub fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||||
module.add("DataPathError", module.py().get_type::<DataPathError>())?;
|
module.add("DataPathError", module.py().get_type::<DataPathError>())?;
|
||||||
module.add("BlobError", module.py().get_type::<BlobError>())?;
|
module.add("BlobError", module.py().get_type::<BlobError>())?;
|
||||||
module.add("SessionError", module.py().get_type::<SessionError>())?;
|
module.add("SessionError", module.py().get_type::<SessionError>())?;
|
||||||
|
module.add("StreamError", module.py().get_type::<StreamError>())?;
|
||||||
module.add_class::<PyDataPlane>()?;
|
module.add_class::<PyDataPlane>()?;
|
||||||
module.add_class::<PyBlob>()?;
|
module.add_class::<PyBlob>()?;
|
||||||
module.add_class::<PyBlobView>()?;
|
module.add_class::<PyBlobView>()?;
|
||||||
module.add_class::<PyStreamWriter>()?;
|
module.add_class::<PyStreamWriter>()?;
|
||||||
|
module.add_class::<PyStreamReader>()?;
|
||||||
module.add_class::<PyContext>()?;
|
module.add_class::<PyContext>()?;
|
||||||
module.add_function(wrap_pyfunction!(run, module)?)?;
|
module.add_function(wrap_pyfunction!(run, module)?)?;
|
||||||
#[cfg(all(debug_assertions, target_os = "linux"))]
|
#[cfg(all(debug_assertions, target_os = "linux"))]
|
||||||
{
|
{
|
||||||
|
module.add_class::<PyTestStreamCapture>()?;
|
||||||
module.add_class::<PyTestDataPlaneHost>()?;
|
module.add_class::<PyTestDataPlaneHost>()?;
|
||||||
module.add_function(wrap_pyfunction!(_test_data_plane_host, module)?)?;
|
module.add_function(wrap_pyfunction!(_test_data_plane_host, module)?)?;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -2,16 +2,15 @@
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import ctypes
|
import ctypes
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import struct
|
import struct
|
||||||
import subprocess
|
import subprocess
|
||||||
import importlib
|
import importlib
|
||||||
import socket
|
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
import threading
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
@ -215,39 +214,29 @@ def test_missing_path_and_authorization_are_typed(monkeypatch, host):
|
||||||
swactor.run(main)
|
swactor.run(main)
|
||||||
|
|
||||||
|
|
||||||
def test_temporary_output_stream_bridge_remains_available(monkeypatch, host, tmp_path):
|
def test_native_stream_round_trip_has_ordered_eof(monkeypatch, host):
|
||||||
install_host_env(monkeypatch, host)
|
install_host_env(monkeypatch, host)
|
||||||
socket_path = tmp_path / "output.sock"
|
|
||||||
listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
|
||||||
listener.bind(str(socket_path))
|
|
||||||
listener.listen(1)
|
|
||||||
monkeypatch.setenv("SWACTOR_DATA_PLANE_OUTPUT", str(socket_path))
|
|
||||||
received = []
|
received = []
|
||||||
|
|
||||||
def receive():
|
|
||||||
connection, _ = listener.accept()
|
|
||||||
with connection:
|
|
||||||
chunks = []
|
|
||||||
while chunk := connection.recv(4096):
|
|
||||||
chunks.append(chunk)
|
|
||||||
received.append(b"".join(chunks))
|
|
||||||
|
|
||||||
receiver = threading.Thread(target=receive)
|
|
||||||
receiver.start()
|
|
||||||
|
|
||||||
async def main(ctx):
|
async def main(ctx):
|
||||||
|
async def receive():
|
||||||
|
reader = await ctx.data.read_stream(
|
||||||
|
"/runs/self/results/predictions"
|
||||||
|
)
|
||||||
|
while (chunk := await reader.read()) is not None:
|
||||||
|
received.append(chunk)
|
||||||
|
|
||||||
|
receiver = asyncio.create_task(receive())
|
||||||
async with ctx.data.write_stream(
|
async with ctx.data.write_stream(
|
||||||
"/runs/self/results/predictions"
|
"/runs/self/results/predictions"
|
||||||
) as stream:
|
) as stream:
|
||||||
await stream.write(b"temporary-result")
|
await stream.write(b"native-")
|
||||||
|
await stream.write(b"result")
|
||||||
|
await receiver
|
||||||
|
|
||||||
try:
|
swactor.run(main)
|
||||||
swactor.run(main)
|
assert b"".join(received) == b"native-result"
|
||||||
receiver.join(timeout=2)
|
assert "SWACTOR_DATA_PLANE_OUTPUT" not in host.env()
|
||||||
assert not receiver.is_alive()
|
|
||||||
assert received == [b"temporary-result"]
|
|
||||||
finally:
|
|
||||||
listener.close()
|
|
||||||
|
|
||||||
|
|
||||||
def test_invalid_capability_prevents_main(monkeypatch, host):
|
def test_invalid_capability_prevents_main(monkeypatch, host):
|
||||||
|
|
@ -350,34 +339,18 @@ def test_real_exec_attachment_and_blob_mapping(host):
|
||||||
not Path("/dev/nvidia0").exists(),
|
not Path("/dev/nvidia0").exists(),
|
||||||
reason="CUDA device is unavailable",
|
reason="CUDA device is unavailable",
|
||||||
)
|
)
|
||||||
def test_real_exec_tinygrad_cuda_scenario(host, tmp_path):
|
def test_real_exec_tinygrad_cuda_scenario(host):
|
||||||
socket_path = tmp_path / "cuda-output.sock"
|
capture = host.capture_stream("/runs/self/results/inference")
|
||||||
listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
|
||||||
listener.bind(str(socket_path))
|
|
||||||
listener.listen(1)
|
|
||||||
received = []
|
|
||||||
|
|
||||||
def receive():
|
|
||||||
connection, _ = listener.accept()
|
|
||||||
with connection:
|
|
||||||
chunks = []
|
|
||||||
while chunk := connection.recv(4096):
|
|
||||||
chunks.append(chunk)
|
|
||||||
received.append(b"".join(chunks))
|
|
||||||
|
|
||||||
receiver = threading.Thread(target=receive)
|
|
||||||
receiver.start()
|
|
||||||
env = {
|
env = {
|
||||||
key: value
|
key: value
|
||||||
for key, value in os.environ.items()
|
for key, value in os.environ.items()
|
||||||
if key not in (*BOOTSTRAP_ENV, *OLD_WAKE_ENV)
|
if key not in (*BOOTSTRAP_ENV, *OLD_WAKE_ENV, "SWACTOR_DATA_PLANE_OUTPUT")
|
||||||
}
|
}
|
||||||
env.update(dict(host.env()))
|
env.update(dict(host.env()))
|
||||||
env.update(
|
env.update(
|
||||||
{
|
{
|
||||||
"CUDA_PTX": "1",
|
"CUDA_PTX": "1",
|
||||||
"DEV": "CUDA",
|
"DEV": "CUDA",
|
||||||
"SWACTOR_DATA_PLANE_OUTPUT": str(socket_path),
|
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
script = (
|
script = (
|
||||||
|
|
@ -387,21 +360,16 @@ def test_real_exec_tinygrad_cuda_scenario(host, tmp_path):
|
||||||
/ "jobs"
|
/ "jobs"
|
||||||
/ "tiny_linear_inference.py"
|
/ "tiny_linear_inference.py"
|
||||||
)
|
)
|
||||||
try:
|
result = subprocess.run(
|
||||||
result = subprocess.run(
|
[sys.executable, str(script)],
|
||||||
[sys.executable, str(script)],
|
env=env,
|
||||||
env=env,
|
pass_fds=(host.arena_fd(),),
|
||||||
pass_fds=(host.arena_fd(),),
|
text=True,
|
||||||
text=True,
|
capture_output=True,
|
||||||
capture_output=True,
|
timeout=45,
|
||||||
timeout=45,
|
check=False,
|
||||||
check=False,
|
)
|
||||||
)
|
assert result.returncode == 0, result.stderr
|
||||||
receiver.join(timeout=2)
|
payload = json.loads(capture.result())
|
||||||
assert result.returncode == 0, result.stderr
|
assert payload["device"].startswith("CUDA")
|
||||||
assert not receiver.is_alive()
|
assert payload["output"] == pytest.approx([2.75, -8.75])
|
||||||
payload = json.loads(received[0])
|
|
||||||
assert payload["device"].startswith("CUDA")
|
|
||||||
assert payload["output"] == pytest.approx([2.75, -8.75])
|
|
||||||
finally:
|
|
||||||
listener.close()
|
|
||||||
|
|
|
||||||
|
|
@ -1282,6 +1282,30 @@ wheels = [
|
||||||
{ url = "https://files.pythonhosted.org/packages/af/33/ee4519fa02ed11a94aef9559552f3b17bb863f2ecfe1a35dc7f548cde231/matplotlib_inline-0.2.1-py3-none-any.whl", hash = "sha256:d56ce5156ba6085e00a9d54fead6ed29a9c47e215cd1bba2e976ef39f5710a76", size = 9516, upload-time = "2025-10-23T09:00:20.675Z" },
|
{ url = "https://files.pythonhosted.org/packages/af/33/ee4519fa02ed11a94aef9559552f3b17bb863f2ecfe1a35dc7f548cde231/matplotlib_inline-0.2.1-py3-none-any.whl", hash = "sha256:d56ce5156ba6085e00a9d54fead6ed29a9c47e215cd1bba2e976ef39f5710a76", size = 9516, upload-time = "2025-10-23T09:00:20.675Z" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "maturin"
|
||||||
|
version = "1.14.1"
|
||||||
|
source = { registry = "https://pypi.org/simple" }
|
||||||
|
dependencies = [
|
||||||
|
{ name = "tomli", marker = "python_full_version < '3.11'" },
|
||||||
|
]
|
||||||
|
sdist = { url = "https://files.pythonhosted.org/packages/e7/b3/addd877f871fb1860d46d3a4f206ecb10b946c85846805e6367631926fd3/maturin-1.14.1.tar.gz", hash = "sha256:9d6577a62cd08e0ceba7a0db06fb098e0c9b1b3429bad747a4f3a18215a1b3df", size = 369637, upload-time = "2026-06-19T05:19:49.774Z" }
|
||||||
|
wheels = [
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/f4/f0/97c5a5bd9c71653a066c0976a484eaaae50b9369557838a4176b7b0bdaa5/maturin-1.14.1-py3-none-linux_armv6l.whl", hash = "sha256:522292398945442cdafa9daeb2271b2340fbde57027b818f923f88eab04174f8", size = 10207496, upload-time = "2026-06-19T05:19:09.321Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/fe/83/294bca639b0e052f1e2f65199b3db258780c7d4e31408b934c9c974a1379/maturin-1.14.1-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:ffe5ad71f21d1e6603c4dd75f7fee34adf5ed5ebcebb692886549888ebb329ed", size = 19680113, upload-time = "2026-06-19T05:19:13.43Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/43/b6/79c881410a3b1c187f7eb3d407aecae646c6a4433d630d72200359015e83/maturin-1.14.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:f3306078070c1508fd715b9116070cbcaff5959024272a9f1e6f5cb29768b86c", size = 10169205, upload-time = "2026-06-19T05:19:16.615Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/93/9d/44b6f26dcb7f7a04c5501ac2dbb6ca1490150682baa525ca5860504f9eab/maturin-1.14.1-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:cd457cd88961156e26379e1155bd287cc0ec1c8b2f1582b0660fb31b87c8842d", size = 10188098, upload-time = "2026-06-19T05:19:19.736Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/1a/bd/9c0d5d6983905ce2c9edaa073a7e89355a9cf7f396988e05d32f1c37785d/maturin-1.14.1-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:dfc54ae32e6fcb18302193ab9a30b0b25eefffba994ae13238974805533ef75e", size = 10627576, upload-time = "2026-06-19T05:19:22.713Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/e5/33/b096412bd6a7cb399652b260666f901adf88a687181a6dbd6a3f89f0a94e/maturin-1.14.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:a131d912b5267e640bc96d70f4914e10590aed64082ec9abacba7cea52004224", size = 10085181, upload-time = "2026-06-19T05:19:25.69Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/56/8d/08c3bf469c38a23c9e6c877e338193001eb604d010fedc08341974e38528/maturin-1.14.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:be18fc568fb76884c0205456336892a75105ec398e6b667cd777c6268bd06d69", size = 10026363, upload-time = "2026-06-19T05:19:28.904Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/3a/a4/c4d1a92839f8745ab4aab988a7db884a79d6d710bd3b286fcf9316dece1a/maturin-1.14.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:994a0c8ba3ad8a92b3a9ee1b02645d200d610216b15cff5102b0fe65e8e08666", size = 13321347, upload-time = "2026-06-19T05:19:32.411Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/b3/fa/170f04624d03fd07d2a8b1b67de83a127af93aef9eaa425839553347297b/maturin-1.14.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:be80866363e605d137991b491a741a84cde9ae350183c4c85f49690ca9aaaa65", size = 10877609, upload-time = "2026-06-19T05:19:35.448Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/61/ad/1ae2e1d0ded282bf2c55ac13f0811d87deb425e200ae64a15785675dede9/maturin-1.14.1-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:5282dffd4b539d2be245f4e5b1a5ab6bc1033b58f4a4872f5833f9d43c954aa4", size = 10417316, upload-time = "2026-06-19T05:19:38.28Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/fb/27/bf677183920718da49cd7982d6a3ffc440aad8919329f571d189f81b7bdf/maturin-1.14.1-py3-none-win32.whl", hash = "sha256:1a04de0a20188f95c721b5702eed18140bdcccb28c386797093eca3f62f4d4e0", size = 8931293, upload-time = "2026-06-19T05:19:41.183Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/63/4b/585adeb9167b08d3cdff0032a938b0e72655c92003df4f52c3f696a1bcc2/maturin-1.14.1-py3-none-win_amd64.whl", hash = "sha256:3c9f94640ecc4895e94abaf834a0684430032c865b2748a36c12461fd9252fdd", size = 10314067, upload-time = "2026-06-19T05:19:44.389Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/51/d4/dac8c0720ae246be1700afb6fbdbbea20fe35b13f6570b2f70faa005df77/maturin-1.14.1-py3-none-win_arm64.whl", hash = "sha256:15cea8fcb3ba47dd636f50092bb34baea8b04ac777392f23e6bf8a9a61efb894", size = 9718943, upload-time = "2026-06-19T05:19:47.49Z" },
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mistune"
|
name = "mistune"
|
||||||
version = "3.2.0"
|
version = "3.2.0"
|
||||||
|
|
@ -1644,6 +1668,19 @@ wheels = [
|
||||||
{ url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" },
|
{ url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "pytest-timeout"
|
||||||
|
version = "2.4.0"
|
||||||
|
source = { registry = "https://pypi.org/simple" }
|
||||||
|
dependencies = [
|
||||||
|
{ name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
|
||||||
|
{ name = "pytest", version = "9.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
|
||||||
|
]
|
||||||
|
sdist = { url = "https://files.pythonhosted.org/packages/ac/82/4c9ecabab13363e72d880f2fb504c5f750433b2b6f16e99f4ec21ada284c/pytest_timeout-2.4.0.tar.gz", hash = "sha256:7e68e90b01f9eff71332b25001f85c75495fc4e3a836701876183c4bcfd0540a", size = 17973, upload-time = "2025-05-05T19:44:34.99Z" }
|
||||||
|
wheels = [
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl", hash = "sha256:c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2", size = 14382, upload-time = "2025-05-05T19:44:33.502Z" },
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "python-dateutil"
|
name = "python-dateutil"
|
||||||
version = "2.9.0.post0"
|
version = "2.9.0.post0"
|
||||||
|
|
@ -2313,8 +2350,10 @@ dev = [
|
||||||
{ name = "ipykernel", version = "6.31.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
|
{ name = "ipykernel", version = "6.31.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
|
||||||
{ name = "ipykernel", version = "7.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
|
{ name = "ipykernel", version = "7.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
|
||||||
{ name = "jupyter" },
|
{ name = "jupyter" },
|
||||||
|
{ name = "maturin" },
|
||||||
{ name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
|
{ name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
|
||||||
{ name = "pytest", version = "9.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
|
{ name = "pytest", version = "9.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
|
||||||
|
{ name = "pytest-timeout" },
|
||||||
]
|
]
|
||||||
|
|
||||||
[package.metadata]
|
[package.metadata]
|
||||||
|
|
@ -2323,7 +2362,9 @@ dev = [
|
||||||
dev = [
|
dev = [
|
||||||
{ name = "ipykernel" },
|
{ name = "ipykernel" },
|
||||||
{ name = "jupyter" },
|
{ name = "jupyter" },
|
||||||
|
{ name = "maturin", specifier = ">=1.7,<2" },
|
||||||
{ name = "pytest" },
|
{ name = "pytest" },
|
||||||
|
{ name = "pytest-timeout" },
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
|
||||||
|
|
@ -11,3 +11,6 @@ crate-type = ["cdylib"]
|
||||||
swactor = { path = "../../..", default-features = false, features = ["wasm", "std"] }
|
swactor = { path = "../../..", default-features = false, features = ["wasm", "std"] }
|
||||||
swactor-engine = { path = "../../engine", default-features = false }
|
swactor-engine = { path = "../../engine", default-features = false }
|
||||||
wasm-bindgen = "0.2"
|
wasm-bindgen = "0.2"
|
||||||
|
|
||||||
|
[lints]
|
||||||
|
workspace = true
|
||||||
|
|
|
||||||
|
|
@ -103,6 +103,12 @@ pub struct WasmRuntime {
|
||||||
backend: SteppingBackend,
|
backend: SteppingBackend,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl Default for WasmRuntime {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[wasm_bindgen]
|
#[wasm_bindgen]
|
||||||
impl WasmRuntime {
|
impl WasmRuntime {
|
||||||
#[wasm_bindgen(constructor)]
|
#[wasm_bindgen(constructor)]
|
||||||
|
|
|
||||||
|
|
@ -23,3 +23,6 @@ demo-control = []
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
proptest = "1"
|
proptest = "1"
|
||||||
tower = { version = "0.5", features = ["util"] }
|
tower = { version = "0.5", features = ["util"] }
|
||||||
|
|
||||||
|
[lints]
|
||||||
|
workspace = true
|
||||||
|
|
|
||||||
|
|
@ -187,9 +187,10 @@ mod properties {
|
||||||
concurrent in any::<bool>(),
|
concurrent in any::<bool>(),
|
||||||
destination_disappears in any::<bool>(),
|
destination_disappears in any::<bool>(),
|
||||||
) {
|
) {
|
||||||
let mut config = RuntimeConfig::default();
|
let parts = RuntimeParts::new(RuntimeConfig {
|
||||||
config.worker_count = 1;
|
worker_count: 1,
|
||||||
let parts = RuntimeParts::new(config);
|
..RuntimeConfig::default()
|
||||||
|
});
|
||||||
let runtime = parts.runtime().clone();
|
let runtime = parts.runtime().clone();
|
||||||
let backend = SteppingBackend::new();
|
let backend = SteppingBackend::new();
|
||||||
let _engine =
|
let _engine =
|
||||||
|
|
|
||||||
|
|
@ -236,7 +236,9 @@ struct NodeCard {
|
||||||
errors: Vec<String>,
|
errors: Vec<String>,
|
||||||
cpu: Option<CpuSnapshot>,
|
cpu: Option<CpuSnapshot>,
|
||||||
gpu: Option<GpuSnapshot>,
|
gpu: Option<GpuSnapshot>,
|
||||||
|
memory: Option<telemetry::hardware::memory::HostMemorySample>,
|
||||||
net: Option<crate::hardware_view::NetSnapshot>,
|
net: Option<crate::hardware_view::NetSnapshot>,
|
||||||
|
storage: Option<telemetry::hardware::storage::HostStorageSample>,
|
||||||
process: Option<crate::hardware_view::ProcessSnapshot>,
|
process: Option<crate::hardware_view::ProcessSnapshot>,
|
||||||
history: Vec<HardwareHistorySnapshot>,
|
history: Vec<HardwareHistorySnapshot>,
|
||||||
actor_summary: ActorSummarySnapshot,
|
actor_summary: ActorSummarySnapshot,
|
||||||
|
|
@ -329,15 +331,9 @@ fn stream_key(stream: &StreamEvent) -> String {
|
||||||
fn node_card(node: &FusedNode, now: Instant) -> NodeCard {
|
fn node_card(node: &FusedNode, now: Instant) -> NodeCard {
|
||||||
let summary = node.hardware.summary();
|
let summary = node.hardware.summary();
|
||||||
let totals = node.actors.totals();
|
let totals = node.actors.totals();
|
||||||
let mut roster: Vec<RosterRow> = node.actors.actors.values().map(roster_row).collect();
|
// Address-keyed map order is the stable default. Volatile telemetry must
|
||||||
// Busiest actors first; ties fall back to address for stable rendering.
|
// not move a row out from under the pointer; the page offers explicit sorts.
|
||||||
roster.sort_by(|left, right| {
|
let roster: Vec<RosterRow> = node.actors.actors.values().map(roster_row).collect();
|
||||||
right
|
|
||||||
.msg_per_sec
|
|
||||||
.partial_cmp(&left.msg_per_sec)
|
|
||||||
.map_or(std::cmp::Ordering::Equal, |order| order)
|
|
||||||
.then_with(|| left.address.cmp(&right.address))
|
|
||||||
});
|
|
||||||
NodeCard {
|
NodeCard {
|
||||||
stream: StreamKeySnapshot {
|
stream: StreamKeySnapshot {
|
||||||
key: stream_key(&node.stream),
|
key: stream_key(&node.stream),
|
||||||
|
|
@ -352,7 +348,9 @@ fn node_card(node: &FusedNode, now: Instant) -> NodeCard {
|
||||||
errors: node.hardware.errors(),
|
errors: node.hardware.errors(),
|
||||||
cpu: node.hardware.cpu.as_ref().map(CpuSnapshot::from),
|
cpu: node.hardware.cpu.as_ref().map(CpuSnapshot::from),
|
||||||
gpu: node.hardware.gpu.as_ref().map(GpuSnapshot::from),
|
gpu: node.hardware.gpu.as_ref().map(GpuSnapshot::from),
|
||||||
|
memory: node.hardware.memory.clone(),
|
||||||
net: node.hardware.net.clone(),
|
net: node.hardware.net.clone(),
|
||||||
|
storage: node.hardware.storage.clone(),
|
||||||
process: node.hardware.process.clone(),
|
process: node.hardware.process.clone(),
|
||||||
history: node
|
history: node
|
||||||
.hardware
|
.hardware
|
||||||
|
|
@ -362,11 +360,16 @@ fn node_card(node: &FusedNode, now: Instant) -> NodeCard {
|
||||||
ms_ago: duration_ms(now.duration_since(sample.at)),
|
ms_ago: duration_ms(now.duration_since(sample.at)),
|
||||||
sample_unix_ms: sample.sample_unix_ms,
|
sample_unix_ms: sample.sample_unix_ms,
|
||||||
cpu_total_percent: sample.cpu_total_percent,
|
cpu_total_percent: sample.cpu_total_percent,
|
||||||
|
cpu_cores_percent: sample.cpu_cores_percent.clone(),
|
||||||
gpu_max_percent: sample.gpu_max_percent,
|
gpu_max_percent: sample.gpu_max_percent,
|
||||||
gpu_memory_used_mib: sample.gpu_memory_used_mib,
|
gpu_memory_used_mib: sample.gpu_memory_used_mib,
|
||||||
gpu_memory_total_mib: sample.gpu_memory_total_mib,
|
gpu_memory_total_mib: sample.gpu_memory_total_mib,
|
||||||
net_rx_bps: sample.net_rx_bps,
|
net_rx_bps: sample.net_rx_bps,
|
||||||
net_tx_bps: sample.net_tx_bps,
|
net_tx_bps: sample.net_tx_bps,
|
||||||
|
memory_used_percent: sample.memory_used_percent,
|
||||||
|
memory_pressure_some_avg10: sample.memory_pressure_some_avg10,
|
||||||
|
storage_used_percent: sample.storage_used_percent,
|
||||||
|
io_pressure_some_avg10: sample.io_pressure_some_avg10,
|
||||||
})
|
})
|
||||||
.collect(),
|
.collect(),
|
||||||
actor_summary: ActorSummarySnapshot {
|
actor_summary: ActorSummarySnapshot {
|
||||||
|
|
@ -667,6 +670,191 @@ mod tests {
|
||||||
assert_eq!(still_live.first(), Some(&"orch"));
|
assert_eq!(still_live.first(), Some(&"orch"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn hardware_channels_fold_into_fleet_snapshot() {
|
||||||
|
let view = ControlPlaneView::default();
|
||||||
|
let stream = StreamId::new(NodeId::new("worker"), Lifetime(1));
|
||||||
|
let cpu = json!({
|
||||||
|
"schema":"host.cpu.v1",
|
||||||
|
"seq":1,
|
||||||
|
"sample_unix_ms":1_000,
|
||||||
|
"query_elapsed_ms":1,
|
||||||
|
"host":{
|
||||||
|
"logical_cpus":8,
|
||||||
|
"total_percent":42.5,
|
||||||
|
"idle_percent":57.5,
|
||||||
|
"iowait_percent":0.0,
|
||||||
|
"steal_percent":0.0,
|
||||||
|
"load1":1.0,
|
||||||
|
"load5":0.5,
|
||||||
|
"load15":0.25
|
||||||
|
},
|
||||||
|
"cores":[
|
||||||
|
{"index":0,"total_percent":25.0,"idle_percent":75.0,"iowait_percent":0.0,"steal_percent":0.0},
|
||||||
|
{"index":1,"total_percent":60.0,"idle_percent":40.0,"iowait_percent":0.0,"steal_percent":0.0}
|
||||||
|
],
|
||||||
|
"processes":[],
|
||||||
|
"error":null
|
||||||
|
});
|
||||||
|
let gpu = json!({
|
||||||
|
"schema":"host.gpu.v1",
|
||||||
|
"seq":1,
|
||||||
|
"sample_unix_ms":1_000,
|
||||||
|
"query_elapsed_ms":2,
|
||||||
|
"gpus":[{
|
||||||
|
"index":0,
|
||||||
|
"uuid":"gpu-0",
|
||||||
|
"name":"test gpu",
|
||||||
|
"memory_used_mib":512,
|
||||||
|
"memory_total_mib":4096,
|
||||||
|
"utilization_gpu_percent":71,
|
||||||
|
"utilization_memory_percent":12,
|
||||||
|
"temperature_c":55,
|
||||||
|
"power_draw_w":25.0
|
||||||
|
}],
|
||||||
|
"processes":[],
|
||||||
|
"error":null
|
||||||
|
});
|
||||||
|
let memory = json!({
|
||||||
|
"schema":"host.memory.v1",
|
||||||
|
"seq":1,
|
||||||
|
"sample_unix_ms":2_000,
|
||||||
|
"query_elapsed_ms":1,
|
||||||
|
"total_bytes":16_000,
|
||||||
|
"available_bytes":4_000,
|
||||||
|
"used_bytes":12_000,
|
||||||
|
"cached_bytes":2_000,
|
||||||
|
"swap_total_bytes":8_000,
|
||||||
|
"swap_used_bytes":1_000,
|
||||||
|
"pressure":{
|
||||||
|
"some_avg10":1.25,
|
||||||
|
"some_avg60":0.75,
|
||||||
|
"some_avg300":0.5,
|
||||||
|
"some_total_us":100,
|
||||||
|
"full_avg10":0.1,
|
||||||
|
"full_avg60":0.05,
|
||||||
|
"full_avg300":0.01,
|
||||||
|
"full_total_us":10
|
||||||
|
},
|
||||||
|
"error":null
|
||||||
|
});
|
||||||
|
let net_sample = |seq, sample_unix_ms, rx_bytes, tx_bytes| {
|
||||||
|
json!({
|
||||||
|
"schema":"host.net.v1",
|
||||||
|
"seq":seq,
|
||||||
|
"sample_unix_ms":sample_unix_ms,
|
||||||
|
"interfaces":[{
|
||||||
|
"name":"eth0",
|
||||||
|
"rx_bytes":rx_bytes,
|
||||||
|
"tx_bytes":tx_bytes,
|
||||||
|
"rx_packets":10,
|
||||||
|
"tx_packets":10,
|
||||||
|
"rx_errors":0,
|
||||||
|
"tx_errors":0,
|
||||||
|
"rx_dropped":0,
|
||||||
|
"tx_dropped":0
|
||||||
|
}],
|
||||||
|
"error":null
|
||||||
|
})
|
||||||
|
};
|
||||||
|
let storage = json!({
|
||||||
|
"schema":"host.storage.v1",
|
||||||
|
"seq":1,
|
||||||
|
"sample_unix_ms":2_000,
|
||||||
|
"query_elapsed_ms":1,
|
||||||
|
"filesystems":[{
|
||||||
|
"mount":"/",
|
||||||
|
"total_bytes":100_000,
|
||||||
|
"used_bytes":80_000,
|
||||||
|
"available_bytes":20_000,
|
||||||
|
"used_percent":80.0
|
||||||
|
}],
|
||||||
|
"pressure":{
|
||||||
|
"some_avg10":2.5,
|
||||||
|
"some_avg60":1.5,
|
||||||
|
"some_avg300":0.5,
|
||||||
|
"some_total_us":200,
|
||||||
|
"full_avg10":0.2,
|
||||||
|
"full_avg60":0.1,
|
||||||
|
"full_avg300":0.05,
|
||||||
|
"full_total_us":20
|
||||||
|
},
|
||||||
|
"error":null
|
||||||
|
});
|
||||||
|
|
||||||
|
for (position, channel, payload) in [
|
||||||
|
(0, "host.cpu", cpu),
|
||||||
|
(1, "host.gpu", gpu),
|
||||||
|
(2, "host.memory", memory),
|
||||||
|
(3, "host.net", net_sample(0, 1_000, 1_000, 2_000)),
|
||||||
|
(4, "host.net", net_sample(1, 2_000, 2_000, 3_500)),
|
||||||
|
(5, "host.storage", storage),
|
||||||
|
] {
|
||||||
|
ingest_json(
|
||||||
|
&view,
|
||||||
|
&stream,
|
||||||
|
position,
|
||||||
|
channel,
|
||||||
|
serde_json::to_vec(&payload).expect("hardware payload"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let snapshot = view.snapshot_json();
|
||||||
|
let node = &snapshot["live"][0];
|
||||||
|
assert_eq!(node["cpu"]["host"]["total_percent"], json!(42.5));
|
||||||
|
assert_eq!(node["gpu"]["gpus"][0]["utilization_gpu_percent"], json!(71));
|
||||||
|
assert_eq!(node["cpu"]["cores"][1]["total_percent"], json!(60.0));
|
||||||
|
assert_eq!(node["memory"]["used_bytes"], json!(12_000));
|
||||||
|
assert_eq!(node["net"]["interfaces"][0]["rx_bps"], json!(1_000.0));
|
||||||
|
assert_eq!(node["net"]["interfaces"][0]["tx_bps"], json!(1_500.0));
|
||||||
|
assert_eq!(
|
||||||
|
node["storage"]["filesystems"][0]["used_percent"],
|
||||||
|
json!(80.0)
|
||||||
|
);
|
||||||
|
assert_eq!(node["history"][0]["cpu_cores_percent"], json!([25.0, 60.0]));
|
||||||
|
assert_eq!(
|
||||||
|
node["history"][0]["memory_pressure_some_avg10"],
|
||||||
|
json!(1.25)
|
||||||
|
);
|
||||||
|
assert_eq!(node["history"][0]["io_pressure_some_avg10"], json!(2.5));
|
||||||
|
assert_eq!(node["last_sample_unix_ms"], json!(2_000));
|
||||||
|
assert!(node["errors"].as_array().expect("errors").is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn roster_default_order_does_not_follow_volatile_throughput() {
|
||||||
|
let view = ControlPlaneView::default();
|
||||||
|
let stream = StreamId::new(NodeId::new("node"), Lifetime(1));
|
||||||
|
ingest_json(
|
||||||
|
&view,
|
||||||
|
&stream,
|
||||||
|
0,
|
||||||
|
"runtime.actors",
|
||||||
|
actors_payload(
|
||||||
|
0,
|
||||||
|
json!([
|
||||||
|
{ "address": "zz", "messages_processed": 100 },
|
||||||
|
{ "address": "aa", "messages_processed": 1 }
|
||||||
|
]),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
{
|
||||||
|
let mut state = view.state.write();
|
||||||
|
let actors = &mut state.streams.get_mut("node#1").expect("node").actors.actors;
|
||||||
|
actors.get_mut("zz").expect("zz actor").msg_per_sec = 10_000.0;
|
||||||
|
actors.get_mut("aa").expect("aa actor").msg_per_sec = 1.0;
|
||||||
|
}
|
||||||
|
|
||||||
|
let snapshot = view.snapshot_json();
|
||||||
|
let addresses: Vec<&str> = snapshot["live"][0]["roster"]
|
||||||
|
.as_array()
|
||||||
|
.expect("roster")
|
||||||
|
.iter()
|
||||||
|
.map(|actor| actor["address"].as_str().expect("address"))
|
||||||
|
.collect();
|
||||||
|
assert_eq!(addresses, vec!["aa", "zz"]);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn newer_life_generation_evicts_superseded_stream() {
|
fn newer_life_generation_evicts_superseded_stream() {
|
||||||
let view = ControlPlaneView::default();
|
let view = ControlPlaneView::default();
|
||||||
|
|
@ -701,7 +889,7 @@ mod tests {
|
||||||
fn stale_pool_is_hard_capped() {
|
fn stale_pool_is_hard_capped() {
|
||||||
let view = ControlPlaneView::default();
|
let view = ControlPlaneView::default();
|
||||||
for index in 0..(STALE_POOL_CAP as u64 + 5) {
|
for index in 0..(STALE_POOL_CAP as u64 + 5) {
|
||||||
let stream = StreamId::new(NodeId::new(&format!("old-{index}")), Lifetime(1));
|
let stream = StreamId::new(NodeId::new(format!("old-{index}")), Lifetime(1));
|
||||||
ingest_json(
|
ingest_json(
|
||||||
&view,
|
&view,
|
||||||
&stream,
|
&stream,
|
||||||
|
|
|
||||||
|
|
@ -87,6 +87,12 @@
|
||||||
table { width: 100%; border-collapse: collapse; font-family: var(--mono); font-size: 12px; }
|
table { width: 100%; border-collapse: collapse; font-family: var(--mono); font-size: 12px; }
|
||||||
th, td { text-align: left; padding: 7px 8px; border-bottom: 1px solid var(--divider); font-variant-numeric: tabular-nums; }
|
th, td { text-align: left; padding: 7px 8px; border-bottom: 1px solid var(--divider); font-variant-numeric: tabular-nums; }
|
||||||
th { color: var(--amber); font-weight: 600; font-size: 10px; text-transform: uppercase; letter-spacing: .08em; }
|
th { color: var(--amber); font-weight: 600; font-size: 10px; text-transform: uppercase; letter-spacing: .08em; }
|
||||||
|
.roster-sort-controls { display: inline-flex; gap: 2px; margin-left: 4px; vertical-align: middle; }
|
||||||
|
.roster-sort-button { position: relative; width: 14px; height: 14px; padding: 0; color: var(--muted); background: transparent; border: 1px solid transparent; border-radius: var(--r); font-size: 0; line-height: 0; cursor: pointer; }
|
||||||
|
.roster-sort-button::before { content: ""; position: absolute; left: 3px; width: 0; height: 0; border-left: 3px solid transparent; border-right: 3px solid transparent; }
|
||||||
|
.roster-sort-button[data-direction="asc"]::before { top: 3px; border-bottom: 5px solid currentColor; }
|
||||||
|
.roster-sort-button[data-direction="desc"]::before { bottom: 3px; border-top: 5px solid currentColor; }
|
||||||
|
.roster-sort-button:hover, .roster-sort-button[aria-pressed="true"] { color: var(--cyan); border-color: var(--cyan); background: var(--panel-hover); }
|
||||||
tbody tr[data-addr] { cursor: pointer; transition: background var(--t); }
|
tbody tr[data-addr] { cursor: pointer; transition: background var(--t); }
|
||||||
tbody tr[data-addr]:hover { background: var(--row-hover); }
|
tbody tr[data-addr]:hover { background: var(--row-hover); }
|
||||||
tbody tr[data-selected="true"] { background: var(--selected); box-shadow: inset 2px 0 0 var(--selected-edge); }
|
tbody tr[data-selected="true"] { background: var(--selected); box-shadow: inset 2px 0 0 var(--selected-edge); }
|
||||||
|
|
@ -102,6 +108,33 @@
|
||||||
.empty { padding: 24px; border: 1px dashed var(--border); border-radius: var(--r); color: var(--muted); font-family: var(--mono); font-size: 12px; }
|
.empty { padding: 24px; border: 1px dashed var(--border); border-radius: var(--r); color: var(--muted); font-family: var(--mono); font-size: 12px; }
|
||||||
.notice { font-size: 12px; color: var(--muted); margin: 6px 0; }
|
.notice { font-size: 12px; color: var(--muted); margin: 6px 0; }
|
||||||
canvas { width: 100%; height: 64px; background: var(--inset); border: 1px solid var(--border); border-radius: var(--r); }
|
canvas { width: 100%; height: 64px; background: var(--inset); border: 1px solid var(--border); border-radius: var(--r); }
|
||||||
|
.hardware-source { margin-bottom: 8px; }
|
||||||
|
.hardware-grid { display: grid; grid-template-columns: repeat(12, minmax(0, 1fr)); gap: 8px; }
|
||||||
|
.hw-card { min-width: 0; padding: 10px; background: var(--inset); border: 1px solid var(--divider); border-radius: var(--r); }
|
||||||
|
.hw-card.cpu { grid-column: span 7; }
|
||||||
|
.hw-card.memory { grid-column: span 5; }
|
||||||
|
.hw-card.storage, .hw-card.gpu, .hw-card.network { grid-column: span 4; }
|
||||||
|
.hw-card-head { display: flex; align-items: baseline; justify-content: space-between; gap: 8px; margin-bottom: 7px; }
|
||||||
|
.hw-card-title { color: var(--muted); font: 700 10px/1.2 var(--mono); text-transform: uppercase; letter-spacing: .09em; }
|
||||||
|
.hw-card-value { color: var(--text); font: 700 16px/1 var(--mono); font-variant-numeric: tabular-nums; }
|
||||||
|
.hw-card-subtitle { margin-top: 5px; color: var(--muted); font: 11px/1.35 var(--mono); }
|
||||||
|
.metric-pair { display: flex; justify-content: space-between; gap: 8px; margin-top: 6px; color: var(--muted); font: 11px/1.3 var(--mono); }
|
||||||
|
.metric-pair strong { color: var(--text); font-weight: 600; }
|
||||||
|
.pressure-value.ok { color: var(--ok); } .pressure-value.warn { color: var(--amber); } .pressure-value.bad { color: var(--bad); }
|
||||||
|
canvas.thread-graph { height: 72px; background: var(--bg); border-color: var(--divider); image-rendering: pixelated; }
|
||||||
|
.thread-strip { display: grid; grid-template-columns: repeat(auto-fit, minmax(26px, 1fr)); gap: 3px; margin-top: 5px; }
|
||||||
|
.thread-cell { min-width: 0; height: 20px; display: grid; place-items: center; border: 1px solid var(--divider); border-radius: var(--r); color: var(--text); font: 9px/1 var(--mono); font-variant-numeric: tabular-nums; }
|
||||||
|
.network-list { display: grid; gap: 6px; }
|
||||||
|
.network-row { display: grid; grid-template-columns: minmax(0, 1fr) auto; gap: 8px; font: 11px/1.25 var(--mono); }
|
||||||
|
.network-row .interface { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; color: var(--muted); }
|
||||||
|
.hw-empty { color: var(--muted); font: 11px/1.35 var(--mono); }
|
||||||
|
@media (max-width: 900px) {
|
||||||
|
.hw-card.cpu { grid-column: span 12; }
|
||||||
|
.hw-card.memory, .hw-card.storage, .hw-card.gpu, .hw-card.network { grid-column: span 6; }
|
||||||
|
}
|
||||||
|
@media (max-width: 600px) {
|
||||||
|
.hw-card.cpu, .hw-card.memory, .hw-card.storage, .hw-card.gpu, .hw-card.network { grid-column: span 12; }
|
||||||
|
}
|
||||||
.err { color: var(--bad); }
|
.err { color: var(--bad); }
|
||||||
@media (prefers-reduced-motion: reduce) { * { transition-duration: 0.01ms !important; animation: none !important; } }
|
@media (prefers-reduced-motion: reduce) { * { transition-duration: 0.01ms !important; animation: none !important; } }
|
||||||
</style>
|
</style>
|
||||||
|
|
@ -134,6 +167,8 @@ const params = new URLSearchParams(window.location.search);
|
||||||
let selectedStream = params.get('stream') || null;
|
let selectedStream = params.get('stream') || null;
|
||||||
let selectedActor = params.get('actor') || null;
|
let selectedActor = params.get('actor') || null;
|
||||||
let rosterFilter = '';
|
let rosterFilter = '';
|
||||||
|
let rosterSort = { key: 'address', direction: 'asc' };
|
||||||
|
let hardwareSource = 'node';
|
||||||
let lastSnapshot = null;
|
let lastSnapshot = null;
|
||||||
let detailTimer = null;
|
let detailTimer = null;
|
||||||
// A1: last-rendered HTML per region. A poll that yields identical markup
|
// A1: last-rendered HTML per region. A poll that yields identical markup
|
||||||
|
|
@ -144,6 +179,14 @@ let lastRosterHtml = null;
|
||||||
let lastMachineHtml = null;
|
let lastMachineHtml = null;
|
||||||
let lastDossierHtml = null;
|
let lastDossierHtml = null;
|
||||||
|
|
||||||
|
window.addEventListener('dashboard-hardware-source', event => {
|
||||||
|
const next = event.detail?.source === 'orchestrator' ? 'orchestrator' : 'node';
|
||||||
|
if (next === hardwareSource) return;
|
||||||
|
hardwareSource = next;
|
||||||
|
lastMachineHtml = null;
|
||||||
|
if (lastSnapshot) render();
|
||||||
|
});
|
||||||
|
|
||||||
// Receipt ages (dossier) render as empty spans carrying an absolute epoch —
|
// Receipt ages (dossier) render as empty spans carrying an absolute epoch —
|
||||||
// stable across polls — and a 1 Hz pass rewrites their textContent. All other
|
// stable across polls — and a 1 Hz pass rewrites their textContent. All other
|
||||||
// ticking text (card seen/stats) is written per-poll by updateCardTexts.
|
// ticking text (card seen/stats) is written per-poll by updateCardTexts.
|
||||||
|
|
@ -173,6 +216,17 @@ function fmtRate(value) {
|
||||||
if (n >= 1e3) return (n / 1e3).toFixed(1) + ' k';
|
if (n >= 1e3) return (n / 1e3).toFixed(1) + ' k';
|
||||||
return n.toFixed(n < 10 ? 1 : 0);
|
return n.toFixed(n < 10 ? 1 : 0);
|
||||||
}
|
}
|
||||||
|
function fmtBytes(value) {
|
||||||
|
const n = Number(value);
|
||||||
|
if (!Number.isFinite(n)) return '—';
|
||||||
|
const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
|
||||||
|
let scaled = Math.max(0, n), unit = 0;
|
||||||
|
while (scaled >= 1024 && unit < units.length - 1) {
|
||||||
|
scaled /= 1024;
|
||||||
|
unit += 1;
|
||||||
|
}
|
||||||
|
return scaled.toFixed(scaled < 10 && unit > 0 ? 1 : 0) + ' ' + units[unit];
|
||||||
|
}
|
||||||
function typeShort(name) {
|
function typeShort(name) {
|
||||||
if (!name) return '—';
|
if (!name) return '—';
|
||||||
const parts = String(name).split('::');
|
const parts = String(name).split('::');
|
||||||
|
|
@ -291,6 +345,14 @@ function updateCardTexts(nodes) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function hardwareNodeFor(node, live, stale) {
|
||||||
|
if (hardwareSource !== 'orchestrator' || node.stream.origin === 'orchestrator') {
|
||||||
|
return { node, mirrored: false };
|
||||||
|
}
|
||||||
|
const orchestrator = live.concat(stale).find(candidate => candidate.stream.origin === 'orchestrator');
|
||||||
|
return orchestrator ? { node: orchestrator, mirrored: true } : { node, mirrored: false };
|
||||||
|
}
|
||||||
|
|
||||||
function renderNode(page, node, live, stale) {
|
function renderNode(page, node, live, stale) {
|
||||||
const summary = node.actor_summary || {};
|
const summary = node.actor_summary || {};
|
||||||
const active = document.activeElement;
|
const active = document.activeElement;
|
||||||
|
|
@ -333,7 +395,8 @@ function renderNode(page, node, live, stale) {
|
||||||
if (actorCount) actorCount.textContent = fmt(summary.actors);
|
if (actorCount) actorCount.textContent = fmt(summary.actors);
|
||||||
const seen = page.querySelector('h2 [data-seen]');
|
const seen = page.querySelector('h2 [data-seen]');
|
||||||
if (seen) seen.textContent = node.live ? 'live' : 'stale ' + ago(node.last_seen_ms_ago) + ' ago';
|
if (seen) seen.textContent = node.live ? 'live' : 'stale ' + ago(node.last_seen_ms_ago) + ' ago';
|
||||||
const machine = machineDetail(node);
|
const machineSource = hardwareNodeFor(node, live, stale);
|
||||||
|
const machine = machineDetail(machineSource.node, machineSource.mirrored);
|
||||||
const machineSlot = document.getElementById('machine-detail');
|
const machineSlot = document.getElementById('machine-detail');
|
||||||
if (machineSlot && machine !== lastMachineHtml) {
|
if (machineSlot && machine !== lastMachineHtml) {
|
||||||
machineSlot.innerHTML = machine;
|
machineSlot.innerHTML = machine;
|
||||||
|
|
@ -347,67 +410,192 @@ function renderNode(page, node, live, stale) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function machineDetail(node) {
|
function percentOf(used, total) {
|
||||||
const cpu = node.cpu, gpu = node.gpu, net = node.net;
|
const numerator = Number(used), denominator = Number(total);
|
||||||
const rows = [];
|
return Number.isFinite(numerator) && Number.isFinite(denominator) && denominator > 0
|
||||||
const cpuTotal = cpu && cpu.host ? cpu.host.total_percent : null;
|
? numerator * 100 / denominator
|
||||||
rows.push(`<div class="bar-row"><span class="name">CPU</span>${bar(cpuTotal)}<span class="num">${cpuTotal == null ? '—' : fmt(cpuTotal, 1) + '%'}</span></div>`);
|
: null;
|
||||||
if (gpu && gpu.gpus) {
|
|
||||||
gpu.gpus.forEach((device, index) => {
|
|
||||||
const util = device.utilization_gpu_percent;
|
|
||||||
const used = device.memory_used_mib || 0, total = device.memory_total_mib || 0;
|
|
||||||
rows.push(`<div class="bar-row"><span class="name">GPU ${fmt(index)}</span>${bar(util)}<span class="num">${util == null ? '—' : fmt(util) + '%'}</span></div>`);
|
|
||||||
if (total) rows.push(`<div class="bar-row"><span class="name"> </span>${bar(used, total)}<span class="num">${fmt(used)}/${fmt(total)} MiB</span></div>`);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
if (net && net.interfaces) {
|
|
||||||
net.interfaces.slice(0, 4).forEach(nic => {
|
|
||||||
rows.push(`<div class="bar-row"><span class="name" title="${esc(nic.name)}">${esc(nic.name.slice(0, 12))}</span><span class="num" style="width:auto;flex:1">${fmtRate(nic.rx_bps)}↓ ${fmtRate(nic.tx_bps)}↑</span></div>`);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
if (node.history && node.history.length) drawMachineHistory(node.history);
|
|
||||||
const errors = (node.errors || []).map(e => `<div class="notice err">${esc(e)}</div>`).join('');
|
|
||||||
return `<div class="bars">${rows.join('')}</div><canvas id="machine-spark" width="600" height="64"></canvas>${errors}`;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function drawMachineHistory(history) {
|
function pressureTone(value, warn = 1, bad = 5) {
|
||||||
|
const pressure = Number(value);
|
||||||
|
if (!Number.isFinite(pressure)) return '';
|
||||||
|
return pressure >= bad ? 'bad' : pressure >= warn ? 'warn' : 'ok';
|
||||||
|
}
|
||||||
|
|
||||||
|
function machineDetail(node, mirrored = false) {
|
||||||
|
const cpu = node.cpu, gpu = node.gpu, memory = node.memory, net = node.net, storage = node.storage;
|
||||||
|
const cpuHost = cpu?.host;
|
||||||
|
const cpuTotal = cpuHost?.total_percent;
|
||||||
|
const cores = [...(cpu?.cores || [])].sort((left, right) => left.index - right.index);
|
||||||
|
const threadHeight = Math.min(92, Math.max(42, cores.length * 4));
|
||||||
|
drawThreadGraph(node.history || [], cores.map(core => core.total_percent));
|
||||||
|
const threadCells = cores.map(core => {
|
||||||
|
const value = core.total_percent;
|
||||||
|
const load = value == null ? 0 : Math.max(0, Math.min(100, Number(value)));
|
||||||
|
const label = value == null ? '—' : fmt(value);
|
||||||
|
return `<span class="thread-cell" title="Thread ${fmt(core.index)} · ${value == null ? 'no sample' : fmt(value, 1) + '%'}" style="background:linear-gradient(to top,var(--ok) 0%,var(--ok) ${load}%,var(--bg) ${load}%,var(--bg) 100%)">${label}</span>`;
|
||||||
|
}).join('');
|
||||||
|
const load = [cpuHost?.load1, cpuHost?.load5, cpuHost?.load15].map(value => fmt(value, 2)).join(' / ');
|
||||||
|
const cpuLabel = cpuTotal == null ? '—' : fmt(cpuTotal, 1) + '%';
|
||||||
|
|
||||||
|
const memoryPercent = percentOf(memory?.used_bytes, memory?.total_bytes);
|
||||||
|
const memoryPressure = memory?.pressure?.some_avg10;
|
||||||
|
const swapPercent = percentOf(memory?.swap_used_bytes, memory?.swap_total_bytes);
|
||||||
|
const memoryLabel = memoryPercent == null ? '—' : fmt(memoryPercent, 1) + '%';
|
||||||
|
|
||||||
|
const filesystem = storage?.filesystems?.[0];
|
||||||
|
const storagePercent = filesystem?.used_percent;
|
||||||
|
const ioPressure = storage?.pressure?.some_avg10;
|
||||||
|
const storageLabel = storagePercent == null ? '—' : fmt(storagePercent, 1) + '%';
|
||||||
|
|
||||||
|
const gpuDevices = gpu?.gpus || [];
|
||||||
|
const gpuMax = gpuDevices.reduce((maximum, device) => {
|
||||||
|
const value = Number(device.utilization_gpu_percent);
|
||||||
|
return Number.isFinite(value) ? Math.max(maximum, value) : maximum;
|
||||||
|
}, 0);
|
||||||
|
const gpuRows = gpuDevices.map((device, index) => {
|
||||||
|
const utilization = device.utilization_gpu_percent;
|
||||||
|
const memoryPercent = percentOf(device.memory_used_mib, device.memory_total_mib);
|
||||||
|
return `<div class="metric-pair"><span>GPU ${fmt(index)}</span><strong>${utilization == null ? '—' : fmt(utilization) + '%'}</strong></div>
|
||||||
|
${bar(utilization)}
|
||||||
|
<div class="metric-pair"><span>Memory</span><strong>${device.memory_total_mib ? fmt(device.memory_used_mib) + ' / ' + fmt(device.memory_total_mib) + ' MiB' : '—'}</strong></div>
|
||||||
|
${device.memory_total_mib ? bar(memoryPercent) : ''}`;
|
||||||
|
}).join('');
|
||||||
|
|
||||||
|
const networkRows = (net?.interfaces || []).slice(0, 4).map(nic => `
|
||||||
|
<div class="network-row">
|
||||||
|
<span class="interface" title="${esc(nic.name)}">${esc(nic.name)}</span>
|
||||||
|
<span>${fmtRate(nic.rx_bps)}↓ ${fmtRate(nic.tx_bps)}↑</span>
|
||||||
|
</div>`).join('');
|
||||||
|
|
||||||
|
const source = mirrored ? '<div class="hardware-source muted">Mock node · local orchestrator hardware</div>' : '';
|
||||||
|
const errors = (node.errors || []).map(error => `<div class="notice err">${esc(error)}</div>`).join('');
|
||||||
|
return `${source}<div class="hardware-grid">
|
||||||
|
<section class="hw-card cpu">
|
||||||
|
<div class="hw-card-head"><span class="hw-card-title">CPU · ${fmt(cores.length)} threads</span><span class="hw-card-value">${cpuLabel}</span></div>
|
||||||
|
<canvas id="thread-graph" class="thread-graph" width="560" height="${threadHeight}" style="height:${threadHeight}px" role="img" aria-label="Per-thread CPU utilization history"></canvas>
|
||||||
|
<div class="thread-strip">${threadCells || '<span class="hw-empty">Waiting for per-thread samples</span>'}</div>
|
||||||
|
<div class="hw-card-subtitle">Load 1 / 5 / 15 min ${load}</div>
|
||||||
|
</section>
|
||||||
|
<section class="hw-card memory">
|
||||||
|
<div class="hw-card-head"><span class="hw-card-title">Memory</span><span class="hw-card-value">${memoryLabel}</span></div>
|
||||||
|
${bar(memoryPercent)}
|
||||||
|
<div class="metric-pair"><span>Used</span><strong>${fmtBytes(memory?.used_bytes)} / ${fmtBytes(memory?.total_bytes)}</strong></div>
|
||||||
|
<div class="metric-pair"><span>Available</span><strong>${fmtBytes(memory?.available_bytes)}</strong></div>
|
||||||
|
<div class="metric-pair"><span>Swap</span><strong>${swapPercent == null ? '—' : fmt(swapPercent, 1) + '%'}</strong></div>
|
||||||
|
<div class="metric-pair"><span>PSI some · 10s</span><strong class="pressure-value ${pressureTone(memoryPressure)}">${memoryPressure == null ? '—' : fmt(memoryPressure, 2) + '%'}</strong></div>
|
||||||
|
</section>
|
||||||
|
<section class="hw-card storage">
|
||||||
|
<div class="hw-card-head"><span class="hw-card-title">Storage · ${esc(filesystem?.mount || '/')}</span><span class="hw-card-value">${storageLabel}</span></div>
|
||||||
|
${bar(storagePercent)}
|
||||||
|
<div class="metric-pair"><span>Used</span><strong>${fmtBytes(filesystem?.used_bytes)} / ${fmtBytes(filesystem?.total_bytes)}</strong></div>
|
||||||
|
<div class="metric-pair"><span>Available</span><strong>${fmtBytes(filesystem?.available_bytes)}</strong></div>
|
||||||
|
<div class="metric-pair"><span>I/O PSI · 10s</span><strong class="pressure-value ${pressureTone(ioPressure)}">${ioPressure == null ? '—' : fmt(ioPressure, 2) + '%'}</strong></div>
|
||||||
|
</section>
|
||||||
|
<section class="hw-card gpu">
|
||||||
|
<div class="hw-card-head"><span class="hw-card-title">GPU · ${fmt(gpuDevices.length)} devices</span><span class="hw-card-value">${gpuDevices.length ? fmt(gpuMax) + '%' : '—'}</span></div>
|
||||||
|
${gpuRows || '<div class="hw-empty">No GPU telemetry</div>'}
|
||||||
|
</section>
|
||||||
|
<section class="hw-card network">
|
||||||
|
<div class="hw-card-head"><span class="hw-card-title">Network</span><span class="hw-card-value">${fmt(net?.interfaces?.length || 0)}</span></div>
|
||||||
|
<div class="network-list">${networkRows || '<div class="hw-empty">No interface telemetry</div>'}</div>
|
||||||
|
</section>
|
||||||
|
</div>${errors}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function drawThreadGraph(history, currentCores) {
|
||||||
requestAnimationFrame(() => {
|
requestAnimationFrame(() => {
|
||||||
const canvas = document.getElementById('machine-spark');
|
const canvas = document.getElementById('thread-graph');
|
||||||
if (!canvas) return;
|
if (!canvas) return;
|
||||||
|
const ratio = Math.max(1, window.devicePixelRatio || 1);
|
||||||
|
const rect = canvas.getBoundingClientRect();
|
||||||
|
const width = Math.max(1, Math.round(rect.width));
|
||||||
|
const height = Math.max(1, Math.round(rect.height));
|
||||||
|
const pixelWidth = Math.round(width * ratio), pixelHeight = Math.round(height * ratio);
|
||||||
|
if (canvas.width !== pixelWidth || canvas.height !== pixelHeight) {
|
||||||
|
canvas.width = pixelWidth;
|
||||||
|
canvas.height = pixelHeight;
|
||||||
|
}
|
||||||
const ctx = canvas.getContext('2d');
|
const ctx = canvas.getContext('2d');
|
||||||
ctx.clearRect(0, 0, canvas.width, canvas.height);
|
ctx.setTransform(ratio, 0, 0, ratio, 0, 0);
|
||||||
const points = history.map(h => ({ cpu: h.cpu_total_percent, gpu: h.gpu_max_percent }));
|
const styles = getComputedStyle(document.documentElement);
|
||||||
drawLine(ctx, points.map(p => p.gpu), '#fbbf24');
|
const background = styles.getPropertyValue('--bg').trim() || '#00060c';
|
||||||
drawLine(ctx, points.map(p => p.cpu), '#34d399');
|
const divider = styles.getPropertyValue('--divider').trim() || '#0d2c4a';
|
||||||
|
const foreground = styles.getPropertyValue('--ok').trim() || '#00d400';
|
||||||
|
ctx.globalAlpha = 1;
|
||||||
|
ctx.fillStyle = background;
|
||||||
|
ctx.fillRect(0, 0, width, height);
|
||||||
|
|
||||||
|
const threadCount = Math.max(currentCores.length, ...history.map(point => point.cpu_cores_percent?.length || 0));
|
||||||
|
if (!threadCount) return;
|
||||||
|
const rowHeight = height / threadCount;
|
||||||
|
ctx.fillStyle = divider;
|
||||||
|
for (let row = 1; row < threadCount; row += 1) {
|
||||||
|
ctx.fillRect(0, Math.floor(row * rowHeight), width, 1);
|
||||||
|
}
|
||||||
|
const slot = 2;
|
||||||
|
const points = history
|
||||||
|
.map(point => point.cpu_cores_percent || [])
|
||||||
|
.slice(-Math.floor(width / slot));
|
||||||
|
if (!points.length) points.push(currentCores);
|
||||||
|
ctx.fillStyle = foreground;
|
||||||
|
points.forEach((values, pointIndex) => {
|
||||||
|
const x = width - (points.length - pointIndex) * slot;
|
||||||
|
values.forEach((value, threadIndex) => {
|
||||||
|
if (value == null || !Number.isFinite(Number(value))) return;
|
||||||
|
const utilization = Math.max(0, Math.min(100, Number(value)));
|
||||||
|
ctx.globalAlpha = 0.10 + utilization * 0.009;
|
||||||
|
const y = Math.ceil(threadIndex * rowHeight);
|
||||||
|
ctx.fillRect(x, y, slot, Math.max(1, Math.floor(rowHeight) - 1));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
ctx.globalAlpha = 1;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function drawLine(ctx, values, color) {
|
function rosterSortControls(key, label) {
|
||||||
const valid = values.filter(v => v != null);
|
return `<span class="roster-sort-controls">
|
||||||
if (valid.length < 2) return;
|
<button type="button" class="roster-sort-button" data-sort="${key}" data-direction="asc" aria-label="Sort ${label} ascending" title="Sort ${label} ascending" aria-pressed="${rosterSort.key === key && rosterSort.direction === 'asc'}">↑</button>
|
||||||
const max = Math.max(100, ...valid);
|
<button type="button" class="roster-sort-button" data-sort="${key}" data-direction="desc" aria-label="Sort ${label} descending" title="Sort ${label} descending" aria-pressed="${rosterSort.key === key && rosterSort.direction === 'desc'}">↓</button>
|
||||||
const step = ctx.canvas.width / (values.length - 1 || 1);
|
</span>`;
|
||||||
ctx.strokeStyle = color;
|
}
|
||||||
ctx.lineWidth = 1.5;
|
|
||||||
ctx.beginPath();
|
function compareRosterValues(left, right) {
|
||||||
let started = false;
|
const leftMissing = left == null || left === '';
|
||||||
values.forEach((value, index) => {
|
const rightMissing = right == null || right === '';
|
||||||
if (value == null) return;
|
if (leftMissing || rightMissing) return leftMissing === rightMissing ? 0 : leftMissing ? 1 : -1;
|
||||||
const x = index * step;
|
if (typeof left === 'number' && typeof right === 'number') return left - right;
|
||||||
const y = ctx.canvas.height - (value / max) * (ctx.canvas.height - 6) - 3;
|
if (typeof left === 'boolean' && typeof right === 'boolean') return Number(left) - Number(right);
|
||||||
if (!started) { ctx.moveTo(x, y); started = true; } else ctx.lineTo(x, y);
|
return String(left).localeCompare(String(right), undefined, { numeric: true, sensitivity: 'base' });
|
||||||
});
|
}
|
||||||
ctx.stroke();
|
|
||||||
|
function isLegacyHardwareSampler(actor) {
|
||||||
|
const actorType = String(actor.actor_type || '');
|
||||||
|
return (actorType.includes('BlockingSamplerActor') && actorType.includes('telemetry::hardware'))
|
||||||
|
|| /Host(?:Cpu|Gpu|Net)SamplerActor/.test(actorType);
|
||||||
}
|
}
|
||||||
|
|
||||||
function renderRoster(node) {
|
function renderRoster(node) {
|
||||||
const wrap = document.getElementById('roster-wrap');
|
const wrap = document.getElementById('roster-wrap');
|
||||||
if (!wrap) return;
|
if (!wrap) return;
|
||||||
let rows = node.roster || [];
|
let rows = [...(node.roster || [])];
|
||||||
|
if (hardwareSource === 'orchestrator') {
|
||||||
|
// Mock containers may come from an older local image. Current samplers are
|
||||||
|
// engine tasks; do not leak legacy sampler actors into the simulated roster.
|
||||||
|
rows = rows.filter(actor => !isLegacyHardwareSampler(actor));
|
||||||
|
const actorCount = document.querySelector('[data-actor-count]');
|
||||||
|
if (actorCount) actorCount.textContent = fmt(rows.length);
|
||||||
|
}
|
||||||
if (rosterFilter) {
|
if (rosterFilter) {
|
||||||
const q = rosterFilter.toLowerCase();
|
const q = rosterFilter.toLowerCase();
|
||||||
rows = rows.filter(a => `${a.name || ''} ${a.actor_type || ''} ${a.address} ${a.worker_id ?? ''}`.toLowerCase().includes(q));
|
rows = rows.filter(a => `${a.name || ''} ${a.actor_type || ''} ${a.address} ${a.worker_id ?? ''}`.toLowerCase().includes(q));
|
||||||
}
|
}
|
||||||
|
rows.sort((left, right) => {
|
||||||
|
const compared = compareRosterValues(left[rosterSort.key], right[rosterSort.key]);
|
||||||
|
return (rosterSort.direction === 'asc' ? compared : -compared)
|
||||||
|
|| String(left.address).localeCompare(String(right.address));
|
||||||
|
});
|
||||||
const capped = rows.slice(0, ROSTER_RENDER_CAP);
|
const capped = rows.slice(0, ROSTER_RENDER_CAP);
|
||||||
const notice = rows.length > capped.length
|
const notice = rows.length > capped.length
|
||||||
? `<div class="notice">showing ${fmt(capped.length)} of ${fmt(rows.length)} — refine the filter to see more</div>`
|
? `<div class="notice">showing ${fmt(capped.length)} of ${fmt(rows.length)} — refine the filter to see more</div>`
|
||||||
|
|
@ -417,7 +605,7 @@ function renderRoster(node) {
|
||||||
html = '<div class="empty">No actors on this stream (or none match the filter).</div>';
|
html = '<div class="empty">No actors on this stream (or none match the filter).</div>';
|
||||||
} else {
|
} else {
|
||||||
html = notice + `<table>
|
html = notice + `<table>
|
||||||
<thead><tr><th>Actor</th><th>Type</th><th>State</th><th>Mailbox</th><th>Msg/s</th><th>Processed</th><th>Worker</th><th>Last message</th></tr></thead>
|
<thead><tr><th>Actor${rosterSortControls('address', 'actor')}</th><th>Type${rosterSortControls('actor_type', 'type')}</th><th>State${rosterSortControls('poisoned', 'state')}</th><th>Mailbox${rosterSortControls('mailbox_depth', 'mailbox')}</th><th>Msg/s${rosterSortControls('msg_per_sec', 'throughput')}</th><th>Processed${rosterSortControls('messages_processed', 'processed count')}</th><th>Worker${rosterSortControls('worker_id', 'worker')}</th><th>Last message${rosterSortControls('last_msg_type', 'last message')}</th></tr></thead>
|
||||||
<tbody>${capped.map(a => `<tr data-addr="${esc(a.address)}" tabindex="0" ${a.address === selectedActor ? 'data-selected="true"' : ''}>
|
<tbody>${capped.map(a => `<tr data-addr="${esc(a.address)}" tabindex="0" ${a.address === selectedActor ? 'data-selected="true"' : ''}>
|
||||||
<td class="mono">${esc(shortAddr(a.address))}${a.name ? `<br><span class="muted">${esc(a.name)}</span>` : ''}</td>
|
<td class="mono">${esc(shortAddr(a.address))}${a.name ? `<br><span class="muted">${esc(a.name)}</span>` : ''}</td>
|
||||||
<td><span class="muted" title="${esc(a.actor_type || '')}">${typeShort(a.actor_type)}</span></td>
|
<td><span class="muted" title="${esc(a.actor_type || '')}">${typeShort(a.actor_type)}</span></td>
|
||||||
|
|
@ -552,6 +740,12 @@ pageEl.addEventListener('click', e => {
|
||||||
render();
|
render();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
const sortButton = e.target.closest('.roster-sort-button');
|
||||||
|
if (sortButton) {
|
||||||
|
rosterSort = { key: sortButton.dataset.sort, direction: sortButton.dataset.direction };
|
||||||
|
render();
|
||||||
|
return;
|
||||||
|
}
|
||||||
const rosterRow = e.target.closest('tr[data-addr]');
|
const rosterRow = e.target.closest('tr[data-addr]');
|
||||||
if (rosterRow) {
|
if (rosterRow) {
|
||||||
selectedActor = rosterRow.getAttribute('data-addr');
|
selectedActor = rosterRow.getAttribute('data-addr');
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,9 @@ use telemetry::hardware::cpu::{
|
||||||
use telemetry::hardware::gpu::{
|
use telemetry::hardware::gpu::{
|
||||||
GpuDeviceSample, GpuProcessSample, HOST_GPU_CHANNEL, HostGpuSample,
|
GpuDeviceSample, GpuProcessSample, HOST_GPU_CHANNEL, HostGpuSample,
|
||||||
};
|
};
|
||||||
|
use telemetry::hardware::memory::{HOST_MEMORY_CHANNEL, HostMemorySample};
|
||||||
use telemetry::hardware::net::{HOST_NET_CHANNEL, HostNetSample, NetInterfaceSample};
|
use telemetry::hardware::net::{HOST_NET_CHANNEL, HostNetSample, NetInterfaceSample};
|
||||||
|
use telemetry::hardware::storage::{HOST_STORAGE_CHANNEL, HostStorageSample};
|
||||||
|
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
|
|
||||||
|
|
@ -27,7 +29,9 @@ pub(crate) struct NodeHardwareState {
|
||||||
decode_errors: BTreeMap<&'static str, String>,
|
decode_errors: BTreeMap<&'static str, String>,
|
||||||
pub(crate) cpu: Option<HostCpuSample>,
|
pub(crate) cpu: Option<HostCpuSample>,
|
||||||
pub(crate) gpu: Option<HostGpuSample>,
|
pub(crate) gpu: Option<HostGpuSample>,
|
||||||
|
pub(crate) memory: Option<HostMemorySample>,
|
||||||
pub(crate) net: Option<NetSnapshot>,
|
pub(crate) net: Option<NetSnapshot>,
|
||||||
|
pub(crate) storage: Option<HostStorageSample>,
|
||||||
pub(crate) process: Option<ProcessSnapshot>,
|
pub(crate) process: Option<ProcessSnapshot>,
|
||||||
pub(crate) history: VecDeque<HardwareHistoryState>,
|
pub(crate) history: VecDeque<HardwareHistoryState>,
|
||||||
}
|
}
|
||||||
|
|
@ -39,7 +43,9 @@ impl NodeHardwareState {
|
||||||
decode_errors: BTreeMap::new(),
|
decode_errors: BTreeMap::new(),
|
||||||
cpu: None,
|
cpu: None,
|
||||||
gpu: None,
|
gpu: None,
|
||||||
|
memory: None,
|
||||||
net: None,
|
net: None,
|
||||||
|
storage: None,
|
||||||
process: None,
|
process: None,
|
||||||
history: VecDeque::with_capacity(HISTORY_CAP),
|
history: VecDeque::with_capacity(HISTORY_CAP),
|
||||||
}
|
}
|
||||||
|
|
@ -64,6 +70,14 @@ impl NodeHardwareState {
|
||||||
}
|
}
|
||||||
Err(error) => self.store_decode_error(HOST_GPU_CHANNEL, error),
|
Err(error) => self.store_decode_error(HOST_GPU_CHANNEL, error),
|
||||||
},
|
},
|
||||||
|
HOST_MEMORY_CHANNEL => match HostMemorySample::decode(payload) {
|
||||||
|
Ok(sample) => {
|
||||||
|
self.memory = Some(sample);
|
||||||
|
self.decode_errors.remove(HOST_MEMORY_CHANNEL);
|
||||||
|
self.update_history(now);
|
||||||
|
}
|
||||||
|
Err(error) => self.store_decode_error(HOST_MEMORY_CHANNEL, error),
|
||||||
|
},
|
||||||
HOST_NET_CHANNEL => match HostNetSample::decode(payload) {
|
HOST_NET_CHANNEL => match HostNetSample::decode(payload) {
|
||||||
Ok(sample) => {
|
Ok(sample) => {
|
||||||
self.net = Some(NetSnapshot::from_sample(sample, self.net.as_ref()));
|
self.net = Some(NetSnapshot::from_sample(sample, self.net.as_ref()));
|
||||||
|
|
@ -72,6 +86,14 @@ impl NodeHardwareState {
|
||||||
}
|
}
|
||||||
Err(error) => self.store_decode_error(HOST_NET_CHANNEL, error),
|
Err(error) => self.store_decode_error(HOST_NET_CHANNEL, error),
|
||||||
},
|
},
|
||||||
|
HOST_STORAGE_CHANNEL => match HostStorageSample::decode(payload) {
|
||||||
|
Ok(sample) => {
|
||||||
|
self.storage = Some(sample);
|
||||||
|
self.decode_errors.remove(HOST_STORAGE_CHANNEL);
|
||||||
|
self.update_history(now);
|
||||||
|
}
|
||||||
|
Err(error) => self.store_decode_error(HOST_STORAGE_CHANNEL, error),
|
||||||
|
},
|
||||||
_ => {
|
_ => {
|
||||||
if channel.starts_with("proc.") && channel.ends_with(".lifecycle") {
|
if channel.starts_with("proc.") && channel.ends_with(".lifecycle") {
|
||||||
self.process = decode_process_snapshot(payload);
|
self.process = decode_process_snapshot(payload);
|
||||||
|
|
@ -99,6 +121,35 @@ impl NodeHardwareState {
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.and_then(|sample| sample.host.as_ref())
|
.and_then(|sample| sample.host.as_ref())
|
||||||
.and_then(|host| host.total_percent);
|
.and_then(|host| host.total_percent);
|
||||||
|
let cpu_cores_percent = self
|
||||||
|
.cpu
|
||||||
|
.as_ref()
|
||||||
|
.map(|sample| {
|
||||||
|
sample
|
||||||
|
.cores
|
||||||
|
.iter()
|
||||||
|
.map(|core| core.total_percent)
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
let memory_used_percent = self.memory.as_ref().and_then(|sample| {
|
||||||
|
Some(sample.used_bytes? as f64 * 100.0 / sample.total_bytes?.max(1) as f64)
|
||||||
|
});
|
||||||
|
let memory_pressure_some_avg10 = self
|
||||||
|
.memory
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|sample| sample.pressure.as_ref())
|
||||||
|
.map(|pressure| pressure.some_avg10);
|
||||||
|
let storage_used_percent = self
|
||||||
|
.storage
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|sample| sample.filesystems.first())
|
||||||
|
.and_then(|filesystem| filesystem.used_percent);
|
||||||
|
let io_pressure_some_avg10 = self
|
||||||
|
.storage
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|sample| sample.pressure.as_ref())
|
||||||
|
.map(|pressure| pressure.some_avg10);
|
||||||
|
|
||||||
let mut gpu_max_percent = None;
|
let mut gpu_max_percent = None;
|
||||||
let mut gpu_memory_used_mib = 0_u64;
|
let mut gpu_memory_used_mib = 0_u64;
|
||||||
|
|
@ -128,17 +179,24 @@ impl NodeHardwareState {
|
||||||
sample_unix_ms: [
|
sample_unix_ms: [
|
||||||
self.cpu.as_ref().map(|sample| sample.sample_unix_ms),
|
self.cpu.as_ref().map(|sample| sample.sample_unix_ms),
|
||||||
self.gpu.as_ref().map(|sample| sample.sample_unix_ms),
|
self.gpu.as_ref().map(|sample| sample.sample_unix_ms),
|
||||||
|
self.memory.as_ref().map(|sample| sample.sample_unix_ms),
|
||||||
self.net.as_ref().map(|sample| sample.sample_unix_ms),
|
self.net.as_ref().map(|sample| sample.sample_unix_ms),
|
||||||
|
self.storage.as_ref().map(|sample| sample.sample_unix_ms),
|
||||||
]
|
]
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.flatten()
|
.flatten()
|
||||||
.max(),
|
.max(),
|
||||||
cpu_total_percent,
|
cpu_total_percent,
|
||||||
|
cpu_cores_percent,
|
||||||
gpu_max_percent,
|
gpu_max_percent,
|
||||||
gpu_memory_used_mib,
|
gpu_memory_used_mib,
|
||||||
gpu_memory_total_mib,
|
gpu_memory_total_mib,
|
||||||
net_rx_bps,
|
net_rx_bps,
|
||||||
net_tx_bps,
|
net_tx_bps,
|
||||||
|
memory_used_percent,
|
||||||
|
memory_pressure_some_avg10,
|
||||||
|
storage_used_percent,
|
||||||
|
io_pressure_some_avg10,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -149,11 +207,16 @@ impl NodeHardwareState {
|
||||||
{
|
{
|
||||||
last.sample_unix_ms = summary.sample_unix_ms;
|
last.sample_unix_ms = summary.sample_unix_ms;
|
||||||
last.cpu_total_percent = summary.cpu_total_percent;
|
last.cpu_total_percent = summary.cpu_total_percent;
|
||||||
|
last.cpu_cores_percent = summary.cpu_cores_percent.clone();
|
||||||
last.gpu_max_percent = summary.gpu_max_percent;
|
last.gpu_max_percent = summary.gpu_max_percent;
|
||||||
last.gpu_memory_used_mib = summary.gpu_memory_used_mib;
|
last.gpu_memory_used_mib = summary.gpu_memory_used_mib;
|
||||||
last.gpu_memory_total_mib = summary.gpu_memory_total_mib;
|
last.gpu_memory_total_mib = summary.gpu_memory_total_mib;
|
||||||
last.net_rx_bps = summary.net_rx_bps;
|
last.net_rx_bps = summary.net_rx_bps;
|
||||||
last.net_tx_bps = summary.net_tx_bps;
|
last.net_tx_bps = summary.net_tx_bps;
|
||||||
|
last.memory_used_percent = summary.memory_used_percent;
|
||||||
|
last.memory_pressure_some_avg10 = summary.memory_pressure_some_avg10;
|
||||||
|
last.storage_used_percent = summary.storage_used_percent;
|
||||||
|
last.io_pressure_some_avg10 = summary.io_pressure_some_avg10;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -164,11 +227,16 @@ impl NodeHardwareState {
|
||||||
at: now,
|
at: now,
|
||||||
sample_unix_ms: summary.sample_unix_ms,
|
sample_unix_ms: summary.sample_unix_ms,
|
||||||
cpu_total_percent: summary.cpu_total_percent,
|
cpu_total_percent: summary.cpu_total_percent,
|
||||||
|
cpu_cores_percent: summary.cpu_cores_percent,
|
||||||
gpu_max_percent: summary.gpu_max_percent,
|
gpu_max_percent: summary.gpu_max_percent,
|
||||||
gpu_memory_used_mib: summary.gpu_memory_used_mib,
|
gpu_memory_used_mib: summary.gpu_memory_used_mib,
|
||||||
gpu_memory_total_mib: summary.gpu_memory_total_mib,
|
gpu_memory_total_mib: summary.gpu_memory_total_mib,
|
||||||
net_rx_bps: summary.net_rx_bps,
|
net_rx_bps: summary.net_rx_bps,
|
||||||
net_tx_bps: summary.net_tx_bps,
|
net_tx_bps: summary.net_tx_bps,
|
||||||
|
memory_used_percent: summary.memory_used_percent,
|
||||||
|
memory_pressure_some_avg10: summary.memory_pressure_some_avg10,
|
||||||
|
storage_used_percent: summary.storage_used_percent,
|
||||||
|
io_pressure_some_avg10: summary.io_pressure_some_avg10,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -180,9 +248,23 @@ impl NodeHardwareState {
|
||||||
if let Some(error) = self.gpu.as_ref().and_then(|sample| sample.error.as_ref()) {
|
if let Some(error) = self.gpu.as_ref().and_then(|sample| sample.error.as_ref()) {
|
||||||
errors.push(format!("{HOST_GPU_CHANNEL}: {error}"));
|
errors.push(format!("{HOST_GPU_CHANNEL}: {error}"));
|
||||||
}
|
}
|
||||||
|
if let Some(error) = self
|
||||||
|
.memory
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|sample| sample.error.as_ref())
|
||||||
|
{
|
||||||
|
errors.push(format!("{HOST_MEMORY_CHANNEL}: {error}"));
|
||||||
|
}
|
||||||
if let Some(error) = self.net.as_ref().and_then(|sample| sample.error.as_ref()) {
|
if let Some(error) = self.net.as_ref().and_then(|sample| sample.error.as_ref()) {
|
||||||
errors.push(format!("{HOST_NET_CHANNEL}: {error}"));
|
errors.push(format!("{HOST_NET_CHANNEL}: {error}"));
|
||||||
}
|
}
|
||||||
|
if let Some(error) = self
|
||||||
|
.storage
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|sample| sample.error.as_ref())
|
||||||
|
{
|
||||||
|
errors.push(format!("{HOST_STORAGE_CHANNEL}: {error}"));
|
||||||
|
}
|
||||||
errors
|
errors
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -308,26 +390,36 @@ impl NetInterfaceSnapshot {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy)]
|
#[derive(Clone)]
|
||||||
pub(crate) struct HardwareSummary {
|
pub(crate) struct HardwareSummary {
|
||||||
pub(crate) sample_unix_ms: Option<u64>,
|
pub(crate) sample_unix_ms: Option<u64>,
|
||||||
pub(crate) cpu_total_percent: Option<f64>,
|
pub(crate) cpu_total_percent: Option<f64>,
|
||||||
|
pub(crate) cpu_cores_percent: Vec<Option<f64>>,
|
||||||
pub(crate) gpu_max_percent: Option<u64>,
|
pub(crate) gpu_max_percent: Option<u64>,
|
||||||
pub(crate) gpu_memory_used_mib: u64,
|
pub(crate) gpu_memory_used_mib: u64,
|
||||||
pub(crate) gpu_memory_total_mib: u64,
|
pub(crate) gpu_memory_total_mib: u64,
|
||||||
pub(crate) net_rx_bps: f64,
|
pub(crate) net_rx_bps: f64,
|
||||||
pub(crate) net_tx_bps: f64,
|
pub(crate) net_tx_bps: f64,
|
||||||
|
pub(crate) memory_used_percent: Option<f64>,
|
||||||
|
pub(crate) memory_pressure_some_avg10: Option<f64>,
|
||||||
|
pub(crate) storage_used_percent: Option<f64>,
|
||||||
|
pub(crate) io_pressure_some_avg10: Option<f64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) struct HardwareHistoryState {
|
pub(crate) struct HardwareHistoryState {
|
||||||
pub(crate) at: Instant,
|
pub(crate) at: Instant,
|
||||||
pub(crate) sample_unix_ms: Option<u64>,
|
pub(crate) sample_unix_ms: Option<u64>,
|
||||||
pub(crate) cpu_total_percent: Option<f64>,
|
pub(crate) cpu_total_percent: Option<f64>,
|
||||||
|
pub(crate) cpu_cores_percent: Vec<Option<f64>>,
|
||||||
pub(crate) gpu_max_percent: Option<u64>,
|
pub(crate) gpu_max_percent: Option<u64>,
|
||||||
pub(crate) gpu_memory_used_mib: u64,
|
pub(crate) gpu_memory_used_mib: u64,
|
||||||
pub(crate) gpu_memory_total_mib: u64,
|
pub(crate) gpu_memory_total_mib: u64,
|
||||||
pub(crate) net_rx_bps: f64,
|
pub(crate) net_rx_bps: f64,
|
||||||
pub(crate) net_tx_bps: f64,
|
pub(crate) net_tx_bps: f64,
|
||||||
|
pub(crate) memory_used_percent: Option<f64>,
|
||||||
|
pub(crate) memory_pressure_some_avg10: Option<f64>,
|
||||||
|
pub(crate) storage_used_percent: Option<f64>,
|
||||||
|
pub(crate) io_pressure_some_avg10: Option<f64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
|
|
@ -383,11 +475,16 @@ pub(crate) struct HardwareHistorySnapshot {
|
||||||
pub(crate) ms_ago: u64,
|
pub(crate) ms_ago: u64,
|
||||||
pub(crate) sample_unix_ms: Option<u64>,
|
pub(crate) sample_unix_ms: Option<u64>,
|
||||||
pub(crate) cpu_total_percent: Option<f64>,
|
pub(crate) cpu_total_percent: Option<f64>,
|
||||||
|
pub(crate) cpu_cores_percent: Vec<Option<f64>>,
|
||||||
pub(crate) gpu_max_percent: Option<u64>,
|
pub(crate) gpu_max_percent: Option<u64>,
|
||||||
pub(crate) gpu_memory_used_mib: u64,
|
pub(crate) gpu_memory_used_mib: u64,
|
||||||
pub(crate) gpu_memory_total_mib: u64,
|
pub(crate) gpu_memory_total_mib: u64,
|
||||||
pub(crate) net_rx_bps: f64,
|
pub(crate) net_rx_bps: f64,
|
||||||
pub(crate) net_tx_bps: f64,
|
pub(crate) net_tx_bps: f64,
|
||||||
|
pub(crate) memory_used_percent: Option<f64>,
|
||||||
|
pub(crate) memory_pressure_some_avg10: Option<f64>,
|
||||||
|
pub(crate) storage_used_percent: Option<f64>,
|
||||||
|
pub(crate) io_pressure_some_avg10: Option<f64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn duration_ms(duration: Duration) -> u64 {
|
pub(crate) fn duration_ms(duration: Duration) -> u64 {
|
||||||
|
|
|
||||||
|
|
@ -469,7 +469,7 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn is_valid_for_route(&self) -> bool {
|
fn is_valid_for_route(&self) -> bool {
|
||||||
self.payload % 5 == 0
|
self.payload.is_multiple_of(5)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -20,6 +20,7 @@
|
||||||
//! out. Receipts are bounded and interval-spaced so noisy actors cannot flood
|
//! out. Receipts are bounded and interval-spaced so noisy actors cannot flood
|
||||||
//! the page.
|
//! the page.
|
||||||
|
|
||||||
|
use std::cmp::Reverse;
|
||||||
use std::collections::{BTreeMap, VecDeque};
|
use std::collections::{BTreeMap, VecDeque};
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
|
@ -172,8 +173,10 @@ impl RuntimeState {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn totals(&self) -> Totals {
|
pub(crate) fn totals(&self) -> Totals {
|
||||||
let mut totals = Totals::default();
|
let mut totals = Totals {
|
||||||
totals.actors = self.actors.len().min(u32::MAX as usize) as u32;
|
actors: self.actors.len().min(u32::MAX as usize) as u32,
|
||||||
|
..Totals::default()
|
||||||
|
};
|
||||||
for actor in self.actors.values() {
|
for actor in self.actors.values() {
|
||||||
totals.mailbox_depth = totals.mailbox_depth.saturating_add(actor.mailbox_depth);
|
totals.mailbox_depth = totals.mailbox_depth.saturating_add(actor.mailbox_depth);
|
||||||
totals.msg_per_sec += actor.msg_per_sec;
|
totals.msg_per_sec += actor.msg_per_sec;
|
||||||
|
|
@ -449,7 +452,7 @@ fn parse_message_type_counts(value: Option<&Value>) -> Option<Vec<(String, u64)>
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|(name, count)| value_to_u64(count).map(|count| (name.clone(), count)))
|
.filter_map(|(name, count)| value_to_u64(count).map(|count| (name.clone(), count)))
|
||||||
.collect();
|
.collect();
|
||||||
out.sort_by(|a, b| b.1.cmp(&a.1));
|
out.sort_by_key(|&(_, count)| Reverse(count));
|
||||||
return Some(out);
|
return Some(out);
|
||||||
}
|
}
|
||||||
None
|
None
|
||||||
|
|
|
||||||
|
|
@ -14,6 +14,7 @@ swactor = { path = "../..", features = ["serde", "transport"] }
|
||||||
swactor-transport = { path = "../transport" }
|
swactor-transport = { path = "../transport" }
|
||||||
swactor-engine = { path = "../engine" }
|
swactor-engine = { path = "../engine" }
|
||||||
|
|
||||||
|
parking_lot = "0.12"
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
parking_lot = "0.12"
|
parking_lot = "0.12"
|
||||||
|
|
|
||||||
|
|
@ -46,6 +46,7 @@
|
||||||
//! - P11 (binding slice) fast path performs no syscalls;
|
//! - P11 (binding slice) fast path performs no syscalls;
|
||||||
//! - P12 one copy per side per byte.
|
//! - P12 one copy per side per byte.
|
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
use std::ptr::NonNull;
|
use std::ptr::NonNull;
|
||||||
use std::sync::atomic::{AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
|
|
||||||
|
|
@ -85,7 +86,7 @@ pub struct ByteRingSpec {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A located, installed ring: what `attach` needs to find it again.
|
/// A located, installed ring: what `attach` needs to find it again.
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct RingHandle {
|
pub struct RingHandle {
|
||||||
/// Lease start (the header lives here).
|
/// Lease start (the header lives here).
|
||||||
pub offset: u64,
|
pub offset: u64,
|
||||||
|
|
@ -93,9 +94,11 @@ pub struct RingHandle {
|
||||||
pub capacity: u64,
|
pub capacity: u64,
|
||||||
/// Ring generation.
|
/// Ring generation.
|
||||||
pub generation: u64,
|
pub generation: u64,
|
||||||
|
/// Arena lease identity used for deterministic release.
|
||||||
|
pub lease_id: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub enum Role {
|
pub enum Role {
|
||||||
Producer,
|
Producer,
|
||||||
Consumer,
|
Consumer,
|
||||||
|
|
@ -112,15 +115,139 @@ pub struct Reservation {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Record framing layered on the byte stream (property P9).
|
/// Record framing layered on the byte stream (property P9).
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub enum RecordKind {
|
pub enum RecordKind {
|
||||||
Data,
|
Data,
|
||||||
Eof,
|
Eof,
|
||||||
Fault,
|
Fault,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub struct RecordMeta {
|
||||||
|
pub kind: RecordKind,
|
||||||
|
pub len: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Borrowed committed record. Dropping the view releases the complete framed
|
||||||
|
/// record back to the producer.
|
||||||
|
pub struct PinnedRecord<'a> {
|
||||||
|
endpoint: &'a mut Endpoint,
|
||||||
|
kind: RecordKind,
|
||||||
|
payload_start: u64,
|
||||||
|
payload_len: u64,
|
||||||
|
record_start: u64,
|
||||||
|
record_len: u64,
|
||||||
|
generation: u64,
|
||||||
|
released: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Debug for PinnedRecord<'_> {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
f.debug_struct("PinnedRecord")
|
||||||
|
.field("kind", &self.kind)
|
||||||
|
.field("payload_len", &self.payload_len)
|
||||||
|
.field("record_start", &self.record_start)
|
||||||
|
.finish_non_exhaustive()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PinnedRecord<'_> {
|
||||||
|
pub fn kind(&self) -> RecordKind {
|
||||||
|
self.kind
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn len(&self) -> usize {
|
||||||
|
self.payload_len as usize
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_empty(&self) -> bool {
|
||||||
|
self.payload_len == 0
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn spans(&self) -> (&[u8], &[u8]) {
|
||||||
|
self.endpoint
|
||||||
|
.read_spans(self.payload_start, self.payload_len)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn release(mut self) -> Result<(), FlowError> {
|
||||||
|
self.release_inner()?;
|
||||||
|
self.released = true;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn release_inner(&mut self) -> Result<(), FlowError> {
|
||||||
|
self.endpoint.check("release_record", Role::Consumer)?;
|
||||||
|
let generation = self.endpoint.field_u64(OFF_GENERATION);
|
||||||
|
if generation != self.generation {
|
||||||
|
return Err(FlowError::StaleReservation {
|
||||||
|
reservation: self.generation,
|
||||||
|
ring: generation,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let consume = self.endpoint.consume_cursor();
|
||||||
|
if consume != self.record_start {
|
||||||
|
return Err(FlowError::PinnedRecordMoved {
|
||||||
|
expected: self.record_start,
|
||||||
|
found: consume,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
self.endpoint
|
||||||
|
.atomic(OFF_CONSUME)
|
||||||
|
.store(self.record_start + self.record_len, Ordering::Release);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for PinnedRecord<'_> {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
if !self.released && self.release_inner().is_ok() {
|
||||||
|
self.released = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Producer reservation for a complete framed record. Payload bytes are
|
||||||
|
/// written directly into the ring and remain invisible until `commit`.
|
||||||
|
pub struct WritableRecord<'a> {
|
||||||
|
endpoint: &'a mut Endpoint,
|
||||||
|
reservation: Option<Reservation>,
|
||||||
|
payload_start: u64,
|
||||||
|
payload_len: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Debug for WritableRecord<'_> {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
f.debug_struct("WritableRecord")
|
||||||
|
.field("payload_len", &self.payload_len)
|
||||||
|
.finish_non_exhaustive()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WritableRecord<'_> {
|
||||||
|
pub fn len(&self) -> usize {
|
||||||
|
self.payload_len as usize
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_empty(&self) -> bool {
|
||||||
|
self.payload_len == 0
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn spans_mut(&mut self) -> (&mut [u8], &mut [u8]) {
|
||||||
|
self.endpoint
|
||||||
|
.write_spans(self.payload_start, self.payload_len)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn commit(mut self) -> Result<(), FlowError> {
|
||||||
|
let reservation = self
|
||||||
|
.reservation
|
||||||
|
.take()
|
||||||
|
.expect("writable record commits at most once");
|
||||||
|
self.endpoint.commit(reservation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl RecordKind {
|
impl RecordKind {
|
||||||
fn to_byte(self) -> u8 {
|
pub fn to_byte(self) -> u8 {
|
||||||
match self {
|
match self {
|
||||||
Self::Data => 1,
|
Self::Data => 1,
|
||||||
Self::Eof => 2,
|
Self::Eof => 2,
|
||||||
|
|
@ -128,7 +255,7 @@ impl RecordKind {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn from_byte(byte: u8) -> Option<Self> {
|
pub fn from_byte(byte: u8) -> Option<Self> {
|
||||||
match byte {
|
match byte {
|
||||||
1 => Some(Self::Data),
|
1 => Some(Self::Data),
|
||||||
2 => Some(Self::Eof),
|
2 => Some(Self::Eof),
|
||||||
|
|
@ -194,6 +321,7 @@ pub enum AttachError {
|
||||||
pub enum RecordError {
|
pub enum RecordError {
|
||||||
LengthExceedsCapacity { len: u64, capacity: u64 },
|
LengthExceedsCapacity { len: u64, capacity: u64 },
|
||||||
InvalidKind(u8),
|
InvalidKind(u8),
|
||||||
|
LengthExceedsWire { len: u64 },
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
|
@ -218,6 +346,10 @@ pub enum FlowError {
|
||||||
},
|
},
|
||||||
/// The header stopped validating mid-protocol (property P5).
|
/// The header stopped validating mid-protocol (property P5).
|
||||||
Corrupt(HeaderError),
|
Corrupt(HeaderError),
|
||||||
|
PinnedRecordMoved {
|
||||||
|
expected: u64,
|
||||||
|
found: u64,
|
||||||
|
},
|
||||||
BadRecord(RecordError),
|
BadRecord(RecordError),
|
||||||
Io,
|
Io,
|
||||||
}
|
}
|
||||||
|
|
@ -230,6 +362,59 @@ pub struct Endpoint {
|
||||||
role: Role,
|
role: Role,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
pub struct RingProbe {
|
||||||
|
header: NonNull<u8>,
|
||||||
|
info: RingHandle,
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe impl Send for RingProbe {}
|
||||||
|
unsafe impl Sync for RingProbe {}
|
||||||
|
|
||||||
|
impl RingProbe {
|
||||||
|
fn atomic(&self, off: u64) -> &AtomicU64 {
|
||||||
|
// SAFETY: probes are created only from a successfully attached,
|
||||||
|
// aligned endpoint and remain bounded by that ring's live lease.
|
||||||
|
unsafe { &*(self.header.as_ptr().add(off as usize) as *const AtomicU64) }
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn positions(&self) -> Result<(u64, u64), FlowError> {
|
||||||
|
let generation = self.atomic(OFF_GENERATION).load(Ordering::Relaxed);
|
||||||
|
if generation != self.info.generation {
|
||||||
|
return Err(FlowError::StaleReservation {
|
||||||
|
reservation: self.info.generation,
|
||||||
|
ring: generation,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let commit = self.atomic(OFF_COMMIT).load(Ordering::Acquire);
|
||||||
|
let consume = self.atomic(OFF_CONSUME).load(Ordering::Acquire);
|
||||||
|
if commit < consume {
|
||||||
|
return Err(FlowError::Corrupt(HeaderError::CommitBelowConsume {
|
||||||
|
commit,
|
||||||
|
consume,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
if commit - consume > self.info.capacity {
|
||||||
|
return Err(FlowError::Corrupt(HeaderError::ReadableExceedsCapacity {
|
||||||
|
commit,
|
||||||
|
consume,
|
||||||
|
capacity: self.info.capacity,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
Ok((commit, consume))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn has_data(&self) -> bool {
|
||||||
|
self.positions()
|
||||||
|
.is_ok_and(|(commit, consume)| commit > consume)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn has_capacity(&self) -> bool {
|
||||||
|
self.positions()
|
||||||
|
.is_ok_and(|(commit, consume)| commit - consume < self.info.capacity)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// SAFETY: an endpoint touches only its role-owned cursor field and the data
|
// SAFETY: an endpoint touches only its role-owned cursor field and the data
|
||||||
// region under the single-writer protocol (properties P3/P4); the raw
|
// region under the single-writer protocol (properties P3/P4); the raw
|
||||||
// pointer is never dereferenced outside `[header, header + DATA_OFFSET +
|
// pointer is never dereferenced outside `[header, header + DATA_OFFSET +
|
||||||
|
|
@ -295,6 +480,7 @@ pub fn install(arena: &mut ArenaManager, spec: ByteRingSpec) -> Result<RingHandl
|
||||||
offset: lease.layout.start_offset,
|
offset: lease.layout.start_offset,
|
||||||
capacity: spec.capacity,
|
capacity: spec.capacity,
|
||||||
generation: spec.generation,
|
generation: spec.generation,
|
||||||
|
lease_id: lease.ring_id.0,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -339,6 +525,38 @@ pub fn attach(
|
||||||
Ok(endpoint)
|
Ok(endpoint)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Attach through a child/host process-local mapping of the same arena.
|
||||||
|
pub fn attach_mapped(
|
||||||
|
arena: &crate::mapped_arena::MappedArena,
|
||||||
|
handle: RingHandle,
|
||||||
|
role: Role,
|
||||||
|
) -> Result<Endpoint, AttachError> {
|
||||||
|
let total = DATA_OFFSET
|
||||||
|
.checked_add(handle.capacity)
|
||||||
|
.ok_or(AttachError::OutOfBounds {
|
||||||
|
end: u64::MAX,
|
||||||
|
arena_len: arena.len() as u64,
|
||||||
|
})?;
|
||||||
|
let range =
|
||||||
|
arena
|
||||||
|
.checked_range(handle.offset, total)
|
||||||
|
.map_err(|_| AttachError::OutOfBounds {
|
||||||
|
end: handle.offset.saturating_add(total),
|
||||||
|
arena_len: arena.len() as u64,
|
||||||
|
})?;
|
||||||
|
let endpoint = Endpoint {
|
||||||
|
header: arena.ptr_at(range.start),
|
||||||
|
info: handle,
|
||||||
|
role,
|
||||||
|
};
|
||||||
|
endpoint.validate_fixed().map_err(AttachError::Header)?;
|
||||||
|
endpoint
|
||||||
|
.validate_generation(handle.generation)
|
||||||
|
.map_err(AttachError::Header)?;
|
||||||
|
endpoint.validate_cursors().map_err(AttachError::Header)?;
|
||||||
|
Ok(endpoint)
|
||||||
|
}
|
||||||
|
|
||||||
fn lease_ring(
|
fn lease_ring(
|
||||||
arena: &mut ArenaManager,
|
arena: &mut ArenaManager,
|
||||||
request_id: u64,
|
request_id: u64,
|
||||||
|
|
@ -360,6 +578,32 @@ fn lease_ring(
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Endpoint {
|
impl Endpoint {
|
||||||
|
pub fn capacity(&self) -> u64 {
|
||||||
|
self.info.capacity
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn role(&self) -> Role {
|
||||||
|
self.role
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Current published producer and consumer positions. This role-neutral
|
||||||
|
/// observation is used only to wait for an already-committed clean close
|
||||||
|
/// to enter the downstream bounded transport.
|
||||||
|
pub fn positions(&self) -> Result<(u64, u64), FlowError> {
|
||||||
|
self.validate_fixed().map_err(FlowError::Corrupt)?;
|
||||||
|
self.validate_generation(self.info.generation)
|
||||||
|
.map_err(FlowError::Corrupt)?;
|
||||||
|
self.validate_cursors().map_err(FlowError::Corrupt)?;
|
||||||
|
Ok((self.commit_cursor(), self.consume_cursor()))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn probe(&self) -> RingProbe {
|
||||||
|
RingProbe {
|
||||||
|
header: self.header,
|
||||||
|
info: self.info,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ─── field access ────────────────────────────────────────────────────────
|
// ─── field access ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/// SAFETY: callers keep `self` alive; the pointer is bounds-checked at
|
/// SAFETY: callers keep `self` alive; the pointer is bounds-checked at
|
||||||
|
|
@ -490,6 +734,36 @@ impl Endpoint {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn read_spans(&self, stream_pos: u64, len: u64) -> (&[u8], &[u8]) {
|
||||||
|
let capacity = self.info.capacity as usize;
|
||||||
|
let start = (stream_pos % self.info.capacity) as usize;
|
||||||
|
let len = len as usize;
|
||||||
|
let first = len.min(capacity - start);
|
||||||
|
// SAFETY: `stream_pos` and `len` describe a validated committed
|
||||||
|
// record no larger than the ring. The split ranges do not overlap.
|
||||||
|
unsafe {
|
||||||
|
(
|
||||||
|
std::slice::from_raw_parts(self.data_ptr().add(start), first),
|
||||||
|
std::slice::from_raw_parts(self.data_ptr(), len - first),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_spans(&mut self, stream_pos: u64, len: u64) -> (&mut [u8], &mut [u8]) {
|
||||||
|
let capacity = self.info.capacity as usize;
|
||||||
|
let start = (stream_pos % self.info.capacity) as usize;
|
||||||
|
let len = len as usize;
|
||||||
|
let first = len.min(capacity - start);
|
||||||
|
// SAFETY: the producer exclusively owns this uncommitted reservation;
|
||||||
|
// it is no larger than the ring and the split ranges do not overlap.
|
||||||
|
unsafe {
|
||||||
|
(
|
||||||
|
std::slice::from_raw_parts_mut(self.data_ptr().add(start), first),
|
||||||
|
std::slice::from_raw_parts_mut(self.data_ptr(), len - first),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn copy_out(&self, stream_pos: u64, len: u64) -> Vec<u8> {
|
fn copy_out(&self, stream_pos: u64, len: u64) -> Vec<u8> {
|
||||||
let capacity = self.info.capacity as usize;
|
let capacity = self.info.capacity as usize;
|
||||||
let start = (stream_pos % self.info.capacity) as usize;
|
let start = (stream_pos % self.info.capacity) as usize;
|
||||||
|
|
@ -610,6 +884,34 @@ impl Endpoint {
|
||||||
|
|
||||||
// ─── records (P9) ────────────────────────────────────────────────────────
|
// ─── records (P9) ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Reserve one complete record for direct producer-side payload writes.
|
||||||
|
pub fn reserve_record(
|
||||||
|
&mut self,
|
||||||
|
kind: RecordKind,
|
||||||
|
len: u64,
|
||||||
|
) -> Result<WritableRecord<'_>, FlowError> {
|
||||||
|
if len > u64::from(u32::MAX) {
|
||||||
|
return Err(FlowError::BadRecord(RecordError::LengthExceedsWire { len }));
|
||||||
|
}
|
||||||
|
if RECORD_HEADER_LEN + len > self.info.capacity {
|
||||||
|
return Err(FlowError::BadRecord(RecordError::LengthExceedsCapacity {
|
||||||
|
len,
|
||||||
|
capacity: self.info.capacity,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
let reservation = self.reserve(RECORD_HEADER_LEN + len)?;
|
||||||
|
let mut prefix = [0_u8; RECORD_HEADER_LEN as usize];
|
||||||
|
prefix[0] = kind.to_byte();
|
||||||
|
prefix[1..5].copy_from_slice(&(len as u32).to_le_bytes());
|
||||||
|
self.copy_into(reservation.start, &prefix);
|
||||||
|
let payload_start = reservation.start + RECORD_HEADER_LEN;
|
||||||
|
Ok(WritableRecord {
|
||||||
|
endpoint: self,
|
||||||
|
reservation: Some(reservation),
|
||||||
|
payload_start,
|
||||||
|
payload_len: len,
|
||||||
|
})
|
||||||
|
}
|
||||||
/// Frame and commit one record in a single step.
|
/// Frame and commit one record in a single step.
|
||||||
pub fn send_record(&mut self, kind: RecordKind, bytes: &[u8]) -> Result<(), FlowError> {
|
pub fn send_record(&mut self, kind: RecordKind, bytes: &[u8]) -> Result<(), FlowError> {
|
||||||
let len = bytes.len() as u64;
|
let len = bytes.len() as u64;
|
||||||
|
|
@ -628,6 +930,72 @@ impl Endpoint {
|
||||||
self.commit(reservation)
|
self.commit(reservation)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Inspect the next complete committed record without pinning or consuming
|
||||||
|
/// it. Used by local direct transfer to reserve destination capacity before
|
||||||
|
/// borrowing the source payload.
|
||||||
|
pub fn next_record_meta(&self) -> Result<Option<RecordMeta>, FlowError> {
|
||||||
|
self.check("next_record_meta", Role::Consumer)?;
|
||||||
|
self.validate_generation(self.info.generation)
|
||||||
|
.map_err(FlowError::Corrupt)?;
|
||||||
|
let (commit, consume) = (self.commit_cursor(), self.consume_cursor());
|
||||||
|
let readable = commit - consume;
|
||||||
|
if readable < RECORD_HEADER_LEN {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
let prefix = self.copy_out(consume, RECORD_HEADER_LEN);
|
||||||
|
let kind = RecordKind::from_byte(prefix[0])
|
||||||
|
.ok_or(FlowError::BadRecord(RecordError::InvalidKind(prefix[0])))?;
|
||||||
|
let len = u64::from(u32::from_le_bytes(prefix[1..5].try_into().unwrap()));
|
||||||
|
if RECORD_HEADER_LEN + len > self.info.capacity {
|
||||||
|
return Err(FlowError::BadRecord(RecordError::LengthExceedsCapacity {
|
||||||
|
len,
|
||||||
|
capacity: self.info.capacity,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
if readable < RECORD_HEADER_LEN + len {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
Ok(Some(RecordMeta { kind, len }))
|
||||||
|
}
|
||||||
|
/// Borrow one complete committed record without copying its payload.
|
||||||
|
/// The record remains pinned until the returned view is released or
|
||||||
|
/// dropped.
|
||||||
|
pub fn peek_record(&mut self) -> Result<Option<PinnedRecord<'_>>, FlowError> {
|
||||||
|
self.check("peek_record", Role::Consumer)?;
|
||||||
|
self.validate_generation(self.info.generation)
|
||||||
|
.map_err(FlowError::Corrupt)?;
|
||||||
|
let (commit, consume) = (self.commit_cursor(), self.consume_cursor());
|
||||||
|
let readable = commit - consume;
|
||||||
|
if readable < RECORD_HEADER_LEN {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
let prefix = self.copy_out(consume, RECORD_HEADER_LEN);
|
||||||
|
let kind = RecordKind::from_byte(prefix[0])
|
||||||
|
.ok_or(FlowError::BadRecord(RecordError::InvalidKind(prefix[0])))?;
|
||||||
|
let len = u64::from(u32::from_le_bytes(prefix[1..5].try_into().unwrap()));
|
||||||
|
if RECORD_HEADER_LEN + len > self.info.capacity {
|
||||||
|
return Err(FlowError::BadRecord(RecordError::LengthExceedsCapacity {
|
||||||
|
len,
|
||||||
|
capacity: self.info.capacity,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
let record_len = RECORD_HEADER_LEN + len;
|
||||||
|
if readable < record_len {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
let generation = self.info.generation;
|
||||||
|
Ok(Some(PinnedRecord {
|
||||||
|
endpoint: self,
|
||||||
|
kind,
|
||||||
|
payload_start: consume + RECORD_HEADER_LEN,
|
||||||
|
payload_len: len,
|
||||||
|
record_start: consume,
|
||||||
|
record_len,
|
||||||
|
generation,
|
||||||
|
released: false,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
/// Receive one complete record; `Ok(None)` when nothing (or only a
|
/// Receive one complete record; `Ok(None)` when nothing (or only a
|
||||||
/// torn, uncommitted prefix) is readable.
|
/// torn, uncommitted prefix) is readable.
|
||||||
pub fn recv_record(&mut self) -> Result<Option<(RecordKind, Vec<u8>)>, FlowError> {
|
pub fn recv_record(&mut self) -> Result<Option<(RecordKind, Vec<u8>)>, FlowError> {
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,6 +1,6 @@
|
||||||
//! Host-side session, binding, and arena-allocation actors.
|
//! Host-side session, binding, and arena-allocation actors.
|
||||||
|
|
||||||
use std::collections::HashSet;
|
use std::collections::{HashMap, HashSet};
|
||||||
use std::os::fd::{FromRawFd, OwnedFd};
|
use std::os::fd::{FromRawFd, OwnedFd};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
|
@ -20,18 +20,27 @@ use crate::blob_transfer::{
|
||||||
BlobTransferEvent, BlobTransferId, BlobTransferOffer, BlobTransferReceiver, BlobTransferSender,
|
BlobTransferEvent, BlobTransferId, BlobTransferOffer, BlobTransferReceiver, BlobTransferSender,
|
||||||
};
|
};
|
||||||
use crate::bootstrap::JobHandoff;
|
use crate::bootstrap::JobHandoff;
|
||||||
|
use crate::byte_ring::{self, ByteRingSpec, RingHandle, Role};
|
||||||
|
use crate::mapped_arena::MappedArena;
|
||||||
use crate::namespace::{
|
use crate::namespace::{
|
||||||
BlobBinding as NamespaceBlobBinding, DataDirectoryOut, NamespaceClient, NamespaceClientIn,
|
BlobBinding as NamespaceBlobBinding, DataDirectoryOut, NamespaceClient, NamespaceClientIn,
|
||||||
NamespaceError, NamespaceRequest, OperationId, SourceRecovery,
|
NamespaceError, NamespaceRequest, OperationId, SourceRecovery, StreamIncarnation, StreamMatch,
|
||||||
|
StreamRole,
|
||||||
};
|
};
|
||||||
use crate::path::{DataPath, JobContext};
|
use crate::path::{DataPath, JobContext};
|
||||||
use crate::protocol::{
|
use crate::protocol::{
|
||||||
AttachmentFailure, ChildSessionIn, DataOperation, DataPlaneError, HostSessionIn, JobCapability,
|
AttachmentFailure, ChildSessionIn, DataOperation, DataPlaneError, HostSessionIn, HostStreamIn,
|
||||||
|
JobCapability,
|
||||||
};
|
};
|
||||||
use crate::source::{BlobSourceIn, BlobSourcePublisher, BlobSourceRetirement, FileBlobSourceActor};
|
use crate::source::{BlobSourceIn, BlobSourcePublisher, BlobSourceRetirement, FileBlobSourceActor};
|
||||||
|
use crate::stream_transport::{
|
||||||
|
StreamPeerDescriptor, StreamSinkRequest, StreamSourceRequest, StreamTransport,
|
||||||
|
StreamTransportEvent, StreamTransportNotifier,
|
||||||
|
};
|
||||||
|
|
||||||
const BLOB_ALIGNMENT: u64 = 64;
|
const BLOB_ALIGNMENT: u64 = 64;
|
||||||
const FIRST_BLOB_REQUEST_ID: u64 = 2;
|
const FIRST_BLOB_REQUEST_ID: u64 = 2;
|
||||||
|
const STREAM_RING_CAPACITY: u64 = 256 * 1024;
|
||||||
|
|
||||||
pub trait HostRouteRegistrar: Send + Sync + 'static {
|
pub trait HostRouteRegistrar: Send + Sync + 'static {
|
||||||
fn register_child(
|
fn register_child(
|
||||||
|
|
@ -53,6 +62,7 @@ pub struct HostDataPlaneConfig {
|
||||||
pub source_sender: Option<Arc<dyn BlobTransferSender>>,
|
pub source_sender: Option<Arc<dyn BlobTransferSender>>,
|
||||||
pub source_publisher: Option<Arc<dyn BlobSourcePublisher>>,
|
pub source_publisher: Option<Arc<dyn BlobSourcePublisher>>,
|
||||||
pub route_registrar: Option<Arc<dyn HostRouteRegistrar>>,
|
pub route_registrar: Option<Arc<dyn HostRouteRegistrar>>,
|
||||||
|
pub stream_transport: Option<Arc<dyn StreamTransport>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
|
@ -75,9 +85,12 @@ pub struct HostDataPlaneSessionActor {
|
||||||
source_sender: Option<Arc<dyn BlobTransferSender>>,
|
source_sender: Option<Arc<dyn BlobTransferSender>>,
|
||||||
source_publisher: Option<Arc<dyn BlobSourcePublisher>>,
|
source_publisher: Option<Arc<dyn BlobSourcePublisher>>,
|
||||||
route_registrar: Option<Arc<dyn HostRouteRegistrar>>,
|
route_registrar: Option<Arc<dyn HostRouteRegistrar>>,
|
||||||
|
stream_arena: Option<Arc<MappedArena>>,
|
||||||
|
stream_transport: Option<Arc<dyn StreamTransport>>,
|
||||||
child_session: Option<ActorAddress>,
|
child_session: Option<ActorAddress>,
|
||||||
allocator: Option<ActorAddress>,
|
allocator: Option<ActorAddress>,
|
||||||
active_bindings: HashSet<ActorAddress>,
|
active_bindings: HashSet<ActorAddress>,
|
||||||
|
stream_bindings: HashMap<ActorAddress, ActorAddress>,
|
||||||
state: HostSessionState,
|
state: HostSessionState,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -92,6 +105,21 @@ impl HostDataPlaneSessionActor {
|
||||||
.job_context
|
.job_context
|
||||||
.validate()
|
.validate()
|
||||||
.map_err(|error| DataPlaneError::InvalidPath(error.to_string()))?;
|
.map_err(|error| DataPlaneError::InvalidPath(error.to_string()))?;
|
||||||
|
let stream_arena = if config.stream_transport.is_some() {
|
||||||
|
let fd = unsafe { libc::dup(config.arena.arena_fd()) };
|
||||||
|
if fd < 0 {
|
||||||
|
return Err(DataPlaneError::SessionFailed(format!(
|
||||||
|
"duplicate arena backing for streams: {}",
|
||||||
|
std::io::Error::last_os_error()
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
let owned = unsafe { OwnedFd::from_raw_fd(fd) };
|
||||||
|
let (mapped, _) = MappedArena::map(owned)
|
||||||
|
.map_err(|error| DataPlaneError::SessionFailed(error.to_string()))?;
|
||||||
|
Some(Arc::new(mapped))
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
arena: Some(config.arena),
|
arena: Some(config.arena),
|
||||||
arena_generation: config.arena_generation,
|
arena_generation: config.arena_generation,
|
||||||
|
|
@ -104,9 +132,12 @@ impl HostDataPlaneSessionActor {
|
||||||
source_sender: config.source_sender,
|
source_sender: config.source_sender,
|
||||||
source_publisher: config.source_publisher,
|
source_publisher: config.source_publisher,
|
||||||
route_registrar: config.route_registrar,
|
route_registrar: config.route_registrar,
|
||||||
|
stream_arena,
|
||||||
|
stream_transport: config.stream_transport,
|
||||||
child_session: None,
|
child_session: None,
|
||||||
allocator: None,
|
allocator: None,
|
||||||
active_bindings: HashSet::new(),
|
active_bindings: HashSet::new(),
|
||||||
|
stream_bindings: HashMap::new(),
|
||||||
state: HostSessionState::AwaitingAttachment,
|
state: HostSessionState::AwaitingAttachment,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
@ -159,7 +190,10 @@ impl HostDataPlaneSessionActor {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn maybe_finish_close(&mut self) {
|
fn maybe_finish_close(&mut self) {
|
||||||
if self.state == HostSessionState::Closing && self.active_bindings.is_empty() {
|
if self.state == HostSessionState::Closing
|
||||||
|
&& self.active_bindings.is_empty()
|
||||||
|
&& self.stream_bindings.is_empty()
|
||||||
|
{
|
||||||
self.state = HostSessionState::Closed;
|
self.state = HostSessionState::Closed;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -329,6 +363,133 @@ impl ActorInterface for HostDataPlaneSessionActor {
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
ref message @ (HostSessionIn::OpenReadStream {
|
||||||
|
ref path,
|
||||||
|
child_session,
|
||||||
|
operation,
|
||||||
|
replace,
|
||||||
|
}
|
||||||
|
| HostSessionIn::OpenWriteStream {
|
||||||
|
ref path,
|
||||||
|
child_session,
|
||||||
|
operation,
|
||||||
|
replace,
|
||||||
|
}) => {
|
||||||
|
let role = match message {
|
||||||
|
HostSessionIn::OpenReadStream { .. } => StreamRole::Sink,
|
||||||
|
HostSessionIn::OpenWriteStream { .. } => StreamRole::Source,
|
||||||
|
_ => unreachable!(),
|
||||||
|
};
|
||||||
|
let data_operation = match role {
|
||||||
|
StreamRole::Source => DataOperation::WriteStream,
|
||||||
|
StreamRole::Sink => DataOperation::ReadStream,
|
||||||
|
};
|
||||||
|
let resolved = match self.validate_open(child_session, path, data_operation) {
|
||||||
|
Ok(path) => path,
|
||||||
|
Err(error) => {
|
||||||
|
self.send_open_failure(ctx, child_session, operation, error);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let (Some(namespace), Some(arena), Some(transport)) = (
|
||||||
|
self.namespace.clone(),
|
||||||
|
self.stream_arena.clone(),
|
||||||
|
self.stream_transport.clone(),
|
||||||
|
) else {
|
||||||
|
self.send_open_failure(
|
||||||
|
ctx,
|
||||||
|
child_session,
|
||||||
|
operation,
|
||||||
|
DataPlaneError::SessionFailed(
|
||||||
|
"stream namespace or transport service is unavailable".to_owned(),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let local_descriptor = match transport.descriptor() {
|
||||||
|
Ok(descriptor) => descriptor,
|
||||||
|
Err(reason) => {
|
||||||
|
self.send_open_failure(
|
||||||
|
ctx,
|
||||||
|
child_session,
|
||||||
|
operation,
|
||||||
|
DataPlaneError::StreamFault(reason),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let binding = HostStreamBindingActor {
|
||||||
|
runtime: self.runtime.clone(),
|
||||||
|
host_session: ctx.self_addr(),
|
||||||
|
allocator: self.allocator.expect("allocator started"),
|
||||||
|
child_session,
|
||||||
|
operation,
|
||||||
|
path: resolved,
|
||||||
|
replace,
|
||||||
|
role,
|
||||||
|
namespace,
|
||||||
|
arena,
|
||||||
|
transport,
|
||||||
|
local_descriptor,
|
||||||
|
ring: None,
|
||||||
|
matched: None,
|
||||||
|
peer_descriptor: None,
|
||||||
|
transport_installed: false,
|
||||||
|
transport_ready: false,
|
||||||
|
transport_quiesced: false,
|
||||||
|
opened: false,
|
||||||
|
terminal: None,
|
||||||
|
data_waiters: Vec::new(),
|
||||||
|
capacity_waiters: Vec::new(),
|
||||||
|
close_waiters: Vec::new(),
|
||||||
|
release_started: false,
|
||||||
|
};
|
||||||
|
match ctx.spawn(binding) {
|
||||||
|
Ok(binding) => {
|
||||||
|
if let Some(publisher) = &self.source_publisher
|
||||||
|
&& let Err(error) = publisher.publish_source(binding)
|
||||||
|
{
|
||||||
|
let _ = ctx.stop_actor(binding);
|
||||||
|
self.send_open_failure(
|
||||||
|
ctx,
|
||||||
|
child_session,
|
||||||
|
operation,
|
||||||
|
DataPlaneError::SessionFailed(format!(
|
||||||
|
"publish stream endpoint: {error}"
|
||||||
|
)),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
self.stream_bindings.insert(operation, binding);
|
||||||
|
}
|
||||||
|
Err(error) => self.send_open_failure(
|
||||||
|
ctx,
|
||||||
|
child_session,
|
||||||
|
operation,
|
||||||
|
DataPlaneError::SessionFailed(error.to_string()),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
HostSessionIn::CancelStream { operation } => {
|
||||||
|
if let Some(binding) = self.stream_bindings.get(&operation).copied() {
|
||||||
|
let _ = ctx.send(
|
||||||
|
binding,
|
||||||
|
HostStreamIn::Close {
|
||||||
|
clean: false,
|
||||||
|
reply_to: None,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
HostSessionIn::StreamControl { binding, message } => {
|
||||||
|
if self
|
||||||
|
.stream_bindings
|
||||||
|
.values()
|
||||||
|
.any(|stream_binding| *stream_binding == binding)
|
||||||
|
{
|
||||||
|
let _ = ctx.send(binding, message);
|
||||||
|
}
|
||||||
|
}
|
||||||
HostSessionIn::ReleaseBlob {
|
HostSessionIn::ReleaseBlob {
|
||||||
binding,
|
binding,
|
||||||
lease_id,
|
lease_id,
|
||||||
|
|
@ -391,6 +552,8 @@ impl ActorInterface for HostDataPlaneSessionActor {
|
||||||
}
|
}
|
||||||
HostSessionIn::BindingDone { binding } | HostSessionIn::BindingDetached { binding } => {
|
HostSessionIn::BindingDone { binding } | HostSessionIn::BindingDetached { binding } => {
|
||||||
self.active_bindings.remove(&binding);
|
self.active_bindings.remove(&binding);
|
||||||
|
self.stream_bindings
|
||||||
|
.retain(|_, stream_binding| *stream_binding != binding);
|
||||||
self.maybe_finish_close();
|
self.maybe_finish_close();
|
||||||
}
|
}
|
||||||
HostSessionIn::ConfigureRun { run_id, reply_to } => {
|
HostSessionIn::ConfigureRun { run_id, reply_to } => {
|
||||||
|
|
@ -422,6 +585,15 @@ impl ActorInterface for HostDataPlaneSessionActor {
|
||||||
for binding in self.active_bindings.iter().copied() {
|
for binding in self.active_bindings.iter().copied() {
|
||||||
let _ = ctx.send(binding, HostBindingIn::SessionClosed);
|
let _ = ctx.send(binding, HostBindingIn::SessionClosed);
|
||||||
}
|
}
|
||||||
|
for binding in self.stream_bindings.values().copied() {
|
||||||
|
let _ = ctx.send(
|
||||||
|
binding,
|
||||||
|
HostStreamIn::Close {
|
||||||
|
clean: false,
|
||||||
|
reply_to: None,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
self.maybe_finish_close();
|
self.maybe_finish_close();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -465,6 +637,14 @@ enum ArenaAllocatorIn {
|
||||||
transfer: ActorAddress,
|
transfer: ActorAddress,
|
||||||
kind: AllocationKind,
|
kind: AllocationKind,
|
||||||
},
|
},
|
||||||
|
AllocateStream {
|
||||||
|
binding: ActorAddress,
|
||||||
|
capacity: u64,
|
||||||
|
},
|
||||||
|
ReleaseStream {
|
||||||
|
binding: ActorAddress,
|
||||||
|
ring: RingHandle,
|
||||||
|
},
|
||||||
ValidateSealed {
|
ValidateSealed {
|
||||||
binding: ActorAddress,
|
binding: ActorAddress,
|
||||||
lease: BlobLease,
|
lease: BlobLease,
|
||||||
|
|
@ -664,6 +844,65 @@ impl ArenaAllocatorActor {
|
||||||
ctx.spawn(source)
|
ctx.spawn(source)
|
||||||
.map_err(|error| DataPlaneError::SessionFailed(error.to_string()))
|
.map_err(|error| DataPlaneError::SessionFailed(error.to_string()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn allocate_stream(&mut self, capacity: u64) -> Result<RingHandle, DataPlaneError> {
|
||||||
|
let request_id = self.next_request_id;
|
||||||
|
self.next_request_id = self.next_request_id.checked_add(1).ok_or_else(|| {
|
||||||
|
DataPlaneError::SessionFailed("stream request id exhausted".to_owned())
|
||||||
|
})?;
|
||||||
|
let generation = self.next_generation;
|
||||||
|
self.next_generation = self
|
||||||
|
.next_generation
|
||||||
|
.checked_add(1)
|
||||||
|
.filter(|next| *next != 0)
|
||||||
|
.ok_or_else(|| {
|
||||||
|
DataPlaneError::SessionFailed("stream generation exhausted".to_owned())
|
||||||
|
})?;
|
||||||
|
byte_ring::install(
|
||||||
|
&mut self.arena,
|
||||||
|
ByteRingSpec {
|
||||||
|
capacity,
|
||||||
|
generation,
|
||||||
|
alignment: BLOB_ALIGNMENT,
|
||||||
|
request_id,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.map_err(|error| match error {
|
||||||
|
byte_ring::InstallError::LeaseRejected(_) | byte_ring::InstallError::LeaseQueued => {
|
||||||
|
DataPlaneError::ArenaExhausted
|
||||||
|
}
|
||||||
|
other => {
|
||||||
|
DataPlaneError::SessionFailed(format!("stream ring installation failed: {other:?}"))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn release_stream(&mut self, ring: RingHandle) -> Result<(), DataPlaneError> {
|
||||||
|
let ring_id = RingId(ring.lease_id);
|
||||||
|
let Some(allocation) = self.arena.lookup_lease(ring_id) else {
|
||||||
|
return Err(DataPlaneError::StreamFault(
|
||||||
|
"stream arena lease is no longer live".to_owned(),
|
||||||
|
));
|
||||||
|
};
|
||||||
|
if allocation.layout.start_offset != ring.offset
|
||||||
|
|| allocation.layout.data_bytes != ring.capacity
|
||||||
|
{
|
||||||
|
return Err(DataPlaneError::StreamFault(
|
||||||
|
"stream arena lease does not match ring handle".to_owned(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let events = self.arena.request(ArenaRequest::ReleaseRing {
|
||||||
|
ring_id,
|
||||||
|
proof: QuiescenceProof::verified(),
|
||||||
|
});
|
||||||
|
if matches!(events.as_slice(), [ArenaEvent::RingReleased { .. }]) {
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(DataPlaneError::SessionFailed(
|
||||||
|
"arena rejected stream release".to_owned(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ActorInterface for ArenaAllocatorActor {
|
impl ActorInterface for ArenaAllocatorActor {
|
||||||
|
|
@ -699,6 +938,14 @@ impl ActorInterface for ArenaAllocatorActor {
|
||||||
let _ = ctx.send(transfer, BlobTransferEvent::AllocatorFailed(error.into()));
|
let _ = ctx.send(transfer, BlobTransferEvent::AllocatorFailed(error.into()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
ArenaAllocatorIn::AllocateStream { binding, capacity } => {
|
||||||
|
let result = self.allocate_stream(capacity);
|
||||||
|
let _ = ctx.send(binding, HostStreamIn::Allocated(result));
|
||||||
|
}
|
||||||
|
ArenaAllocatorIn::ReleaseStream { binding, ring } => {
|
||||||
|
let result = self.release_stream(ring);
|
||||||
|
let _ = ctx.send(binding, HostStreamIn::ReleaseComplete(result));
|
||||||
|
}
|
||||||
ArenaAllocatorIn::SealTransfer {
|
ArenaAllocatorIn::SealTransfer {
|
||||||
transfer,
|
transfer,
|
||||||
lease,
|
lease,
|
||||||
|
|
@ -1056,7 +1303,6 @@ impl ActorInterface for NamespacePublishActor {
|
||||||
Err(error) => HostBindingIn::PublicationRejected(namespace_error(error)),
|
Err(error) => HostBindingIn::PublicationRejected(namespace_error(error)),
|
||||||
};
|
};
|
||||||
let _ = ctx.send(self.binding, response);
|
let _ = ctx.send(self.binding, response);
|
||||||
ctx.stop_self();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1118,9 +1364,557 @@ impl ActorInterface for NamespaceUnpublishActor {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct NamespaceStreamOpenActor {
|
||||||
|
proxy: ActorAddress,
|
||||||
|
parent: ActorAddress,
|
||||||
|
path: DataPath,
|
||||||
|
role: StreamRole,
|
||||||
|
replace: bool,
|
||||||
|
descriptor: Vec<u8>,
|
||||||
|
operation_id: OperationId,
|
||||||
|
completed: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ActorInterface for NamespaceStreamOpenActor {
|
||||||
|
type Incoming = DataDirectoryOut;
|
||||||
|
type Response = ();
|
||||||
|
|
||||||
|
fn on_start(&mut self, ctx: &Ctx<'_>) {
|
||||||
|
let _ = ctx.send(
|
||||||
|
self.proxy,
|
||||||
|
NamespaceClientIn::Request {
|
||||||
|
request: NamespaceRequest::OpenStream {
|
||||||
|
path: self.path.clone(),
|
||||||
|
role: self.role,
|
||||||
|
endpoint: self.parent,
|
||||||
|
descriptor: self.descriptor.clone(),
|
||||||
|
replace: self.replace,
|
||||||
|
operation_id: self.operation_id,
|
||||||
|
},
|
||||||
|
reply_to: ctx.self_addr(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn handle(&mut self, ctx: &Ctx<'_>, message: DataDirectoryOut) {
|
||||||
|
let result = match message {
|
||||||
|
DataDirectoryOut::StreamOpened { result, .. } => result,
|
||||||
|
other => Err(NamespaceError::Protocol(format!(
|
||||||
|
"expected stream-open reply, received {other:?}"
|
||||||
|
))),
|
||||||
|
};
|
||||||
|
self.completed = true;
|
||||||
|
let _ = ctx.send(self.parent, HostStreamIn::NamespaceMatched(result));
|
||||||
|
ctx.stop_self();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn on_stop(&mut self, ctx: &Ctx<'_>) {
|
||||||
|
if !self.completed {
|
||||||
|
let _ = ctx.send(
|
||||||
|
self.proxy,
|
||||||
|
NamespaceClientIn::Cancel {
|
||||||
|
reply_to: ctx.self_addr(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct NamespaceStreamCloseActor {
|
||||||
|
proxy: ActorAddress,
|
||||||
|
path: DataPath,
|
||||||
|
incarnation: StreamIncarnation,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ActorInterface for NamespaceStreamCloseActor {
|
||||||
|
type Incoming = DataDirectoryOut;
|
||||||
|
type Response = ();
|
||||||
|
|
||||||
|
fn on_start(&mut self, ctx: &Ctx<'_>) {
|
||||||
|
let _ = ctx.send(
|
||||||
|
self.proxy,
|
||||||
|
NamespaceClientIn::Request {
|
||||||
|
request: NamespaceRequest::CloseStream {
|
||||||
|
path: self.path.clone(),
|
||||||
|
incarnation: self.incarnation,
|
||||||
|
},
|
||||||
|
reply_to: ctx.self_addr(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn handle(&mut self, ctx: &Ctx<'_>, _message: DataDirectoryOut) {
|
||||||
|
ctx.stop_self();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct RuntimeStreamNotifier {
|
||||||
|
runtime: Runtime,
|
||||||
|
target: ActorAddress,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StreamTransportNotifier for RuntimeStreamNotifier {
|
||||||
|
fn notify(&self, event: StreamTransportEvent) {
|
||||||
|
let _ = self
|
||||||
|
.runtime
|
||||||
|
.send_to(self.target, HostStreamIn::Transport(event));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct HostStreamBindingActor {
|
||||||
|
runtime: Runtime,
|
||||||
|
host_session: ActorAddress,
|
||||||
|
allocator: ActorAddress,
|
||||||
|
child_session: ActorAddress,
|
||||||
|
operation: ActorAddress,
|
||||||
|
path: DataPath,
|
||||||
|
replace: bool,
|
||||||
|
role: StreamRole,
|
||||||
|
namespace: NamespaceClient,
|
||||||
|
arena: Arc<MappedArena>,
|
||||||
|
transport: Arc<dyn StreamTransport>,
|
||||||
|
local_descriptor: StreamPeerDescriptor,
|
||||||
|
ring: Option<RingHandle>,
|
||||||
|
matched: Option<StreamMatch>,
|
||||||
|
peer_descriptor: Option<StreamPeerDescriptor>,
|
||||||
|
transport_installed: bool,
|
||||||
|
transport_ready: bool,
|
||||||
|
opened: bool,
|
||||||
|
transport_quiesced: bool,
|
||||||
|
terminal: Option<DataPlaneError>,
|
||||||
|
data_waiters: Vec<ActorAddress>,
|
||||||
|
capacity_waiters: Vec<ActorAddress>,
|
||||||
|
close_waiters: Vec<ActorAddress>,
|
||||||
|
release_started: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl HostStreamBindingActor {
|
||||||
|
fn operation_id(address: ActorAddress) -> OperationId {
|
||||||
|
let mut bytes = [0_u8; 16];
|
||||||
|
bytes.copy_from_slice(&address.0[..16]);
|
||||||
|
OperationId::from_u128(u128::from_le_bytes(bytes))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn fail_open(&mut self, ctx: &Ctx<'_>, error: DataPlaneError) {
|
||||||
|
if !self.opened {
|
||||||
|
let _ = ctx.send(
|
||||||
|
self.child_session,
|
||||||
|
ChildSessionIn::OperationFailed {
|
||||||
|
operation: self.operation,
|
||||||
|
error: error.clone(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
self.begin_terminal(ctx, error, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn try_install_transport(&mut self, ctx: &Ctx<'_>) {
|
||||||
|
if self.transport_installed || self.terminal.is_some() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let (Some(ring), Some(matched)) = (self.ring, self.matched.clone()) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let endpoint_role = match self.role {
|
||||||
|
StreamRole::Source => Role::Consumer,
|
||||||
|
StreamRole::Sink => Role::Producer,
|
||||||
|
};
|
||||||
|
let endpoint = match byte_ring::attach_mapped(&self.arena, ring, endpoint_role) {
|
||||||
|
Ok(endpoint) => endpoint,
|
||||||
|
Err(error) => {
|
||||||
|
self.fail_open(
|
||||||
|
ctx,
|
||||||
|
DataPlaneError::StreamFault(format!("attach host stream ring: {error:?}")),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let notifier: Arc<dyn StreamTransportNotifier> = Arc::new(RuntimeStreamNotifier {
|
||||||
|
runtime: self.runtime.clone(),
|
||||||
|
target: ctx.self_addr(),
|
||||||
|
});
|
||||||
|
let install = match self.role {
|
||||||
|
StreamRole::Source => {
|
||||||
|
let Some(peer) = self.peer_descriptor.clone() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
self.transport.install_source(StreamSourceRequest {
|
||||||
|
incarnation: matched.incarnation,
|
||||||
|
peer,
|
||||||
|
endpoint,
|
||||||
|
notifier,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
StreamRole::Sink => self.transport.install_sink(StreamSinkRequest {
|
||||||
|
incarnation: matched.incarnation,
|
||||||
|
endpoint,
|
||||||
|
notifier,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
match install {
|
||||||
|
Ok(()) => {
|
||||||
|
self.transport_installed = true;
|
||||||
|
if self.role == StreamRole::Sink && matched.sink_descriptor.is_empty() {
|
||||||
|
let _ = ctx.send(
|
||||||
|
matched.source,
|
||||||
|
HostStreamIn::PeerOffer {
|
||||||
|
incarnation: matched.incarnation,
|
||||||
|
descriptor: self.local_descriptor.clone(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(reason) => self.fail_open(ctx, DataPlaneError::StreamFault(reason)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn open_if_ready(&mut self, ctx: &Ctx<'_>) {
|
||||||
|
if self.opened || !self.transport_ready || self.terminal.is_some() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let Some(ring) = self.ring else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
self.opened = true;
|
||||||
|
let _ = ctx.send(
|
||||||
|
self.child_session,
|
||||||
|
ChildSessionIn::StreamOpened {
|
||||||
|
operation: self.operation,
|
||||||
|
host_binding: ctx.self_addr(),
|
||||||
|
ring,
|
||||||
|
role: match self.role {
|
||||||
|
StreamRole::Source => Role::Producer,
|
||||||
|
StreamRole::Sink => Role::Consumer,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn wake_waiters(
|
||||||
|
ctx: &Ctx<'_>,
|
||||||
|
child_session: ActorAddress,
|
||||||
|
waiters: &mut Vec<ActorAddress>,
|
||||||
|
result: Result<(), DataPlaneError>,
|
||||||
|
) {
|
||||||
|
for reply_to in waiters.drain(..) {
|
||||||
|
let _ = ctx.send(
|
||||||
|
child_session,
|
||||||
|
ChildSessionIn::StreamWake {
|
||||||
|
reply_to,
|
||||||
|
result: result.clone(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn begin_terminal(&mut self, ctx: &Ctx<'_>, error: DataPlaneError, notify_peer: bool) {
|
||||||
|
if self.terminal.is_some() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
self.terminal = Some(error.clone());
|
||||||
|
Self::wake_waiters(
|
||||||
|
ctx,
|
||||||
|
self.child_session,
|
||||||
|
&mut self.data_waiters,
|
||||||
|
Err(error.clone()),
|
||||||
|
);
|
||||||
|
Self::wake_waiters(
|
||||||
|
ctx,
|
||||||
|
self.child_session,
|
||||||
|
&mut self.capacity_waiters,
|
||||||
|
Err(error.clone()),
|
||||||
|
);
|
||||||
|
if let Some(matched) = &self.matched {
|
||||||
|
if notify_peer {
|
||||||
|
let peer = match self.role {
|
||||||
|
StreamRole::Source => matched.sink,
|
||||||
|
StreamRole::Sink => matched.source,
|
||||||
|
};
|
||||||
|
let peer_error = if matches!(error, DataPlaneError::StreamClosed) {
|
||||||
|
DataPlaneError::StreamClosed
|
||||||
|
} else {
|
||||||
|
DataPlaneError::PeerLost
|
||||||
|
};
|
||||||
|
let _ = ctx.send(
|
||||||
|
peer,
|
||||||
|
HostStreamIn::PeerTerminated {
|
||||||
|
incarnation: matched.incarnation,
|
||||||
|
error: peer_error,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let _ = ctx.spawn(NamespaceStreamCloseActor {
|
||||||
|
proxy: self.namespace.proxy(),
|
||||||
|
path: self.path.clone(),
|
||||||
|
incarnation: matched.incarnation,
|
||||||
|
});
|
||||||
|
if self.transport_installed {
|
||||||
|
self.transport.terminate(matched.incarnation);
|
||||||
|
if !self.transport_quiesced {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.release_ring(ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn complete_release(&mut self, ctx: &Ctx<'_>, result: Result<(), DataPlaneError>) {
|
||||||
|
for reply_to in self.close_waiters.drain(..) {
|
||||||
|
let _ = ctx.send(
|
||||||
|
self.child_session,
|
||||||
|
ChildSessionIn::StreamWake {
|
||||||
|
reply_to,
|
||||||
|
result: result.clone(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let _ = ctx.send(
|
||||||
|
self.host_session,
|
||||||
|
HostSessionIn::BindingDone {
|
||||||
|
binding: ctx.self_addr(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
ctx.stop_self();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn release_ring(&mut self, ctx: &Ctx<'_>) {
|
||||||
|
if self.release_started {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
self.release_started = true;
|
||||||
|
if let Some(ring) = self.ring {
|
||||||
|
let _ = ctx.send(
|
||||||
|
self.allocator,
|
||||||
|
ArenaAllocatorIn::ReleaseStream {
|
||||||
|
binding: ctx.self_addr(),
|
||||||
|
ring,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
self.complete_release(ctx, Ok(()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ActorInterface for HostStreamBindingActor {
|
||||||
|
type Incoming = HostStreamIn;
|
||||||
|
type Response = ();
|
||||||
|
|
||||||
|
fn on_start(&mut self, ctx: &Ctx<'_>) {
|
||||||
|
let _ = ctx.send(
|
||||||
|
self.allocator,
|
||||||
|
ArenaAllocatorIn::AllocateStream {
|
||||||
|
binding: ctx.self_addr(),
|
||||||
|
capacity: STREAM_RING_CAPACITY,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let _ = ctx.spawn(NamespaceStreamOpenActor {
|
||||||
|
proxy: self.namespace.proxy(),
|
||||||
|
parent: ctx.self_addr(),
|
||||||
|
path: self.path.clone(),
|
||||||
|
role: self.role,
|
||||||
|
descriptor: self.local_descriptor.0.clone(),
|
||||||
|
replace: self.replace,
|
||||||
|
operation_id: Self::operation_id(ctx.self_addr()),
|
||||||
|
completed: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
fn handle(&mut self, ctx: &Ctx<'_>, message: HostStreamIn) {
|
||||||
|
match message {
|
||||||
|
HostStreamIn::NamespaceMatched(result) => match result {
|
||||||
|
Ok(matched) => {
|
||||||
|
let expected = match self.role {
|
||||||
|
StreamRole::Source => matched.source,
|
||||||
|
StreamRole::Sink => matched.sink,
|
||||||
|
};
|
||||||
|
if expected != ctx.self_addr() {
|
||||||
|
self.fail_open(
|
||||||
|
ctx,
|
||||||
|
DataPlaneError::StreamFault(
|
||||||
|
"namespace matched the wrong stream endpoint".to_owned(),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if self.role == StreamRole::Source && !matched.sink_descriptor.is_empty() {
|
||||||
|
self.peer_descriptor =
|
||||||
|
Some(StreamPeerDescriptor(matched.sink_descriptor.clone()));
|
||||||
|
}
|
||||||
|
self.matched = Some(matched);
|
||||||
|
self.try_install_transport(ctx);
|
||||||
|
}
|
||||||
|
Err(error) => self.fail_open(ctx, namespace_error(error)),
|
||||||
|
},
|
||||||
|
HostStreamIn::Allocated(result) => match result {
|
||||||
|
Ok(ring) => {
|
||||||
|
self.ring = Some(ring);
|
||||||
|
self.try_install_transport(ctx);
|
||||||
|
}
|
||||||
|
Err(error) => self.fail_open(ctx, error),
|
||||||
|
},
|
||||||
|
HostStreamIn::PeerOffer {
|
||||||
|
incarnation,
|
||||||
|
descriptor,
|
||||||
|
} => {
|
||||||
|
if self.role != StreamRole::Source {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if self
|
||||||
|
.matched
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|matched| matched.incarnation == incarnation)
|
||||||
|
{
|
||||||
|
self.peer_descriptor = Some(descriptor);
|
||||||
|
self.try_install_transport(ctx);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
HostStreamIn::Transport(StreamTransportEvent::Ready) => {
|
||||||
|
self.transport_ready = true;
|
||||||
|
self.open_if_ready(ctx);
|
||||||
|
}
|
||||||
|
HostStreamIn::Transport(StreamTransportEvent::DataAvailable) => {
|
||||||
|
Self::wake_waiters(ctx, self.child_session, &mut self.data_waiters, Ok(()));
|
||||||
|
}
|
||||||
|
HostStreamIn::Transport(StreamTransportEvent::CapacityAvailable) => {
|
||||||
|
Self::wake_waiters(ctx, self.child_session, &mut self.capacity_waiters, Ok(()));
|
||||||
|
}
|
||||||
|
HostStreamIn::Transport(StreamTransportEvent::Fault(reason)) => {
|
||||||
|
self.fail_open(ctx, DataPlaneError::StreamFault(reason));
|
||||||
|
}
|
||||||
|
HostStreamIn::Transport(StreamTransportEvent::Quiesced) => {
|
||||||
|
self.transport_quiesced = true;
|
||||||
|
if self.terminal.is_some() {
|
||||||
|
self.release_ring(ctx);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
HostStreamIn::DataAvailable => {
|
||||||
|
if let Some(matched) = &self.matched {
|
||||||
|
self.transport.source_progress(matched.incarnation);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
HostStreamIn::CapacityAvailable => {
|
||||||
|
if let Some(matched) = &self.matched {
|
||||||
|
self.transport.sink_progress(matched.incarnation);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
HostStreamIn::WaitData { reply_to } => {
|
||||||
|
let result = self
|
||||||
|
.terminal
|
||||||
|
.as_ref()
|
||||||
|
.map(|error| Err(error.clone()))
|
||||||
|
.or_else(|| {
|
||||||
|
self.matched
|
||||||
|
.as_ref()
|
||||||
|
.filter(|matched| self.transport.sink_has_data(matched.incarnation))
|
||||||
|
.map(|_| Ok(()))
|
||||||
|
});
|
||||||
|
if let Some(result) = result {
|
||||||
|
let _ = ctx.send(
|
||||||
|
self.child_session,
|
||||||
|
ChildSessionIn::StreamWake { reply_to, result },
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
self.data_waiters.push(reply_to);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
HostStreamIn::WaitCapacity { reply_to } => {
|
||||||
|
let result = self
|
||||||
|
.terminal
|
||||||
|
.as_ref()
|
||||||
|
.map(|error| Err(error.clone()))
|
||||||
|
.or_else(|| {
|
||||||
|
self.matched
|
||||||
|
.as_ref()
|
||||||
|
.filter(|matched| {
|
||||||
|
self.transport.source_has_capacity(matched.incarnation)
|
||||||
|
})
|
||||||
|
.map(|_| Ok(()))
|
||||||
|
});
|
||||||
|
if let Some(result) = result {
|
||||||
|
let _ = ctx.send(
|
||||||
|
self.child_session,
|
||||||
|
ChildSessionIn::StreamWake { reply_to, result },
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
self.capacity_waiters.push(reply_to);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
HostStreamIn::Close { clean, reply_to } => {
|
||||||
|
if let Some(reply_to) = reply_to {
|
||||||
|
self.close_waiters.push(reply_to);
|
||||||
|
}
|
||||||
|
let error = if clean {
|
||||||
|
DataPlaneError::StreamClosed
|
||||||
|
} else {
|
||||||
|
DataPlaneError::OperationCancelled
|
||||||
|
};
|
||||||
|
self.fail_open(ctx, error);
|
||||||
|
}
|
||||||
|
HostStreamIn::PeerTerminated { incarnation, error } => {
|
||||||
|
if self
|
||||||
|
.matched
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|matched| matched.incarnation == incarnation)
|
||||||
|
{
|
||||||
|
if !self.opened {
|
||||||
|
let _ = ctx.send(
|
||||||
|
self.child_session,
|
||||||
|
ChildSessionIn::OperationFailed {
|
||||||
|
operation: self.operation,
|
||||||
|
error: error.clone(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
self.begin_terminal(ctx, error, false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
HostStreamIn::ReleaseComplete(result) => {
|
||||||
|
if let Err(error) = &result
|
||||||
|
&& !self.opened
|
||||||
|
{
|
||||||
|
let _ = ctx.send(
|
||||||
|
self.child_session,
|
||||||
|
ChildSessionIn::OperationFailed {
|
||||||
|
operation: self.operation,
|
||||||
|
error: error.clone(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
self.complete_release(ctx, result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn on_stop(&mut self, _ctx: &Ctx<'_>) {
|
||||||
|
if let Some(matched) = &self.matched {
|
||||||
|
self.transport.terminate(matched.incarnation);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn namespace_error(error: NamespaceError) -> DataPlaneError {
|
fn namespace_error(error: NamespaceError) -> DataPlaneError {
|
||||||
match error {
|
match error {
|
||||||
NamespaceError::PathNotFound(path) => DataPlaneError::PathNotFound(path),
|
NamespaceError::PathNotFound(path) => DataPlaneError::PathNotFound(path),
|
||||||
|
NamespaceError::WrongEntryType {
|
||||||
|
path,
|
||||||
|
expected,
|
||||||
|
found,
|
||||||
|
} => DataPlaneError::WrongEntryType {
|
||||||
|
path,
|
||||||
|
expected,
|
||||||
|
found,
|
||||||
|
},
|
||||||
|
NamespaceError::PathReplaced(path) => DataPlaneError::PathReplaced(path),
|
||||||
|
NamespaceError::DuplicateStreamRole { path, role } => {
|
||||||
|
DataPlaneError::SessionFailed(format!("stream path {path} already has a {role:?}"))
|
||||||
|
}
|
||||||
|
NamespaceError::StaleIncarnation { path, incarnation } => {
|
||||||
|
DataPlaneError::SessionFailed(format!(
|
||||||
|
"stream path {path} no longer names incarnation {}:{}",
|
||||||
|
incarnation.authority_epoch, incarnation.revision
|
||||||
|
))
|
||||||
|
}
|
||||||
NamespaceError::SourceRecovery(reason) => DataPlaneError::SourceFailure(reason),
|
NamespaceError::SourceRecovery(reason) => DataPlaneError::SourceFailure(reason),
|
||||||
NamespaceError::DirectoryUnavailable(reason)
|
NamespaceError::DirectoryUnavailable(reason)
|
||||||
| NamespaceError::Storage(reason)
|
| NamespaceError::Storage(reason)
|
||||||
|
|
@ -1345,6 +2139,7 @@ impl HostBlobBindingActor {
|
||||||
|
|
||||||
fn finish_without_lease(&mut self, ctx: &Ctx<'_>, outcome: ReleaseOutcome) {
|
fn finish_without_lease(&mut self, ctx: &Ctx<'_>, outcome: ReleaseOutcome) {
|
||||||
self.state = HostBindingState::Released;
|
self.state = HostBindingState::Released;
|
||||||
|
let read_released = matches!(outcome, ReleaseOutcome::ReadReleased);
|
||||||
if let Some(auxiliary) = self.auxiliary.take() {
|
if let Some(auxiliary) = self.auxiliary.take() {
|
||||||
let _ = ctx.stop_actor(auxiliary);
|
let _ = ctx.stop_actor(auxiliary);
|
||||||
}
|
}
|
||||||
|
|
@ -1354,6 +2149,9 @@ impl HostBlobBindingActor {
|
||||||
ChildSessionIn::WriteAborted { operation },
|
ChildSessionIn::WriteAborted { operation },
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
if read_released {
|
||||||
|
let _ = ctx.send(self.child_session, ChildSessionIn::BlobReleased);
|
||||||
|
}
|
||||||
let _ = ctx.send(
|
let _ = ctx.send(
|
||||||
self.host_session,
|
self.host_session,
|
||||||
HostSessionIn::BindingDone {
|
HostSessionIn::BindingDone {
|
||||||
|
|
|
||||||
|
|
@ -25,3 +25,4 @@ pub mod path;
|
||||||
pub mod protocol;
|
pub mod protocol;
|
||||||
pub mod ring;
|
pub mod ring;
|
||||||
pub mod source;
|
pub mod source;
|
||||||
|
pub mod stream_transport;
|
||||||
|
|
|
||||||
|
|
@ -33,9 +33,51 @@ pub struct BlobBinding {
|
||||||
pub revision: u64,
|
pub revision: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub enum EntryKind {
|
||||||
|
Blob,
|
||||||
|
Stream,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub enum StreamRole {
|
||||||
|
Source,
|
||||||
|
Sink,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||||
|
pub struct StreamIncarnation {
|
||||||
|
pub authority_epoch: u64,
|
||||||
|
pub revision: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct StreamMatch {
|
||||||
|
pub incarnation: StreamIncarnation,
|
||||||
|
pub source: ActorAddress,
|
||||||
|
pub source_descriptor: Vec<u8>,
|
||||||
|
pub sink_descriptor: Vec<u8>,
|
||||||
|
pub sink: ActorAddress,
|
||||||
|
pub revision: u64,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub enum NamespaceError {
|
pub enum NamespaceError {
|
||||||
PathNotFound(DataPath),
|
PathNotFound(DataPath),
|
||||||
|
WrongEntryType {
|
||||||
|
path: DataPath,
|
||||||
|
expected: EntryKind,
|
||||||
|
found: EntryKind,
|
||||||
|
},
|
||||||
|
PathReplaced(DataPath),
|
||||||
|
DuplicateStreamRole {
|
||||||
|
path: DataPath,
|
||||||
|
role: StreamRole,
|
||||||
|
},
|
||||||
|
StaleIncarnation {
|
||||||
|
path: DataPath,
|
||||||
|
incarnation: StreamIncarnation,
|
||||||
|
},
|
||||||
OperationConflict(OperationId),
|
OperationConflict(OperationId),
|
||||||
Storage(String),
|
Storage(String),
|
||||||
SourceRecovery(String),
|
SourceRecovery(String),
|
||||||
|
|
@ -47,6 +89,25 @@ impl fmt::Display for NamespaceError {
|
||||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
match self {
|
match self {
|
||||||
Self::PathNotFound(path) => write!(f, "data path not found: {path}"),
|
Self::PathNotFound(path) => write!(f, "data path not found: {path}"),
|
||||||
|
Self::WrongEntryType {
|
||||||
|
path,
|
||||||
|
expected,
|
||||||
|
found,
|
||||||
|
} => write!(
|
||||||
|
f,
|
||||||
|
"data path {path} has entry kind {found:?}, expected {expected:?}"
|
||||||
|
),
|
||||||
|
Self::PathReplaced(path) => {
|
||||||
|
write!(f, "pending data path was replaced: {path}")
|
||||||
|
}
|
||||||
|
Self::DuplicateStreamRole { path, role } => {
|
||||||
|
write!(f, "stream path {path} already has a {role:?}")
|
||||||
|
}
|
||||||
|
Self::StaleIncarnation { path, incarnation } => write!(
|
||||||
|
f,
|
||||||
|
"stream path {path} no longer names incarnation {}:{}",
|
||||||
|
incarnation.authority_epoch, incarnation.revision
|
||||||
|
),
|
||||||
Self::OperationConflict(operation) => write!(
|
Self::OperationConflict(operation) => write!(
|
||||||
f,
|
f,
|
||||||
"namespace operation ID {:02x?} was reused for a different request",
|
"namespace operation ID {:02x?} was reused for a different request",
|
||||||
|
|
@ -92,6 +153,26 @@ pub enum DataDirectoryIn {
|
||||||
operation_id: OperationId,
|
operation_id: OperationId,
|
||||||
reply_to: ActorAddress,
|
reply_to: ActorAddress,
|
||||||
},
|
},
|
||||||
|
OpenStream {
|
||||||
|
request_id: DirectoryRequestId,
|
||||||
|
path: DataPath,
|
||||||
|
role: StreamRole,
|
||||||
|
descriptor: Vec<u8>,
|
||||||
|
endpoint: ActorAddress,
|
||||||
|
replace: bool,
|
||||||
|
operation_id: OperationId,
|
||||||
|
reply_to: ActorAddress,
|
||||||
|
},
|
||||||
|
CancelStream {
|
||||||
|
path: DataPath,
|
||||||
|
operation_id: OperationId,
|
||||||
|
},
|
||||||
|
CloseStream {
|
||||||
|
request_id: DirectoryRequestId,
|
||||||
|
path: DataPath,
|
||||||
|
incarnation: StreamIncarnation,
|
||||||
|
reply_to: ActorAddress,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
impl NetworkMessage for DataDirectoryIn {
|
impl NetworkMessage for DataDirectoryIn {
|
||||||
|
|
@ -117,14 +198,25 @@ pub enum DataDirectoryOut {
|
||||||
authority_epoch: u64,
|
authority_epoch: u64,
|
||||||
result: Result<MutationReceipt, NamespaceError>,
|
result: Result<MutationReceipt, NamespaceError>,
|
||||||
},
|
},
|
||||||
|
StreamOpened {
|
||||||
|
request_id: DirectoryRequestId,
|
||||||
|
authority_epoch: u64,
|
||||||
|
result: Result<StreamMatch, NamespaceError>,
|
||||||
|
},
|
||||||
|
StreamClosed {
|
||||||
|
request_id: DirectoryRequestId,
|
||||||
|
authority_epoch: u64,
|
||||||
|
result: Result<(), NamespaceError>,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
impl DataDirectoryOut {
|
impl DataDirectoryOut {
|
||||||
pub fn request_id(&self) -> DirectoryRequestId {
|
pub fn request_id(&self) -> DirectoryRequestId {
|
||||||
match self {
|
match self {
|
||||||
Self::Registered { request_id, .. }
|
Self::Registered { request_id, .. }
|
||||||
| Self::Resolved { request_id, .. }
|
| Self::Resolved { request_id, .. }
|
||||||
| Self::Unregistered { request_id, .. } => *request_id,
|
| Self::Unregistered { request_id, .. }
|
||||||
|
| Self::StreamOpened { request_id, .. }
|
||||||
|
| Self::StreamClosed { request_id, .. } => *request_id,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -145,8 +237,19 @@ pub enum NamespaceRequest {
|
||||||
path: DataPath,
|
path: DataPath,
|
||||||
operation_id: OperationId,
|
operation_id: OperationId,
|
||||||
},
|
},
|
||||||
|
OpenStream {
|
||||||
|
path: DataPath,
|
||||||
|
role: StreamRole,
|
||||||
|
endpoint: ActorAddress,
|
||||||
|
descriptor: Vec<u8>,
|
||||||
|
replace: bool,
|
||||||
|
operation_id: OperationId,
|
||||||
|
},
|
||||||
|
CloseStream {
|
||||||
|
path: DataPath,
|
||||||
|
incarnation: StreamIncarnation,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||||
pub enum NamespaceClientIn {
|
pub enum NamespaceClientIn {
|
||||||
Request {
|
Request {
|
||||||
|
|
@ -177,9 +280,42 @@ enum RuntimeSource {
|
||||||
Unavailable(String),
|
Unavailable(String),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct PendingStream {
|
||||||
|
role: StreamRole,
|
||||||
|
endpoint: ActorAddress,
|
||||||
|
operation_id: OperationId,
|
||||||
|
request_id: DirectoryRequestId,
|
||||||
|
descriptor: Vec<u8>,
|
||||||
|
reply_to: ActorAddress,
|
||||||
|
revision: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct StreamOpenRequest {
|
||||||
|
request_id: DirectoryRequestId,
|
||||||
|
path: DataPath,
|
||||||
|
role: StreamRole,
|
||||||
|
endpoint: ActorAddress,
|
||||||
|
replace: bool,
|
||||||
|
descriptor: Vec<u8>,
|
||||||
|
operation_id: OperationId,
|
||||||
|
reply_to: ActorAddress,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ActiveStream {
|
||||||
|
binding: StreamMatch,
|
||||||
|
source_operation: OperationId,
|
||||||
|
sink_operation: OperationId,
|
||||||
|
}
|
||||||
|
|
||||||
|
enum RuntimeStream {
|
||||||
|
Pending(PendingStream),
|
||||||
|
Active(ActiveStream),
|
||||||
|
}
|
||||||
|
|
||||||
pub struct DataDirectoryActor {
|
pub struct DataDirectoryActor {
|
||||||
store: NamespaceStore,
|
store: NamespaceStore,
|
||||||
sources: BTreeMap<DataPath, RuntimeSource>,
|
sources: BTreeMap<DataPath, RuntimeSource>,
|
||||||
|
streams: BTreeMap<DataPath, RuntimeStream>,
|
||||||
authority_epoch: u64,
|
authority_epoch: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -202,6 +338,7 @@ impl DataDirectoryActor {
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
store,
|
store,
|
||||||
sources,
|
sources,
|
||||||
|
streams: BTreeMap::new(),
|
||||||
authority_epoch,
|
authority_epoch,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
@ -232,6 +369,257 @@ impl DataDirectoryActor {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn bind_stream(
|
||||||
|
&mut self,
|
||||||
|
path: DataPath,
|
||||||
|
operation_id: OperationId,
|
||||||
|
retired: Option<ActorAddress>,
|
||||||
|
) -> Result<MutationReceipt, NamespaceError> {
|
||||||
|
let request = MutationRequest::BindStream { path: path.clone() };
|
||||||
|
if let Some(replayed) = self.replay(operation_id, &request) {
|
||||||
|
return replayed;
|
||||||
|
}
|
||||||
|
let revision = self.store.snapshot().next_revision;
|
||||||
|
let next_revision = revision
|
||||||
|
.checked_add(1)
|
||||||
|
.filter(|revision| *revision != 0)
|
||||||
|
.ok_or(NamespaceStoreError::RevisionExhausted)?;
|
||||||
|
let receipt = MutationReceipt { revision };
|
||||||
|
let mut next = self.store.snapshot().clone();
|
||||||
|
next.next_revision = next_revision;
|
||||||
|
next.bindings.remove(&path);
|
||||||
|
if let Some(retired) = retired
|
||||||
|
&& !next.retirements.contains(&retired)
|
||||||
|
{
|
||||||
|
next.retirements.push(retired);
|
||||||
|
}
|
||||||
|
next.operations.insert(
|
||||||
|
operation_id,
|
||||||
|
PersistedOperation {
|
||||||
|
request,
|
||||||
|
result: PersistedMutationResult::Committed(receipt),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
self.store.commit(next)?;
|
||||||
|
self.sources.remove(&path);
|
||||||
|
Ok(receipt)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn send_stream_result(
|
||||||
|
&self,
|
||||||
|
ctx: &Ctx<'_>,
|
||||||
|
request_id: DirectoryRequestId,
|
||||||
|
reply_to: ActorAddress,
|
||||||
|
result: Result<StreamMatch, NamespaceError>,
|
||||||
|
) {
|
||||||
|
let _ = ctx.send(
|
||||||
|
reply_to,
|
||||||
|
NamespaceClientIn::DirectoryReply(DataDirectoryOut::StreamOpened {
|
||||||
|
request_id,
|
||||||
|
authority_epoch: self.authority_epoch,
|
||||||
|
result,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn displace_stream(&mut self, ctx: &Ctx<'_>, path: &DataPath) {
|
||||||
|
if let Some(RuntimeStream::Pending(pending)) = self.streams.remove(path) {
|
||||||
|
self.send_stream_result(
|
||||||
|
ctx,
|
||||||
|
pending.request_id,
|
||||||
|
pending.reply_to,
|
||||||
|
Err(NamespaceError::PathReplaced(path.clone())),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn open_stream(&mut self, ctx: &Ctx<'_>, request: StreamOpenRequest) {
|
||||||
|
let StreamOpenRequest {
|
||||||
|
request_id,
|
||||||
|
path,
|
||||||
|
role,
|
||||||
|
endpoint,
|
||||||
|
replace,
|
||||||
|
descriptor,
|
||||||
|
operation_id,
|
||||||
|
reply_to,
|
||||||
|
} = request;
|
||||||
|
if let Some(RuntimeStream::Pending(pending)) = self.streams.get_mut(&path)
|
||||||
|
&& pending.operation_id == operation_id
|
||||||
|
{
|
||||||
|
if pending.role != role || pending.endpoint != endpoint {
|
||||||
|
self.send_stream_result(
|
||||||
|
ctx,
|
||||||
|
request_id,
|
||||||
|
reply_to,
|
||||||
|
Err(NamespaceError::OperationConflict(operation_id)),
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
pending.request_id = request_id;
|
||||||
|
pending.reply_to = reply_to;
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if let Some(RuntimeStream::Active(active)) = self.streams.get(&path)
|
||||||
|
&& (active.source_operation == operation_id || active.sink_operation == operation_id)
|
||||||
|
{
|
||||||
|
self.send_stream_result(ctx, request_id, reply_to, Ok(active.binding.clone()));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let compatible_pending = matches!(
|
||||||
|
self.streams.get(&path),
|
||||||
|
Some(RuntimeStream::Pending(pending)) if pending.role != role
|
||||||
|
);
|
||||||
|
if replace && self.streams.contains_key(&path) && !compatible_pending {
|
||||||
|
self.displace_stream(ctx, &path);
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(RuntimeStream::Pending(pending)) = self.streams.remove(&path) {
|
||||||
|
if pending.role == role {
|
||||||
|
self.streams
|
||||||
|
.insert(path.clone(), RuntimeStream::Pending(pending));
|
||||||
|
self.send_stream_result(
|
||||||
|
ctx,
|
||||||
|
request_id,
|
||||||
|
reply_to,
|
||||||
|
Err(NamespaceError::DuplicateStreamRole { path, role }),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let (
|
||||||
|
source,
|
||||||
|
sink,
|
||||||
|
source_descriptor,
|
||||||
|
sink_descriptor,
|
||||||
|
source_operation,
|
||||||
|
sink_operation,
|
||||||
|
) = match role {
|
||||||
|
StreamRole::Source => (
|
||||||
|
endpoint,
|
||||||
|
pending.endpoint,
|
||||||
|
descriptor,
|
||||||
|
pending.descriptor,
|
||||||
|
operation_id,
|
||||||
|
pending.operation_id,
|
||||||
|
),
|
||||||
|
StreamRole::Sink => (
|
||||||
|
pending.endpoint,
|
||||||
|
endpoint,
|
||||||
|
pending.descriptor,
|
||||||
|
descriptor,
|
||||||
|
pending.operation_id,
|
||||||
|
operation_id,
|
||||||
|
),
|
||||||
|
};
|
||||||
|
let binding = StreamMatch {
|
||||||
|
incarnation: StreamIncarnation {
|
||||||
|
authority_epoch: self.authority_epoch,
|
||||||
|
revision: pending.revision,
|
||||||
|
},
|
||||||
|
source,
|
||||||
|
source_descriptor,
|
||||||
|
sink_descriptor,
|
||||||
|
sink,
|
||||||
|
revision: pending.revision,
|
||||||
|
};
|
||||||
|
self.send_stream_result(
|
||||||
|
ctx,
|
||||||
|
pending.request_id,
|
||||||
|
pending.reply_to,
|
||||||
|
Ok(binding.clone()),
|
||||||
|
);
|
||||||
|
self.send_stream_result(ctx, request_id, reply_to, Ok(binding.clone()));
|
||||||
|
self.streams.insert(
|
||||||
|
path,
|
||||||
|
RuntimeStream::Active(ActiveStream {
|
||||||
|
binding,
|
||||||
|
source_operation,
|
||||||
|
sink_operation,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if self.streams.contains_key(&path) {
|
||||||
|
self.send_stream_result(
|
||||||
|
ctx,
|
||||||
|
request_id,
|
||||||
|
reply_to,
|
||||||
|
Err(NamespaceError::DuplicateStreamRole { path, role }),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if self.store.snapshot().bindings.contains_key(&path) && !replace {
|
||||||
|
self.send_stream_result(
|
||||||
|
ctx,
|
||||||
|
request_id,
|
||||||
|
reply_to,
|
||||||
|
Err(NamespaceError::WrongEntryType {
|
||||||
|
path,
|
||||||
|
expected: EntryKind::Stream,
|
||||||
|
found: EntryKind::Blob,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let retired = self.sources.get(&path).and_then(|source| match source {
|
||||||
|
RuntimeSource::Available(actor) => Some(*actor),
|
||||||
|
RuntimeSource::Unavailable(_) => None,
|
||||||
|
});
|
||||||
|
match self.bind_stream(path.clone(), operation_id, retired) {
|
||||||
|
Ok(receipt) => {
|
||||||
|
if let Some(retired) = retired {
|
||||||
|
let _ = ctx.send(retired, BlobSourceIn::Retire);
|
||||||
|
}
|
||||||
|
self.streams.insert(
|
||||||
|
path,
|
||||||
|
RuntimeStream::Pending(PendingStream {
|
||||||
|
role,
|
||||||
|
descriptor,
|
||||||
|
endpoint,
|
||||||
|
operation_id,
|
||||||
|
request_id,
|
||||||
|
reply_to,
|
||||||
|
revision: receipt.revision,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Err(error) => self.send_stream_result(ctx, request_id, reply_to, Err(error)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cancel_stream(&mut self, path: &DataPath, operation_id: OperationId) {
|
||||||
|
let should_remove = matches!(
|
||||||
|
self.streams.get(path),
|
||||||
|
Some(RuntimeStream::Pending(pending)) if pending.operation_id == operation_id
|
||||||
|
);
|
||||||
|
if should_remove {
|
||||||
|
self.streams.remove(path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn close_stream(
|
||||||
|
&mut self,
|
||||||
|
path: &DataPath,
|
||||||
|
incarnation: StreamIncarnation,
|
||||||
|
) -> Result<(), NamespaceError> {
|
||||||
|
let matches = matches!(
|
||||||
|
self.streams.get(path),
|
||||||
|
Some(RuntimeStream::Active(active)) if active.binding.incarnation == incarnation
|
||||||
|
);
|
||||||
|
if matches {
|
||||||
|
self.streams.remove(path);
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(NamespaceError::StaleIncarnation {
|
||||||
|
path: path.clone(),
|
||||||
|
incarnation,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn register(
|
fn register(
|
||||||
&mut self,
|
&mut self,
|
||||||
path: DataPath,
|
path: DataPath,
|
||||||
|
|
@ -283,6 +671,13 @@ impl DataDirectoryActor {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn resolve(&self, path: &DataPath) -> Result<BlobBinding, NamespaceError> {
|
fn resolve(&self, path: &DataPath) -> Result<BlobBinding, NamespaceError> {
|
||||||
|
if self.streams.contains_key(path) {
|
||||||
|
return Err(NamespaceError::WrongEntryType {
|
||||||
|
path: path.clone(),
|
||||||
|
expected: EntryKind::Blob,
|
||||||
|
found: EntryKind::Stream,
|
||||||
|
});
|
||||||
|
}
|
||||||
let persisted = self
|
let persisted = self
|
||||||
.store
|
.store
|
||||||
.snapshot()
|
.snapshot()
|
||||||
|
|
@ -376,6 +771,7 @@ impl ActorInterface for DataDirectoryActor {
|
||||||
operation_id,
|
operation_id,
|
||||||
reply_to,
|
reply_to,
|
||||||
} => {
|
} => {
|
||||||
|
let logical = path.clone();
|
||||||
let replayed = self.store.snapshot().operations.contains_key(&operation_id);
|
let replayed = self.store.snapshot().operations.contains_key(&operation_id);
|
||||||
let retired = (!replayed)
|
let retired = (!replayed)
|
||||||
.then(|| self.sources.get(&path))
|
.then(|| self.sources.get(&path))
|
||||||
|
|
@ -390,6 +786,9 @@ impl ActorInterface for DataDirectoryActor {
|
||||||
{
|
{
|
||||||
let _ = ctx.send(retired, BlobSourceIn::Retire);
|
let _ = ctx.send(retired, BlobSourceIn::Retire);
|
||||||
}
|
}
|
||||||
|
if result.is_ok() {
|
||||||
|
self.displace_stream(ctx, &logical);
|
||||||
|
}
|
||||||
let _ = ctx.send(
|
let _ = ctx.send(
|
||||||
reply_to,
|
reply_to,
|
||||||
NamespaceClientIn::DirectoryReply(DataDirectoryOut::Registered {
|
NamespaceClientIn::DirectoryReply(DataDirectoryOut::Registered {
|
||||||
|
|
@ -420,6 +819,21 @@ impl ActorInterface for DataDirectoryActor {
|
||||||
operation_id,
|
operation_id,
|
||||||
reply_to,
|
reply_to,
|
||||||
} => {
|
} => {
|
||||||
|
if self.streams.contains_key(&path) {
|
||||||
|
let _ = ctx.send(
|
||||||
|
reply_to,
|
||||||
|
NamespaceClientIn::DirectoryReply(DataDirectoryOut::Unregistered {
|
||||||
|
request_id,
|
||||||
|
authority_epoch: self.authority_epoch,
|
||||||
|
result: Err(NamespaceError::WrongEntryType {
|
||||||
|
path,
|
||||||
|
expected: EntryKind::Blob,
|
||||||
|
found: EntryKind::Stream,
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
let replayed = self.store.snapshot().operations.contains_key(&operation_id);
|
let replayed = self.store.snapshot().operations.contains_key(&operation_id);
|
||||||
let retired = (!replayed)
|
let retired = (!replayed)
|
||||||
.then(|| self.sources.get(&path))
|
.then(|| self.sources.get(&path))
|
||||||
|
|
@ -443,6 +857,47 @@ impl ActorInterface for DataDirectoryActor {
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
DataDirectoryIn::OpenStream {
|
||||||
|
request_id,
|
||||||
|
path,
|
||||||
|
role,
|
||||||
|
endpoint,
|
||||||
|
descriptor,
|
||||||
|
replace,
|
||||||
|
operation_id,
|
||||||
|
reply_to,
|
||||||
|
} => self.open_stream(
|
||||||
|
ctx,
|
||||||
|
StreamOpenRequest {
|
||||||
|
request_id,
|
||||||
|
path,
|
||||||
|
role,
|
||||||
|
endpoint,
|
||||||
|
replace,
|
||||||
|
descriptor,
|
||||||
|
operation_id,
|
||||||
|
reply_to,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
DataDirectoryIn::CancelStream { path, operation_id } => {
|
||||||
|
self.cancel_stream(&path, operation_id);
|
||||||
|
}
|
||||||
|
DataDirectoryIn::CloseStream {
|
||||||
|
request_id,
|
||||||
|
path,
|
||||||
|
incarnation,
|
||||||
|
reply_to,
|
||||||
|
} => {
|
||||||
|
let result = self.close_stream(&path, incarnation);
|
||||||
|
let _ = ctx.send(
|
||||||
|
reply_to,
|
||||||
|
NamespaceClientIn::DirectoryReply(DataDirectoryOut::StreamClosed {
|
||||||
|
request_id,
|
||||||
|
authority_epoch: self.authority_epoch,
|
||||||
|
result,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -513,6 +968,29 @@ impl NamespaceClientActor {
|
||||||
operation_id: *operation_id,
|
operation_id: *operation_id,
|
||||||
reply_to: ctx.self_addr(),
|
reply_to: ctx.self_addr(),
|
||||||
},
|
},
|
||||||
|
NamespaceRequest::OpenStream {
|
||||||
|
path,
|
||||||
|
role,
|
||||||
|
endpoint,
|
||||||
|
descriptor,
|
||||||
|
replace,
|
||||||
|
operation_id,
|
||||||
|
} => DataDirectoryIn::OpenStream {
|
||||||
|
request_id,
|
||||||
|
path: path.clone(),
|
||||||
|
role: *role,
|
||||||
|
endpoint: *endpoint,
|
||||||
|
descriptor: descriptor.clone(),
|
||||||
|
replace: *replace,
|
||||||
|
operation_id: *operation_id,
|
||||||
|
reply_to: ctx.self_addr(),
|
||||||
|
},
|
||||||
|
NamespaceRequest::CloseStream { path, incarnation } => DataDirectoryIn::CloseStream {
|
||||||
|
request_id,
|
||||||
|
path: path.clone(),
|
||||||
|
incarnation: *incarnation,
|
||||||
|
reply_to: ctx.self_addr(),
|
||||||
|
},
|
||||||
};
|
};
|
||||||
let _ = ctx.send(directory, message);
|
let _ = ctx.send(directory, message);
|
||||||
}
|
}
|
||||||
|
|
@ -558,6 +1036,25 @@ impl ActorInterface for NamespaceClientActor {
|
||||||
.insert(request_id, PendingRequest { request, reply_to });
|
.insert(request_id, PendingRequest { request, reply_to });
|
||||||
}
|
}
|
||||||
NamespaceClientIn::Cancel { reply_to } => {
|
NamespaceClientIn::Cancel { reply_to } => {
|
||||||
|
let cancelled: Vec<(DataPath, OperationId)> = self
|
||||||
|
.pending
|
||||||
|
.values()
|
||||||
|
.filter(|pending| pending.reply_to == reply_to)
|
||||||
|
.filter_map(|pending| match &pending.request {
|
||||||
|
NamespaceRequest::OpenStream {
|
||||||
|
path, operation_id, ..
|
||||||
|
} => Some((path.clone(), *operation_id)),
|
||||||
|
_ => None,
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
if let Some(directory) = self.discovery.current_directory() {
|
||||||
|
for (path, operation_id) in cancelled {
|
||||||
|
let _ = ctx.send(
|
||||||
|
directory,
|
||||||
|
DataDirectoryIn::CancelStream { path, operation_id },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
self.pending
|
self.pending
|
||||||
.retain(|_, pending| pending.reply_to != reply_to);
|
.retain(|_, pending| pending.reply_to != reply_to);
|
||||||
}
|
}
|
||||||
|
|
@ -692,6 +1189,92 @@ impl NamespaceClient {
|
||||||
))),
|
))),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn open_stream_inner(
|
||||||
|
&self,
|
||||||
|
path: DataPath,
|
||||||
|
role: StreamRole,
|
||||||
|
endpoint: ActorAddress,
|
||||||
|
replace: bool,
|
||||||
|
operation_id: OperationId,
|
||||||
|
) -> Result<StreamMatch, NamespaceError> {
|
||||||
|
match self
|
||||||
|
.request(NamespaceRequest::OpenStream {
|
||||||
|
path,
|
||||||
|
role,
|
||||||
|
endpoint,
|
||||||
|
replace,
|
||||||
|
descriptor: Vec::new(),
|
||||||
|
operation_id,
|
||||||
|
})
|
||||||
|
.await?
|
||||||
|
{
|
||||||
|
DataDirectoryOut::StreamOpened { result, .. } => result,
|
||||||
|
other => Err(NamespaceError::Protocol(format!(
|
||||||
|
"expected stream-open reply, received {other:?}"
|
||||||
|
))),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn open_stream(
|
||||||
|
&self,
|
||||||
|
path: DataPath,
|
||||||
|
role: StreamRole,
|
||||||
|
endpoint: ActorAddress,
|
||||||
|
operation_id: OperationId,
|
||||||
|
) -> Result<StreamMatch, NamespaceError> {
|
||||||
|
self.open_stream_inner(path, role, endpoint, false, operation_id)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn replace_with_stream(
|
||||||
|
&self,
|
||||||
|
path: DataPath,
|
||||||
|
role: StreamRole,
|
||||||
|
endpoint: ActorAddress,
|
||||||
|
operation_id: OperationId,
|
||||||
|
) -> Result<StreamMatch, NamespaceError> {
|
||||||
|
self.open_stream_inner(path, role, endpoint, true, operation_id)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn close_stream(
|
||||||
|
&self,
|
||||||
|
path: DataPath,
|
||||||
|
incarnation: StreamIncarnation,
|
||||||
|
) -> Result<(), NamespaceError> {
|
||||||
|
match self
|
||||||
|
.request(NamespaceRequest::CloseStream { path, incarnation })
|
||||||
|
.await?
|
||||||
|
{
|
||||||
|
DataDirectoryOut::StreamClosed { result, .. } => result,
|
||||||
|
other => Err(NamespaceError::Protocol(format!(
|
||||||
|
"expected stream-close reply, received {other:?}"
|
||||||
|
))),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct DirectoryStreamCancellation {
|
||||||
|
runtime: Runtime,
|
||||||
|
directory: ActorAddress,
|
||||||
|
path: DataPath,
|
||||||
|
operation_id: OperationId,
|
||||||
|
armed: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for DirectoryStreamCancellation {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
if self.armed {
|
||||||
|
let _ = self.runtime.send_to(
|
||||||
|
self.directory,
|
||||||
|
DataDirectoryIn::CancelStream {
|
||||||
|
path: self.path.clone(),
|
||||||
|
operation_id: self.operation_id,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
|
|
@ -814,6 +1397,100 @@ impl DirectoryClient {
|
||||||
))),
|
))),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn open_stream_inner(
|
||||||
|
&self,
|
||||||
|
path: DataPath,
|
||||||
|
role: StreamRole,
|
||||||
|
endpoint: ActorAddress,
|
||||||
|
replace: bool,
|
||||||
|
operation_id: OperationId,
|
||||||
|
) -> Result<StreamMatch, NamespaceError> {
|
||||||
|
let inbox = self
|
||||||
|
.runtime
|
||||||
|
.new_inbox::<NamespaceClientIn>()
|
||||||
|
.map_err(|error| NamespaceError::DirectoryUnavailable(error.to_string()))?;
|
||||||
|
let mut cancellation = DirectoryStreamCancellation {
|
||||||
|
runtime: self.runtime.clone(),
|
||||||
|
directory: self.directory,
|
||||||
|
path: path.clone(),
|
||||||
|
operation_id,
|
||||||
|
armed: true,
|
||||||
|
};
|
||||||
|
self.runtime
|
||||||
|
.send_to(
|
||||||
|
self.directory,
|
||||||
|
DataDirectoryIn::OpenStream {
|
||||||
|
request_id: self.request_id(),
|
||||||
|
path,
|
||||||
|
role,
|
||||||
|
endpoint,
|
||||||
|
replace,
|
||||||
|
operation_id,
|
||||||
|
descriptor: Vec::new(),
|
||||||
|
reply_to: *inbox.addr(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.map_err(|error| NamespaceError::DirectoryUnavailable(error.to_string()))?;
|
||||||
|
let result = match self.receive(&inbox).await? {
|
||||||
|
DataDirectoryOut::StreamOpened { result, .. } => result,
|
||||||
|
other => Err(NamespaceError::Protocol(format!(
|
||||||
|
"expected stream-open reply, received {other:?}"
|
||||||
|
))),
|
||||||
|
};
|
||||||
|
cancellation.armed = false;
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn open_stream(
|
||||||
|
&self,
|
||||||
|
path: DataPath,
|
||||||
|
role: StreamRole,
|
||||||
|
endpoint: ActorAddress,
|
||||||
|
operation_id: OperationId,
|
||||||
|
) -> Result<StreamMatch, NamespaceError> {
|
||||||
|
self.open_stream_inner(path, role, endpoint, false, operation_id)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn replace_with_stream(
|
||||||
|
&self,
|
||||||
|
path: DataPath,
|
||||||
|
role: StreamRole,
|
||||||
|
endpoint: ActorAddress,
|
||||||
|
operation_id: OperationId,
|
||||||
|
) -> Result<StreamMatch, NamespaceError> {
|
||||||
|
self.open_stream_inner(path, role, endpoint, true, operation_id)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn close_stream(
|
||||||
|
&self,
|
||||||
|
path: DataPath,
|
||||||
|
incarnation: StreamIncarnation,
|
||||||
|
) -> Result<(), NamespaceError> {
|
||||||
|
let inbox = self
|
||||||
|
.runtime
|
||||||
|
.new_inbox::<NamespaceClientIn>()
|
||||||
|
.map_err(|error| NamespaceError::DirectoryUnavailable(error.to_string()))?;
|
||||||
|
self.runtime
|
||||||
|
.send_to(
|
||||||
|
self.directory,
|
||||||
|
DataDirectoryIn::CloseStream {
|
||||||
|
request_id: self.request_id(),
|
||||||
|
path,
|
||||||
|
incarnation,
|
||||||
|
reply_to: *inbox.addr(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.map_err(|error| NamespaceError::DirectoryUnavailable(error.to_string()))?;
|
||||||
|
match self.receive(&inbox).await? {
|
||||||
|
DataDirectoryOut::StreamClosed { result, .. } => result,
|
||||||
|
other => Err(NamespaceError::Protocol(format!(
|
||||||
|
"expected stream-close reply, received {other:?}"
|
||||||
|
))),
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn register_namespace_codecs(registry: &mut CodecRegistry) {
|
pub fn register_namespace_codecs(registry: &mut CodecRegistry) {
|
||||||
|
|
|
||||||
|
|
@ -72,6 +72,9 @@ pub enum MutationRequest {
|
||||||
length: u64,
|
length: u64,
|
||||||
recovery: SourceRecovery,
|
recovery: SourceRecovery,
|
||||||
},
|
},
|
||||||
|
BindStream {
|
||||||
|
path: DataPath,
|
||||||
|
},
|
||||||
Unregister {
|
Unregister {
|
||||||
path: DataPath,
|
path: DataPath,
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -7,8 +7,11 @@ use swactor::actor::ActorAddress;
|
||||||
use swactor_transport::{CodecRegistry, JsonCodec, NetworkMessage};
|
use swactor_transport::{CodecRegistry, JsonCodec, NetworkMessage};
|
||||||
|
|
||||||
use crate::blob::{BlobError, BlobLease, BlobMetadata};
|
use crate::blob::{BlobError, BlobLease, BlobMetadata};
|
||||||
|
use crate::byte_ring::{RingHandle, Role};
|
||||||
use crate::ids::BlobLeaseId;
|
use crate::ids::BlobLeaseId;
|
||||||
|
use crate::namespace::{EntryKind, NamespaceError, StreamIncarnation, StreamMatch};
|
||||||
use crate::path::DataPath;
|
use crate::path::DataPath;
|
||||||
|
use crate::stream_transport::{StreamPeerDescriptor, StreamTransportEvent};
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct JobCapability([u8; 32]);
|
pub struct JobCapability([u8; 32]);
|
||||||
|
|
@ -108,7 +111,15 @@ pub enum DataPlaneError {
|
||||||
ArenaExhausted,
|
ArenaExhausted,
|
||||||
Blob(BlobFailure),
|
Blob(BlobFailure),
|
||||||
OperationCancelled,
|
OperationCancelled,
|
||||||
StreamsDeferred,
|
WrongEntryType {
|
||||||
|
path: DataPath,
|
||||||
|
expected: EntryKind,
|
||||||
|
found: EntryKind,
|
||||||
|
},
|
||||||
|
PathReplaced(DataPath),
|
||||||
|
PeerLost,
|
||||||
|
StreamFault(String),
|
||||||
|
StreamClosed,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl fmt::Display for DataPlaneError {
|
impl fmt::Display for DataPlaneError {
|
||||||
|
|
@ -127,7 +138,18 @@ impl fmt::Display for DataPlaneError {
|
||||||
Self::ArenaExhausted => f.write_str("data-plane arena is exhausted"),
|
Self::ArenaExhausted => f.write_str("data-plane arena is exhausted"),
|
||||||
Self::Blob(reason) => write!(f, "blob lease failure: {reason:?}"),
|
Self::Blob(reason) => write!(f, "blob lease failure: {reason:?}"),
|
||||||
Self::OperationCancelled => f.write_str("data-plane operation was cancelled"),
|
Self::OperationCancelled => f.write_str("data-plane operation was cancelled"),
|
||||||
Self::StreamsDeferred => f.write_str("actor-driven streams are not installed"),
|
Self::WrongEntryType {
|
||||||
|
path,
|
||||||
|
expected,
|
||||||
|
found,
|
||||||
|
} => write!(
|
||||||
|
f,
|
||||||
|
"data path {path} has entry kind {found:?}, expected {expected:?}"
|
||||||
|
),
|
||||||
|
Self::PathReplaced(path) => write!(f, "data path was replaced: {path}"),
|
||||||
|
Self::PeerLost => f.write_str("stream peer was lost"),
|
||||||
|
Self::StreamFault(reason) => write!(f, "stream fault: {reason}"),
|
||||||
|
Self::StreamClosed => f.write_str("stream is closed"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -162,6 +184,25 @@ pub enum HostSessionIn {
|
||||||
child_session: ActorAddress,
|
child_session: ActorAddress,
|
||||||
operation: ActorAddress,
|
operation: ActorAddress,
|
||||||
},
|
},
|
||||||
|
OpenReadStream {
|
||||||
|
path: DataPath,
|
||||||
|
child_session: ActorAddress,
|
||||||
|
operation: ActorAddress,
|
||||||
|
replace: bool,
|
||||||
|
},
|
||||||
|
OpenWriteStream {
|
||||||
|
path: DataPath,
|
||||||
|
child_session: ActorAddress,
|
||||||
|
operation: ActorAddress,
|
||||||
|
replace: bool,
|
||||||
|
},
|
||||||
|
CancelStream {
|
||||||
|
operation: ActorAddress,
|
||||||
|
},
|
||||||
|
StreamControl {
|
||||||
|
binding: ActorAddress,
|
||||||
|
message: HostStreamIn,
|
||||||
|
},
|
||||||
ReleaseBlob {
|
ReleaseBlob {
|
||||||
binding: ActorAddress,
|
binding: ActorAddress,
|
||||||
lease_id: BlobLeaseId,
|
lease_id: BlobLeaseId,
|
||||||
|
|
@ -224,6 +265,33 @@ pub enum ChildSessionIn {
|
||||||
length: u64,
|
length: u64,
|
||||||
reply_to: ActorAddress,
|
reply_to: ActorAddress,
|
||||||
},
|
},
|
||||||
|
OpenReadStream {
|
||||||
|
path: DataPath,
|
||||||
|
reply_to: ActorAddress,
|
||||||
|
replace: bool,
|
||||||
|
},
|
||||||
|
OpenWriteStream {
|
||||||
|
path: DataPath,
|
||||||
|
reply_to: ActorAddress,
|
||||||
|
replace: bool,
|
||||||
|
},
|
||||||
|
CancelStream {
|
||||||
|
reply_to: ActorAddress,
|
||||||
|
},
|
||||||
|
StreamWake {
|
||||||
|
reply_to: ActorAddress,
|
||||||
|
result: Result<(), DataPlaneError>,
|
||||||
|
},
|
||||||
|
StreamControl {
|
||||||
|
binding: ActorAddress,
|
||||||
|
message: HostStreamIn,
|
||||||
|
},
|
||||||
|
ReleaseBlob {
|
||||||
|
binding: ActorAddress,
|
||||||
|
lease_id: BlobLeaseId,
|
||||||
|
generation: u64,
|
||||||
|
},
|
||||||
|
BlobReleased,
|
||||||
BlobOpened {
|
BlobOpened {
|
||||||
operation: ActorAddress,
|
operation: ActorAddress,
|
||||||
host_binding: ActorAddress,
|
host_binding: ActorAddress,
|
||||||
|
|
@ -236,6 +304,12 @@ pub enum ChildSessionIn {
|
||||||
lease: BlobLease,
|
lease: BlobLease,
|
||||||
metadata: BlobMetadata,
|
metadata: BlobMetadata,
|
||||||
},
|
},
|
||||||
|
StreamOpened {
|
||||||
|
operation: ActorAddress,
|
||||||
|
host_binding: ActorAddress,
|
||||||
|
ring: RingHandle,
|
||||||
|
role: Role,
|
||||||
|
},
|
||||||
OperationFailed {
|
OperationFailed {
|
||||||
operation: ActorAddress,
|
operation: ActorAddress,
|
||||||
error: DataPlaneError,
|
error: DataPlaneError,
|
||||||
|
|
@ -258,9 +332,44 @@ impl NetworkMessage for ChildSessionIn {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub enum HostStreamIn {
|
||||||
|
NamespaceMatched(Result<StreamMatch, NamespaceError>),
|
||||||
|
Allocated(Result<RingHandle, DataPlaneError>),
|
||||||
|
PeerOffer {
|
||||||
|
incarnation: StreamIncarnation,
|
||||||
|
descriptor: StreamPeerDescriptor,
|
||||||
|
},
|
||||||
|
Transport(StreamTransportEvent),
|
||||||
|
DataAvailable,
|
||||||
|
CapacityAvailable,
|
||||||
|
WaitData {
|
||||||
|
reply_to: ActorAddress,
|
||||||
|
},
|
||||||
|
WaitCapacity {
|
||||||
|
reply_to: ActorAddress,
|
||||||
|
},
|
||||||
|
Close {
|
||||||
|
clean: bool,
|
||||||
|
reply_to: Option<ActorAddress>,
|
||||||
|
},
|
||||||
|
PeerTerminated {
|
||||||
|
incarnation: StreamIncarnation,
|
||||||
|
error: DataPlaneError,
|
||||||
|
},
|
||||||
|
ReleaseComplete(Result<(), DataPlaneError>),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl NetworkMessage for HostStreamIn {
|
||||||
|
fn type_tag() -> &'static str {
|
||||||
|
"data-plane.host-stream.v1"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn register_data_plane_codecs(registry: &mut CodecRegistry) {
|
pub fn register_data_plane_codecs(registry: &mut CodecRegistry) {
|
||||||
registry.register::<HostSessionIn, _>(JsonCodec::default());
|
registry.register::<HostSessionIn, _>(JsonCodec::default());
|
||||||
registry.register::<ChildSessionIn, _>(JsonCodec::default());
|
registry.register::<ChildSessionIn, _>(JsonCodec::default());
|
||||||
|
registry.register::<HostStreamIn, _>(JsonCodec::default());
|
||||||
crate::namespace::register_namespace_codecs(registry);
|
crate::namespace::register_namespace_codecs(registry);
|
||||||
crate::blob_transfer::register_blob_transfer_codecs(registry);
|
crate::blob_transfer::register_blob_transfer_codecs(registry);
|
||||||
crate::source::register_blob_source_codecs(registry);
|
crate::source::register_blob_source_codecs(registry);
|
||||||
|
|
|
||||||
311
crates/data-plane/src/stream_transport.rs
Normal file
311
crates/data-plane/src/stream_transport.rs
Normal file
|
|
@ -0,0 +1,311 @@
|
||||||
|
//! Replaceable bounded transport port for SPSC stream incarnations.
|
||||||
|
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
|
|
||||||
|
use parking_lot::Mutex;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use crate::byte_ring::{Endpoint, FlowError};
|
||||||
|
use crate::namespace::StreamIncarnation;
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct StreamPeerDescriptor(pub Vec<u8>);
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub enum StreamTransportEvent {
|
||||||
|
Ready,
|
||||||
|
DataAvailable,
|
||||||
|
CapacityAvailable,
|
||||||
|
Quiesced,
|
||||||
|
Fault(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
pub trait StreamTransportNotifier: Send + Sync + 'static {
|
||||||
|
fn notify(&self, event: StreamTransportEvent);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct StreamSourceRequest {
|
||||||
|
pub incarnation: StreamIncarnation,
|
||||||
|
pub peer: StreamPeerDescriptor,
|
||||||
|
pub endpoint: Endpoint,
|
||||||
|
pub notifier: Arc<dyn StreamTransportNotifier>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct StreamSinkRequest {
|
||||||
|
pub incarnation: StreamIncarnation,
|
||||||
|
pub endpoint: Endpoint,
|
||||||
|
pub notifier: Arc<dyn StreamTransportNotifier>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Transport effect boundary. Namespace, terminal, reconnection, and lease
|
||||||
|
/// policy stay in the data-plane actors that invoke this port.
|
||||||
|
pub trait StreamTransport: Send + Sync + 'static {
|
||||||
|
fn descriptor(&self) -> Result<StreamPeerDescriptor, String>;
|
||||||
|
fn install_source(&self, request: StreamSourceRequest) -> Result<(), String>;
|
||||||
|
fn install_sink(&self, request: StreamSinkRequest) -> Result<(), String>;
|
||||||
|
fn source_progress(&self, incarnation: StreamIncarnation);
|
||||||
|
fn sink_progress(&self, incarnation: StreamIncarnation);
|
||||||
|
fn source_has_capacity(&self, _incarnation: StreamIncarnation) -> bool {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
fn sink_has_data(&self, _incarnation: StreamIncarnation) -> bool {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
fn terminate(&self, incarnation: StreamIncarnation);
|
||||||
|
}
|
||||||
|
|
||||||
|
struct LocalSource {
|
||||||
|
endpoint: Endpoint,
|
||||||
|
notifier: Arc<dyn StreamTransportNotifier>,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct LocalSink {
|
||||||
|
endpoint: Endpoint,
|
||||||
|
notifier: Arc<dyn StreamTransportNotifier>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct LocalTransfer {
|
||||||
|
source: Option<LocalSource>,
|
||||||
|
sink: Option<LocalSink>,
|
||||||
|
ready: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct LocalState {
|
||||||
|
transfers: BTreeMap<StreamIncarnation, LocalTransfer>,
|
||||||
|
}
|
||||||
|
|
||||||
|
static NEXT_LOCAL_TRANSPORT: AtomicU64 = AtomicU64::new(1);
|
||||||
|
|
||||||
|
/// Direct in-process adapter. Host composition shares one instance between
|
||||||
|
/// sessions on a node. Payload bytes copy once from the source ring spans to
|
||||||
|
/// the destination ring spans; no payload-sized staging allocation exists.
|
||||||
|
pub struct LocalStreamTransport {
|
||||||
|
id: u64,
|
||||||
|
state: Mutex<LocalState>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for LocalStreamTransport {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LocalStreamTransport {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
id: NEXT_LOCAL_TRANSPORT.fetch_add(1, Ordering::Relaxed),
|
||||||
|
state: Mutex::new(LocalState::default()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn drive(&self, incarnation: StreamIncarnation) {
|
||||||
|
let mut notifications = Vec::new();
|
||||||
|
let mut fault = None;
|
||||||
|
{
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
let Some(transfer) = state.transfers.get_mut(&incarnation) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let (Some(source), Some(sink)) = (&mut transfer.source, &mut transfer.sink) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
if !transfer.ready {
|
||||||
|
transfer.ready = true;
|
||||||
|
notifications.push((Arc::clone(&source.notifier), StreamTransportEvent::Ready));
|
||||||
|
notifications.push((Arc::clone(&sink.notifier), StreamTransportEvent::Ready));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut moved = false;
|
||||||
|
loop {
|
||||||
|
let meta = match source.endpoint.next_record_meta() {
|
||||||
|
Ok(Some(meta)) => meta,
|
||||||
|
Ok(None) => break,
|
||||||
|
Err(error) => {
|
||||||
|
fault = Some(format_flow_error(error));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let mut destination = match sink.endpoint.reserve_record(meta.kind, meta.len) {
|
||||||
|
Ok(destination) => destination,
|
||||||
|
Err(FlowError::InsufficientSpace { .. }) => break,
|
||||||
|
Err(error) => {
|
||||||
|
fault = Some(format_flow_error(error));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let source_view = match source.endpoint.peek_record() {
|
||||||
|
Ok(Some(view)) => view,
|
||||||
|
Ok(None) => {
|
||||||
|
fault = Some("source record disappeared after inspection".to_owned());
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
fault = Some(format_flow_error(error));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let (source_first, source_second) = source_view.spans();
|
||||||
|
let (destination_first, destination_second) = destination.spans_mut();
|
||||||
|
copy_spans(
|
||||||
|
source_first,
|
||||||
|
source_second,
|
||||||
|
destination_first,
|
||||||
|
destination_second,
|
||||||
|
);
|
||||||
|
if let Err(error) = destination.commit() {
|
||||||
|
fault = Some(format_flow_error(error));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if let Err(error) = source_view.release() {
|
||||||
|
fault = Some(format_flow_error(error));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
moved = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if moved {
|
||||||
|
notifications.push((
|
||||||
|
Arc::clone(&source.notifier),
|
||||||
|
StreamTransportEvent::CapacityAvailable,
|
||||||
|
));
|
||||||
|
notifications.push((
|
||||||
|
Arc::clone(&sink.notifier),
|
||||||
|
StreamTransportEvent::DataAvailable,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if let Some(reason) = &fault {
|
||||||
|
notifications.push((
|
||||||
|
Arc::clone(&source.notifier),
|
||||||
|
StreamTransportEvent::Fault(reason.clone()),
|
||||||
|
));
|
||||||
|
notifications.push((
|
||||||
|
Arc::clone(&sink.notifier),
|
||||||
|
StreamTransportEvent::Fault(reason.clone()),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (notifier, event) in notifications {
|
||||||
|
notifier.notify(event);
|
||||||
|
}
|
||||||
|
if fault.is_some() {
|
||||||
|
self.state.lock().transfers.remove(&incarnation);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StreamTransport for LocalStreamTransport {
|
||||||
|
fn descriptor(&self) -> Result<StreamPeerDescriptor, String> {
|
||||||
|
Ok(StreamPeerDescriptor(self.id.to_le_bytes().to_vec()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn install_source(&self, request: StreamSourceRequest) -> Result<(), String> {
|
||||||
|
if request.peer != self.descriptor()? {
|
||||||
|
return Err("local stream peer belongs to a different transport instance".to_owned());
|
||||||
|
}
|
||||||
|
let incarnation = request.incarnation;
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
let transfer = state.transfers.entry(incarnation).or_default();
|
||||||
|
if transfer.source.is_some() {
|
||||||
|
return Err("stream source is already installed".to_owned());
|
||||||
|
}
|
||||||
|
transfer.source = Some(LocalSource {
|
||||||
|
endpoint: request.endpoint,
|
||||||
|
notifier: request.notifier,
|
||||||
|
});
|
||||||
|
drop(state);
|
||||||
|
self.drive(incarnation);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn install_sink(&self, request: StreamSinkRequest) -> Result<(), String> {
|
||||||
|
let incarnation = request.incarnation;
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
let transfer = state.transfers.entry(incarnation).or_default();
|
||||||
|
if transfer.sink.is_some() {
|
||||||
|
return Err("stream sink is already installed".to_owned());
|
||||||
|
}
|
||||||
|
transfer.sink = Some(LocalSink {
|
||||||
|
endpoint: request.endpoint,
|
||||||
|
notifier: request.notifier,
|
||||||
|
});
|
||||||
|
drop(state);
|
||||||
|
self.drive(incarnation);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn source_progress(&self, incarnation: StreamIncarnation) {
|
||||||
|
self.drive(incarnation);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sink_progress(&self, incarnation: StreamIncarnation) {
|
||||||
|
self.drive(incarnation);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn source_has_capacity(&self, incarnation: StreamIncarnation) -> bool {
|
||||||
|
self.state
|
||||||
|
.lock()
|
||||||
|
.transfers
|
||||||
|
.get(&incarnation)
|
||||||
|
.and_then(|transfer| transfer.source.as_ref())
|
||||||
|
.is_some_and(|source| source.endpoint.probe().has_capacity())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sink_has_data(&self, incarnation: StreamIncarnation) -> bool {
|
||||||
|
self.state
|
||||||
|
.lock()
|
||||||
|
.transfers
|
||||||
|
.get(&incarnation)
|
||||||
|
.and_then(|transfer| transfer.sink.as_ref())
|
||||||
|
.is_some_and(|sink| sink.endpoint.probe().has_data())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn terminate(&self, incarnation: StreamIncarnation) {
|
||||||
|
let transfer = self.state.lock().transfers.remove(&incarnation);
|
||||||
|
if let Some(transfer) = transfer {
|
||||||
|
if let Some(source) = transfer.source {
|
||||||
|
source.notifier.notify(StreamTransportEvent::Quiesced);
|
||||||
|
}
|
||||||
|
if let Some(sink) = transfer.sink {
|
||||||
|
sink.notifier.notify(StreamTransportEvent::Quiesced);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn copy_spans(
|
||||||
|
source_first: &[u8],
|
||||||
|
source_second: &[u8],
|
||||||
|
destination_first: &mut [u8],
|
||||||
|
destination_second: &mut [u8],
|
||||||
|
) {
|
||||||
|
debug_assert_eq!(
|
||||||
|
source_first.len() + source_second.len(),
|
||||||
|
destination_first.len() + destination_second.len()
|
||||||
|
);
|
||||||
|
let sources = [source_first, source_second];
|
||||||
|
let mut source_index = 0;
|
||||||
|
let mut source_offset = 0;
|
||||||
|
for destination in [destination_first, destination_second] {
|
||||||
|
let mut destination_offset = 0;
|
||||||
|
while destination_offset < destination.len() {
|
||||||
|
while source_index < sources.len() && source_offset == sources[source_index].len() {
|
||||||
|
source_index += 1;
|
||||||
|
source_offset = 0;
|
||||||
|
}
|
||||||
|
let source = sources[source_index];
|
||||||
|
let take = (destination.len() - destination_offset).min(source.len() - source_offset);
|
||||||
|
destination[destination_offset..destination_offset + take]
|
||||||
|
.copy_from_slice(&source[source_offset..source_offset + take]);
|
||||||
|
destination_offset += take;
|
||||||
|
source_offset += take;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn format_flow_error(error: FlowError) -> String {
|
||||||
|
format!("byte ring transport fault: {error:?}")
|
||||||
|
}
|
||||||
|
|
@ -24,6 +24,7 @@ const CAPABILITY: JobCapability = JobCapability::new([9; 32]);
|
||||||
const ARENA_GENERATION: u64 = 17;
|
const ARENA_GENERATION: u64 = 17;
|
||||||
const SESSION_GENERATION: u64 = 29;
|
const SESSION_GENERATION: u64 = 29;
|
||||||
const WEIGHTS: &[u8] = b"0123456789abcdefghijklmn";
|
const WEIGHTS: &[u8] = b"0123456789abcdefghijklmn";
|
||||||
|
static STREAM_TEST_LOCK: parking_lot::Mutex<()> = parking_lot::Mutex::new(());
|
||||||
|
|
||||||
struct DirectRuntimeSink {
|
struct DirectRuntimeSink {
|
||||||
destination: Runtime,
|
destination: Runtime,
|
||||||
|
|
@ -167,7 +168,54 @@ impl data_plane::source::BlobSourcePublisher for NoopSourceRegistrar {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct RejectingStreamTransport;
|
||||||
|
|
||||||
|
impl data_plane::stream_transport::StreamTransport for RejectingStreamTransport {
|
||||||
|
fn descriptor(&self) -> Result<data_plane::stream_transport::StreamPeerDescriptor, String> {
|
||||||
|
Ok(data_plane::stream_transport::StreamPeerDescriptor(vec![1]))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn install_source(
|
||||||
|
&self,
|
||||||
|
_request: data_plane::stream_transport::StreamSourceRequest,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
Err("injected source transport failure".to_owned())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn install_sink(
|
||||||
|
&self,
|
||||||
|
_request: data_plane::stream_transport::StreamSinkRequest,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
Err("injected sink transport failure".to_owned())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn source_progress(&self, _incarnation: data_plane::namespace::StreamIncarnation) {}
|
||||||
|
|
||||||
|
fn sink_progress(&self, _incarnation: data_plane::namespace::StreamIncarnation) {}
|
||||||
|
|
||||||
|
fn terminate(&self, _incarnation: data_plane::namespace::StreamIncarnation) {}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct CollectBytes(Arc<parking_lot::Mutex<Vec<u8>>>);
|
||||||
|
|
||||||
|
impl data_plane::data_plane::StreamConsumer for CollectBytes {
|
||||||
|
fn consume(&self, bytes: &[u8]) -> Result<(), String> {
|
||||||
|
self.0.lock().extend_from_slice(bytes);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn harness(arena_bytes: u64) -> Harness {
|
fn harness(arena_bytes: u64) -> Harness {
|
||||||
|
harness_with_transport(
|
||||||
|
arena_bytes,
|
||||||
|
Arc::new(data_plane::stream_transport::LocalStreamTransport::new()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn harness_with_transport(
|
||||||
|
arena_bytes: u64,
|
||||||
|
stream_transport: Arc<dyn data_plane::stream_transport::StreamTransport>,
|
||||||
|
) -> Harness {
|
||||||
let temp = TempState::new();
|
let temp = TempState::new();
|
||||||
let mut arena = ArenaManager::boot(ArenaConfig {
|
let mut arena = ArenaManager::boot(ArenaConfig {
|
||||||
node_id: NodeId(1),
|
node_id: NodeId(1),
|
||||||
|
|
@ -194,12 +242,20 @@ fn harness(arena_bytes: u64) -> Harness {
|
||||||
}));
|
}));
|
||||||
let host_engine = Engine::new(
|
let host_engine = Engine::new(
|
||||||
host_parts,
|
host_parts,
|
||||||
TokioBackend::new(TokioConfig::default()).expect("host backend"),
|
TokioBackend::new(TokioConfig {
|
||||||
|
worker_threads: 1,
|
||||||
|
..TokioConfig::default()
|
||||||
|
})
|
||||||
|
.expect("host backend"),
|
||||||
)
|
)
|
||||||
.expect("host engine");
|
.expect("host engine");
|
||||||
let child_engine = Engine::new(
|
let child_engine = Engine::new(
|
||||||
child_parts,
|
child_parts,
|
||||||
TokioBackend::new(TokioConfig::default()).expect("child backend"),
|
TokioBackend::new(TokioConfig {
|
||||||
|
worker_threads: 1,
|
||||||
|
..TokioConfig::default()
|
||||||
|
})
|
||||||
|
.expect("child backend"),
|
||||||
)
|
)
|
||||||
.expect("child engine");
|
.expect("child engine");
|
||||||
|
|
||||||
|
|
@ -269,6 +325,7 @@ fn harness(arena_bytes: u64) -> Harness {
|
||||||
source_sender: Some(sender),
|
source_sender: Some(sender),
|
||||||
source_publisher: Some(Arc::new(NoopSourceRegistrar)),
|
source_publisher: Some(Arc::new(NoopSourceRegistrar)),
|
||||||
route_registrar: None,
|
route_registrar: None,
|
||||||
|
stream_transport: Some(stream_transport),
|
||||||
})
|
})
|
||||||
.expect("host session config"),
|
.expect("host session config"),
|
||||||
)
|
)
|
||||||
|
|
@ -314,7 +371,15 @@ fn attachment_without_a_host_reply_fails_on_actor_deadline() {
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let (parts, runtime) = runtime_parts();
|
let (parts, runtime) = runtime_parts();
|
||||||
runtime.set_remote_sink(Arc::new(BlackHoleSink));
|
runtime.set_remote_sink(Arc::new(BlackHoleSink));
|
||||||
let engine = Engine::new(parts, TokioBackend::new(TokioConfig::default()).unwrap()).unwrap();
|
let engine = Engine::new(
|
||||||
|
parts,
|
||||||
|
TokioBackend::new(TokioConfig {
|
||||||
|
worker_threads: 1,
|
||||||
|
..TokioConfig::default()
|
||||||
|
})
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
let (mapped, resolved) = DataPlaneBootstrap::map_arena(handoff.arena_fd).unwrap();
|
let (mapped, resolved) = DataPlaneBootstrap::map_arena(handoff.arena_fd).unwrap();
|
||||||
let result = future::block_on(DataPlaneBootstrap::attach_mapped_with_deadline(
|
let result = future::block_on(DataPlaneBootstrap::attach_mapped_with_deadline(
|
||||||
mapped,
|
mapped,
|
||||||
|
|
@ -618,3 +683,194 @@ fn write_blob_seals_once_and_abort_publishes_nothing() {
|
||||||
Err(DataPlaneError::PathNotFound(_))
|
Err(DataPlaneError::PathNotFound(_))
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn stream_endpoints_open_only_after_match_and_deliver_eof_in_order() {
|
||||||
|
let _stream_test = STREAM_TEST_LOCK.lock();
|
||||||
|
let harness = harness(2 << 20);
|
||||||
|
let data_plane = harness.bootstrap.data_plane.clone();
|
||||||
|
let logical = path("/runs/self/results/inference");
|
||||||
|
|
||||||
|
future::block_on(async {
|
||||||
|
let mut reader_open = Box::pin(data_plane.read_stream(&logical));
|
||||||
|
assert!(
|
||||||
|
future::poll_once(reader_open.as_mut()).await.is_none(),
|
||||||
|
"reader open waits for its source"
|
||||||
|
);
|
||||||
|
let mut writer_open = Box::pin(data_plane.write_stream(&logical));
|
||||||
|
let mut writer = writer_open.as_mut().await.expect("writer opens");
|
||||||
|
let mut reader = reader_open.await.expect("reader opens");
|
||||||
|
|
||||||
|
writer.write(b"first").await.expect("write first");
|
||||||
|
writer.write(b"second").await.expect("write second");
|
||||||
|
assert_eq!(
|
||||||
|
reader.read().await.expect("read first"),
|
||||||
|
Some(b"first".to_vec())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
reader.read().await.expect("read second"),
|
||||||
|
Some(b"second".to_vec())
|
||||||
|
);
|
||||||
|
writer.close().await.expect("clean writer close");
|
||||||
|
assert!(matches!(
|
||||||
|
writer.write(b"late").await,
|
||||||
|
Err(DataPlaneError::StreamClosed)
|
||||||
|
));
|
||||||
|
assert_eq!(reader.read().await.expect("read eof"), None);
|
||||||
|
assert_eq!(reader.read().await.expect("sticky eof"), None);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn stream_writer_suspends_until_reader_releases_bounded_capacity() {
|
||||||
|
let _stream_test = STREAM_TEST_LOCK.lock();
|
||||||
|
let harness = harness(2 << 20);
|
||||||
|
let data_plane = harness.bootstrap.data_plane.clone();
|
||||||
|
let logical = path("/runs/self/results/backpressure");
|
||||||
|
|
||||||
|
future::block_on(async {
|
||||||
|
let mut reader_open = Box::pin(data_plane.read_stream(&logical));
|
||||||
|
assert!(future::poll_once(reader_open.as_mut()).await.is_none());
|
||||||
|
let mut writer = data_plane
|
||||||
|
.write_stream(&logical)
|
||||||
|
.await
|
||||||
|
.expect("writer opens");
|
||||||
|
let mut reader = reader_open.await.expect("reader opens");
|
||||||
|
|
||||||
|
let capacity = writer.capacity() as usize;
|
||||||
|
let payload: Vec<u8> = (0..(capacity * 2 + 97))
|
||||||
|
.map(|index| (index % 251) as u8)
|
||||||
|
.collect();
|
||||||
|
let mut writing = Box::pin(writer.write(&payload));
|
||||||
|
assert!(
|
||||||
|
future::poll_once(writing.as_mut()).await.is_none(),
|
||||||
|
"bounded source and destination rings must eventually suspend the writer"
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut observed = reader
|
||||||
|
.read()
|
||||||
|
.await
|
||||||
|
.expect("read releases destination capacity")
|
||||||
|
.expect("first data");
|
||||||
|
writing.await.expect("writer resumes");
|
||||||
|
while observed.len() < payload.len() {
|
||||||
|
observed.extend(
|
||||||
|
reader
|
||||||
|
.read()
|
||||||
|
.await
|
||||||
|
.expect("read remaining")
|
||||||
|
.expect("remaining data"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert_eq!(observed, payload);
|
||||||
|
writer.close().await.expect("close");
|
||||||
|
assert_eq!(reader.read().await.expect("eof"), None);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
#[test]
|
||||||
|
fn transport_startup_failure_faults_both_pending_opens() {
|
||||||
|
let _stream_test = STREAM_TEST_LOCK.lock();
|
||||||
|
let harness = harness_with_transport(2 << 20, Arc::new(RejectingStreamTransport));
|
||||||
|
let data_plane = harness.bootstrap.data_plane.clone();
|
||||||
|
let logical = path("/runs/self/results/faulted");
|
||||||
|
|
||||||
|
future::block_on(async {
|
||||||
|
let mut reader_open = Box::pin(data_plane.read_stream(&logical));
|
||||||
|
assert!(future::poll_once(reader_open.as_mut()).await.is_none());
|
||||||
|
let writer_error = match data_plane.write_stream(&logical).await {
|
||||||
|
Ok(_) => panic!("writer must not open when transport setup fails"),
|
||||||
|
Err(error) => error,
|
||||||
|
};
|
||||||
|
let reader_error = match reader_open.await {
|
||||||
|
Ok(_) => panic!("reader must not open when transport setup fails"),
|
||||||
|
Err(error) => error,
|
||||||
|
};
|
||||||
|
assert!(matches!(
|
||||||
|
writer_error,
|
||||||
|
DataPlaneError::PeerLost | DataPlaneError::StreamFault(_)
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
reader_error,
|
||||||
|
DataPlaneError::PeerLost | DataPlaneError::StreamFault(_)
|
||||||
|
));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn peer_replacement_requires_and_supports_a_fresh_incarnation() {
|
||||||
|
let _stream_test = STREAM_TEST_LOCK.lock();
|
||||||
|
let harness = harness(2 << 20);
|
||||||
|
let data_plane = harness.bootstrap.data_plane.clone();
|
||||||
|
let logical = path("/runs/self/results/failover");
|
||||||
|
|
||||||
|
future::block_on(async {
|
||||||
|
let mut first_reader_open = Box::pin(data_plane.read_stream(&logical));
|
||||||
|
assert!(
|
||||||
|
future::poll_once(first_reader_open.as_mut())
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
let mut first_writer = data_plane
|
||||||
|
.write_stream(&logical)
|
||||||
|
.await
|
||||||
|
.expect("first writer");
|
||||||
|
let mut first_reader = first_reader_open.await.expect("first reader");
|
||||||
|
first_writer.write(b"old").await.expect("old write");
|
||||||
|
assert_eq!(
|
||||||
|
first_reader.read().await.expect("old read"),
|
||||||
|
Some(b"old".to_vec())
|
||||||
|
);
|
||||||
|
first_writer.abort().expect("abort first incarnation");
|
||||||
|
assert!(matches!(
|
||||||
|
first_reader.read().await,
|
||||||
|
Err(DataPlaneError::PeerLost)
|
||||||
|
));
|
||||||
|
|
||||||
|
let mut replacement_writer_open = Box::pin(data_plane.write_stream_replacing(&logical));
|
||||||
|
assert!(
|
||||||
|
future::poll_once(replacement_writer_open.as_mut())
|
||||||
|
.await
|
||||||
|
.is_none(),
|
||||||
|
"replacement writer waits for an explicit new reader"
|
||||||
|
);
|
||||||
|
let mut replacement_reader = data_plane
|
||||||
|
.read_stream(&logical)
|
||||||
|
.await
|
||||||
|
.expect("replacement reader");
|
||||||
|
let mut replacement_writer = replacement_writer_open.await.expect("replacement writer");
|
||||||
|
replacement_writer.write(b"new").await.expect("new write");
|
||||||
|
assert_eq!(
|
||||||
|
replacement_reader.read().await.expect("new read"),
|
||||||
|
Some(b"new".to_vec())
|
||||||
|
);
|
||||||
|
replacement_writer.close().await.expect("new close");
|
||||||
|
assert_eq!(replacement_reader.read().await.expect("new eof"), None);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn actor_stream_consumer_registers_before_writer_and_collects_to_eof() {
|
||||||
|
let _stream_test = STREAM_TEST_LOCK.lock();
|
||||||
|
let harness = harness(2 << 20);
|
||||||
|
let data_plane = harness.bootstrap.data_plane.clone();
|
||||||
|
let logical = path("/runs/self/results/collector");
|
||||||
|
let observed = Arc::new(parking_lot::Mutex::new(Vec::new()));
|
||||||
|
let consumer: Arc<dyn data_plane::data_plane::StreamConsumer> =
|
||||||
|
Arc::new(CollectBytes(Arc::clone(&observed)));
|
||||||
|
let completion = data_plane
|
||||||
|
.collect_stream(logical.clone(), consumer)
|
||||||
|
.expect("spawn collector");
|
||||||
|
|
||||||
|
future::block_on(async {
|
||||||
|
let mut writer = data_plane
|
||||||
|
.write_stream(&logical)
|
||||||
|
.await
|
||||||
|
.expect("writer matches collector");
|
||||||
|
writer.write(b"actor-").await.expect("first write");
|
||||||
|
writer.write(b"consumer").await.expect("second write");
|
||||||
|
writer.close().await.expect("close");
|
||||||
|
});
|
||||||
|
|
||||||
|
completion.wait().expect("collector completes");
|
||||||
|
assert_eq!(&*observed.lock(), b"actor-consumer");
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -470,3 +470,90 @@ fn operations_revalidate_cursors_and_never_panic() {
|
||||||
})
|
})
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pinned_record_blocks_capacity_until_release() {
|
||||||
|
let (arena, handle) = installed(32, 1);
|
||||||
|
let mut producer = attach(&arena, handle, Role::Producer).expect("producer");
|
||||||
|
let mut consumer = attach(&arena, handle, Role::Consumer).expect("consumer");
|
||||||
|
|
||||||
|
producer
|
||||||
|
.send_record(RecordKind::Data, &[7; 20])
|
||||||
|
.expect("send");
|
||||||
|
let view = consumer
|
||||||
|
.peek_record()
|
||||||
|
.expect("peek")
|
||||||
|
.expect("record must be visible");
|
||||||
|
assert_eq!(view.kind(), RecordKind::Data);
|
||||||
|
assert_eq!(view.len(), 20);
|
||||||
|
assert!(view.spans().1.is_empty());
|
||||||
|
assert_eq!(
|
||||||
|
producer.reserve(8).unwrap_err(),
|
||||||
|
FlowError::InsufficientSpace {
|
||||||
|
requested: 8,
|
||||||
|
free: 7,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
drop(view);
|
||||||
|
assert_eq!(producer.reserve(8).expect("capacity released").len, 8);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pinned_record_exposes_wrapped_payload_as_two_spans() {
|
||||||
|
let (arena, handle) = installed(32, 1);
|
||||||
|
let mut producer = attach(&arena, handle, Role::Producer).expect("producer");
|
||||||
|
let mut consumer = attach(&arena, handle, Role::Consumer).expect("consumer");
|
||||||
|
|
||||||
|
producer
|
||||||
|
.send_record(RecordKind::Data, &[1; 18])
|
||||||
|
.expect("first");
|
||||||
|
assert_eq!(
|
||||||
|
consumer.recv_record().expect("consume first"),
|
||||||
|
Some((RecordKind::Data, vec![1; 18]))
|
||||||
|
);
|
||||||
|
let expected: Vec<u8> = (0..15).collect();
|
||||||
|
producer
|
||||||
|
.send_record(RecordKind::Data, &expected)
|
||||||
|
.expect("wrapped record");
|
||||||
|
|
||||||
|
let view = consumer
|
||||||
|
.peek_record()
|
||||||
|
.expect("peek")
|
||||||
|
.expect("wrapped record visible");
|
||||||
|
let (first, second) = view.spans();
|
||||||
|
assert!(!first.is_empty());
|
||||||
|
assert!(!second.is_empty());
|
||||||
|
let observed: Vec<u8> = first.iter().chain(second).copied().collect();
|
||||||
|
assert_eq!(observed, expected);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn writable_record_is_invisible_until_commit() {
|
||||||
|
let (arena, handle) = installed(64, 1);
|
||||||
|
let mut producer = attach(&arena, handle, Role::Producer).expect("producer");
|
||||||
|
let mut consumer = attach(&arena, handle, Role::Consumer).expect("consumer");
|
||||||
|
|
||||||
|
let mut reservation = producer
|
||||||
|
.reserve_record(RecordKind::Data, 17)
|
||||||
|
.expect("reserve record");
|
||||||
|
let (first, second) = reservation.spans_mut();
|
||||||
|
for (index, byte) in first.iter_mut().chain(second).enumerate() {
|
||||||
|
*byte = index as u8;
|
||||||
|
}
|
||||||
|
assert!(consumer.peek_record().expect("peek uncommitted").is_none());
|
||||||
|
reservation.commit().expect("commit");
|
||||||
|
|
||||||
|
let view = consumer
|
||||||
|
.peek_record()
|
||||||
|
.expect("peek")
|
||||||
|
.expect("committed record");
|
||||||
|
let observed: Vec<u8> = view
|
||||||
|
.spans()
|
||||||
|
.0
|
||||||
|
.iter()
|
||||||
|
.chain(view.spans().1)
|
||||||
|
.copied()
|
||||||
|
.collect();
|
||||||
|
assert_eq!(observed, (0..17).collect::<Vec<_>>());
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -5,8 +5,8 @@ use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use data_plane::namespace::{
|
use data_plane::namespace::{
|
||||||
DataDirectoryActor, DirectoryClient, NamespaceClient, NamespaceClientActor, NamespaceClientIn,
|
DataDirectoryActor, DirectoryClient, EntryKind, NamespaceClient, NamespaceClientActor,
|
||||||
NamespaceDiscovery, NamespaceError, OperationId, SourceRecovery,
|
NamespaceClientIn, NamespaceDiscovery, NamespaceError, OperationId, SourceRecovery, StreamRole,
|
||||||
};
|
};
|
||||||
use data_plane::path::DataPath;
|
use data_plane::path::DataPath;
|
||||||
use futures_lite::future;
|
use futures_lite::future;
|
||||||
|
|
@ -80,7 +80,11 @@ fn spawn_directory(store: &Path) -> DirectoryHarness {
|
||||||
let directory = runtime.spawn(actor).expect("spawn directory actor");
|
let directory = runtime.spawn(actor).expect("spawn directory actor");
|
||||||
let engine = Engine::new(
|
let engine = Engine::new(
|
||||||
parts,
|
parts,
|
||||||
TokioBackend::new(TokioConfig::default()).expect("tokio backend"),
|
TokioBackend::new(TokioConfig {
|
||||||
|
worker_threads: 1,
|
||||||
|
..TokioConfig::default()
|
||||||
|
})
|
||||||
|
.expect("tokio backend"),
|
||||||
)
|
)
|
||||||
.expect("directory engine");
|
.expect("directory engine");
|
||||||
DirectoryHarness {
|
DirectoryHarness {
|
||||||
|
|
@ -168,6 +172,222 @@ fn namespace_mutations_are_linearizable_and_durable() {
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn stream_rendezvous_is_symmetric_and_incarnations_are_isolated() {
|
||||||
|
let state = TempState::new("stream-rendezvous");
|
||||||
|
let directory = spawn_directory(&state.store());
|
||||||
|
let logical = path("/runs/7/results");
|
||||||
|
|
||||||
|
future::block_on(async {
|
||||||
|
let mut source_open = Box::pin(directory.client.open_stream(
|
||||||
|
logical.clone(),
|
||||||
|
StreamRole::Source,
|
||||||
|
source(10),
|
||||||
|
OperationId::from_u128(10),
|
||||||
|
));
|
||||||
|
assert!(future::poll_once(source_open.as_mut()).await.is_none());
|
||||||
|
|
||||||
|
let sink_match = directory
|
||||||
|
.client
|
||||||
|
.open_stream(
|
||||||
|
logical.clone(),
|
||||||
|
StreamRole::Sink,
|
||||||
|
source(11),
|
||||||
|
OperationId::from_u128(11),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("sink matches source");
|
||||||
|
let source_match = source_open.await.expect("source matches sink");
|
||||||
|
assert_eq!(source_match, sink_match);
|
||||||
|
assert_eq!(source_match.source, source(10));
|
||||||
|
assert_eq!(source_match.sink, source(11));
|
||||||
|
|
||||||
|
directory
|
||||||
|
.client
|
||||||
|
.close_stream(logical.clone(), source_match.incarnation)
|
||||||
|
.await
|
||||||
|
.expect("close first incarnation");
|
||||||
|
|
||||||
|
let mut sink_open = Box::pin(directory.client.open_stream(
|
||||||
|
logical.clone(),
|
||||||
|
StreamRole::Sink,
|
||||||
|
source(12),
|
||||||
|
OperationId::from_u128(12),
|
||||||
|
));
|
||||||
|
assert!(future::poll_once(sink_open.as_mut()).await.is_none());
|
||||||
|
let second_source = directory
|
||||||
|
.client
|
||||||
|
.open_stream(
|
||||||
|
logical,
|
||||||
|
StreamRole::Source,
|
||||||
|
source(13),
|
||||||
|
OperationId::from_u128(13),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("source matches waiting sink");
|
||||||
|
let second_sink = sink_open.await.expect("sink matches source");
|
||||||
|
assert_eq!(second_source, second_sink);
|
||||||
|
assert_ne!(source_match.incarnation, second_source.incarnation);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn typed_paths_require_explicit_rebinding() {
|
||||||
|
let state = TempState::new("typed-path");
|
||||||
|
let directory = spawn_directory(&state.store());
|
||||||
|
let logical = path("/typed/value");
|
||||||
|
let blob_source = source(20);
|
||||||
|
|
||||||
|
future::block_on(async {
|
||||||
|
directory
|
||||||
|
.client
|
||||||
|
.register(
|
||||||
|
logical.clone(),
|
||||||
|
blob_source,
|
||||||
|
4,
|
||||||
|
recovery(blob_source),
|
||||||
|
OperationId::from_u128(20),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("register blob");
|
||||||
|
|
||||||
|
assert!(matches!(
|
||||||
|
directory
|
||||||
|
.client
|
||||||
|
.open_stream(
|
||||||
|
logical.clone(),
|
||||||
|
StreamRole::Source,
|
||||||
|
source(21),
|
||||||
|
OperationId::from_u128(21),
|
||||||
|
)
|
||||||
|
.await,
|
||||||
|
Err(NamespaceError::WrongEntryType {
|
||||||
|
expected: EntryKind::Stream,
|
||||||
|
found: EntryKind::Blob,
|
||||||
|
..
|
||||||
|
})
|
||||||
|
));
|
||||||
|
|
||||||
|
let mut source_open = Box::pin(directory.client.replace_with_stream(
|
||||||
|
logical.clone(),
|
||||||
|
StreamRole::Source,
|
||||||
|
source(22),
|
||||||
|
OperationId::from_u128(22),
|
||||||
|
));
|
||||||
|
assert!(future::poll_once(source_open.as_mut()).await.is_none());
|
||||||
|
assert!(matches!(
|
||||||
|
directory.client.resolve(logical.clone()).await,
|
||||||
|
Err(NamespaceError::WrongEntryType {
|
||||||
|
expected: EntryKind::Blob,
|
||||||
|
found: EntryKind::Stream,
|
||||||
|
..
|
||||||
|
})
|
||||||
|
));
|
||||||
|
let sink_match = directory
|
||||||
|
.client
|
||||||
|
.open_stream(
|
||||||
|
logical,
|
||||||
|
StreamRole::Sink,
|
||||||
|
source(23),
|
||||||
|
OperationId::from_u128(23),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("match rebound stream");
|
||||||
|
assert_eq!(
|
||||||
|
source_open.await.expect("rebound source matched"),
|
||||||
|
sink_match
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn replacing_waiting_stream_displaces_old_open() {
|
||||||
|
let state = TempState::new("stream-displacement");
|
||||||
|
let directory = spawn_directory(&state.store());
|
||||||
|
let logical = path("/replace/waiting");
|
||||||
|
|
||||||
|
future::block_on(async {
|
||||||
|
let mut old_open = Box::pin(directory.client.open_stream(
|
||||||
|
logical.clone(),
|
||||||
|
StreamRole::Source,
|
||||||
|
source(30),
|
||||||
|
OperationId::from_u128(30),
|
||||||
|
));
|
||||||
|
assert!(future::poll_once(old_open.as_mut()).await.is_none());
|
||||||
|
|
||||||
|
let mut replacement = Box::pin(directory.client.replace_with_stream(
|
||||||
|
logical.clone(),
|
||||||
|
StreamRole::Source,
|
||||||
|
source(31),
|
||||||
|
OperationId::from_u128(31),
|
||||||
|
));
|
||||||
|
assert!(future::poll_once(replacement.as_mut()).await.is_none());
|
||||||
|
assert!(matches!(
|
||||||
|
old_open.await,
|
||||||
|
Err(NamespaceError::PathReplaced(found)) if found == logical
|
||||||
|
));
|
||||||
|
|
||||||
|
let sink_match = directory
|
||||||
|
.client
|
||||||
|
.open_stream(
|
||||||
|
logical,
|
||||||
|
StreamRole::Sink,
|
||||||
|
source(32),
|
||||||
|
OperationId::from_u128(32),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("sink matches replacement");
|
||||||
|
assert_eq!(
|
||||||
|
replacement.await.expect("replacement source matched"),
|
||||||
|
sink_match
|
||||||
|
);
|
||||||
|
assert_eq!(sink_match.source, source(31));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn duplicate_stream_role_fails_without_replacing_the_waiter() {
|
||||||
|
let state = TempState::new("duplicate-stream-role");
|
||||||
|
let directory = spawn_directory(&state.store());
|
||||||
|
let logical = path("/duplicate/source");
|
||||||
|
|
||||||
|
future::block_on(async {
|
||||||
|
let mut first = Box::pin(directory.client.open_stream(
|
||||||
|
logical.clone(),
|
||||||
|
StreamRole::Source,
|
||||||
|
source(51),
|
||||||
|
OperationId::from_u128(51),
|
||||||
|
));
|
||||||
|
assert!(future::poll_once(first.as_mut()).await.is_none());
|
||||||
|
assert!(matches!(
|
||||||
|
directory
|
||||||
|
.client
|
||||||
|
.open_stream(
|
||||||
|
logical.clone(),
|
||||||
|
StreamRole::Source,
|
||||||
|
source(52),
|
||||||
|
OperationId::from_u128(52),
|
||||||
|
)
|
||||||
|
.await,
|
||||||
|
Err(NamespaceError::DuplicateStreamRole {
|
||||||
|
role: StreamRole::Source,
|
||||||
|
..
|
||||||
|
})
|
||||||
|
));
|
||||||
|
let matched = directory
|
||||||
|
.client
|
||||||
|
.open_stream(
|
||||||
|
logical,
|
||||||
|
StreamRole::Sink,
|
||||||
|
source(53),
|
||||||
|
OperationId::from_u128(53),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("sink matches original source");
|
||||||
|
assert_eq!(matched.source, source(51));
|
||||||
|
assert_eq!(first.await.expect("original source survives"), matched);
|
||||||
|
});
|
||||||
|
}
|
||||||
#[test]
|
#[test]
|
||||||
fn committed_mutation_retry_has_at_most_once_effect() {
|
fn committed_mutation_retry_has_at_most_once_effect() {
|
||||||
let state = TempState::new("idempotent");
|
let state = TempState::new("idempotent");
|
||||||
|
|
@ -369,9 +589,15 @@ struct ModelBinding {
|
||||||
revision: u64,
|
revision: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
enum TypedModelEntry {
|
||||||
|
Blob(ModelBinding),
|
||||||
|
Stream(data_plane::namespace::StreamMatch),
|
||||||
|
}
|
||||||
|
|
||||||
proptest! {
|
proptest! {
|
||||||
#![proptest_config(ProptestConfig {
|
#![proptest_config(ProptestConfig {
|
||||||
cases: 16,
|
cases: 8,
|
||||||
max_shrink_iters: 128,
|
max_shrink_iters: 128,
|
||||||
..ProptestConfig::default()
|
..ProptestConfig::default()
|
||||||
})]
|
})]
|
||||||
|
|
@ -434,9 +660,98 @@ proptest! {
|
||||||
let observed = future::block_on(directory.client.resolve(path.clone()))
|
let observed = future::block_on(directory.client.resolve(path.clone()))
|
||||||
.expect("all model bindings remain resolvable");
|
.expect("all model bindings remain resolvable");
|
||||||
prop_assert_eq!(observed.source, expected.source);
|
prop_assert_eq!(observed.source, expected.source);
|
||||||
|
|
||||||
prop_assert_eq!(observed.length, expected.length);
|
prop_assert_eq!(observed.length, expected.length);
|
||||||
prop_assert_eq!(observed.revision, expected.revision);
|
prop_assert_eq!(observed.revision, expected.revision);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
#[test]
|
||||||
|
fn typed_binding_action_strings_match_reference_model(actions in prop::collection::vec(any::<u8>(), 1..64)) {
|
||||||
|
let state = TempState::new("typed-stateful");
|
||||||
|
let directory = spawn_directory(&state.store());
|
||||||
|
let paths = [path("/state/a"), path("/state/b"), path("/state/c")];
|
||||||
|
let mut model = BTreeMap::<DataPath, TypedModelEntry>::new();
|
||||||
|
let mut next_operation = 10_000_u128;
|
||||||
|
|
||||||
|
for (step, action) in actions.into_iter().enumerate() {
|
||||||
|
let logical = paths[usize::from(action) % paths.len()].clone();
|
||||||
|
match action % 4 {
|
||||||
|
0 => {
|
||||||
|
let actor = source(action.wrapping_add(step as u8).wrapping_add(1));
|
||||||
|
let length = u64::from(action) + 1;
|
||||||
|
let receipt = future::block_on(directory.client.register(
|
||||||
|
logical.clone(),
|
||||||
|
actor,
|
||||||
|
length,
|
||||||
|
recovery(actor),
|
||||||
|
OperationId::from_u128(next_operation),
|
||||||
|
)).expect("blob rebind");
|
||||||
|
next_operation += 1;
|
||||||
|
model.insert(logical, TypedModelEntry::Blob(ModelBinding {
|
||||||
|
source: actor,
|
||||||
|
length,
|
||||||
|
revision: receipt.revision,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
1 => {
|
||||||
|
let source_actor = source(action.wrapping_add(41));
|
||||||
|
let sink_actor = source(action.wrapping_add(97));
|
||||||
|
let mut source_open = Box::pin(directory.client.replace_with_stream(
|
||||||
|
logical.clone(),
|
||||||
|
StreamRole::Source,
|
||||||
|
source_actor,
|
||||||
|
OperationId::from_u128(next_operation),
|
||||||
|
));
|
||||||
|
next_operation += 1;
|
||||||
|
prop_assert!(future::block_on(future::poll_once(source_open.as_mut())).is_none());
|
||||||
|
let sink_match = future::block_on(directory.client.open_stream(
|
||||||
|
logical.clone(),
|
||||||
|
StreamRole::Sink,
|
||||||
|
sink_actor,
|
||||||
|
OperationId::from_u128(next_operation),
|
||||||
|
)).expect("sink match");
|
||||||
|
next_operation += 1;
|
||||||
|
let source_match = future::block_on(source_open).expect("source match");
|
||||||
|
prop_assert_eq!(&source_match, &sink_match);
|
||||||
|
model.insert(logical, TypedModelEntry::Stream(sink_match));
|
||||||
|
}
|
||||||
|
2 => {
|
||||||
|
if let Some(TypedModelEntry::Stream(binding)) = model.get(&logical) {
|
||||||
|
future::block_on(directory.client.close_stream(
|
||||||
|
logical.clone(),
|
||||||
|
binding.incarnation,
|
||||||
|
)).expect("close current stream");
|
||||||
|
model.remove(&logical);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
let observed = future::block_on(directory.client.resolve(logical.clone()));
|
||||||
|
match model.get(&logical) {
|
||||||
|
Some(TypedModelEntry::Blob(expected)) => {
|
||||||
|
let observed = observed.expect("blob resolves");
|
||||||
|
prop_assert_eq!(observed.source, expected.source);
|
||||||
|
prop_assert_eq!(observed.length, expected.length);
|
||||||
|
prop_assert_eq!(observed.revision, expected.revision);
|
||||||
|
}
|
||||||
|
Some(TypedModelEntry::Stream(_)) => {
|
||||||
|
let wrong_type = matches!(
|
||||||
|
observed,
|
||||||
|
Err(NamespaceError::WrongEntryType {
|
||||||
|
expected: EntryKind::Blob,
|
||||||
|
found: EntryKind::Stream,
|
||||||
|
..
|
||||||
|
})
|
||||||
|
);
|
||||||
|
prop_assert!(wrong_type, "blob lookup must reject a stream binding");
|
||||||
|
}
|
||||||
|
None => prop_assert!(matches!(
|
||||||
|
observed,
|
||||||
|
Err(NamespaceError::PathNotFound(found)) if found == logical
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
14
crates/data-plane/tests/namespace_host_read_guarantees.rs
Executable file → Normal file
14
crates/data-plane/tests/namespace_host_read_guarantees.rs
Executable file → Normal file
|
|
@ -202,12 +202,20 @@ fn host_read_resolves_file_source_and_seals_final_arena_lease() {
|
||||||
}));
|
}));
|
||||||
let host_engine = Engine::new(
|
let host_engine = Engine::new(
|
||||||
host_parts,
|
host_parts,
|
||||||
TokioBackend::new(TokioConfig::default()).unwrap(),
|
TokioBackend::new(TokioConfig {
|
||||||
|
worker_threads: 1,
|
||||||
|
..TokioConfig::default()
|
||||||
|
})
|
||||||
|
.unwrap(),
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let child_engine = Engine::new(
|
let child_engine = Engine::new(
|
||||||
child_parts,
|
child_parts,
|
||||||
TokioBackend::new(TokioConfig::default()).unwrap(),
|
TokioBackend::new(TokioConfig {
|
||||||
|
worker_threads: 1,
|
||||||
|
..TokioConfig::default()
|
||||||
|
})
|
||||||
|
.unwrap(),
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
|
|
@ -264,6 +272,7 @@ fn host_read_resolves_file_source_and_seals_final_arena_lease() {
|
||||||
source_sender: Some(Arc::clone(&sender)),
|
source_sender: Some(Arc::clone(&sender)),
|
||||||
source_publisher: Some(Arc::clone(&source_publisher)),
|
source_publisher: Some(Arc::clone(&source_publisher)),
|
||||||
route_registrar: None,
|
route_registrar: None,
|
||||||
|
stream_transport: None,
|
||||||
})
|
})
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
)
|
)
|
||||||
|
|
@ -296,6 +305,7 @@ fn host_read_resolves_file_source_and_seals_final_arena_lease() {
|
||||||
source_sender: Some(Arc::clone(&sender)),
|
source_sender: Some(Arc::clone(&sender)),
|
||||||
source_publisher: Some(source_publisher),
|
source_publisher: Some(source_publisher),
|
||||||
route_registrar: None,
|
route_registrar: None,
|
||||||
|
stream_transport: None,
|
||||||
})
|
})
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
)
|
)
|
||||||
|
|
|
||||||
260
crates/data-plane/tests/stream_transport_guarantees.rs
Executable file
260
crates/data-plane/tests/stream_transport_guarantees.rs
Executable file
|
|
@ -0,0 +1,260 @@
|
||||||
|
#![cfg(target_os = "linux")]
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use data_plane::arena::{ArenaConfig, ArenaManager, NodeId};
|
||||||
|
use data_plane::byte_ring::{ByteRingSpec, RecordKind, Role, attach, install};
|
||||||
|
use data_plane::namespace::StreamIncarnation;
|
||||||
|
use data_plane::stream_transport::{
|
||||||
|
LocalStreamTransport, StreamSinkRequest, StreamSourceRequest, StreamTransport,
|
||||||
|
StreamTransportEvent, StreamTransportNotifier,
|
||||||
|
};
|
||||||
|
use parking_lot::Mutex;
|
||||||
|
use proptest::prelude::*;
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct Events(Mutex<Vec<StreamTransportEvent>>);
|
||||||
|
|
||||||
|
impl StreamTransportNotifier for Events {
|
||||||
|
fn notify(&self, event: StreamTransportEvent) {
|
||||||
|
self.0.lock().push(event);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn arena(node: u64) -> ArenaManager {
|
||||||
|
ArenaManager::boot(ArenaConfig {
|
||||||
|
node_id: NodeId(node),
|
||||||
|
reservation_ceiling: 1 << 20,
|
||||||
|
base_alignment: 64,
|
||||||
|
})
|
||||||
|
.expect("arena")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn local_transport_preserves_order_backpressure_and_eof() {
|
||||||
|
let mut source_arena = arena(1);
|
||||||
|
let source_handle = install(
|
||||||
|
&mut source_arena,
|
||||||
|
ByteRingSpec {
|
||||||
|
capacity: 32,
|
||||||
|
generation: 1,
|
||||||
|
alignment: 64,
|
||||||
|
request_id: 1,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("source ring");
|
||||||
|
let mut destination_arena = arena(2);
|
||||||
|
let destination_handle = install(
|
||||||
|
&mut destination_arena,
|
||||||
|
ByteRingSpec {
|
||||||
|
capacity: 16,
|
||||||
|
generation: 2,
|
||||||
|
alignment: 64,
|
||||||
|
request_id: 2,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("destination ring");
|
||||||
|
|
||||||
|
let mut writer = attach(&source_arena, source_handle, Role::Producer).expect("writer");
|
||||||
|
let source_pump = attach(&source_arena, source_handle, Role::Consumer).expect("source pump");
|
||||||
|
let destination_pump =
|
||||||
|
attach(&destination_arena, destination_handle, Role::Producer).expect("destination pump");
|
||||||
|
let mut reader =
|
||||||
|
attach(&destination_arena, destination_handle, Role::Consumer).expect("reader");
|
||||||
|
|
||||||
|
let transport = LocalStreamTransport::new();
|
||||||
|
let incarnation = StreamIncarnation {
|
||||||
|
authority_epoch: 7,
|
||||||
|
revision: 11,
|
||||||
|
};
|
||||||
|
let source_events = Arc::new(Events::default());
|
||||||
|
let sink_events = Arc::new(Events::default());
|
||||||
|
transport
|
||||||
|
.install_sink(StreamSinkRequest {
|
||||||
|
incarnation,
|
||||||
|
endpoint: destination_pump,
|
||||||
|
notifier: sink_events.clone(),
|
||||||
|
})
|
||||||
|
.expect("install sink");
|
||||||
|
transport
|
||||||
|
.install_source(StreamSourceRequest {
|
||||||
|
incarnation,
|
||||||
|
peer: transport.descriptor().expect("descriptor"),
|
||||||
|
endpoint: source_pump,
|
||||||
|
notifier: source_events.clone(),
|
||||||
|
})
|
||||||
|
.expect("install source");
|
||||||
|
assert!(
|
||||||
|
source_events
|
||||||
|
.0
|
||||||
|
.lock()
|
||||||
|
.contains(&StreamTransportEvent::Ready)
|
||||||
|
);
|
||||||
|
assert!(sink_events.0.lock().contains(&StreamTransportEvent::Ready));
|
||||||
|
|
||||||
|
writer
|
||||||
|
.send_record(RecordKind::Data, b"first")
|
||||||
|
.expect("write first");
|
||||||
|
transport.source_progress(incarnation);
|
||||||
|
writer
|
||||||
|
.send_record(RecordKind::Data, b"next!")
|
||||||
|
.expect("write second");
|
||||||
|
transport.source_progress(incarnation);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
reader.recv_record().expect("read first"),
|
||||||
|
Some((RecordKind::Data, b"first".to_vec()))
|
||||||
|
);
|
||||||
|
assert_eq!(reader.recv_record().expect("second remains upstream"), None);
|
||||||
|
transport.sink_progress(incarnation);
|
||||||
|
assert_eq!(
|
||||||
|
reader.recv_record().expect("read second"),
|
||||||
|
Some((RecordKind::Data, b"next!".to_vec()))
|
||||||
|
);
|
||||||
|
|
||||||
|
writer.send_record(RecordKind::Eof, b"").expect("write eof");
|
||||||
|
transport.source_progress(incarnation);
|
||||||
|
assert_eq!(
|
||||||
|
reader.recv_record().expect("read eof"),
|
||||||
|
Some((RecordKind::Eof, Vec::new()))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn local_transport_rejects_a_descriptor_from_another_backend() {
|
||||||
|
let first = LocalStreamTransport::new();
|
||||||
|
let second = LocalStreamTransport::new();
|
||||||
|
assert_ne!(
|
||||||
|
first.descriptor().expect("first descriptor"),
|
||||||
|
second.descriptor().expect("second descriptor")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
proptest! {
|
||||||
|
#![proptest_config(ProptestConfig {
|
||||||
|
cases: 32,
|
||||||
|
max_shrink_iters: 256,
|
||||||
|
..ProptestConfig::default()
|
||||||
|
})]
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn randomized_payloads_preserve_exact_bytes(
|
||||||
|
payload in prop::collection::vec(any::<u8>(), 0..4096),
|
||||||
|
raw_capacity in 16_u8..128,
|
||||||
|
chunk_seeds in prop::collection::vec(1_u8..=255, 1..32),
|
||||||
|
) {
|
||||||
|
let capacity = u64::from(raw_capacity);
|
||||||
|
let mut source_arena = arena(21);
|
||||||
|
let source_handle = install(
|
||||||
|
&mut source_arena,
|
||||||
|
ByteRingSpec {
|
||||||
|
capacity,
|
||||||
|
generation: 1,
|
||||||
|
alignment: 64,
|
||||||
|
request_id: 1,
|
||||||
|
},
|
||||||
|
).expect("source ring");
|
||||||
|
let mut destination_arena = arena(22);
|
||||||
|
let destination_handle = install(
|
||||||
|
&mut destination_arena,
|
||||||
|
ByteRingSpec {
|
||||||
|
capacity,
|
||||||
|
generation: 2,
|
||||||
|
alignment: 64,
|
||||||
|
request_id: 2,
|
||||||
|
},
|
||||||
|
).expect("destination ring");
|
||||||
|
let mut writer = attach(&source_arena, source_handle, Role::Producer).expect("writer");
|
||||||
|
let source_pump = attach(&source_arena, source_handle, Role::Consumer).expect("source pump");
|
||||||
|
let destination_pump =
|
||||||
|
attach(&destination_arena, destination_handle, Role::Producer).expect("destination pump");
|
||||||
|
let mut reader =
|
||||||
|
attach(&destination_arena, destination_handle, Role::Consumer).expect("reader");
|
||||||
|
let transport = LocalStreamTransport::new();
|
||||||
|
let incarnation = StreamIncarnation {
|
||||||
|
authority_epoch: 3,
|
||||||
|
revision: 9,
|
||||||
|
};
|
||||||
|
transport.install_sink(StreamSinkRequest {
|
||||||
|
incarnation,
|
||||||
|
endpoint: destination_pump,
|
||||||
|
notifier: Arc::new(Events::default()),
|
||||||
|
}).expect("sink");
|
||||||
|
transport.install_source(StreamSourceRequest {
|
||||||
|
incarnation,
|
||||||
|
peer: transport.descriptor().expect("descriptor"),
|
||||||
|
endpoint: source_pump,
|
||||||
|
notifier: Arc::new(Events::default()),
|
||||||
|
}).expect("source");
|
||||||
|
|
||||||
|
let max_chunk = capacity as usize - 5;
|
||||||
|
let mut offset = 0;
|
||||||
|
let mut seed_index = 0;
|
||||||
|
let mut observed = Vec::new();
|
||||||
|
while offset < payload.len() {
|
||||||
|
let chunk_len = usize::from(chunk_seeds[seed_index % chunk_seeds.len()])
|
||||||
|
.min(max_chunk)
|
||||||
|
.min(payload.len() - offset);
|
||||||
|
seed_index += 1;
|
||||||
|
loop {
|
||||||
|
match writer.send_record(
|
||||||
|
RecordKind::Data,
|
||||||
|
&payload[offset..offset + chunk_len],
|
||||||
|
) {
|
||||||
|
Ok(()) => break,
|
||||||
|
Err(data_plane::byte_ring::FlowError::InsufficientSpace { .. }) => {
|
||||||
|
if let Some((RecordKind::Data, bytes)) =
|
||||||
|
reader.recv_record().expect("drain destination")
|
||||||
|
{
|
||||||
|
observed.extend(bytes);
|
||||||
|
transport.sink_progress(incarnation);
|
||||||
|
} else {
|
||||||
|
transport.source_progress(incarnation);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(error) => panic!("unexpected source error: {error:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
offset += chunk_len;
|
||||||
|
transport.source_progress(incarnation);
|
||||||
|
}
|
||||||
|
|
||||||
|
loop {
|
||||||
|
match writer.send_record(RecordKind::Eof, &[]) {
|
||||||
|
Ok(()) => break,
|
||||||
|
Err(data_plane::byte_ring::FlowError::InsufficientSpace { .. }) => {
|
||||||
|
if let Some((RecordKind::Data, bytes)) =
|
||||||
|
reader.recv_record().expect("drain for eof")
|
||||||
|
{
|
||||||
|
observed.extend(bytes);
|
||||||
|
transport.sink_progress(incarnation);
|
||||||
|
} else {
|
||||||
|
transport.source_progress(incarnation);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(error) => panic!("unexpected eof error: {error:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
transport.source_progress(incarnation);
|
||||||
|
|
||||||
|
let mut steps = 0;
|
||||||
|
loop {
|
||||||
|
steps += 1;
|
||||||
|
prop_assert!(steps < payload.len() + 1024, "transfer made no bounded progress");
|
||||||
|
match reader.recv_record().expect("receive") {
|
||||||
|
Some((RecordKind::Data, bytes)) => {
|
||||||
|
observed.extend(bytes);
|
||||||
|
transport.sink_progress(incarnation);
|
||||||
|
}
|
||||||
|
Some((RecordKind::Eof, _)) => break,
|
||||||
|
Some((RecordKind::Fault, bytes)) => {
|
||||||
|
return Err(TestCaseError::fail(format!(
|
||||||
|
"unexpected fault record: {bytes:?}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
None => transport.source_progress(incarnation),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
prop_assert_eq!(observed, payload);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -15,3 +15,6 @@ tokio = { workspace = true, optional = true }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
proptest = "1"
|
proptest = "1"
|
||||||
|
|
||||||
|
[lints]
|
||||||
|
workspace = true
|
||||||
|
|
|
||||||
|
|
@ -18,8 +18,7 @@ use swactor::runtime::{ExternalSender, Runtime, RuntimeParts};
|
||||||
pub struct Engine {
|
pub struct Engine {
|
||||||
/// Retained so the engine owns the runtime handle it drives for its full
|
/// Retained so the engine owns the runtime handle it drives for its full
|
||||||
/// lifetime. Core workers are moved into substrate tasks at construction.
|
/// lifetime. Core workers are moved into substrate tasks at construction.
|
||||||
#[allow(dead_code)]
|
_runtime: Runtime,
|
||||||
runtime: Runtime,
|
|
||||||
backend: Arc<dyn ExecutionBackend>,
|
backend: Arc<dyn ExecutionBackend>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -41,7 +40,10 @@ impl Engine {
|
||||||
// no Tokio feature gate — so core progression does not silently
|
// no Tokio feature gate — so core progression does not silently
|
||||||
// disappear when an alternate backend is used (ENGINE_SPEC.md).
|
// disappear when an alternate backend is used (ENGINE_SPEC.md).
|
||||||
crate::core_driver::install(workers, &backend);
|
crate::core_driver::install(workers, &backend);
|
||||||
Ok(Engine { runtime, backend })
|
Ok(Engine {
|
||||||
|
_runtime: runtime,
|
||||||
|
backend,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Return a clonable handle for scheduling engine work.
|
/// Return a clonable handle for scheduling engine work.
|
||||||
|
|
|
||||||
|
|
@ -3,8 +3,6 @@
|
||||||
//! Imports only public `swactor` APIs and exposes no private engine state. See
|
//! Imports only public `swactor` APIs and exposes no private engine state. See
|
||||||
//! `ENGINE_SPEC.md`.
|
//! `ENGINE_SPEC.md`.
|
||||||
|
|
||||||
#![allow(dead_code)]
|
|
||||||
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
@ -23,9 +21,10 @@ pub fn default_runtime_parts() -> (RuntimeParts, Runtime) {
|
||||||
runtime_parts(RuntimeConfig::default())
|
runtime_parts(RuntimeConfig::default())
|
||||||
}
|
}
|
||||||
pub fn runtime_parts_with_workers(worker_count: usize) -> (RuntimeParts, Runtime) {
|
pub fn runtime_parts_with_workers(worker_count: usize) -> (RuntimeParts, Runtime) {
|
||||||
let mut config = RuntimeConfig::default();
|
runtime_parts(RuntimeConfig {
|
||||||
config.worker_count = worker_count;
|
worker_count,
|
||||||
runtime_parts(config)
|
..RuntimeConfig::default()
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn default_parts() -> RuntimeParts {
|
pub fn default_parts() -> RuntimeParts {
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@
|
||||||
//! These tests exercise the native Tokio backend specifically; the
|
//! These tests exercise the native Tokio backend specifically; the
|
||||||
//! non-Tokio portability proof lives in `engine_unit.rs`.
|
//! non-Tokio portability proof lives in `engine_unit.rs`.
|
||||||
|
|
||||||
mod common;
|
pub mod common;
|
||||||
use common::*;
|
use common::*;
|
||||||
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@
|
||||||
//! These tests exercise internal logic directly and use the [`SteppingBackend`]
|
//! These tests exercise internal logic directly and use the [`SteppingBackend`]
|
||||||
//! to prove substrate independence without Tokio (ENGINE_SPEC.md).
|
//! to prove substrate independence without Tokio (ENGINE_SPEC.md).
|
||||||
|
|
||||||
mod common;
|
pub mod common;
|
||||||
use common::*;
|
use common::*;
|
||||||
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
@ -1318,8 +1318,7 @@ fn engine_instant_is_ordered() {
|
||||||
/// A tasks-only probe backend that shares a sentinel `Arc<()>` so the test can
|
/// A tasks-only probe backend that shares a sentinel `Arc<()>` so the test can
|
||||||
/// observe exactly when the engine's strong backend reference is released.
|
/// observe exactly when the engine's strong backend reference is released.
|
||||||
struct SentinelBackend {
|
struct SentinelBackend {
|
||||||
#[allow(dead_code)]
|
_sentinel: Arc<()>,
|
||||||
sentinel: Arc<()>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ExecutionBackend for SentinelBackend {
|
impl ExecutionBackend for SentinelBackend {
|
||||||
|
|
@ -1347,7 +1346,7 @@ fn dropping_engine_releases_backend_even_with_live_handles() {
|
||||||
let engine = Engine::new(
|
let engine = Engine::new(
|
||||||
parts,
|
parts,
|
||||||
SentinelBackend {
|
SentinelBackend {
|
||||||
sentinel: sentinel.clone(),
|
_sentinel: sentinel.clone(),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.expect("tasks capability present");
|
.expect("tasks capability present");
|
||||||
|
|
|
||||||
|
|
@ -33,6 +33,7 @@ use distribution::types::NodeId;
|
||||||
|
|
||||||
use crate::edge_transport::spawn_edge_send_pump as spawn_edge_sender_task;
|
use crate::edge_transport::spawn_edge_send_pump as spawn_edge_sender_task;
|
||||||
use crate::edge_transport::{EDGE_ALPN, EdgeSendHandle, spawn_edge_recv_pump};
|
use crate::edge_transport::{EDGE_ALPN, EdgeSendHandle, spawn_edge_recv_pump};
|
||||||
|
use crate::stream_transport::{IrohStreamTransport, STREAM_ALPN};
|
||||||
use crate::telemetry_transport::{
|
use crate::telemetry_transport::{
|
||||||
TELEMETRY_ALPN, TelemetryQuicHeader, TelemetryQuicRead, read_events_from_stream,
|
TELEMETRY_ALPN, TelemetryQuicHeader, TelemetryQuicRead, read_events_from_stream,
|
||||||
spawn_subscription_writer,
|
spawn_subscription_writer,
|
||||||
|
|
@ -327,6 +328,7 @@ pub struct IrohDriver {
|
||||||
/// iroh work is scheduled through this handle; it never exposes the raw
|
/// iroh work is scheduled through this handle; it never exposes the raw
|
||||||
/// Tokio runtime (ENGINE_SPEC.md §7).
|
/// Tokio runtime (ENGINE_SPEC.md §7).
|
||||||
engine: EngineHandle,
|
engine: EngineHandle,
|
||||||
|
stream_transport: Arc<IrohStreamTransport>,
|
||||||
conns: Arc<Mutex<ConnCache>>,
|
conns: Arc<Mutex<ConnCache>>,
|
||||||
peer_auth: Option<Arc<Mutex<PeerAllowList>>>,
|
peer_auth: Option<Arc<Mutex<PeerAllowList>>>,
|
||||||
/// Collects connections from background join tasks.
|
/// Collects connections from background join tasks.
|
||||||
|
|
@ -450,7 +452,7 @@ impl IrohDriver {
|
||||||
let secret_key = config.secret_key;
|
let secret_key = config.secret_key;
|
||||||
let (endpoint_tx, endpoint_rx) = std::sync::mpsc::channel::<Result<Endpoint, String>>();
|
let (endpoint_tx, endpoint_rx) = std::sync::mpsc::channel::<Result<Endpoint, String>>();
|
||||||
engine.spawn(async move {
|
engine.spawn(async move {
|
||||||
let mut all_alpns = vec![ALPN.to_vec()];
|
let mut all_alpns = vec![ALPN.to_vec(), STREAM_ALPN.to_vec()];
|
||||||
all_alpns.extend(additional_alpns);
|
all_alpns.extend(additional_alpns);
|
||||||
let mut builder = Endpoint::builder(iroh::endpoint::presets::Minimal)
|
let mut builder = Endpoint::builder(iroh::endpoint::presets::Minimal)
|
||||||
.relay_mode(effective_relay_mode)
|
.relay_mode(effective_relay_mode)
|
||||||
|
|
@ -481,6 +483,7 @@ impl IrohDriver {
|
||||||
format!("engine endpoint-bind task dropped: {e}").into()
|
format!("engine endpoint-bind task dropped: {e}").into()
|
||||||
})?
|
})?
|
||||||
.map_err(|e| -> Box<dyn std::error::Error> { e.into() })?;
|
.map_err(|e| -> Box<dyn std::error::Error> { e.into() })?;
|
||||||
|
let stream_transport = Arc::new(IrohStreamTransport::new(engine.clone(), endpoint.clone()));
|
||||||
let relay_url = endpoint
|
let relay_url = endpoint
|
||||||
.addr()
|
.addr()
|
||||||
.relay_urls()
|
.relay_urls()
|
||||||
|
|
@ -504,6 +507,7 @@ impl IrohDriver {
|
||||||
let peer_auth = config.peer_auth.clone();
|
let peer_auth = config.peer_auth.clone();
|
||||||
let swim_buf = Arc::clone(&accepted_conns);
|
let swim_buf = Arc::clone(&accepted_conns);
|
||||||
let other_buf = Arc::clone(&other_accepted_conns);
|
let other_buf = Arc::clone(&other_accepted_conns);
|
||||||
|
let accepted_streams = Arc::clone(&stream_transport);
|
||||||
engine.spawn(async move {
|
engine.spawn(async move {
|
||||||
while let Some(incoming) = ep.accept().await {
|
while let Some(incoming) = ep.accept().await {
|
||||||
if let Ok(conn) = incoming.await {
|
if let Ok(conn) = incoming.await {
|
||||||
|
|
@ -522,6 +526,8 @@ impl IrohDriver {
|
||||||
let negotiated_alpn = conn.alpn().to_vec();
|
let negotiated_alpn = conn.alpn().to_vec();
|
||||||
if negotiated_alpn == ALPN {
|
if negotiated_alpn == ALPN {
|
||||||
swim_buf.lock().push((node_id, conn));
|
swim_buf.lock().push((node_id, conn));
|
||||||
|
} else if negotiated_alpn == STREAM_ALPN {
|
||||||
|
accepted_streams.accept_connection(conn);
|
||||||
} else {
|
} else {
|
||||||
other_buf.lock().push((node_id, negotiated_alpn, conn));
|
other_buf.lock().push((node_id, negotiated_alpn, conn));
|
||||||
}
|
}
|
||||||
|
|
@ -534,6 +540,7 @@ impl IrohDriver {
|
||||||
keypair,
|
keypair,
|
||||||
endpoint,
|
endpoint,
|
||||||
engine,
|
engine,
|
||||||
|
stream_transport,
|
||||||
conns: Arc::new(Mutex::new(ConnCache {
|
conns: Arc::new(Mutex::new(ConnCache {
|
||||||
connections: HashMap::new(),
|
connections: HashMap::new(),
|
||||||
next_generation: 1,
|
next_generation: 1,
|
||||||
|
|
@ -561,6 +568,10 @@ impl IrohDriver {
|
||||||
self.endpoint.clone()
|
self.endpoint.clone()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn stream_transport(&self) -> Arc<IrohStreamTransport> {
|
||||||
|
Arc::clone(&self.stream_transport)
|
||||||
|
}
|
||||||
|
|
||||||
/// Drain accepted connections whose negotiated ALPN exactly matches `alpn`.
|
/// Drain accepted connections whose negotiated ALPN exactly matches `alpn`.
|
||||||
pub fn drain_accepted_for_alpn(&self, alpn: &[u8]) -> Vec<(NodeId, Connection)> {
|
pub fn drain_accepted_for_alpn(&self, alpn: &[u8]) -> Vec<(NodeId, Connection)> {
|
||||||
let mut pending = self.other_accepted_conns.lock();
|
let mut pending = self.other_accepted_conns.lock();
|
||||||
|
|
|
||||||
|
|
@ -13,6 +13,7 @@ pub mod blob_transfer;
|
||||||
pub mod edge_transport;
|
pub mod edge_transport;
|
||||||
pub mod endpoint_advertisement;
|
pub mod endpoint_advertisement;
|
||||||
pub mod iroh_driver;
|
pub mod iroh_driver;
|
||||||
|
pub mod stream_transport;
|
||||||
pub mod telemetry_transport;
|
pub mod telemetry_transport;
|
||||||
|
|
||||||
pub use blob_transfer::{IrohBlobTransferReceiver, IrohBlobTransferSender};
|
pub use blob_transfer::{IrohBlobTransferReceiver, IrohBlobTransferSender};
|
||||||
|
|
@ -25,6 +26,7 @@ pub use iroh_driver::{
|
||||||
};
|
};
|
||||||
|
|
||||||
pub use edge_transport::{EDGE_ALPN, EdgeSendHandle};
|
pub use edge_transport::{EDGE_ALPN, EdgeSendHandle};
|
||||||
|
pub use stream_transport::{IrohStreamTransport, STREAM_ALPN};
|
||||||
|
|
||||||
pub use telemetry_transport::{
|
pub use telemetry_transport::{
|
||||||
PullCollectorConfig, PullCollectorHandle, TELEMETRY_ALPN, TelemetryQuicHeader,
|
PullCollectorConfig, PullCollectorHandle, TELEMETRY_ALPN, TelemetryQuicHeader,
|
||||||
|
|
|
||||||
431
crates/iroh-driver/src/stream_transport.rs
Normal file
431
crates/iroh-driver/src/stream_transport.rs
Normal file
|
|
@ -0,0 +1,431 @@
|
||||||
|
//! Iroh-specific implementation of the data-plane SPSC stream transport port.
|
||||||
|
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
|
|
||||||
|
use data_plane::byte_ring::{Endpoint as RingEndpoint, FlowError, RecordKind, RingProbe};
|
||||||
|
use data_plane::namespace::StreamIncarnation;
|
||||||
|
use data_plane::stream_transport::{
|
||||||
|
StreamPeerDescriptor, StreamSinkRequest, StreamSourceRequest, StreamTransport,
|
||||||
|
StreamTransportEvent, StreamTransportNotifier,
|
||||||
|
};
|
||||||
|
use iroh::endpoint::{Connection, RecvStream, SendStream};
|
||||||
|
use iroh::{Endpoint as IrohEndpoint, EndpointAddr};
|
||||||
|
use parking_lot::Mutex;
|
||||||
|
use swactor_engine::EngineHandle;
|
||||||
|
use tokio::io::AsyncWriteExt;
|
||||||
|
use tokio::sync::mpsc;
|
||||||
|
|
||||||
|
pub const STREAM_ALPN: &[u8] = b"swactor/data-plane-spsc/1";
|
||||||
|
const PREAMBLE_LEN: usize = 16;
|
||||||
|
const RECORD_HEADER_LEN: usize = 5;
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct TaskControl {
|
||||||
|
wake: mpsc::Sender<()>,
|
||||||
|
progress_pending: Arc<AtomicBool>,
|
||||||
|
cancelled: Arc<AtomicBool>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TaskControl {
|
||||||
|
fn pair() -> (Self, mpsc::Receiver<()>) {
|
||||||
|
let (wake, receiver) = mpsc::channel(1);
|
||||||
|
(
|
||||||
|
Self {
|
||||||
|
wake,
|
||||||
|
progress_pending: Arc::new(AtomicBool::new(false)),
|
||||||
|
cancelled: Arc::new(AtomicBool::new(false)),
|
||||||
|
},
|
||||||
|
receiver,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn progress(&self) {
|
||||||
|
if !self.progress_pending.swap(true, Ordering::AcqRel) {
|
||||||
|
let _ = self.wake.try_send(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cancel(&self) {
|
||||||
|
self.cancelled.store(true, Ordering::Release);
|
||||||
|
self.progress();
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn wait(&self, receiver: &mut mpsc::Receiver<()>) -> bool {
|
||||||
|
if self.cancelled.load(Ordering::Acquire) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if receiver.recv().await.is_none() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
self.progress_pending.store(false, Ordering::Release);
|
||||||
|
!self.cancelled.load(Ordering::Acquire)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct PendingSink {
|
||||||
|
endpoint: RingEndpoint,
|
||||||
|
notifier: Arc<dyn StreamTransportNotifier>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct TransportState {
|
||||||
|
pending_sinks: BTreeMap<StreamIncarnation, PendingSink>,
|
||||||
|
controls: BTreeMap<StreamIncarnation, Vec<TaskControl>>,
|
||||||
|
source_probes: BTreeMap<StreamIncarnation, RingProbe>,
|
||||||
|
sink_probes: BTreeMap<StreamIncarnation, RingProbe>,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Inner {
|
||||||
|
engine: EngineHandle,
|
||||||
|
endpoint: IrohEndpoint,
|
||||||
|
state: Mutex<TransportState>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Cloneable adapter capability installed into `HostDataPlaneConfig`.
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct IrohStreamTransport {
|
||||||
|
inner: Arc<Inner>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl IrohStreamTransport {
|
||||||
|
pub fn new(engine: EngineHandle, endpoint: IrohEndpoint) -> Self {
|
||||||
|
Self {
|
||||||
|
inner: Arc::new(Inner {
|
||||||
|
engine,
|
||||||
|
endpoint,
|
||||||
|
state: Mutex::new(TransportState::default()),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn accept_connection(&self, connection: Connection) {
|
||||||
|
let transport = self.clone();
|
||||||
|
self.inner.engine.spawn(async move {
|
||||||
|
while let Ok(mut recv) = connection.accept_uni().await {
|
||||||
|
let mut preamble = [0_u8; PREAMBLE_LEN];
|
||||||
|
if recv.read_exact(&mut preamble).await.is_err() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let incarnation = decode_incarnation(preamble);
|
||||||
|
let pending = transport
|
||||||
|
.inner
|
||||||
|
.state
|
||||||
|
.lock()
|
||||||
|
.pending_sinks
|
||||||
|
.remove(&incarnation);
|
||||||
|
let Some(pending) = pending else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let (control, receiver) = TaskControl::pair();
|
||||||
|
transport
|
||||||
|
.inner
|
||||||
|
.state
|
||||||
|
.lock()
|
||||||
|
.controls
|
||||||
|
.entry(incarnation)
|
||||||
|
.or_default()
|
||||||
|
.push(control.clone());
|
||||||
|
transport.inner.engine.spawn(run_sink(
|
||||||
|
incarnation,
|
||||||
|
recv,
|
||||||
|
pending.endpoint,
|
||||||
|
pending.notifier,
|
||||||
|
control,
|
||||||
|
receiver,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
fn register_control(&self, incarnation: StreamIncarnation, control: TaskControl) {
|
||||||
|
self.inner
|
||||||
|
.state
|
||||||
|
.lock()
|
||||||
|
.controls
|
||||||
|
.entry(incarnation)
|
||||||
|
.or_default()
|
||||||
|
.push(control);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn progress_controls(&self, incarnation: StreamIncarnation) {
|
||||||
|
if let Some(controls) = self.inner.state.lock().controls.get(&incarnation) {
|
||||||
|
for control in controls {
|
||||||
|
control.progress();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StreamTransport for IrohStreamTransport {
|
||||||
|
fn descriptor(&self) -> Result<StreamPeerDescriptor, String> {
|
||||||
|
serde_json::to_vec(&self.inner.endpoint.addr())
|
||||||
|
.map(StreamPeerDescriptor)
|
||||||
|
.map_err(|error| format!("encode iroh stream endpoint: {error}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn install_source(&self, request: StreamSourceRequest) -> Result<(), String> {
|
||||||
|
let peer: EndpointAddr = serde_json::from_slice(&request.peer.0)
|
||||||
|
.map_err(|error| format!("decode iroh stream endpoint: {error}"))?;
|
||||||
|
let (control, receiver) = TaskControl::pair();
|
||||||
|
self.register_control(request.incarnation, control.clone());
|
||||||
|
let endpoint = self.inner.endpoint.clone();
|
||||||
|
let probe = request.endpoint.probe();
|
||||||
|
self.inner
|
||||||
|
.state
|
||||||
|
.lock()
|
||||||
|
.source_probes
|
||||||
|
.insert(request.incarnation, probe);
|
||||||
|
self.inner.engine.spawn(run_source(
|
||||||
|
request.incarnation,
|
||||||
|
endpoint,
|
||||||
|
peer,
|
||||||
|
request.endpoint,
|
||||||
|
request.notifier,
|
||||||
|
control,
|
||||||
|
receiver,
|
||||||
|
));
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn install_sink(&self, request: StreamSinkRequest) -> Result<(), String> {
|
||||||
|
let mut state = self.inner.state.lock();
|
||||||
|
if state.pending_sinks.contains_key(&request.incarnation) {
|
||||||
|
return Err("iroh stream sink is already installed".to_owned());
|
||||||
|
}
|
||||||
|
let probe = request.endpoint.probe();
|
||||||
|
state.sink_probes.insert(request.incarnation, probe);
|
||||||
|
state.pending_sinks.insert(
|
||||||
|
request.incarnation,
|
||||||
|
PendingSink {
|
||||||
|
endpoint: request.endpoint,
|
||||||
|
notifier: request.notifier,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn source_progress(&self, incarnation: StreamIncarnation) {
|
||||||
|
self.progress_controls(incarnation);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sink_progress(&self, incarnation: StreamIncarnation) {
|
||||||
|
self.progress_controls(incarnation);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn source_has_capacity(&self, incarnation: StreamIncarnation) -> bool {
|
||||||
|
self.inner
|
||||||
|
.state
|
||||||
|
.lock()
|
||||||
|
.source_probes
|
||||||
|
.get(&incarnation)
|
||||||
|
.is_some_and(RingProbe::has_capacity)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sink_has_data(&self, incarnation: StreamIncarnation) -> bool {
|
||||||
|
self.inner
|
||||||
|
.state
|
||||||
|
.lock()
|
||||||
|
.sink_probes
|
||||||
|
.get(&incarnation)
|
||||||
|
.is_some_and(RingProbe::has_data)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn terminate(&self, incarnation: StreamIncarnation) {
|
||||||
|
let (pending, controls) = {
|
||||||
|
let mut state = self.inner.state.lock();
|
||||||
|
let pending = state.pending_sinks.remove(&incarnation);
|
||||||
|
let controls = state.controls.remove(&incarnation).unwrap_or_default();
|
||||||
|
state.source_probes.remove(&incarnation);
|
||||||
|
state.sink_probes.remove(&incarnation);
|
||||||
|
(pending, controls)
|
||||||
|
};
|
||||||
|
if let Some(pending) = pending {
|
||||||
|
pending.notifier.notify(StreamTransportEvent::Quiesced);
|
||||||
|
}
|
||||||
|
for control in controls {
|
||||||
|
control.cancel();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn run_source(
|
||||||
|
incarnation: StreamIncarnation,
|
||||||
|
endpoint: IrohEndpoint,
|
||||||
|
peer: EndpointAddr,
|
||||||
|
mut source: RingEndpoint,
|
||||||
|
notifier: Arc<dyn StreamTransportNotifier>,
|
||||||
|
control: TaskControl,
|
||||||
|
mut receiver: mpsc::Receiver<()>,
|
||||||
|
) {
|
||||||
|
let result = async {
|
||||||
|
let connection = endpoint
|
||||||
|
.connect(peer, STREAM_ALPN)
|
||||||
|
.await
|
||||||
|
.map_err(|error| format!("connect stream incarnation: {error}"))?;
|
||||||
|
let mut send = connection
|
||||||
|
.open_uni()
|
||||||
|
.await
|
||||||
|
.map_err(|error| format!("open stream incarnation: {error}"))?;
|
||||||
|
send.write_all(&encode_incarnation(incarnation))
|
||||||
|
.await
|
||||||
|
.map_err(|error| format!("write stream preamble: {error}"))?;
|
||||||
|
send.flush()
|
||||||
|
.await
|
||||||
|
.map_err(|error| format!("flush stream preamble: {error}"))?;
|
||||||
|
notifier.notify(StreamTransportEvent::Ready);
|
||||||
|
|
||||||
|
loop {
|
||||||
|
if control.cancelled.load(Ordering::Acquire) {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let mut moved = false;
|
||||||
|
while let Some(meta) = source
|
||||||
|
.next_record_meta()
|
||||||
|
.map_err(|error| format!("inspect source ring: {error:?}"))?
|
||||||
|
{
|
||||||
|
let view = source
|
||||||
|
.peek_record()
|
||||||
|
.map_err(|error| format!("pin source ring: {error:?}"))?
|
||||||
|
.ok_or_else(|| "source record disappeared after inspection".to_owned())?;
|
||||||
|
write_record(&mut send, meta.kind, view.spans())
|
||||||
|
.await
|
||||||
|
.map_err(|error| format!("write stream record: {error}"))?;
|
||||||
|
view.release()
|
||||||
|
.map_err(|error| format!("release source ring: {error:?}"))?;
|
||||||
|
notifier.notify(StreamTransportEvent::CapacityAvailable);
|
||||||
|
moved = true;
|
||||||
|
if matches!(meta.kind, RecordKind::Eof | RecordKind::Fault) {
|
||||||
|
send.finish()
|
||||||
|
.map_err(|error| format!("finish stream incarnation: {error}"))?;
|
||||||
|
match send
|
||||||
|
.stopped()
|
||||||
|
.await
|
||||||
|
.map_err(|error| format!("await stream finish: {error}"))?
|
||||||
|
{
|
||||||
|
Some(code) => {
|
||||||
|
return Err(format!("peer stopped stream incarnation: {code}"));
|
||||||
|
}
|
||||||
|
None => return Ok(()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !moved && !control.wait(&mut receiver).await {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.await;
|
||||||
|
if let Err(reason) = result {
|
||||||
|
notifier.notify(StreamTransportEvent::Fault(reason));
|
||||||
|
}
|
||||||
|
notifier.notify(StreamTransportEvent::Quiesced);
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn write_record(
|
||||||
|
send: &mut SendStream,
|
||||||
|
kind: RecordKind,
|
||||||
|
spans: (&[u8], &[u8]),
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let len = spans.0.len() + spans.1.len();
|
||||||
|
let len = u32::try_from(len).map_err(|_| "stream record exceeds u32 framing".to_owned())?;
|
||||||
|
let mut header = [0_u8; RECORD_HEADER_LEN];
|
||||||
|
header[0] = kind.to_byte();
|
||||||
|
header[1..].copy_from_slice(&len.to_le_bytes());
|
||||||
|
send.write_all(&header)
|
||||||
|
.await
|
||||||
|
.map_err(|error| error.to_string())?;
|
||||||
|
if !spans.0.is_empty() {
|
||||||
|
send.write_all(spans.0)
|
||||||
|
.await
|
||||||
|
.map_err(|error| error.to_string())?;
|
||||||
|
}
|
||||||
|
if !spans.1.is_empty() {
|
||||||
|
send.write_all(spans.1)
|
||||||
|
.await
|
||||||
|
.map_err(|error| error.to_string())?;
|
||||||
|
}
|
||||||
|
send.flush().await.map_err(|error| error.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn run_sink(
|
||||||
|
_incarnation: StreamIncarnation,
|
||||||
|
mut recv: RecvStream,
|
||||||
|
mut sink: RingEndpoint,
|
||||||
|
notifier: Arc<dyn StreamTransportNotifier>,
|
||||||
|
control: TaskControl,
|
||||||
|
mut receiver: mpsc::Receiver<()>,
|
||||||
|
) {
|
||||||
|
notifier.notify(StreamTransportEvent::Ready);
|
||||||
|
let result = async {
|
||||||
|
loop {
|
||||||
|
if control.cancelled.load(Ordering::Acquire) {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let mut header = [0_u8; RECORD_HEADER_LEN];
|
||||||
|
recv.read_exact(&mut header)
|
||||||
|
.await
|
||||||
|
.map_err(|error| format!("read stream record header: {error}"))?;
|
||||||
|
let kind = RecordKind::from_byte(header[0])
|
||||||
|
.ok_or_else(|| format!("invalid stream record kind {}", header[0]))?;
|
||||||
|
let len = u64::from(u32::from_le_bytes(header[1..].try_into().unwrap()));
|
||||||
|
let mut reservation = loop {
|
||||||
|
match sink.reserve_record(kind, len) {
|
||||||
|
Ok(reservation) => break reservation,
|
||||||
|
Err(FlowError::InsufficientSpace { .. }) => {
|
||||||
|
if !control.wait(&mut receiver).await {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(error) => return Err(format!("reserve sink ring: {error:?}")),
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let (first, second) = reservation.spans_mut();
|
||||||
|
if !first.is_empty() {
|
||||||
|
recv.read_exact(first)
|
||||||
|
.await
|
||||||
|
.map_err(|error| format!("read first stream span: {error}"))?;
|
||||||
|
}
|
||||||
|
if !second.is_empty() {
|
||||||
|
recv.read_exact(second)
|
||||||
|
.await
|
||||||
|
.map_err(|error| format!("read second stream span: {error}"))?;
|
||||||
|
}
|
||||||
|
reservation
|
||||||
|
.commit()
|
||||||
|
.map_err(|error| format!("commit sink ring: {error:?}"))?;
|
||||||
|
notifier.notify(StreamTransportEvent::DataAvailable);
|
||||||
|
if matches!(kind, RecordKind::Eof | RecordKind::Fault) {
|
||||||
|
let mut trailing = [0_u8; 1];
|
||||||
|
match recv
|
||||||
|
.read(&mut trailing)
|
||||||
|
.await
|
||||||
|
.map_err(|error| format!("read stream finish: {error}"))?
|
||||||
|
{
|
||||||
|
None | Some(0) => return Ok(()),
|
||||||
|
Some(_) => return Err("bytes followed terminal stream record".to_owned()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.await;
|
||||||
|
if let Err(reason) = result {
|
||||||
|
notifier.notify(StreamTransportEvent::Fault(reason));
|
||||||
|
}
|
||||||
|
notifier.notify(StreamTransportEvent::Quiesced);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn encode_incarnation(incarnation: StreamIncarnation) -> [u8; PREAMBLE_LEN] {
|
||||||
|
let mut encoded = [0_u8; PREAMBLE_LEN];
|
||||||
|
encoded[..8].copy_from_slice(&incarnation.authority_epoch.to_le_bytes());
|
||||||
|
encoded[8..].copy_from_slice(&incarnation.revision.to_le_bytes());
|
||||||
|
encoded
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode_incarnation(encoded: [u8; PREAMBLE_LEN]) -> StreamIncarnation {
|
||||||
|
StreamIncarnation {
|
||||||
|
authority_epoch: u64::from_le_bytes(encoded[..8].try_into().unwrap()),
|
||||||
|
revision: u64::from_le_bytes(encoded[8..].try_into().unwrap()),
|
||||||
|
}
|
||||||
|
}
|
||||||
137
crates/iroh-driver/tests/stream_transport.rs
Normal file
137
crates/iroh-driver/tests/stream_transport.rs
Normal file
|
|
@ -0,0 +1,137 @@
|
||||||
|
pub mod common;
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::mpsc::{self, Receiver, Sender};
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use data_plane::arena::{ArenaConfig, ArenaManager, NodeId};
|
||||||
|
use data_plane::byte_ring::{ByteRingSpec, RecordKind, Role, attach, install};
|
||||||
|
use data_plane::namespace::StreamIncarnation;
|
||||||
|
use data_plane::stream_transport::{
|
||||||
|
StreamSinkRequest, StreamSourceRequest, StreamTransport, StreamTransportEvent,
|
||||||
|
StreamTransportNotifier,
|
||||||
|
};
|
||||||
|
|
||||||
|
use common::iroh::make_driver;
|
||||||
|
|
||||||
|
struct ChannelNotifier(Sender<StreamTransportEvent>);
|
||||||
|
|
||||||
|
impl StreamTransportNotifier for ChannelNotifier {
|
||||||
|
fn notify(&self, event: StreamTransportEvent) {
|
||||||
|
let _ = self.0.send(event);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn arena(node: u64) -> ArenaManager {
|
||||||
|
ArenaManager::boot(ArenaConfig {
|
||||||
|
node_id: NodeId(node),
|
||||||
|
reservation_ceiling: 1 << 20,
|
||||||
|
base_alignment: 64,
|
||||||
|
})
|
||||||
|
.expect("arena")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn recv_until(receiver: &Receiver<StreamTransportEvent>, expected: StreamTransportEvent) {
|
||||||
|
loop {
|
||||||
|
let event = receiver
|
||||||
|
.recv_timeout(Duration::from_secs(10))
|
||||||
|
.expect("transport event deadline");
|
||||||
|
if event == expected {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if let StreamTransportEvent::Fault(reason) = event {
|
||||||
|
panic!("unexpected stream fault: {reason}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn iroh_adapter_satisfies_ordering_and_terminal_contract() {
|
||||||
|
let source_node = make_driver();
|
||||||
|
let sink_node = make_driver();
|
||||||
|
let source_transport = source_node.driver.stream_transport();
|
||||||
|
let sink_transport = sink_node.driver.stream_transport();
|
||||||
|
|
||||||
|
let mut source_arena = arena(1);
|
||||||
|
let source_handle = install(
|
||||||
|
&mut source_arena,
|
||||||
|
ByteRingSpec {
|
||||||
|
capacity: 128,
|
||||||
|
generation: 1,
|
||||||
|
alignment: 64,
|
||||||
|
request_id: 1,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("source ring");
|
||||||
|
let mut sink_arena = arena(2);
|
||||||
|
let sink_handle = install(
|
||||||
|
&mut sink_arena,
|
||||||
|
ByteRingSpec {
|
||||||
|
capacity: 128,
|
||||||
|
generation: 2,
|
||||||
|
alignment: 64,
|
||||||
|
request_id: 2,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("sink ring");
|
||||||
|
let mut writer = attach(&source_arena, source_handle, Role::Producer).expect("writer");
|
||||||
|
let source_pump = attach(&source_arena, source_handle, Role::Consumer).expect("source pump");
|
||||||
|
let sink_pump = attach(&sink_arena, sink_handle, Role::Producer).expect("sink pump");
|
||||||
|
let mut reader = attach(&sink_arena, sink_handle, Role::Consumer).expect("reader");
|
||||||
|
|
||||||
|
let incarnation = StreamIncarnation {
|
||||||
|
authority_epoch: 17,
|
||||||
|
revision: 23,
|
||||||
|
};
|
||||||
|
let (source_tx, source_rx) = mpsc::channel();
|
||||||
|
let (sink_tx, sink_rx) = mpsc::channel();
|
||||||
|
sink_transport
|
||||||
|
.install_sink(StreamSinkRequest {
|
||||||
|
incarnation,
|
||||||
|
endpoint: sink_pump,
|
||||||
|
notifier: Arc::new(ChannelNotifier(sink_tx)),
|
||||||
|
})
|
||||||
|
.expect("install sink");
|
||||||
|
source_transport
|
||||||
|
.install_source(StreamSourceRequest {
|
||||||
|
incarnation,
|
||||||
|
peer: sink_transport.descriptor().expect("sink descriptor"),
|
||||||
|
endpoint: source_pump,
|
||||||
|
notifier: Arc::new(ChannelNotifier(source_tx)),
|
||||||
|
})
|
||||||
|
.expect("install source");
|
||||||
|
recv_until(&source_rx, StreamTransportEvent::Ready);
|
||||||
|
recv_until(&sink_rx, StreamTransportEvent::Ready);
|
||||||
|
|
||||||
|
writer
|
||||||
|
.send_record(RecordKind::Data, b"one")
|
||||||
|
.expect("write one");
|
||||||
|
writer
|
||||||
|
.send_record(RecordKind::Data, b"two")
|
||||||
|
.expect("write two");
|
||||||
|
source_transport.source_progress(incarnation);
|
||||||
|
recv_until(&sink_rx, StreamTransportEvent::DataAvailable);
|
||||||
|
recv_until(&sink_rx, StreamTransportEvent::DataAvailable);
|
||||||
|
assert_eq!(
|
||||||
|
reader.recv_record().expect("read one"),
|
||||||
|
Some((RecordKind::Data, b"one".to_vec()))
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
reader.recv_record().expect("read two"),
|
||||||
|
Some((RecordKind::Data, b"two".to_vec()))
|
||||||
|
);
|
||||||
|
sink_transport.sink_progress(incarnation);
|
||||||
|
|
||||||
|
writer.send_record(RecordKind::Eof, &[]).expect("write eof");
|
||||||
|
source_transport.source_progress(incarnation);
|
||||||
|
recv_until(&sink_rx, StreamTransportEvent::DataAvailable);
|
||||||
|
assert_eq!(
|
||||||
|
reader.recv_record().expect("read eof"),
|
||||||
|
Some((RecordKind::Eof, Vec::new()))
|
||||||
|
);
|
||||||
|
recv_until(&source_rx, StreamTransportEvent::Quiesced);
|
||||||
|
recv_until(&sink_rx, StreamTransportEvent::Quiesced);
|
||||||
|
|
||||||
|
source_transport.terminate(incarnation);
|
||||||
|
sink_transport.terminate(incarnation);
|
||||||
|
}
|
||||||
|
|
@ -7,6 +7,8 @@ license = "AGPL-3.0-only"
|
||||||
[dependencies]
|
[dependencies]
|
||||||
swactor = { path = "../..", features = ["serde", "transport"] }
|
swactor = { path = "../..", features = ["serde", "transport"] }
|
||||||
swactor-transport = { path = "../transport" }
|
swactor-transport = { path = "../transport" }
|
||||||
|
swactor-engine = { path = "../engine" }
|
||||||
|
futures-channel = "0.3"
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
iroh = "0.98"
|
iroh = "0.98"
|
||||||
|
|
@ -14,3 +16,6 @@ crossbeam-channel = "0.5"
|
||||||
|
|
||||||
[target.'cfg(target_os = "linux")'.dependencies]
|
[target.'cfg(target_os = "linux")'.dependencies]
|
||||||
libc = "0.2"
|
libc = "0.2"
|
||||||
|
|
||||||
|
[lints]
|
||||||
|
workspace = true
|
||||||
|
|
|
||||||
|
|
@ -17,6 +17,7 @@ pub trait FrameSink: Send {
|
||||||
/// Ship one positioned frame for `stream`. Best-effort: a sink may drop.
|
/// Ship one positioned frame for `stream`. Best-effort: a sink may drop.
|
||||||
fn ship(&mut self, stream: &StreamId, frame: &Frame);
|
fn ship(&mut self, stream: &StreamId, frame: &Frame);
|
||||||
}
|
}
|
||||||
|
pub type ProcessChannelRouter = dyn Fn(&str, bool) -> ChannelId + Send + Sync;
|
||||||
|
|
||||||
/// Static identity a node needs to build its mux.
|
/// Static identity a node needs to build its mux.
|
||||||
pub struct EmitterConfig {
|
pub struct EmitterConfig {
|
||||||
|
|
@ -27,7 +28,7 @@ pub struct EmitterConfig {
|
||||||
|
|
||||||
struct MuxProcObserver {
|
struct MuxProcObserver {
|
||||||
mux: Arc<Mux>,
|
mux: Arc<Mux>,
|
||||||
channel_for: Arc<dyn Fn(&str, bool) -> ChannelId + Send + Sync>,
|
channel_for: Arc<ProcessChannelRouter>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ProcessOutputObserver for MuxProcObserver {
|
impl ProcessOutputObserver for MuxProcObserver {
|
||||||
|
|
|
||||||
|
|
@ -16,6 +16,7 @@ use serde::{Deserialize, Serialize};
|
||||||
use swactor::process_observer::ProcessOutputObserver;
|
use swactor::process_observer::ProcessOutputObserver;
|
||||||
use swactor::stats::{ActorSnapshot, StatsHook};
|
use swactor::stats::{ActorSnapshot, StatsHook};
|
||||||
|
|
||||||
|
use crate::emit::ProcessChannelRouter;
|
||||||
use crate::frame::{
|
use crate::frame::{
|
||||||
ChannelContent, ChannelDescriptor, ChannelFilter, ChannelId, ChannelRef, FrameDelivery,
|
ChannelContent, ChannelDescriptor, ChannelFilter, ChannelId, ChannelRef, FrameDelivery,
|
||||||
SourceFilter, StreamDescriptor, StreamId, StreamOrigin, SubscriptionRequest, TelemetryEvent,
|
SourceFilter, StreamDescriptor, StreamId, StreamOrigin, SubscriptionRequest, TelemetryEvent,
|
||||||
|
|
@ -721,7 +722,7 @@ fn register_channel(
|
||||||
/// Legacy/custom process-output observer adapter that submits stdout/stderr chunks as frames.
|
/// Legacy/custom process-output observer adapter that submits stdout/stderr chunks as frames.
|
||||||
pub struct TelemetryProcessObserver {
|
pub struct TelemetryProcessObserver {
|
||||||
producer: TelemetryProducer,
|
producer: TelemetryProducer,
|
||||||
channel_for: Arc<dyn Fn(&str, bool) -> ChannelId + Send + Sync>,
|
channel_for: Arc<ProcessChannelRouter>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ProcessOutputObserver for TelemetryProcessObserver {
|
impl ProcessOutputObserver for TelemetryProcessObserver {
|
||||||
|
|
|
||||||
|
|
@ -324,10 +324,9 @@ fn parse_cpu_snapshot(raw: &str) -> Option<CpuSnapshot> {
|
||||||
} else if let Some(index) = label
|
} else if let Some(index) = label
|
||||||
.strip_prefix("cpu")
|
.strip_prefix("cpu")
|
||||||
.and_then(|suffix| suffix.parse::<u32>().ok())
|
.and_then(|suffix| suffix.parse::<u32>().ok())
|
||||||
|
&& let Some(times) = parse_cpu_times(line)
|
||||||
{
|
{
|
||||||
if let Some(times) = parse_cpu_times(line) {
|
cores.push(CpuCoreTimes { index, times });
|
||||||
cores.push(CpuCoreTimes { index, times });
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
151
crates/telemetry/src/hardware/memory.rs
Normal file
151
crates/telemetry/src/hardware/memory.rs
Normal file
|
|
@ -0,0 +1,151 @@
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::fs;
|
||||||
|
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use super::pressure::{self, PressureSample};
|
||||||
|
use crate::record::Record;
|
||||||
|
|
||||||
|
pub const HOST_MEMORY_CHANNEL: &str = "host.memory";
|
||||||
|
pub const MEMORY_SAMPLE_INTERVAL: Duration = Duration::from_secs(1);
|
||||||
|
|
||||||
|
const SCHEMA: &str = "host.memory.v1";
|
||||||
|
const KIB: u64 = 1024;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||||
|
pub struct HostMemorySample {
|
||||||
|
pub schema: String,
|
||||||
|
pub seq: u64,
|
||||||
|
pub sample_unix_ms: u64,
|
||||||
|
pub query_elapsed_ms: Option<u64>,
|
||||||
|
pub total_bytes: Option<u64>,
|
||||||
|
pub available_bytes: Option<u64>,
|
||||||
|
pub used_bytes: Option<u64>,
|
||||||
|
pub cached_bytes: Option<u64>,
|
||||||
|
pub swap_total_bytes: Option<u64>,
|
||||||
|
pub swap_used_bytes: Option<u64>,
|
||||||
|
pub pressure: Option<PressureSample>,
|
||||||
|
pub error: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Record for HostMemorySample {
|
||||||
|
const CHANNEL: &'static str = HOST_MEMORY_CHANNEL;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn sample(seq: u64) -> HostMemorySample {
|
||||||
|
let started = Instant::now();
|
||||||
|
let sample_unix_ms = unix_ms_now();
|
||||||
|
let memory = match fs::read_to_string("/proc/meminfo") {
|
||||||
|
Ok(raw) => match parse_meminfo(&raw) {
|
||||||
|
Some(memory) => memory,
|
||||||
|
None => return HostMemorySample::error(seq, "parse /proc/meminfo"),
|
||||||
|
},
|
||||||
|
Err(error) => return HostMemorySample::error(seq, format!("read /proc/meminfo: {error}")),
|
||||||
|
};
|
||||||
|
HostMemorySample {
|
||||||
|
schema: SCHEMA.to_owned(),
|
||||||
|
seq,
|
||||||
|
sample_unix_ms,
|
||||||
|
query_elapsed_ms: Some(elapsed_ms(started)),
|
||||||
|
total_bytes: Some(memory.total_bytes),
|
||||||
|
available_bytes: Some(memory.available_bytes),
|
||||||
|
used_bytes: Some(memory.total_bytes.saturating_sub(memory.available_bytes)),
|
||||||
|
cached_bytes: Some(memory.cached_bytes),
|
||||||
|
swap_total_bytes: Some(memory.swap_total_bytes),
|
||||||
|
swap_used_bytes: Some(
|
||||||
|
memory
|
||||||
|
.swap_total_bytes
|
||||||
|
.saturating_sub(memory.swap_free_bytes),
|
||||||
|
),
|
||||||
|
pressure: pressure::read("memory").ok(),
|
||||||
|
error: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl HostMemorySample {
|
||||||
|
fn error(seq: u64, error: impl Into<String>) -> Self {
|
||||||
|
Self {
|
||||||
|
schema: SCHEMA.to_owned(),
|
||||||
|
seq,
|
||||||
|
sample_unix_ms: unix_ms_now(),
|
||||||
|
query_elapsed_ms: None,
|
||||||
|
total_bytes: None,
|
||||||
|
available_bytes: None,
|
||||||
|
used_bytes: None,
|
||||||
|
cached_bytes: None,
|
||||||
|
swap_total_bytes: None,
|
||||||
|
swap_used_bytes: None,
|
||||||
|
pressure: pressure::read("memory").ok(),
|
||||||
|
error: Some(error.into()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
struct MemoryCounters {
|
||||||
|
total_bytes: u64,
|
||||||
|
available_bytes: u64,
|
||||||
|
cached_bytes: u64,
|
||||||
|
swap_total_bytes: u64,
|
||||||
|
swap_free_bytes: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_meminfo(raw: &str) -> Option<MemoryCounters> {
|
||||||
|
let mut values = BTreeMap::new();
|
||||||
|
for line in raw.lines() {
|
||||||
|
let (name, rest) = line.split_once(':')?;
|
||||||
|
let value_kib = rest.split_whitespace().next()?.parse::<u64>().ok()?;
|
||||||
|
values.insert(name, value_kib.saturating_mul(KIB));
|
||||||
|
}
|
||||||
|
let cached_bytes = values
|
||||||
|
.get("Cached")
|
||||||
|
.copied()
|
||||||
|
.unwrap_or(0)
|
||||||
|
.saturating_add(values.get("SReclaimable").copied().unwrap_or(0));
|
||||||
|
Some(MemoryCounters {
|
||||||
|
total_bytes: *values.get("MemTotal")?,
|
||||||
|
available_bytes: *values.get("MemAvailable")?,
|
||||||
|
cached_bytes,
|
||||||
|
swap_total_bytes: values.get("SwapTotal").copied().unwrap_or(0),
|
||||||
|
swap_free_bytes: values.get("SwapFree").copied().unwrap_or(0),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unix_ms_now() -> u64 {
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_millis()
|
||||||
|
.try_into()
|
||||||
|
.unwrap_or(u64::MAX)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn elapsed_ms(started: Instant) -> u64 {
|
||||||
|
started.elapsed().as_millis().try_into().unwrap_or(u64::MAX)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::{parse_meminfo, sample};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn derives_used_cached_and_swap_memory() {
|
||||||
|
let counters = parse_meminfo(
|
||||||
|
"MemTotal: 1000 kB\nMemAvailable: 400 kB\nCached: 100 kB\nSReclaimable: 20 kB\nSwapTotal: 200 kB\nSwapFree: 150 kB\n",
|
||||||
|
)
|
||||||
|
.expect("memory counters");
|
||||||
|
assert_eq!(counters.total_bytes, 1_024_000);
|
||||||
|
assert_eq!(counters.available_bytes, 409_600);
|
||||||
|
assert_eq!(counters.cached_bytes, 122_880);
|
||||||
|
assert_eq!(counters.swap_total_bytes - counters.swap_free_bytes, 51_200);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn samples_live_memory() {
|
||||||
|
let sample = sample(7);
|
||||||
|
assert_eq!(sample.seq, 7);
|
||||||
|
assert!(sample.total_bytes.is_some_and(|total| total > 0));
|
||||||
|
assert!(sample.error.is_none());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,3 +1,103 @@
|
||||||
pub mod cpu;
|
pub mod cpu;
|
||||||
pub mod gpu;
|
pub mod gpu;
|
||||||
|
pub mod memory;
|
||||||
pub mod net;
|
pub mod net;
|
||||||
|
pub mod pressure;
|
||||||
|
pub mod storage;
|
||||||
|
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use swactor_engine::EngineHandle;
|
||||||
|
|
||||||
|
/// Run a stateful blocking sampler on the engine without creating an actor.
|
||||||
|
///
|
||||||
|
/// Sampling never overlaps: the next interval is armed only after the previous
|
||||||
|
/// blocking sample has returned and `observed` has consumed its result.
|
||||||
|
pub fn spawn_blocking_sampler<State, Sample, Started, Observed>(
|
||||||
|
engine: EngineHandle,
|
||||||
|
period: Duration,
|
||||||
|
state: State,
|
||||||
|
sample: fn(State, u64) -> (State, Sample),
|
||||||
|
started: Started,
|
||||||
|
mut observed: Observed,
|
||||||
|
) where
|
||||||
|
State: Send + 'static,
|
||||||
|
Sample: Send + 'static,
|
||||||
|
Started: FnOnce() + Send + 'static,
|
||||||
|
Observed: FnMut(u64, Sample) + Send + 'static,
|
||||||
|
{
|
||||||
|
let task_engine = engine.clone();
|
||||||
|
engine.spawn(async move {
|
||||||
|
started();
|
||||||
|
let blocking_work = task_engine.blocking_work_sender();
|
||||||
|
let mut interval = task_engine.interval(period);
|
||||||
|
let mut state = state;
|
||||||
|
let mut seq = 0_u64;
|
||||||
|
|
||||||
|
loop {
|
||||||
|
(&mut interval).await;
|
||||||
|
let current_state = state;
|
||||||
|
let (sample_tx, sample_rx) = futures_channel::oneshot::channel();
|
||||||
|
let work = Box::new(move || {
|
||||||
|
let _ = sample_tx.send(sample(current_state, seq));
|
||||||
|
});
|
||||||
|
if blocking_work.submit(work).is_err() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let Ok((next_state, result)) = sample_rx.await else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
state = next_state;
|
||||||
|
observed(seq, result);
|
||||||
|
seq = seq.saturating_add(1);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use std::sync::mpsc;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use swactor::config::RuntimeConfig;
|
||||||
|
use swactor::runtime::RuntimeParts;
|
||||||
|
use swactor_engine::{Engine, TokioBackend, TokioConfig};
|
||||||
|
|
||||||
|
use super::spawn_blocking_sampler;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn blocking_sampler_runs_sequentially_with_monotonic_sequences() {
|
||||||
|
let engine = Engine::new(
|
||||||
|
RuntimeParts::new(RuntimeConfig::default()),
|
||||||
|
TokioBackend::new(TokioConfig::default()).expect("Tokio backend"),
|
||||||
|
)
|
||||||
|
.expect("engine");
|
||||||
|
let (observed_tx, observed_rx) = mpsc::channel();
|
||||||
|
|
||||||
|
spawn_blocking_sampler(
|
||||||
|
engine.handle(),
|
||||||
|
Duration::from_millis(1),
|
||||||
|
0_u64,
|
||||||
|
|state, seq| (state + 1, (seq, state)),
|
||||||
|
|| {},
|
||||||
|
move |seq, result| {
|
||||||
|
observed_tx
|
||||||
|
.send((seq, result))
|
||||||
|
.expect("observation receiver")
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
observed_rx
|
||||||
|
.recv_timeout(Duration::from_secs(1))
|
||||||
|
.expect("first sample"),
|
||||||
|
(0, (0, 0)),
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
observed_rx
|
||||||
|
.recv_timeout(Duration::from_secs(1))
|
||||||
|
.expect("second sample"),
|
||||||
|
(1, (1, 1)),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
95
crates/telemetry/src/hardware/pressure.rs
Normal file
95
crates/telemetry/src/hardware/pressure.rs
Normal file
|
|
@ -0,0 +1,95 @@
|
||||||
|
use std::fs;
|
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||||
|
pub struct PressureSample {
|
||||||
|
pub some_avg10: f64,
|
||||||
|
pub some_avg60: f64,
|
||||||
|
pub some_avg300: f64,
|
||||||
|
pub some_total_us: u64,
|
||||||
|
pub full_avg10: Option<f64>,
|
||||||
|
pub full_avg60: Option<f64>,
|
||||||
|
pub full_avg300: Option<f64>,
|
||||||
|
pub full_total_us: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn read(resource: &str) -> Result<PressureSample, String> {
|
||||||
|
let path = format!("/proc/pressure/{resource}");
|
||||||
|
let raw = fs::read_to_string(&path).map_err(|error| format!("read {path}: {error}"))?;
|
||||||
|
parse(&raw).ok_or_else(|| format!("parse {path}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse(raw: &str) -> Option<PressureSample> {
|
||||||
|
let some = parse_row(raw.lines().find(|line| line.starts_with("some "))?)?;
|
||||||
|
let full = raw
|
||||||
|
.lines()
|
||||||
|
.find(|line| line.starts_with("full "))
|
||||||
|
.and_then(parse_row);
|
||||||
|
Some(PressureSample {
|
||||||
|
some_avg10: some.avg10,
|
||||||
|
some_avg60: some.avg60,
|
||||||
|
some_avg300: some.avg300,
|
||||||
|
some_total_us: some.total_us,
|
||||||
|
full_avg10: full.as_ref().map(|row| row.avg10),
|
||||||
|
full_avg60: full.as_ref().map(|row| row.avg60),
|
||||||
|
full_avg300: full.as_ref().map(|row| row.avg300),
|
||||||
|
full_total_us: full.map(|row| row.total_us),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
struct PressureRow {
|
||||||
|
avg10: f64,
|
||||||
|
avg60: f64,
|
||||||
|
avg300: f64,
|
||||||
|
total_us: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_row(line: &str) -> Option<PressureRow> {
|
||||||
|
let mut avg10 = None;
|
||||||
|
let mut avg60 = None;
|
||||||
|
let mut avg300 = None;
|
||||||
|
let mut total_us = None;
|
||||||
|
for field in line.split_whitespace().skip(1) {
|
||||||
|
let (name, value) = field.split_once('=')?;
|
||||||
|
match name {
|
||||||
|
"avg10" => avg10 = value.parse().ok(),
|
||||||
|
"avg60" => avg60 = value.parse().ok(),
|
||||||
|
"avg300" => avg300 = value.parse().ok(),
|
||||||
|
"total" => total_us = value.parse().ok(),
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Some(PressureRow {
|
||||||
|
avg10: avg10?,
|
||||||
|
avg60: avg60?,
|
||||||
|
avg300: avg300?,
|
||||||
|
total_us: total_us?,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::parse;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_some_and_full_pressure_rows() {
|
||||||
|
let sample = parse(
|
||||||
|
"some avg10=1.25 avg60=2.50 avg300=3.75 total=1234\nfull avg10=0.10 avg60=0.20 avg300=0.30 total=42\n",
|
||||||
|
)
|
||||||
|
.expect("pressure sample");
|
||||||
|
assert_eq!(sample.some_avg10, 1.25);
|
||||||
|
assert_eq!(sample.some_total_us, 1234);
|
||||||
|
assert_eq!(sample.full_avg10, Some(0.10));
|
||||||
|
assert_eq!(sample.full_total_us, Some(42));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn accepts_cpu_pressure_without_full_row() {
|
||||||
|
let sample =
|
||||||
|
parse("some avg10=0.00 avg60=0.01 avg300=0.02 total=99\n").expect("pressure sample");
|
||||||
|
assert_eq!(sample.some_total_us, 99);
|
||||||
|
assert_eq!(sample.full_avg10, None);
|
||||||
|
}
|
||||||
|
}
|
||||||
123
crates/telemetry/src/hardware/storage.rs
Normal file
123
crates/telemetry/src/hardware/storage.rs
Normal file
|
|
@ -0,0 +1,123 @@
|
||||||
|
use std::ffi::CString;
|
||||||
|
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use super::pressure::{self, PressureSample};
|
||||||
|
use crate::record::Record;
|
||||||
|
|
||||||
|
pub const HOST_STORAGE_CHANNEL: &str = "host.storage";
|
||||||
|
pub const STORAGE_SAMPLE_INTERVAL: Duration = Duration::from_secs(1);
|
||||||
|
|
||||||
|
const SCHEMA: &str = "host.storage.v1";
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||||
|
pub struct HostStorageSample {
|
||||||
|
pub schema: String,
|
||||||
|
pub seq: u64,
|
||||||
|
pub sample_unix_ms: u64,
|
||||||
|
pub query_elapsed_ms: Option<u64>,
|
||||||
|
pub filesystems: Vec<FilesystemSample>,
|
||||||
|
pub pressure: Option<PressureSample>,
|
||||||
|
pub error: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Record for HostStorageSample {
|
||||||
|
const CHANNEL: &'static str = HOST_STORAGE_CHANNEL;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||||
|
pub struct FilesystemSample {
|
||||||
|
pub mount: String,
|
||||||
|
pub total_bytes: u64,
|
||||||
|
pub used_bytes: u64,
|
||||||
|
pub available_bytes: u64,
|
||||||
|
pub used_percent: Option<f64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn sample(seq: u64) -> HostStorageSample {
|
||||||
|
let started = Instant::now();
|
||||||
|
match read_filesystem("/") {
|
||||||
|
Ok(filesystem) => HostStorageSample {
|
||||||
|
schema: SCHEMA.to_owned(),
|
||||||
|
seq,
|
||||||
|
sample_unix_ms: unix_ms_now(),
|
||||||
|
query_elapsed_ms: Some(elapsed_ms(started)),
|
||||||
|
filesystems: vec![filesystem],
|
||||||
|
pressure: pressure::read("io").ok(),
|
||||||
|
error: None,
|
||||||
|
},
|
||||||
|
Err(error) => HostStorageSample {
|
||||||
|
schema: SCHEMA.to_owned(),
|
||||||
|
seq,
|
||||||
|
sample_unix_ms: unix_ms_now(),
|
||||||
|
query_elapsed_ms: Some(elapsed_ms(started)),
|
||||||
|
filesystems: Vec::new(),
|
||||||
|
pressure: pressure::read("io").ok(),
|
||||||
|
error: Some(error),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
fn read_filesystem(mount: &str) -> Result<FilesystemSample, String> {
|
||||||
|
let path = CString::new(mount).map_err(|error| format!("filesystem path: {error}"))?;
|
||||||
|
let mut stats = std::mem::MaybeUninit::<libc::statvfs>::uninit();
|
||||||
|
// SAFETY: `path` is a live NUL-terminated string and `stats` points to writable storage.
|
||||||
|
let result = unsafe { libc::statvfs(path.as_ptr(), stats.as_mut_ptr()) };
|
||||||
|
if result != 0 {
|
||||||
|
return Err(format!(
|
||||||
|
"statvfs {mount}: {}",
|
||||||
|
std::io::Error::last_os_error()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
// SAFETY: successful `statvfs` initialized the output structure.
|
||||||
|
let stats = unsafe { stats.assume_init() };
|
||||||
|
let fragment_size = stats.f_frsize;
|
||||||
|
let total_bytes = stats.f_blocks.saturating_mul(fragment_size);
|
||||||
|
let free_bytes = stats.f_bfree.saturating_mul(fragment_size);
|
||||||
|
let available_bytes = stats.f_bavail.saturating_mul(fragment_size);
|
||||||
|
let used_bytes = total_bytes.saturating_sub(free_bytes);
|
||||||
|
let used_percent = (total_bytes > 0).then_some(used_bytes as f64 * 100.0 / total_bytes as f64);
|
||||||
|
Ok(FilesystemSample {
|
||||||
|
mount: mount.to_owned(),
|
||||||
|
total_bytes,
|
||||||
|
used_bytes,
|
||||||
|
available_bytes,
|
||||||
|
used_percent,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(target_os = "linux"))]
|
||||||
|
fn read_filesystem(mount: &str) -> Result<FilesystemSample, String> {
|
||||||
|
Err(format!("filesystem sampling unsupported for {mount}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unix_ms_now() -> u64 {
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_millis()
|
||||||
|
.try_into()
|
||||||
|
.unwrap_or(u64::MAX)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn elapsed_ms(started: Instant) -> u64 {
|
||||||
|
started.elapsed().as_millis().try_into().unwrap_or(u64::MAX)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::sample;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn samples_root_filesystem_capacity() {
|
||||||
|
let sample = sample(9);
|
||||||
|
assert_eq!(sample.seq, 9);
|
||||||
|
let root = sample.filesystems.first().expect("root filesystem");
|
||||||
|
assert_eq!(root.mount, "/");
|
||||||
|
assert!(root.total_bytes > 0);
|
||||||
|
assert!(root.used_bytes <= root.total_bytes);
|
||||||
|
assert!(sample.error.is_none());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -4,7 +4,7 @@
|
||||||
//! payloads into a bounded queue first, then assigns a single monotonic position
|
//! payloads into a bounded queue first, then assigns a single monotonic position
|
||||||
//! sequence while draining accepted payloads.
|
//! sequence while draining accepted payloads.
|
||||||
|
|
||||||
use crossbeam_channel::{Receiver, Sender, TryRecvError, TrySendError, bounded};
|
use crossbeam_channel::{Receiver, Sender, TrySendError, bounded};
|
||||||
use std::sync::atomic::{AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
|
|
||||||
use crate::frame::{ChannelId, Frame, Position, StreamId};
|
use crate::frame::{ChannelId, Frame, Position, StreamId};
|
||||||
|
|
@ -26,7 +26,7 @@ pub struct Mux {
|
||||||
impl Mux {
|
impl Mux {
|
||||||
/// Create a mux for `stream` with a bounded outgoing queue.
|
/// Create a mux for `stream` with a bounded outgoing queue.
|
||||||
pub fn new(stream: StreamId, capacity: usize) -> Self {
|
pub fn new(stream: StreamId, capacity: usize) -> Self {
|
||||||
let capacity = capacity.max(1).min(1_048_576);
|
let capacity = capacity.clamp(1, 1_048_576);
|
||||||
let (tx, rx) = bounded(capacity);
|
let (tx, rx) = bounded(capacity);
|
||||||
Mux {
|
Mux {
|
||||||
stream,
|
stream,
|
||||||
|
|
@ -61,20 +61,15 @@ impl Mux {
|
||||||
/// Pull all currently queued frames in mux queue order.
|
/// Pull all currently queued frames in mux queue order.
|
||||||
pub fn drain(&self) -> Vec<Frame> {
|
pub fn drain(&self) -> Vec<Frame> {
|
||||||
let mut frames = Vec::new();
|
let mut frames = Vec::new();
|
||||||
loop {
|
while let Ok(pending) = self.rx.try_recv() {
|
||||||
match self.rx.try_recv() {
|
// Position is consumed only after a pending frame has left
|
||||||
Ok(pending) => {
|
// the queue; failed submit never reaches this point.
|
||||||
// Position is consumed only after a pending frame has left
|
let position = Position(self.next.fetch_add(1, Ordering::Relaxed));
|
||||||
// the queue; failed submit never reaches this point.
|
frames.push(Frame {
|
||||||
let position = Position(self.next.fetch_add(1, Ordering::Relaxed));
|
channel: pending.channel,
|
||||||
frames.push(Frame {
|
position,
|
||||||
channel: pending.channel,
|
payload: pending.payload,
|
||||||
position,
|
});
|
||||||
payload: pending.payload,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
Err(TryRecvError::Empty) | Err(TryRecvError::Disconnected) => break,
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
frames
|
frames
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -27,17 +27,6 @@ impl Record for ResourceSample {
|
||||||
const CHANNEL: &'static str = "host.resource";
|
const CHANNEL: &'static str = "host.resource";
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
struct MembershipTransition {
|
|
||||||
peer: String,
|
|
||||||
from: String,
|
|
||||||
to: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Record for MembershipTransition {
|
|
||||||
const CHANNEL: &'static str = "membership";
|
|
||||||
}
|
|
||||||
|
|
||||||
fn stream() -> StreamId {
|
fn stream() -> StreamId {
|
||||||
StreamId::new(NodeId::new("node-alpha"), Lifetime(1))
|
StreamId::new(NodeId::new("node-alpha"), Lifetime(1))
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@ use telemetry::ingest::Consumer;
|
||||||
use telemetry::mux::Mux;
|
use telemetry::mux::Mux;
|
||||||
use telemetry::transport::Delivery;
|
use telemetry::transport::Delivery;
|
||||||
use telemetry::wire::{decode_delivery, encode_delivery};
|
use telemetry::wire::{decode_delivery, encode_delivery};
|
||||||
use telemetry::{ChannelId, Lifetime, NodeId, Position, Record, StreamId};
|
use telemetry::{ChannelId, Lifetime, NodeId, Record, StreamId};
|
||||||
|
|
||||||
const RESOURCE_CHANNEL: ChannelId = ChannelId(1);
|
const RESOURCE_CHANNEL: ChannelId = ChannelId(1);
|
||||||
const LOG_CHANNEL: ChannelId = ChannelId(2);
|
const LOG_CHANNEL: ChannelId = ChannelId(2);
|
||||||
|
|
|
||||||
|
|
@ -14,3 +14,6 @@ ed25519-dalek = { version = "2", features = ["std", "rand_core", "serde"] }
|
||||||
rand_core = { version = "0.6", features = ["getrandom"] }
|
rand_core = { version = "0.6", features = ["getrandom"] }
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
|
|
||||||
|
[lints]
|
||||||
|
workspace = true
|
||||||
|
|
|
||||||
|
|
@ -51,7 +51,7 @@ pub fn hex_encode(bytes: &[u8]) -> String {
|
||||||
|
|
||||||
/// Hex-decode a string into bytes. Returns `None` on invalid input.
|
/// Hex-decode a string into bytes. Returns `None` on invalid input.
|
||||||
pub fn hex_decode(hex: &str) -> Option<Vec<u8>> {
|
pub fn hex_decode(hex: &str) -> Option<Vec<u8>> {
|
||||||
if hex.len() % 2 != 0 {
|
if !hex.len().is_multiple_of(2) {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
let mut bytes = Vec::with_capacity(hex.len() / 2);
|
let mut bytes = Vec::with_capacity(hex.len() / 2);
|
||||||
|
|
|
||||||
|
|
@ -19,7 +19,7 @@ pub fn hex_encode(bytes: &[u8]) -> String {
|
||||||
|
|
||||||
/// Hex-decode a string into bytes. Returns `None` on invalid input.
|
/// Hex-decode a string into bytes. Returns `None` on invalid input.
|
||||||
pub fn hex_decode(hex: &str) -> Option<Vec<u8>> {
|
pub fn hex_decode(hex: &str) -> Option<Vec<u8>> {
|
||||||
if hex.len() % 2 != 0 {
|
if !hex.len().is_multiple_of(2) {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
let mut out = Vec::with_capacity(hex.len() / 2);
|
let mut out = Vec::with_capacity(hex.len() / 2);
|
||||||
|
|
|
||||||
|
|
@ -352,7 +352,9 @@ fn round_trip_across_two_runtimes() {
|
||||||
|
|
||||||
// Verify all 3 replies
|
// Verify all 3 replies
|
||||||
for i in 0..3u32 {
|
for i in 0..3u32 {
|
||||||
let pong = inbox_a.try_recv().expect(&format!("missing pong #{i}"));
|
let pong = inbox_a
|
||||||
|
.try_recv()
|
||||||
|
.unwrap_or_else(|| panic!("missing pong #{i}"));
|
||||||
assert_eq!(pong, Pong { value: i * 10 + 1 });
|
assert_eq!(pong, Pong { value: i * 10 + 1 });
|
||||||
}
|
}
|
||||||
assert!(inbox_a.try_recv().is_none(), "no extra messages");
|
assert!(inbox_a.try_recv().is_none(), "no extra messages");
|
||||||
|
|
|
||||||
|
|
@ -611,7 +611,6 @@ pub(crate) struct ResumeSignal;
|
||||||
/// Minimal core interface: send, spawn, stop, and extension access.
|
/// Minimal core interface: send, spawn, stop, and extension access.
|
||||||
/// Registry methods (naming, monitoring, groups) and timer scheduling
|
/// Registry methods (naming, monitoring, groups) and timer scheduling
|
||||||
/// are provided by extension traits in `swactor-std`.
|
/// are provided by extension traits in `swactor-std`.
|
||||||
#[allow(private_interfaces)]
|
|
||||||
pub trait ContextInner {
|
pub trait ContextInner {
|
||||||
fn send_any(&self, addr: ActorAddress, msg: Box<dyn Any + Send>) -> Result<(), Error>;
|
fn send_any(&self, addr: ActorAddress, msg: Box<dyn Any + Send>) -> Result<(), Error>;
|
||||||
fn spawn_any(&self, request: SpawnRequest);
|
fn spawn_any(&self, request: SpawnRequest);
|
||||||
|
|
|
||||||
12
src/admin.rs
12
src/admin.rs
|
|
@ -118,6 +118,10 @@ impl<T: Message> Admin<T> {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) type AdminBoxedReply = Box<dyn Any + Send>;
|
pub(crate) type AdminBoxedReply = Box<dyn Any + Send>;
|
||||||
|
pub(crate) type AdminGetState =
|
||||||
|
Box<dyn FnOnce(ActorAddress, &dyn AnyActor, ActorTypeMetadata) -> AdminBoxedReply + Send>;
|
||||||
|
pub(crate) type AdminReplaceState =
|
||||||
|
Box<dyn FnOnce(&mut dyn AnyActor, ActorTypeMetadata) -> AdminResult<OperationResult> + Send>;
|
||||||
|
|
||||||
pub(crate) enum AdminCommand {
|
pub(crate) enum AdminCommand {
|
||||||
ListActors {
|
ListActors {
|
||||||
|
|
@ -130,17 +134,13 @@ pub(crate) enum AdminCommand {
|
||||||
GetActorState {
|
GetActorState {
|
||||||
actor: ActorAddress,
|
actor: ActorAddress,
|
||||||
reply_to: ActorAddress,
|
reply_to: ActorAddress,
|
||||||
get: Box<
|
get: AdminGetState,
|
||||||
dyn FnOnce(ActorAddress, &dyn AnyActor, ActorTypeMetadata) -> AdminBoxedReply + Send,
|
|
||||||
>,
|
|
||||||
not_found: Box<dyn FnOnce(ActorAddress) -> AdminBoxedReply + Send>,
|
not_found: Box<dyn FnOnce(ActorAddress) -> AdminBoxedReply + Send>,
|
||||||
},
|
},
|
||||||
ReplaceActorState {
|
ReplaceActorState {
|
||||||
actor: ActorAddress,
|
actor: ActorAddress,
|
||||||
reply_to: ActorAddress,
|
reply_to: ActorAddress,
|
||||||
replace: Box<
|
replace: AdminReplaceState,
|
||||||
dyn FnOnce(&mut dyn AnyActor, ActorTypeMetadata) -> AdminResult<OperationResult> + Send,
|
|
||||||
>,
|
|
||||||
},
|
},
|
||||||
StopActor {
|
StopActor {
|
||||||
actor: ActorAddress,
|
actor: ActorAddress,
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
use crate::Instant;
|
use crate::Instant;
|
||||||
use std::any::Any;
|
use std::any::Any;
|
||||||
use std::cell::RefCell;
|
use std::cell::RefCell;
|
||||||
|
use std::cmp::Reverse;
|
||||||
use std::collections::{HashMap, VecDeque};
|
use std::collections::{HashMap, VecDeque};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::sync::atomic::Ordering;
|
use std::sync::atomic::Ordering;
|
||||||
|
|
@ -124,7 +125,7 @@ impl Worker {
|
||||||
|| self
|
|| self
|
||||||
.worker_ext
|
.worker_ext
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map_or(false, |e| e.has_pending_work())
|
.is_some_and(|extension| extension.has_pending_work())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Run one synchronous worker pass. Returns `true` if any work was done.
|
/// Run one synchronous worker pass. Returns `true` if any work was done.
|
||||||
|
|
@ -928,7 +929,7 @@ impl ActorPool {
|
||||||
let snap_depth = slot.mailbox.len();
|
let snap_depth = slot.mailbox.len();
|
||||||
let mut snap_type_counts: Vec<(&'static str, u64)> =
|
let mut snap_type_counts: Vec<(&'static str, u64)> =
|
||||||
slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect();
|
slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect();
|
||||||
snap_type_counts.sort_by(|a, b| b.1.cmp(&a.1));
|
snap_type_counts.sort_by_key(|&(_, count)| Reverse(count));
|
||||||
|
|
||||||
let ctx = Ctx::new(
|
let ctx = Ctx::new(
|
||||||
wctx,
|
wctx,
|
||||||
|
|
@ -1132,7 +1133,7 @@ impl ActorPool {
|
||||||
if is_on_stop_eligible(slot.stopping, slot.poisoned) {
|
if is_on_stop_eligible(slot.stopping, slot.poisoned) {
|
||||||
let mut type_counts: Vec<(&'static str, u64)> =
|
let mut type_counts: Vec<(&'static str, u64)> =
|
||||||
slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect();
|
slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect();
|
||||||
type_counts.sort_by(|a, b| b.1.cmp(&a.1));
|
type_counts.sort_by_key(|&(_, count)| Reverse(count));
|
||||||
let ctx = Ctx::new(
|
let ctx = Ctx::new(
|
||||||
inner,
|
inner,
|
||||||
addr,
|
addr,
|
||||||
|
|
@ -1159,7 +1160,7 @@ impl ActorPool {
|
||||||
out.extend(self.actors.iter().map(|(&addr, slot)| {
|
out.extend(self.actors.iter().map(|(&addr, slot)| {
|
||||||
let mut type_counts: Vec<(&'static str, u64)> =
|
let mut type_counts: Vec<(&'static str, u64)> =
|
||||||
slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect();
|
slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect();
|
||||||
type_counts.sort_by(|a, b| b.1.cmp(&a.1));
|
type_counts.sort_by_key(|&(_, count)| Reverse(count));
|
||||||
let metadata = slot.actor.metadata();
|
let metadata = slot.actor.metadata();
|
||||||
ActorSnapshot {
|
ActorSnapshot {
|
||||||
address: addr,
|
address: addr,
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@
|
||||||
//! Covers: spawning, on_start, lifecycle decision paths, parent-child delegation,
|
//! Covers: spawning, on_start, lifecycle decision paths, parent-child delegation,
|
||||||
//! graceful stop, panic isolation, dead actor cleanup, and watching (ActorExited).
|
//! graceful stop, panic isolation, dead actor cleanup, and watching (ActorExited).
|
||||||
|
|
||||||
mod common;
|
pub mod common;
|
||||||
use common::*;
|
use common::*;
|
||||||
|
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,5 @@
|
||||||
// Shared types and helpers for runtime test files.
|
// Shared types and helpers for runtime test files.
|
||||||
|
|
||||||
#![allow(dead_code, unused_imports)]
|
|
||||||
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -64,10 +64,6 @@ impl RuntimeExtension for SeamExtension {
|
||||||
&self,
|
&self,
|
||||||
dead: &[(ActorAddress, StopReason, Option<ExitValue>)],
|
dead: &[(ActorAddress, StopReason, Option<ExitValue>)],
|
||||||
) -> Vec<(ActorAddress, Box<dyn Any + Send>)> {
|
) -> Vec<(ActorAddress, Box<dyn Any + Send>)> {
|
||||||
let _ = dead
|
|
||||||
.iter()
|
|
||||||
.map(|(_, reason, value)| (reason, value))
|
|
||||||
.count();
|
|
||||||
let Some(report_to) = *self.state.death_report_to.lock() else {
|
let Some(report_to) = *self.state.death_report_to.lock() else {
|
||||||
return Vec::new();
|
return Vec::new();
|
||||||
};
|
};
|
||||||
|
|
|
||||||
205
tests/lint_policy.rs
Normal file
205
tests/lint_policy.rs
Normal file
|
|
@ -0,0 +1,205 @@
|
||||||
|
use std::collections::BTreeSet;
|
||||||
|
use std::fs;
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
|
use syn::visit::Visit;
|
||||||
|
use syn::{Attribute, Meta};
|
||||||
|
|
||||||
|
const SOURCE_ROOTS: &[&str] = &["src", "crates", "apps", "xtask", "tools", "tests"];
|
||||||
|
const FORBIDDEN_ATTRIBUTES: &[&str] = &["allow", "expect"];
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct SuppressionVisitor {
|
||||||
|
found: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'ast> Visit<'ast> for SuppressionVisitor {
|
||||||
|
fn visit_attribute(&mut self, attribute: &'ast Attribute) {
|
||||||
|
if FORBIDDEN_ATTRIBUTES
|
||||||
|
.iter()
|
||||||
|
.any(|name| attribute.path().is_ident(name))
|
||||||
|
|| matches!(
|
||||||
|
&attribute.meta,
|
||||||
|
Meta::List(list)
|
||||||
|
if list.path.is_ident("cfg_attr") && tokens_contain_suppression(&list.tokens)
|
||||||
|
)
|
||||||
|
{
|
||||||
|
self.found = true;
|
||||||
|
}
|
||||||
|
syn::visit::visit_attribute(self, attribute);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn workspace_sources_do_not_suppress_lints() {
|
||||||
|
let workspace = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
|
||||||
|
let mut rust_sources = Vec::new();
|
||||||
|
for root in SOURCE_ROOTS {
|
||||||
|
collect_rust_sources(&workspace.join(root), &mut rust_sources);
|
||||||
|
}
|
||||||
|
rust_sources.sort();
|
||||||
|
|
||||||
|
let mut violations = Vec::new();
|
||||||
|
for source in rust_sources {
|
||||||
|
let text = fs::read_to_string(&source)
|
||||||
|
.unwrap_or_else(|error| panic!("read {}: {error}", source.display()));
|
||||||
|
let syntax = syn::parse_file(&text)
|
||||||
|
.unwrap_or_else(|error| panic!("parse {}: {error}", source.display()));
|
||||||
|
let mut visitor = SuppressionVisitor::default();
|
||||||
|
visitor.visit_file(&syntax);
|
||||||
|
if visitor.found {
|
||||||
|
violations.push(
|
||||||
|
source
|
||||||
|
.strip_prefix(&workspace)
|
||||||
|
.unwrap_or(&source)
|
||||||
|
.display()
|
||||||
|
.to_string(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
violations.is_empty(),
|
||||||
|
"lint suppression attributes are forbidden; fix the warning instead:\n{}",
|
||||||
|
violations.join("\n")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn suppression_visitor_detects_direct_and_conditional_attributes() {
|
||||||
|
for source in [
|
||||||
|
"#[allow(dead_code)] fn hidden() {}",
|
||||||
|
"#![expect(unused_imports)]",
|
||||||
|
"#[cfg_attr(test, allow(clippy::too_many_arguments))] fn hidden() {}",
|
||||||
|
"#![cfg_attr(feature = \"strict\", expect(dead_code))]",
|
||||||
|
] {
|
||||||
|
let syntax = syn::parse_file(source).expect("valid suppression probe");
|
||||||
|
let mut visitor = SuppressionVisitor::default();
|
||||||
|
visitor.visit_file(&syntax);
|
||||||
|
assert!(visitor.found, "suppression escaped detection: {source}");
|
||||||
|
}
|
||||||
|
|
||||||
|
let syntax = syn::parse_file("#[derive(Clone)] struct Clean;").unwrap();
|
||||||
|
let mut visitor = SuppressionVisitor::default();
|
||||||
|
visitor.visit_file(&syntax);
|
||||||
|
assert!(!visitor.found);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn source_tree_test_modules_are_wired() {
|
||||||
|
let workspace = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
|
||||||
|
let mut test_directories = Vec::new();
|
||||||
|
for root in SOURCE_ROOTS {
|
||||||
|
collect_source_test_directories(&workspace.join(root), &mut test_directories);
|
||||||
|
}
|
||||||
|
test_directories.sort();
|
||||||
|
|
||||||
|
let mut unwired = Vec::new();
|
||||||
|
for directory in test_directories {
|
||||||
|
let module_path = directory.join("mod.rs");
|
||||||
|
let module_source = fs::read_to_string(&module_path)
|
||||||
|
.unwrap_or_else(|error| panic!("read {}: {error}", module_path.display()));
|
||||||
|
let module = syn::parse_file(&module_source)
|
||||||
|
.unwrap_or_else(|error| panic!("parse {}: {error}", module_path.display()));
|
||||||
|
let declared = module
|
||||||
|
.items
|
||||||
|
.iter()
|
||||||
|
.filter_map(|item| match item {
|
||||||
|
syn::Item::Mod(item) => Some(item.ident.to_string()),
|
||||||
|
_ => None,
|
||||||
|
})
|
||||||
|
.collect::<BTreeSet<_>>();
|
||||||
|
|
||||||
|
for entry in fs::read_dir(&directory)
|
||||||
|
.unwrap_or_else(|error| panic!("read directory {}: {error}", directory.display()))
|
||||||
|
{
|
||||||
|
let path = entry.expect("read test-module entry").path();
|
||||||
|
if path.extension().is_some_and(|extension| extension == "rs")
|
||||||
|
&& path.file_stem().is_some_and(|stem| stem != "mod")
|
||||||
|
{
|
||||||
|
let module_name = path
|
||||||
|
.file_stem()
|
||||||
|
.expect("test module stem")
|
||||||
|
.to_string_lossy()
|
||||||
|
.to_string();
|
||||||
|
if !declared.contains(&module_name) {
|
||||||
|
unwired.push(
|
||||||
|
path.strip_prefix(&workspace)
|
||||||
|
.unwrap_or(&path)
|
||||||
|
.display()
|
||||||
|
.to_string(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
unwired.is_empty(),
|
||||||
|
"source-tree test modules must be declared by their adjacent mod.rs:\n{}",
|
||||||
|
unwired.join("\n")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect_source_test_directories(directory: &Path, directories: &mut Vec<PathBuf>) {
|
||||||
|
if !directory.exists() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if directory.file_name().is_some_and(|name| name == "tests")
|
||||||
|
&& directory
|
||||||
|
.parent()
|
||||||
|
.and_then(Path::file_name)
|
||||||
|
.is_some_and(|name| name == "src")
|
||||||
|
{
|
||||||
|
directories.push(directory.to_path_buf());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let entries = fs::read_dir(directory)
|
||||||
|
.unwrap_or_else(|error| panic!("read directory {}: {error}", directory.display()));
|
||||||
|
for entry in entries {
|
||||||
|
let path = entry.expect("read source-tree entry").path();
|
||||||
|
if path.is_dir() {
|
||||||
|
collect_source_test_directories(&path, directories);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect_rust_sources(directory: &Path, sources: &mut Vec<PathBuf>) {
|
||||||
|
if !directory.exists() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let entries = fs::read_dir(directory)
|
||||||
|
.unwrap_or_else(|error| panic!("read directory {}: {error}", directory.display()));
|
||||||
|
for entry in entries {
|
||||||
|
let entry = entry.unwrap_or_else(|error| panic!("read directory entry: {error}"));
|
||||||
|
let path = entry.path();
|
||||||
|
if path.is_dir() {
|
||||||
|
collect_rust_sources(&path, sources);
|
||||||
|
} else if path.extension().is_some_and(|extension| extension == "rs") {
|
||||||
|
sources.push(path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn tokens_contain_suppression(tokens: &proc_macro2::TokenStream) -> bool {
|
||||||
|
let mut token_trees = tokens.clone().into_iter().peekable();
|
||||||
|
while let Some(token) = token_trees.next() {
|
||||||
|
match token {
|
||||||
|
proc_macro2::TokenTree::Ident(identifier)
|
||||||
|
if FORBIDDEN_ATTRIBUTES.iter().any(|name| identifier == *name)
|
||||||
|
&& matches!(
|
||||||
|
token_trees.peek(),
|
||||||
|
Some(proc_macro2::TokenTree::Group(group))
|
||||||
|
if group.delimiter() == proc_macro2::Delimiter::Parenthesis
|
||||||
|
) =>
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
proc_macro2::TokenTree::Group(group) if tokens_contain_suppression(&group.stream()) => {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
@ -3,7 +3,7 @@
|
||||||
//! Covers: routing correctness at scale, send-from-within-handler patterns,
|
//! Covers: routing correctness at scale, send-from-within-handler patterns,
|
||||||
//! address error handling, and fairness/budgets.
|
//! address error handling, and fairness/budgets.
|
||||||
|
|
||||||
mod common;
|
pub mod common;
|
||||||
use common::*;
|
use common::*;
|
||||||
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@
|
||||||
//! `docs/specs/drafts/MULTICORE_SPEC.md`. They observe behavior through public
|
//! `docs/specs/drafts/MULTICORE_SPEC.md`. They observe behavior through public
|
||||||
//! APIs only — never inspecting source layout.
|
//! APIs only — never inspecting source layout.
|
||||||
|
|
||||||
mod common;
|
pub mod common;
|
||||||
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
|
@ -157,9 +157,10 @@ fn worker_of(stats: &swactor::stats::RuntimeStats, addr: ActorAddress) -> usize
|
||||||
}
|
}
|
||||||
|
|
||||||
fn config_with(workers: usize) -> RuntimeConfig {
|
fn config_with(workers: usize) -> RuntimeConfig {
|
||||||
let mut c = RuntimeConfig::default();
|
RuntimeConfig {
|
||||||
c.worker_count = workers;
|
worker_count: workers,
|
||||||
c
|
..RuntimeConfig::default()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─── Phase 1: single-thread host advances every worker once ─────────────────
|
// ─── Phase 1: single-thread host advances every worker once ─────────────────
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@
|
||||||
//! Uses proptest for randomized testing and proptest-state-machine for
|
//! Uses proptest for randomized testing and proptest-state-machine for
|
||||||
//! stateful property testing with automatic shrinking of failing sequences.
|
//! stateful property testing with automatic shrinking of failing sequences.
|
||||||
|
|
||||||
mod common;
|
pub mod common;
|
||||||
use common::plain_host;
|
use common::plain_host;
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
//! Runtime Admin API tests — inventory, typed actor state, lifecycle control, and scheduling.
|
//! Runtime Admin API tests — inventory, typed actor state, lifecycle control, and scheduling.
|
||||||
|
|
||||||
mod common;
|
pub mod common;
|
||||||
use common::*;
|
use common::*;
|
||||||
|
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@
|
||||||
//! Covers: high-volume delivery, panic isolation under load, and sustained
|
//! Covers: high-volume delivery, panic isolation under load, and sustained
|
||||||
//! throughput with no message loss. All tests are tick-driven (single worker).
|
//! throughput with no message loss. All tests are tick-driven (single worker).
|
||||||
|
|
||||||
mod common;
|
pub mod common;
|
||||||
use common::*;
|
use common::*;
|
||||||
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@
|
||||||
//! Tests only the std APIs used by production crates: runtime naming,
|
//! Tests only the std APIs used by production crates: runtime naming,
|
||||||
//! runtime groups, actor-side watch, actor-side group join, and extension install.
|
//! runtime groups, actor-side watch, actor-side group join, and extension install.
|
||||||
|
|
||||||
mod common;
|
pub mod common;
|
||||||
use common::*;
|
use common::*;
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
|
|
|
||||||
|
|
@ -7,3 +7,6 @@ publish = false
|
||||||
|
|
||||||
[lib]
|
[lib]
|
||||||
path = "src/lib.rs"
|
path = "src/lib.rs"
|
||||||
|
|
||||||
|
[lints]
|
||||||
|
workspace = true
|
||||||
|
|
|
||||||
7
tools/actor-control-flow-lint/tests/fixtures/fail-domain-capabilities/Cargo.lock
generated
vendored
7
tools/actor-control-flow-lint/tests/fixtures/fail-domain-capabilities/Cargo.lock
generated
vendored
|
|
@ -10,6 +10,12 @@ dependencies = [
|
||||||
"tokio",
|
"tokio",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "atomic-waker"
|
||||||
|
version = "1.1.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bitflags"
|
name = "bitflags"
|
||||||
version = "2.13.1"
|
version = "2.13.1"
|
||||||
|
|
@ -182,6 +188,7 @@ dependencies = [
|
||||||
name = "swactor"
|
name = "swactor"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"atomic-waker",
|
||||||
"crossbeam-queue",
|
"crossbeam-queue",
|
||||||
"crossbeam-utils",
|
"crossbeam-utils",
|
||||||
"getrandom",
|
"getrandom",
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,3 @@
|
||||||
#![allow(clippy::disallowed_methods, dead_code, unused_must_use)]
|
|
||||||
|
|
||||||
use std::thread::sleep as renamed_sleep;
|
use std::thread::sleep as renamed_sleep;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
|
||||||
7
tools/actor-control-flow-lint/tests/fixtures/pass-actor-domain/Cargo.lock
generated
vendored
7
tools/actor-control-flow-lint/tests/fixtures/pass-actor-domain/Cargo.lock
generated
vendored
|
|
@ -10,6 +10,12 @@ dependencies = [
|
||||||
"swactor-engine",
|
"swactor-engine",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "atomic-waker"
|
||||||
|
version = "1.1.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bitflags"
|
name = "bitflags"
|
||||||
version = "2.13.1"
|
version = "2.13.1"
|
||||||
|
|
@ -182,6 +188,7 @@ dependencies = [
|
||||||
name = "swactor"
|
name = "swactor"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"atomic-waker",
|
||||||
"crossbeam-queue",
|
"crossbeam-queue",
|
||||||
"crossbeam-utils",
|
"crossbeam-utils",
|
||||||
"getrandom",
|
"getrandom",
|
||||||
|
|
|
||||||
|
|
@ -18,3 +18,6 @@ urlencoding = "2"
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
wiremock = "0.6"
|
wiremock = "0.6"
|
||||||
|
|
||||||
|
[lints]
|
||||||
|
workspace = true
|
||||||
|
|
|
||||||
|
|
@ -48,50 +48,53 @@ fn env_optional_positive_f64(name: &str) -> Option<f64> {
|
||||||
impl SelectionPolicy {
|
impl SelectionPolicy {
|
||||||
/// Build selection policy from the historical `PP_*` environment knobs.
|
/// Build selection policy from the historical `PP_*` environment knobs.
|
||||||
pub fn from_env() -> Self {
|
pub fn from_env() -> Self {
|
||||||
let mut policy = Self::default();
|
let defaults = Self::default();
|
||||||
policy.min_gpu_ram_mb = env_positive_u64(ENV_GPU_MIN_RAM_MB);
|
let mut blacklist_hosts = defaults.blacklist_hosts;
|
||||||
if let Some(min_compute_cap) = env_positive_u64(ENV_MIN_COMPUTE_CAP) {
|
|
||||||
policy.min_compute_cap = Some(min_compute_cap);
|
|
||||||
}
|
|
||||||
policy.min_down_mbps = env_nonnegative_f64(ENV_MIN_INET_DOWN_MBPS, 100.0);
|
|
||||||
policy.min_reliability = std::env::var(ENV_MIN_RELIABILITY)
|
|
||||||
.ok()
|
|
||||||
.and_then(|s| s.trim().parse::<f64>().ok())
|
|
||||||
.filter(|&v| (0.0..=1.0).contains(&v))
|
|
||||||
.unwrap_or(0.95);
|
|
||||||
policy.require_verified = truthy_env(ENV_REQUIRE_VERIFIED);
|
|
||||||
policy.min_up_mbps = env_optional_positive_f64(ENV_MIN_INET_UP_MBPS);
|
|
||||||
policy.max_dph_total = env_optional_positive_f64(ENV_MAX_DPH_TOTAL);
|
|
||||||
policy.drop_cheap_frac = std::env::var(ENV_DROP_CHEAP_FRAC)
|
|
||||||
.ok()
|
|
||||||
.and_then(|s| s.trim().parse::<f64>().ok())
|
|
||||||
.filter(|v| v.is_finite())
|
|
||||||
.map(|v| v.clamp(0.0, 0.99))
|
|
||||||
.unwrap_or(0.30);
|
|
||||||
policy.image_size_gb = env_optional_positive_f64(ENV_IMAGE_SIZE_GB);
|
|
||||||
if let Ok(raw) = std::env::var(ENV_BLACKLIST_HOSTS) {
|
if let Ok(raw) = std::env::var(ENV_BLACKLIST_HOSTS) {
|
||||||
policy
|
blacklist_hosts.extend(
|
||||||
.blacklist_hosts
|
raw.split(',')
|
||||||
.extend(raw.split(',').filter_map(|s| s.trim().parse::<u64>().ok()));
|
.filter_map(|value| value.trim().parse::<u64>().ok()),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Self {
|
||||||
|
gpu_name: defaults.gpu_name,
|
||||||
|
min_gpu_ram_mb: env_positive_u64(ENV_GPU_MIN_RAM_MB),
|
||||||
|
min_compute_cap: env_positive_u64(ENV_MIN_COMPUTE_CAP).or(defaults.min_compute_cap),
|
||||||
|
min_down_mbps: env_nonnegative_f64(ENV_MIN_INET_DOWN_MBPS, 100.0),
|
||||||
|
min_reliability: std::env::var(ENV_MIN_RELIABILITY)
|
||||||
|
.ok()
|
||||||
|
.and_then(|value| value.trim().parse::<f64>().ok())
|
||||||
|
.filter(|&value| (0.0..=1.0).contains(&value))
|
||||||
|
.unwrap_or(0.95),
|
||||||
|
require_verified: truthy_env(ENV_REQUIRE_VERIFIED),
|
||||||
|
min_up_mbps: env_optional_positive_f64(ENV_MIN_INET_UP_MBPS),
|
||||||
|
max_dph_total: env_optional_positive_f64(ENV_MAX_DPH_TOTAL),
|
||||||
|
blacklist_hosts,
|
||||||
|
drop_cheap_frac: std::env::var(ENV_DROP_CHEAP_FRAC)
|
||||||
|
.ok()
|
||||||
|
.and_then(|value| value.trim().parse::<f64>().ok())
|
||||||
|
.filter(|value| value.is_finite())
|
||||||
|
.map(|value| value.clamp(0.0, 0.99))
|
||||||
|
.unwrap_or(0.30),
|
||||||
|
image_size_gb: env_optional_positive_f64(ENV_IMAGE_SIZE_GB),
|
||||||
}
|
}
|
||||||
policy
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl LifecyclePolicy {
|
impl LifecyclePolicy {
|
||||||
/// Build lifecycle policy from environment, using caller-provided poll cadence.
|
/// Build lifecycle policy from environment, using caller-provided poll cadence.
|
||||||
pub fn from_env(poll_interval: Duration) -> Self {
|
pub fn from_env(poll_interval: Duration) -> Self {
|
||||||
let mut policy = Self::default();
|
Self {
|
||||||
policy.lease_pace = Duration::from_millis(
|
lease_pace: Duration::from_millis(
|
||||||
std::env::var(ENV_LEASE_PACE_MS)
|
std::env::var(ENV_LEASE_PACE_MS)
|
||||||
.ok()
|
.ok()
|
||||||
.and_then(|s| s.trim().parse::<u64>().ok())
|
.and_then(|value| value.trim().parse::<u64>().ok())
|
||||||
.unwrap_or(600),
|
.unwrap_or(600),
|
||||||
);
|
),
|
||||||
policy.poll_interval = poll_interval;
|
poll_interval,
|
||||||
if let Some(state_timeout_secs) = env_positive_u64(ENV_STATE_TIMEOUT_SECS) {
|
state_timeout: env_positive_u64(ENV_STATE_TIMEOUT_SECS)
|
||||||
policy.state_timeout = Duration::from_secs(state_timeout_secs);
|
.map(Duration::from_secs)
|
||||||
|
.unwrap_or_else(|| Self::default().state_timeout),
|
||||||
}
|
}
|
||||||
policy
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -9,9 +9,9 @@ pub(crate) fn reachable_offers(offers: Vec<Offer>, policy: &SelectionPolicy) ->
|
||||||
.filter(|o| {
|
.filter(|o| {
|
||||||
o.geolocation
|
o.geolocation
|
||||||
.as_deref()
|
.as_deref()
|
||||||
.map_or(false, |g| !g.to_uppercase().contains("CN"))
|
.is_some_and(|geolocation| !geolocation.to_uppercase().contains("CN"))
|
||||||
})
|
})
|
||||||
.filter(|o| o.host_id.map_or(true, |h| !blacklist.contains(&h)))
|
.filter(|offer| offer.host_id.is_none_or(|host| !blacklist.contains(&host)))
|
||||||
.filter(|o| o.verification.as_deref() != Some("deverified"))
|
.filter(|o| o.verification.as_deref() != Some("deverified"))
|
||||||
.filter(|o| policy.max_dph_total.is_none_or(|max| o.dph_total <= max))
|
.filter(|o| policy.max_dph_total.is_none_or(|max| o.dph_total <= max))
|
||||||
.filter(|o| {
|
.filter(|o| {
|
||||||
|
|
|
||||||
|
|
@ -83,14 +83,14 @@ fn next_eligible_offer<'a>(
|
||||||
failed_host_ids: &HashSet<u64>,
|
failed_host_ids: &HashSet<u64>,
|
||||||
preferred_offer_id: Option<u64>,
|
preferred_offer_id: Option<u64>,
|
||||||
) -> Option<&'a Offer> {
|
) -> Option<&'a Offer> {
|
||||||
if let Some(offer_id) = preferred_offer_id {
|
if let Some(offer_id) = preferred_offer_id
|
||||||
if let Some(offer) = pool.iter().find(|o| o.id == offer_id) {
|
&& let Some(offer) = pool.iter().find(|offer| offer.id == offer_id)
|
||||||
if !tried_offer_ids.contains(&offer.id)
|
&& !tried_offer_ids.contains(&offer.id)
|
||||||
&& offer.host_id.is_none_or(|h| !failed_host_ids.contains(&h))
|
&& offer
|
||||||
{
|
.host_id
|
||||||
return Some(offer);
|
.is_none_or(|host| !failed_host_ids.contains(&host))
|
||||||
}
|
{
|
||||||
}
|
return Some(offer);
|
||||||
}
|
}
|
||||||
|
|
||||||
pool.iter().find(|o| {
|
pool.iter().find(|o| {
|
||||||
|
|
@ -108,18 +108,32 @@ fn env_for_index(req: &ProvisionRequest, index: u32) -> BTreeMap<String, String>
|
||||||
env
|
env
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn provision_one(
|
struct ProvisionOneRequest<'a> {
|
||||||
client: &reqwest::Client,
|
client: &'a reqwest::Client,
|
||||||
base_url: &str,
|
base_url: &'a str,
|
||||||
api_key: &str,
|
api_key: &'a str,
|
||||||
req: &ProvisionRequest,
|
request: &'a ProvisionRequest,
|
||||||
pool: &[Offer],
|
pool: &'a [Offer],
|
||||||
index: u32,
|
index: u32,
|
||||||
tried_offer_ids: &mut Vec<u64>,
|
tried_offer_ids: &'a mut Vec<u64>,
|
||||||
used_host_ids: &mut HashSet<u64>,
|
used_host_ids: &'a mut HashSet<u64>,
|
||||||
failed_host_ids: &mut HashSet<u64>,
|
failed_host_ids: &'a mut HashSet<u64>,
|
||||||
preferred_offer_id: Option<u64>,
|
preferred_offer_id: Option<u64>,
|
||||||
) -> Result<ProvisionedInstance, String> {
|
}
|
||||||
|
|
||||||
|
async fn provision_one(request: ProvisionOneRequest<'_>) -> Result<ProvisionedInstance, String> {
|
||||||
|
let ProvisionOneRequest {
|
||||||
|
client,
|
||||||
|
base_url,
|
||||||
|
api_key,
|
||||||
|
request: req,
|
||||||
|
pool,
|
||||||
|
index,
|
||||||
|
tried_offer_ids,
|
||||||
|
used_host_ids,
|
||||||
|
failed_host_ids,
|
||||||
|
preferred_offer_id,
|
||||||
|
} = request;
|
||||||
let mut attempt = 1_u64;
|
let mut attempt = 1_u64;
|
||||||
loop {
|
loop {
|
||||||
let offer = match next_eligible_offer(
|
let offer = match next_eligible_offer(
|
||||||
|
|
@ -230,20 +244,21 @@ pub async fn provision_fleet(
|
||||||
let mut used_host_ids = HashSet::new();
|
let mut used_host_ids = HashSet::new();
|
||||||
let mut failed_host_ids = HashSet::new();
|
let mut failed_host_ids = HashSet::new();
|
||||||
for index in 0..req.count {
|
for index in 0..req.count {
|
||||||
match provision_one(
|
match provision_one(ProvisionOneRequest {
|
||||||
client,
|
client,
|
||||||
base_url,
|
base_url,
|
||||||
api_key,
|
api_key,
|
||||||
&req,
|
request: &req,
|
||||||
&pool,
|
pool: &pool,
|
||||||
index,
|
index,
|
||||||
&mut tried_offer_ids,
|
tried_offer_ids: &mut tried_offer_ids,
|
||||||
&mut used_host_ids,
|
used_host_ids: &mut used_host_ids,
|
||||||
&mut failed_host_ids,
|
failed_host_ids: &mut failed_host_ids,
|
||||||
req.preferred_offer_id
|
preferred_offer_id: req
|
||||||
|
.preferred_offer_id
|
||||||
.filter(|_| req.count == 1)
|
.filter(|_| req.count == 1)
|
||||||
.or_else(|| first_wave.get(index as usize).map(|offer| offer.id)),
|
.or_else(|| first_wave.get(index as usize).map(|offer| offer.id)),
|
||||||
)
|
})
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(info) => created.push(info),
|
Ok(info) => created.push(info),
|
||||||
|
|
@ -279,18 +294,18 @@ pub async fn provision_fleet(
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
eprintln!("lease_chain: replacing index {index}");
|
eprintln!("lease_chain: replacing index {index}");
|
||||||
match provision_one(
|
match provision_one(ProvisionOneRequest {
|
||||||
client,
|
client,
|
||||||
base_url,
|
base_url,
|
||||||
api_key,
|
api_key,
|
||||||
&req,
|
request: &req,
|
||||||
&pool,
|
pool: &pool,
|
||||||
index,
|
index,
|
||||||
&mut tried_offer_ids,
|
tried_offer_ids: &mut tried_offer_ids,
|
||||||
&mut used_host_ids,
|
used_host_ids: &mut used_host_ids,
|
||||||
&mut failed_host_ids,
|
failed_host_ids: &mut failed_host_ids,
|
||||||
None,
|
preferred_offer_id: None,
|
||||||
)
|
})
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(info) => created[idx] = info,
|
Ok(info) => created[idx] = info,
|
||||||
|
|
|
||||||
|
|
@ -151,10 +151,10 @@ pub fn plan_distinct_host_first_wave(
|
||||||
if selected.len() == target {
|
if selected.len() == target {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
if let Some(host_id) = offer.host_id {
|
if let Some(host_id) = offer.host_id
|
||||||
if !selected_hosts.insert(host_id) {
|
&& !selected_hosts.insert(host_id)
|
||||||
continue;
|
{
|
||||||
}
|
continue;
|
||||||
}
|
}
|
||||||
selected_ids.insert(offer.id);
|
selected_ids.insert(offer.id);
|
||||||
selected.push(offer.clone());
|
selected.push(offer.clone());
|
||||||
|
|
|
||||||
|
|
@ -25,3 +25,6 @@ libc = "0.2"
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
proptest = "1"
|
proptest = "1"
|
||||||
|
|
||||||
|
[lints]
|
||||||
|
workspace = true
|
||||||
|
|
|
||||||
|
|
@ -107,9 +107,10 @@ mod properties {
|
||||||
codes in prop::collection::vec(any::<u8>(), 0..=32),
|
codes in prop::collection::vec(any::<u8>(), 0..=32),
|
||||||
registration in 0_usize..=32,
|
registration in 0_usize..=32,
|
||||||
) {
|
) {
|
||||||
let mut config = RuntimeConfig::default();
|
let parts = RuntimeParts::new(RuntimeConfig {
|
||||||
config.worker_count = 1;
|
worker_count: 1,
|
||||||
let parts = RuntimeParts::new(config);
|
..RuntimeConfig::default()
|
||||||
|
});
|
||||||
let runtime = parts.runtime().clone();
|
let runtime = parts.runtime().clone();
|
||||||
let backend = SteppingBackend::new();
|
let backend = SteppingBackend::new();
|
||||||
let _engine =
|
let _engine =
|
||||||
|
|
|
||||||
|
|
@ -503,7 +503,6 @@ pub struct NodeEdgeAgent {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl NodeEdgeAgent {
|
impl NodeEdgeAgent {
|
||||||
#[allow(clippy::too_many_arguments)]
|
|
||||||
pub fn new(
|
pub fn new(
|
||||||
attempt: u64,
|
attempt: u64,
|
||||||
logical_node: String,
|
logical_node: String,
|
||||||
|
|
@ -756,7 +755,7 @@ impl swactor::actor::ActorInterface for EdgeAckRelay {
|
||||||
if let Some(addr) = self.supervisor.get() {
|
if let Some(addr) = self.supervisor.get() {
|
||||||
let _ = self
|
let _ = self
|
||||||
.sender
|
.sender
|
||||||
.send_to(addr.clone(), crate::demo::feed::SupervisorMsg::EdgeAck(ack));
|
.send_to(*addr, crate::demo::feed::SupervisorMsg::EdgeAck(ack));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -215,25 +215,43 @@ pub struct SupervisorActor {
|
||||||
shutdown: Option<ShutdownState>,
|
shutdown: Option<ShutdownState>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub struct SupervisorConfig {
|
||||||
|
pub driver: ClusterDriver,
|
||||||
|
pub executor: IdempotentEffectExecutor<DemoBackend, EngineSpawner>,
|
||||||
|
pub manager: NodeManager,
|
||||||
|
pub driver_handle: std::sync::Arc<crate::demo::DemoDriverHandle>,
|
||||||
|
pub telemetry: SupervisorTelemetry,
|
||||||
|
pub dashboard: dashboard::DashboardHandle,
|
||||||
|
pub sender: swactor::runtime::ExternalSender,
|
||||||
|
pub registry: provisioning::BootstrapRegistry,
|
||||||
|
pub collector: std::sync::Arc<dyn provisioning::NodeTelemetryCollector>,
|
||||||
|
pub engine: EngineHandle,
|
||||||
|
pub remote_sub: telemetry::TelemetrySubscription,
|
||||||
|
pub initial_slots: Vec<String>,
|
||||||
|
pub run_id: RunId,
|
||||||
|
pub launch: crate::demo::LaunchStyle,
|
||||||
|
pub edge_actor: ActorAddress,
|
||||||
|
}
|
||||||
|
|
||||||
impl SupervisorActor {
|
impl SupervisorActor {
|
||||||
#[allow(clippy::too_many_arguments)]
|
pub fn new(config: SupervisorConfig) -> Self {
|
||||||
pub fn new(
|
let SupervisorConfig {
|
||||||
driver: ClusterDriver,
|
driver,
|
||||||
executor: IdempotentEffectExecutor<DemoBackend, EngineSpawner>,
|
executor,
|
||||||
manager: NodeManager,
|
manager,
|
||||||
driver_handle: std::sync::Arc<crate::demo::DemoDriverHandle>,
|
driver_handle,
|
||||||
mut telemetry: SupervisorTelemetry,
|
mut telemetry,
|
||||||
dashboard: dashboard::DashboardHandle,
|
dashboard,
|
||||||
sender: swactor::runtime::ExternalSender,
|
sender,
|
||||||
registry: provisioning::BootstrapRegistry,
|
registry,
|
||||||
collector: std::sync::Arc<dyn provisioning::NodeTelemetryCollector>,
|
collector,
|
||||||
engine: EngineHandle,
|
engine,
|
||||||
remote_sub: telemetry::TelemetrySubscription,
|
remote_sub,
|
||||||
initial_slots: Vec<String>,
|
initial_slots,
|
||||||
run_id: RunId,
|
run_id,
|
||||||
launch: crate::demo::LaunchStyle,
|
launch,
|
||||||
edge_actor: ActorAddress,
|
edge_actor,
|
||||||
) -> Self {
|
} = config;
|
||||||
let events_channel = telemetry.register("prov.reconciler.events");
|
let events_channel = telemetry.register("prov.reconciler.events");
|
||||||
let snapshot_channel = telemetry.register("prov.reconciler.snapshot");
|
let snapshot_channel = telemetry.register("prov.reconciler.snapshot");
|
||||||
Self {
|
Self {
|
||||||
|
|
@ -814,7 +832,7 @@ impl SupervisorActor {
|
||||||
/// cards and the control view stay live between lifecycle transitions.
|
/// cards and the control view stay live between lifecycle transitions.
|
||||||
fn emit_node_status(&mut self, now: SystemTime) {
|
fn emit_node_status(&mut self, now: SystemTime) {
|
||||||
self.status_tick = self.status_tick.wrapping_add(1);
|
self.status_tick = self.status_tick.wrapping_add(1);
|
||||||
if self.status_tick % 4 != 0 {
|
if !self.status_tick.is_multiple_of(4) {
|
||||||
return; // 250ms ticks → heartbeat every second
|
return; // 250ms ticks → heartbeat every second
|
||||||
}
|
}
|
||||||
let attempts: Vec<u64> = self.nodes.keys().copied().collect();
|
let attempts: Vec<u64> = self.nodes.keys().copied().collect();
|
||||||
|
|
@ -968,7 +986,6 @@ impl SupervisorActor {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
|
||||||
fn publish_frame(
|
fn publish_frame(
|
||||||
dashboard: &dashboard::DashboardHandle,
|
dashboard: &dashboard::DashboardHandle,
|
||||||
stream: &telemetry::frame::StreamId,
|
stream: &telemetry::frame::StreamId,
|
||||||
|
|
@ -1489,7 +1506,7 @@ mod properties {
|
||||||
|
|
||||||
fn control_command(kind: u8, attempt: u64, index: usize) -> dashboard::control::ControlCommand {
|
fn control_command(kind: u8, attempt: u64, index: usize) -> dashboard::control::ControlCommand {
|
||||||
let command_id = format!("generated-command-{index}");
|
let command_id = format!("generated-command-{index}");
|
||||||
if kind % 2 == 0 {
|
if kind.is_multiple_of(2) {
|
||||||
dashboard::control::ControlCommand::Kill {
|
dashboard::control::ControlCommand::Kill {
|
||||||
command_id,
|
command_id,
|
||||||
node: format!("node-{attempt}"),
|
node: format!("node-{attempt}"),
|
||||||
|
|
@ -1568,9 +1585,10 @@ mod properties {
|
||||||
fn generated_supervisor_transitions_are_once_only_nonblocking_and_clean(
|
fn generated_supervisor_transitions_are_once_only_nonblocking_and_clean(
|
||||||
actions in supervisor_actions(),
|
actions in supervisor_actions(),
|
||||||
) {
|
) {
|
||||||
let mut config = RuntimeConfig::default();
|
let parts = RuntimeParts::new(RuntimeConfig {
|
||||||
config.worker_count = 1;
|
worker_count: 1,
|
||||||
let parts = RuntimeParts::new(config);
|
..RuntimeConfig::default()
|
||||||
|
});
|
||||||
let runtime = parts.runtime().clone();
|
let runtime = parts.runtime().clone();
|
||||||
let sender = runtime.create_sender();
|
let sender = runtime.create_sender();
|
||||||
let backend = SteppingBackend::new();
|
let backend = SteppingBackend::new();
|
||||||
|
|
@ -1644,25 +1662,25 @@ mod properties {
|
||||||
.spawn(AnnounceActor::new(manager.clone(), sender.clone()))
|
.spawn(AnnounceActor::new(manager.clone(), sender.clone()))
|
||||||
.expect("spawn generated announce actor");
|
.expect("spawn generated announce actor");
|
||||||
let supervisor = runtime
|
let supervisor = runtime
|
||||||
.spawn(SupervisorActor::new(
|
.spawn(SupervisorActor::new(SupervisorConfig {
|
||||||
driver,
|
driver,
|
||||||
executor,
|
executor,
|
||||||
manager.clone(),
|
manager: manager.clone(),
|
||||||
driver_handle,
|
driver_handle,
|
||||||
telemetry,
|
telemetry,
|
||||||
dashboard,
|
dashboard,
|
||||||
sender.clone(),
|
sender: sender.clone(),
|
||||||
registry,
|
registry,
|
||||||
Arc::new(evidence.clone()),
|
collector: Arc::new(evidence.clone()),
|
||||||
engine.handle(),
|
engine: engine.handle(),
|
||||||
remote_sub,
|
remote_sub,
|
||||||
Vec::new(),
|
initial_slots: Vec::new(),
|
||||||
RunId(1),
|
run_id: RunId(1),
|
||||||
crate::demo::LaunchStyle::Process {
|
launch: crate::demo::LaunchStyle::Process {
|
||||||
exe: "generated-demo-node".into(),
|
exe: "generated-demo-node".into(),
|
||||||
},
|
},
|
||||||
*edge_inbox.addr(),
|
edge_actor: *edge_inbox.addr(),
|
||||||
))
|
}))
|
||||||
.expect("spawn generated supervisor actor");
|
.expect("spawn generated supervisor actor");
|
||||||
drive(&backend, 8);
|
drive(&backend, 8);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -50,8 +50,8 @@ use provisioning::reconciler::ClusterDriver;
|
||||||
use provisioning::{ClusterShape, RunId};
|
use provisioning::{ClusterShape, RunId};
|
||||||
|
|
||||||
use feed::{
|
use feed::{
|
||||||
EngineSpawner, SupervisorActor, SupervisorMsg, SupervisorTelemetry, demo_retry_policy,
|
EngineSpawner, SupervisorActor, SupervisorConfig, SupervisorMsg, SupervisorTelemetry,
|
||||||
initial_slots,
|
demo_retry_policy, initial_slots,
|
||||||
};
|
};
|
||||||
use provider::{DemoBackend, DemoProvider, NodeManager};
|
use provider::{DemoBackend, DemoProvider, NodeManager};
|
||||||
|
|
||||||
|
|
@ -68,9 +68,10 @@ pub const DEFAULT_NODES: u64 = 3;
|
||||||
pub(super) fn shared_test_driver() -> Arc<IrohDriver> {
|
pub(super) fn shared_test_driver() -> Arc<IrohDriver> {
|
||||||
thread_local! {
|
thread_local! {
|
||||||
static DRIVER: (Arc<IrohDriver>, Engine) = {
|
static DRIVER: (Arc<IrohDriver>, Engine) = {
|
||||||
let mut config = RuntimeConfig::default();
|
let parts = RuntimeParts::new(RuntimeConfig {
|
||||||
config.worker_count = 1;
|
worker_count: 1,
|
||||||
let parts = RuntimeParts::new(config);
|
..RuntimeConfig::default()
|
||||||
|
});
|
||||||
let backend = TokioBackend::new(TokioConfig::default())
|
let backend = TokioBackend::new(TokioConfig::default())
|
||||||
.expect("create shared demo test backend");
|
.expect("create shared demo test backend");
|
||||||
let engine =
|
let engine =
|
||||||
|
|
@ -99,10 +100,10 @@ pub(super) fn shared_test_driver() -> Arc<IrohDriver> {
|
||||||
/// replaced by a rebuild while this process runs, so fall back through
|
/// replaced by a rebuild while this process runs, so fall back through
|
||||||
/// argv[0] and PATH.
|
/// argv[0] and PATH.
|
||||||
fn resolve_exe() -> std::path::PathBuf {
|
fn resolve_exe() -> std::path::PathBuf {
|
||||||
if let Ok(path) = std::env::current_exe() {
|
if let Ok(path) = std::env::current_exe()
|
||||||
if !path.to_string_lossy().ends_with(" (deleted)") {
|
&& !path.to_string_lossy().ends_with(" (deleted)")
|
||||||
return path;
|
{
|
||||||
}
|
return path;
|
||||||
}
|
}
|
||||||
if let Some(arg0) = std::env::args_os().next() {
|
if let Some(arg0) = std::env::args_os().next() {
|
||||||
let candidate = std::path::PathBuf::from(&arg0);
|
let candidate = std::path::PathBuf::from(&arg0);
|
||||||
|
|
@ -189,12 +190,14 @@ impl provisioning::NodeTelemetryCollector for DemoTelemetryCollector {
|
||||||
};
|
};
|
||||||
iroh_driver::spawn_pull_collector_to_actor(
|
iroh_driver::spawn_pull_collector_to_actor(
|
||||||
&self.engine,
|
&self.engine,
|
||||||
self.endpoint.clone(),
|
iroh_driver::PullCollectorConfig {
|
||||||
addr,
|
endpoint: self.endpoint.clone(),
|
||||||
flow_id,
|
peer: addr,
|
||||||
Vec::new(),
|
flow_id,
|
||||||
telemetry::SubscriptionRequest::all(),
|
token: Vec::new(),
|
||||||
Arc::clone(&self.fanout),
|
request: telemetry::SubscriptionRequest::all(),
|
||||||
|
fanout: Arc::clone(&self.fanout),
|
||||||
|
},
|
||||||
self.sender.clone(),
|
self.sender.clone(),
|
||||||
header_actor,
|
header_actor,
|
||||||
);
|
);
|
||||||
|
|
@ -347,15 +350,15 @@ fn run_supervisor(args: &[String]) -> Result<(), String> {
|
||||||
let route_view: RouteView =
|
let route_view: RouteView =
|
||||||
Arc::new(std::sync::RwLock::new(std::collections::HashMap::new()));
|
Arc::new(std::sync::RwLock::new(std::collections::HashMap::new()));
|
||||||
let outbox: Outbox = Arc::new(std::sync::Mutex::new(Vec::new()));
|
let outbox: Outbox = Arc::new(std::sync::Mutex::new(Vec::new()));
|
||||||
driver.enable_actor_bridge(
|
driver.enable_actor_bridge(iroh_driver::ActorBridgeConfig {
|
||||||
runtime.clone(),
|
runtime: runtime.clone(),
|
||||||
Arc::new(codec),
|
codec: Arc::new(codec),
|
||||||
routes,
|
routes,
|
||||||
announce,
|
swim: announce,
|
||||||
relay_mirror,
|
relay_mirror,
|
||||||
route_view,
|
route_view,
|
||||||
outbox,
|
outbox,
|
||||||
);
|
});
|
||||||
driver.install_actor_bridge_pump(Duration::from_millis(250));
|
driver.install_actor_bridge_pump(Duration::from_millis(250));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -458,23 +461,23 @@ fn run_supervisor(args: &[String]) -> Result<(), String> {
|
||||||
sender.clone(),
|
sender.clone(),
|
||||||
supervisor_slot.clone(),
|
supervisor_slot.clone(),
|
||||||
)?;
|
)?;
|
||||||
let supervisor = SupervisorActor::new(
|
let supervisor = SupervisorActor::new(SupervisorConfig {
|
||||||
cluster_driver,
|
driver: cluster_driver,
|
||||||
executor,
|
executor,
|
||||||
manager.clone(),
|
manager: manager.clone(),
|
||||||
driver_handle,
|
driver_handle,
|
||||||
telemetry,
|
telemetry,
|
||||||
dashboard.clone(),
|
dashboard: dashboard.clone(),
|
||||||
sender.clone(),
|
sender: sender.clone(),
|
||||||
bootstrap_registry,
|
registry: bootstrap_registry,
|
||||||
collector,
|
collector,
|
||||||
engine.handle(),
|
engine: engine.handle(),
|
||||||
remote_sub,
|
remote_sub,
|
||||||
slots,
|
initial_slots: slots,
|
||||||
RunId(1),
|
run_id: RunId(1),
|
||||||
launch.clone(),
|
launch: launch.clone(),
|
||||||
edge_actor,
|
edge_actor,
|
||||||
);
|
});
|
||||||
|
|
||||||
// Control plane: dashboard → supervisor.
|
// Control plane: dashboard → supervisor.
|
||||||
control::install(&runtime, supervisor_slot.clone());
|
control::install(&runtime, supervisor_slot.clone());
|
||||||
|
|
@ -483,7 +486,7 @@ fn run_supervisor(args: &[String]) -> Result<(), String> {
|
||||||
.spawn(supervisor)
|
.spawn(supervisor)
|
||||||
.map_err(|e| format!("spawn supervisor actor: {e}"))?;
|
.map_err(|e| format!("spawn supervisor actor: {e}"))?;
|
||||||
supervisor_slot
|
supervisor_slot
|
||||||
.set(supervisor_addr.clone())
|
.set(supervisor_addr)
|
||||||
.expect("supervisor address slot set once");
|
.expect("supervisor address slot set once");
|
||||||
manager.set_spawn_actor(sender.clone(), supervisor_slot.clone());
|
manager.set_spawn_actor(sender.clone(), supervisor_slot.clone());
|
||||||
|
|
||||||
|
|
@ -650,9 +653,10 @@ mod properties {
|
||||||
let pid = child.id() as i32;
|
let pid = child.id() as i32;
|
||||||
let stdout = child.stdout.take().expect("capture demo supervisor stdout");
|
let stdout = child.stdout.take().expect("capture demo supervisor stdout");
|
||||||
|
|
||||||
let mut config = RuntimeConfig::default();
|
let parts = RuntimeParts::new(RuntimeConfig {
|
||||||
config.worker_count = 1;
|
worker_count: 1,
|
||||||
let parts = RuntimeParts::new(config);
|
..RuntimeConfig::default()
|
||||||
|
});
|
||||||
let runtime = parts.runtime().clone();
|
let runtime = parts.runtime().clone();
|
||||||
let engine = Engine::new(
|
let engine = Engine::new(
|
||||||
parts,
|
parts,
|
||||||
|
|
|
||||||
|
|
@ -179,15 +179,15 @@ pub fn run_node_role(supervisor_addr_json: &str, attempt: u64) -> Result<(), Str
|
||||||
let route_view: RouteView =
|
let route_view: RouteView =
|
||||||
Arc::new(std::sync::RwLock::new(std::collections::HashMap::new()));
|
Arc::new(std::sync::RwLock::new(std::collections::HashMap::new()));
|
||||||
let outbox: Outbox = Arc::new(std::sync::Mutex::new(Vec::new()));
|
let outbox: Outbox = Arc::new(std::sync::Mutex::new(Vec::new()));
|
||||||
driver.enable_actor_bridge(
|
driver.enable_actor_bridge(iroh_driver::ActorBridgeConfig {
|
||||||
runtime.clone(),
|
runtime: runtime.clone(),
|
||||||
Arc::new(codec),
|
codec: Arc::new(codec),
|
||||||
routes,
|
routes,
|
||||||
edge_agent,
|
swim: edge_agent,
|
||||||
relay_mirror,
|
relay_mirror,
|
||||||
route_view,
|
route_view,
|
||||||
outbox,
|
outbox,
|
||||||
);
|
});
|
||||||
driver.install_actor_bridge_pump(Duration::from_millis(250));
|
driver.install_actor_bridge_pump(Duration::from_millis(250));
|
||||||
}
|
}
|
||||||
let driver = Arc::new(driver);
|
let driver = Arc::new(driver);
|
||||||
|
|
@ -199,7 +199,7 @@ pub fn run_node_role(supervisor_addr_json: &str, attempt: u64) -> Result<(), Str
|
||||||
|
|
||||||
// Join, then announce identity + advertised address to the supervisor's
|
// Join, then announce identity + advertised address to the supervisor's
|
||||||
// bootstrap actor over the control plane (readiness + telemetry dial).
|
// bootstrap actor over the control plane (readiness + telemetry dial).
|
||||||
driver.join(&[supervisor_addr.clone()]);
|
driver.join(std::slice::from_ref(&supervisor_addr));
|
||||||
let addr_json =
|
let addr_json =
|
||||||
serde_json::to_string(&driver.endpoint_addr()).map_err(|e| format!("addr: {e}"))?;
|
serde_json::to_string(&driver.endpoint_addr()).map_err(|e| format!("addr: {e}"))?;
|
||||||
|
|
||||||
|
|
@ -546,9 +546,10 @@ mod properties {
|
||||||
actions in node_actions(),
|
actions in node_actions(),
|
||||||
attempt in any::<u64>(),
|
attempt in any::<u64>(),
|
||||||
) {
|
) {
|
||||||
let mut config = RuntimeConfig::default();
|
let parts = RuntimeParts::new(RuntimeConfig {
|
||||||
config.worker_count = 1;
|
worker_count: 1,
|
||||||
let parts = RuntimeParts::new(config);
|
..RuntimeConfig::default()
|
||||||
|
});
|
||||||
let runtime = parts.runtime().clone();
|
let runtime = parts.runtime().clone();
|
||||||
let backend = SteppingBackend::new();
|
let backend = SteppingBackend::new();
|
||||||
let engine =
|
let engine =
|
||||||
|
|
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue