Compare commits

...

3 commits

Author SHA1 Message Date
0411c20b94 feat(data-plane): add routed SPSC stream transport
Implement namespace-owned stream rendezvous with incarnation isolation, cancellation, recovery, bounded local byte rings, and host/child actor protocols.

Add a replaceable stream transport boundary plus an Iroh ALPN adapter with framed records, backpressure, reconnect handling, and terminal propagation.

Wire stream sources and sinks through Myelin job deployment and Python bindings, with guarantee coverage across blob, namespace, job, byte-ring, and transport paths.

Sanitize nested Cargo build context in xtask so lint, Nextest, doctest, and maturin invocations retain stable fingerprints.
2026-08-23 15:51:24 +04:00
df72b4ff60 chore(test): enforce complete local test barrier
Deny workspace warnings and lint suppressions, consolidate Rust and Python coverage under cargo xtask test with 60-second per-test limits, and remove stale flaky, stateful, Docker, and orphaned test artifacts.
2026-08-23 15:51:24 +04:00
4854d85906 fix(dashboard): improve control and hardware views
Refine fleet and provisioning interactions, expand control-plane projections, and add CPU, memory, pressure, and storage telemetry for the local dashboard surfaces.
2026-08-23 15:51:24 +04:00
102 changed files with 7176 additions and 3462 deletions

View file

@ -3,4 +3,4 @@ rustc-workspace-wrapper = "tools/actor-control-flow-lint/rustc-wrapper.py"
[alias]
xtask = "run --package xtask --"
myelin-chat = "run --package xtask -- myelin-chat"
lint = "clippy --workspace --all-targets --all-features"

2
.config/nextest.toml Normal file
View file

@ -0,0 +1,2 @@
[profile.default]
slow-timeout = { period = "60s", terminate-after = 1 }

View file

@ -1,119 +0,0 @@
name: Myelin properties
on:
pull_request:
schedule:
- cron: "17 3 * * *"
workflow_dispatch:
permissions:
contents: read
jobs:
component-properties:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install pinned Rust toolchain
run: rustup toolchain install nightly-2026-02-07 --profile minimal --component rustfmt --component rustc-dev --component rust-src --component llvm-tools-preview
- name: Check formatting
run: cargo fmt --all -- --check
- name: Check default workspace members
run: cargo check
- name: Check compiler-policy contracts
run: cargo test -p actor-control-flow-lint-tests --test contracts compiler_policy_contracts -- --exact
- name: Engine scheduling properties
run: |
cargo test -p swactor-engine --test engine_unit generated_actor_timers_and_completion_are_bounded -- --exact
cargo test -p swactor-engine --test engine_unit lifecycle_invariant_detects_injected_duplicate_completion -- --exact
cargo test -p swactor-engine --test engine_unit lifecycle_invariant_detects_injected_uncancelled_periodic_timer -- --exact
- name: Process adapter properties
run: |
cargo test -p swactor-process --lib operations::properties::generated_stream_observations_close_once_and_stay_closed -- --exact
cargo test -p swactor-process --lib operations::properties::generated_lifecycle_actions_make_stop_idempotent_and_exit_terminal -- --exact
cargo test -p swactor-process --lib operations::properties::generated_stop_notifications_are_delivered_at_most_once -- --exact
cargo test -p swactor-process --lib operations::properties::generated_stdin_commands_and_eof_notify_once -- --exact
cargo test -p swactor-process --lib operations::properties::property_invariants_reject_controlled_defects -- --exact
cargo test -p swactor-process --lib operations::properties::trivial_real_child_exit_has_a_hard_timeout -- --exact
- name: Myelin component properties (exclude job and reconciler tests)
run: |
cargo test -p myelin --lib orchestration::manual_control::tests::aggressive_random_event_stream_preserves_control_invariants -- --exact
cargo test -p myelin --lib orchestration::manual_control::tests::rental_free_end_to_end_sequences_converge -- --exact
cargo test -p myelin --lib orchestration::manual_control::tests::manual_actor_generated_public_actions_and_callbacks_are_bounded -- --exact
cargo test -p myelin --lib orchestration::manual_control::tests::fixed_helper_cardinality_invariant_detects_controlled_extra_spawn -- --exact
cargo test -p myelin --lib orchestration::manual_control::tests::callback_panic_reports_typed_failure_without_poisoning_work_actor -- --exact
cargo test -p myelin --lib orchestration::manual_control::tests::callback_panic_invariant_detects_controlled_unguarded_panic -- --exact
cargo test -p myelin --lib provisioning::tests::mock_vastai_handle_state_survives_random_create_and_stop_sequences -- --exact
cargo test -p myelin --lib provisioning::tests::docker_generated_attempt_lifecycles_are_idempotent_and_bounded -- --exact
cargo test -p myelin --lib provisioning::tests::docker_duplicate_resource_detector_rejects_controlled_fault -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::offer_status_classes_are_offers_or_typed_rejections -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::malformed_offer_bodies_are_typed_rejections -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::wrong_or_missing_offer_fields_are_typed_rejections -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::duplicate_offer_records_remain_explicit_values -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::provider_monitor_preserves_contract_identity_and_cardinality -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::provider_monitor_terminal_polling_stops_after_one_typed_outcome -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::provider_monitor_poll_stop_orderings_cease_polling -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::duplicate_terminal_detector_rejects_controlled_fault -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_output_lines_preserve_stream_and_protocol -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_malformed_protocol_is_data_not_poison -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_eof_orderings_stop_relay_and_actor -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_child_failures_have_typed_attempt_outcomes -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_timeout_is_typed_and_stops_polling -- --exact
cargo test -p myelin --lib orchestration::provider_adapters::vastai::tests::ssh_bootstrap_stop_orderings_emit_one_terminal_and_stop_all_actors -- --exact
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::runtime_actors_generated_transitions_complete_once_on_one_worker -- --exact
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::runtime_invariant_checker_rejects_duplicate_readiness_publication -- --exact
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::helper_wait_generated_terminal_sequences_complete_once_on_one_worker -- --exact
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::helper_invariant_checker_rejects_expected_output_after_terminal_error -- --exact
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::stage_fetch_generated_observations_complete_once_on_one_worker -- --exact
cargo test -p myelin --lib node::worker_node_runtime::control_flow_properties::stage_invariant_checker_rejects_wrong_terminal_classification -- --exact
cargo test -p myelin --lib orchestration::app::serve_cluster_properties::serve_cluster_production_transitions_converge_once_without_growth -- --exact
cargo test -p myelin --lib orchestration::app::serve_cluster_properties::serve_cluster_lifecycle_invariants_reject_injected_duplicate_and_growth -- --exact
cargo test -p myelin --lib orchestration::control::properties::generated_http_bridge_sequences_terminate_without_control_actor_growth -- --exact
cargo test -p myelin --lib orchestration::control::properties::generated_duplicate_control_replies_deliver_first_once_and_remove_observer -- --exact
cargo test -p myelin --lib orchestration::control::properties::reply_observer_disappearance_returns_a_bounded_terminal_http_response -- --exact
cargo test -p myelin --lib orchestration::control::properties::http_bridge_invariant_rejects_a_controlled_duplicate_forward -- --exact
- name: Controlled E2E oracle contract
run: cargo test -p myelin --features test-support --test stateful_vastai e2e_oracle_rejects_controlled_lifecycle_faults -- --exact
- name: Dashboard bridge properties
run: |
cargo test -p dashboard --features demo-control --lib control::properties::generated_concurrent_bridge_commands_forward_once_and_shutdown -- --exact
cargo test -p dashboard --features demo-control --lib control::properties::bridge_invariant_rejects_a_controlled_duplicate_delivery -- --exact
cargo test -p dashboard --features demo-control --lib server::tests::generated_control_http_sequences_are_bounded_and_typed -- --exact
cargo test -p dashboard --features demo-control --lib server::tests::control_http_invariant_rejects_a_controlled_server_error -- --exact
- name: Demo actor properties
run: |
cargo test -p xtask --bin xtask demo::control::properties::generated_control_commands_forward_only_after_supervisor_registration -- --exact
cargo test -p xtask --bin xtask demo::control::properties::control_transition_oracle_rejects_duplicate_forwarding -- --exact
cargo test -p xtask --bin xtask demo::feed::properties::generated_supervisor_transitions_are_once_only_nonblocking_and_clean -- --exact
cargo test -p xtask --bin xtask demo::feed::properties::supervisor_transition_oracle_rejects_duplicate_identity_resources -- --exact
cargo test -p xtask --bin xtask demo::node::properties::generated_node_runtime_transitions_emit_heartbeats_and_stop_once -- --exact
cargo test -p xtask --bin xtask demo::node::properties::node_transition_oracle_rejects_duplicate_resources -- --exact
cargo test -p xtask --bin xtask demo::provider::properties::generated_process_reports_complete_exit_watchers_once_and_preserve_last_state -- --exact
cargo test -p xtask --bin xtask demo::provider::properties::process_relay_oracle_rejects_lost_exit -- --exact
cargo test -p xtask --bin xtask demo::properties::direct_binary_signal_smoke_has_a_hard_timeout -- --exact
process-e2e:
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
needs: component-properties
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
- name: Install pinned Rust toolchain
run: rustup toolchain install nightly-2026-02-07 --profile minimal --component rustc-dev --component rust-src --component llvm-tools-preview
- name: Stateful VastAI process E2E
env:
PROPTEST_CASES: "4"
run: |
set -o pipefail
mkdir -p artifacts
cargo test -p myelin --features test-support --test stateful_vastai stateful_vastai_dashboard_control_survives_restarts -- --ignored --exact --nocapture 2>&1 | tee artifacts/stateful-vastai.log
- name: Preserve E2E failure artifacts
if: failure()
uses: actions/upload-artifact@v4
with:
name: stateful-vastai-failure-${{ github.run_id }}
path: |
artifacts/stateful-vastai.log
apps/myelin/proptest-regressions/tests/e2e_vastai.txt
if-no-files-found: warn

6
Cargo.lock generated
View file

@ -962,7 +962,9 @@ dependencies = [
"futures-lite",
"libc",
"parking_lot",
"proptest",
"serde",
"serde_json",
"sha2 0.10.9",
"swactor",
"swactor-engine",
@ -4335,9 +4337,11 @@ dependencies = [
"getrandom 0.2.17",
"loom",
"parking_lot",
"proc-macro2",
"proptest",
"proptest-state-machine",
"serde",
"syn",
"tracing",
"web-time 0.2.4",
]
@ -4489,11 +4493,13 @@ name = "telemetry"
version = "0.1.0"
dependencies = [
"crossbeam-channel",
"futures-channel",
"iroh",
"libc",
"serde",
"serde_json",
"swactor",
"swactor-engine",
"swactor-transport",
]

View file

@ -75,4 +75,15 @@ criterion = { version = "0.5", features = ["html_reports"] }
proptest = "1"
proptest-state-machine = "0.3"
loom = "0.7"
proc-macro2 = "1"
syn = { version = "2", features = ["full", "visit"] }
[workspace.lints.rust]
warnings = "deny"
[workspace.lints.clippy]
allow_attributes = "forbid"
[lints]
workspace = true

View file

@ -1,54 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
BASE_IMAGE=${BASE_IMAGE:-myelin-node-base:cuda12.6}
IMAGE=${IMAGE:-myelin-node:latest}
CONTAINER=${CONTAINER:-myelin-node-e2e-$$}
GPUS=${MYELIN_CUDA_GPUS:-all}
PROMPT=${MYELIN_NODE_SELF_TEST_PROMPT:-ping}
TIMEOUT_SECS=${MYELIN_NODE_E2E_TIMEOUT_SECS:-1800}
FRAME_LOG=${MYELIN_TELEMETRY_FRAME_LOG:-/var/log/myelin-telemetry.ndjson}
cleanup() {
docker rm -f "$CONTAINER" >/dev/null 2>&1 || true
}
trap cleanup EXIT
cargo build --release -p myelin --bin myelin-worker
docker build -f apps/myelin/node-image/Dockerfile.base -t "$BASE_IMAGE" .
docker build -f apps/myelin/node-image/Dockerfile --build-arg BASE_IMAGE="$BASE_IMAGE" -t "$IMAGE" .
docker run -d \
--name "$CONTAINER" \
--gpus "$GPUS" \
-e MYELIN_NODE_SELF_TEST_PROMPT="$PROMPT" \
-e MYELIN_NODE_MAX_RUNTIME_SECS=1 \
-e MYELIN_SELF_TEST_MAX_TOKENS="${MYELIN_SELF_TEST_MAX_TOKENS:-1}" \
-e MYELIN_MODEL_CACHE_DIR=/var/cache/myelin-models \
-e MYELIN_TELEMETRY_FRAME_LOG="$FRAME_LOG" \
${HF_TOKEN:+-e HF_TOKEN="$HF_TOKEN"} \
"$IMAGE" >/dev/null
deadline=$((SECONDS + TIMEOUT_SECS))
while (( SECONDS < deadline )); do
logs=$(docker logs "$CONTAINER" 2>&1 || true)
if grep -q '"type":"ready"' <<<"$logs" && grep -q '"type":"self_test_completed"' <<<"$logs"; then
frames=$(docker exec "$CONTAINER" cat "$FRAME_LOG" 2>/dev/null || true)
if grep -q '"channel":"myelin.node.ready"' <<<"$frames" &&
grep -q '"channel":"myelin.worker.weights"' <<<"$frames" &&
grep -q '"channel":"myelin.worker.prompt"' <<<"$frames"; then
printf '%s\n' "$logs"
printf '%s\n' "$frames"
exit 0
fi
fi
if grep -q 'WorkerFatal\|myelin-node: .*failed\|ModelLoadFailed\|GgufDownloadFailed' <<<"$logs"; then
printf '%s\n' "$logs" >&2
exit 1
fi
sleep 5
done
docker logs "$CONTAINER" 2>&1 || true
echo "myelin-node Docker E2E timed out after ${TIMEOUT_SECS}s" >&2
exit 1

View file

@ -1,5 +0,0 @@
# Seeds for stateful VastAI E2E failures. Proptest replays these before generated cases.
cc 26c3cd5944cb25de70a984389373e7380a0c6e6ba1c8f3e6eff5a0f78dbef168 # current-strategy replay; E2eCase is derived from e2e_case()
cc 5efb0a6c5c348475c859a2c4267d5177aaecd5adefcc5e8b7fc2e6d728deecd1 # shrinks to case = E2eCase { seed: 7099259626237328177, node_seed: 245, kill_mask: 24, offer_offset: 3, actions: [ConcurrentQueries, Search { count: 2 }, Restart { mode: FlushSafeAbrupt }, Query, Kill { node_slot: 218, command_slot: 5 }, Query, Provision { command_slot: 5, use_searched_offers: true }, EndpointProbe { node_slot: 217 }, Restart { mode: Graceful }, Flush, Kill { node_slot: 217, command_slot: 5 }] }
cc 6d005687858520ed65af20ec5b2de056efcda15771b6420d7f67e3cb6c246d5b # shrinks to case = E2eCase { seed: 17649392557414661864, node_seed: 249, kill_mask: 159, offer_offset: 5, actions: [Kill { node_slot: 88, command_slot: 242 }, Flush, ConcurrentQueries, EndpointProbe { node_slot: 88 }, Query, Search { count: 2 }, Provision { command_slot: 242, use_searched_offers: true }, Restart { mode: FlushSafeAbrupt }, Kill { node_slot: 89, command_slot: 242 }, Query, Restart { mode: Graceful }] }
cc 2e36f51e7cb347b571ac838f81638748f8414a32ec91cb9f4b2730363d72be61 # shrinks to case = E2eCase { seed: 13301618846512983428, node_seed: 99, kill_mask: 30, offer_offset: 2, actions: [Query, Restart { mode: FlushSafeAbrupt }, Kill { node_slot: 136, command_slot: 187 }, ConcurrentQueries, Restart { mode: Graceful }, Query, EndpointProbe { node_slot: 136 }, Kill { node_slot: 137, command_slot: 187 }, Provision { command_slot: 187, use_searched_offers: false }, Flush, Search { count: 0 }] }

View file

@ -1,9 +1,11 @@
use std::collections::BTreeMap;
use std::os::fd::{FromRawFd, OwnedFd};
use std::sync::Arc;
use data_plane::arena::{ArenaConfig, ArenaManager, NodeId as ArenaNodeId};
use data_plane::blob_transfer::{BlobTransferReceiver, BlobTransferSender};
use data_plane::bootstrap::{self, BootstrapSpec, ENV_DATA_PLANE_ENDPOINT, JobHandoff};
use data_plane::data_plane::{DataPlane, DataPlaneBootstrap};
use data_plane::host::{
HostDataPlaneConfig, HostDataPlaneSessionActor, HostRouteRegistrar, install_session_env,
};
@ -11,6 +13,7 @@ use data_plane::namespace::NamespaceClient;
use data_plane::path::JobContext;
use data_plane::protocol::JobCapability;
use data_plane::source::BlobSourcePublisher;
use data_plane::stream_transport::StreamTransport;
use distribution::transport_bridge::{OutboxRouteBinder, RouteBinder, RouteView};
use distribution::types::NodeId;
use swactor::actor::ActorAddress;
@ -76,6 +79,7 @@ const ARENA_ALIGNMENT: u64 = 64;
pub(crate) struct ActorJobDataPlane {
handoff: JobHandoff,
host_session: ActorAddress,
capability: JobCapability,
runtime: Runtime,
}
@ -90,6 +94,7 @@ pub(crate) struct ActorJobDataPlaneConfig {
pub(crate) source_sender: Option<Arc<dyn BlobTransferSender>>,
pub(crate) source_publisher: Option<Arc<dyn BlobSourcePublisher>>,
pub(crate) route_registrar: Option<Arc<dyn HostRouteRegistrar>>,
pub(crate) stream_transport: Option<Arc<dyn StreamTransport>>,
}
impl ActorJobDataPlane {
@ -105,6 +110,7 @@ impl ActorJobDataPlane {
source_sender,
source_publisher,
route_registrar,
stream_transport,
} = config;
let mut arena = ArenaManager::boot(ArenaConfig {
node_id: ArenaNodeId(1),
@ -134,6 +140,7 @@ impl ActorJobDataPlane {
source_sender,
source_publisher,
route_registrar,
stream_transport,
})
.map_err(|error| format!("configure host data-plane session: {error}"))?,
)
@ -141,11 +148,31 @@ impl ActorJobDataPlane {
install_session_env(&mut handoff, host_session, capability);
Ok(Self {
handoff,
runtime: runtime.clone(),
host_session,
capability,
runtime: runtime.clone(),
})
}
pub(crate) fn attach_local(&self) -> Result<DataPlane, String> {
let fd = unsafe { libc::dup(std::os::fd::AsRawFd::as_raw_fd(&self.handoff.arena_fd)) };
if fd < 0 {
return Err(format!(
"duplicate local data-plane arena: {}",
std::io::Error::last_os_error()
));
}
let owned = unsafe { OwnedFd::from_raw_fd(fd) };
futures_lite::future::block_on(DataPlaneBootstrap::attach(
owned,
self.runtime.clone(),
self.host_session,
self.capability,
))
.map(|bootstrap| bootstrap.data_plane)
.map_err(|error| format!("attach local data-plane client: {error}"))
}
pub(crate) fn configure_run(&self, run_id: String) -> Result<(), String> {
futures_lite::future::block_on(async {
self.runtime

View file

@ -4,6 +4,7 @@
//! in the directory, and exchanges its `EndpointAddr` + actor address
//! out-of-band so each side can route to the other over the iroh actor plane.
use crate::data_namespace::{DataNamespaceAuthority, install_namespace_client};
use crate::job_data_plane::{
ActorJobDataPlane, ActorJobDataPlaneConfig, MyelinChildRouteRegistrar,
};
@ -11,7 +12,7 @@ use parking_lot::Mutex;
use std::collections::BTreeMap;
use std::env;
use std::io::{BufRead, BufReader};
use std::path::{Path, PathBuf};
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, Instant};
@ -21,25 +22,25 @@ use swactor::actor::{ActorAddress, ActorInterface};
use swactor::runtime::{Ctx, ExternalSender};
use swactor_engine::{ActorCompletion, Engine, EngineHandle, TokioBackend, TokioConfig};
use swactor_job_runner::{
INFERENCE_RESULTS_EDGE_ID, Job, JobDataPlanePort, JobDone, NodeJobActor, OUTPUTS_EDGE_ID,
OrchestratorJobActor, OrchestratorJobMsg, WORKSPACE_EDGE_ID, register_job_codecs,
Job, JobDataPlanePort, JobDone, NodeJobActor, OUTPUTS_EDGE_ID, OrchestratorJobActor,
OrchestratorJobMsg, WORKSPACE_EDGE_ID, register_job_codecs,
};
use swactor_transport::hex_encode;
use data_plane::blob_transfer::{BlobTransferReceiver, BlobTransferSender};
use data_plane::data_plane::StreamConsumer;
use data_plane::edge_wire::WireEvent;
use data_plane::namespace::NamespaceClient;
use data_plane::path::{DataPath, JobContext};
use data_plane::protocol::DataPlaneError;
use data_plane::protocol::{JobCapability, register_data_plane_codecs};
use data_plane::source::BlobSourcePublisher;
use distribution::node::DistributedNodeConfig;
use iroh::{EndpointAddr, RelayMode};
use iroh_driver::{
EDGE_ALPN, EdgeConnector, EdgeSendHandle, EndpointAddrMask, IrohDriver, IrohDriverConfig,
MVP_IROH_ENDPOINT_ADDR_MASK_ENV, advertised_endpoint,
EDGE_ALPN, EndpointAddrMask, IrohDriver, IrohDriverConfig, MVP_IROH_ENDPOINT_ADDR_MASK_ENV,
advertised_endpoint,
};
use tokio::io::AsyncReadExt;
use tokio::sync::Notify;
use crate::orchestration::distribution_stack::DistributionRuntimeStack;
@ -50,57 +51,37 @@ const RELAY_WAIT_DEADLINE: Duration = Duration::from_secs(30);
const MYELIN_IROH_RELAY_MODE_ENV: &str = "MYELIN_IROH_RELAY_MODE";
const MYELIN_IROH_RELAY_URL_ENV: &str = "MYELIN_IROH_RELAY_URL";
const SWACTOR_IROH_RELAY_URL_ENV: &str = "SWACTOR_IROH_RELAY_URL";
const JOB_OUTPUT_SOCKET: &str = "inference-results.sock";
const DATA_PLANE_CONNECT_DEADLINE: Duration = Duration::from_secs(30);
const JOB_ARENA_BYTES: u64 = 1 << 20;
#[derive(Clone, Debug, Serialize, Deserialize)]
struct EmbeddedDataPlaneAssignment {
result_endpoint: EndpointAddr,
}
/// Actor-driven finite-blob ingress plus the retained temporary Unix output
/// stream bridge. Remote bytes remain on `EDGE_ALPN`.
/// Actor-driven finite-blob ingress and namespace-addressed result streams.
#[derive(Clone)]
pub(crate) struct EmbeddedJobDataPlane {
result_sink: Arc<Mutex<Option<EdgeSendHandle>>>,
result_ready: Arc<Notify>,
connector: EdgeConnector,
actor_plane: Arc<ActorJobDataPlane>,
host_endpoint_json: String,
output_path: PathBuf,
}
pub(crate) struct EmbeddedJobDataPlaneConfig<'a> {
pub(crate) engine: EngineHandle,
pub(crate) connector: EdgeConnector,
pub(crate) root: &'a Path,
pub(crate) struct EmbeddedJobDataPlaneConfig {
pub(crate) host_endpoint: EndpointAddr,
pub(crate) namespace: NamespaceClient,
pub(crate) transfer_receiver: Arc<dyn BlobTransferReceiver>,
pub(crate) source_sender: Arc<dyn BlobTransferSender>,
pub(crate) source_publisher: Arc<dyn BlobSourcePublisher>,
pub(crate) stream_transport: Arc<dyn data_plane::stream_transport::StreamTransport>,
}
impl EmbeddedJobDataPlane {
pub(crate) fn start(
stack: &DistributionRuntimeStack,
config: EmbeddedJobDataPlaneConfig<'_>,
config: EmbeddedJobDataPlaneConfig,
) -> Result<Self, String> {
let EmbeddedJobDataPlaneConfig {
engine,
connector,
root,
host_endpoint,
namespace,
transfer_receiver,
source_sender,
source_publisher,
stream_transport,
} = config;
std::fs::create_dir_all(root)
.map_err(|error| format!("create job data-plane root {}: {error}", root.display()))?;
let output_path = root.join(JOB_OUTPUT_SOCKET);
remove_stale_socket(&output_path)?;
let capability = JobCapability::new(ActorAddress::new_random().0);
let route_registrar = Arc::new(MyelinChildRouteRegistrar::new(
stack.route_view.clone(),
@ -124,49 +105,13 @@ impl EmbeddedJobDataPlane {
source_sender: Some(source_sender),
source_publisher: Some(source_publisher),
route_registrar: Some(route_registrar),
stream_transport: Some(stream_transport),
},
)?);
let host_endpoint_json = serde_json::to_string(&host_endpoint)
.map_err(|error| format!("serialize host data-plane endpoint: {error}"))?;
let result_sink: Arc<Mutex<Option<EdgeSendHandle>>> = Arc::new(Mutex::new(None));
let result_ready = Arc::new(Notify::new());
let output_slot = Arc::clone(&result_sink);
let output_ready = Arc::clone(&result_ready);
swactor_process::spawn_unix_stream_listener(engine, &output_path, move |mut stream| {
let output_slot = Arc::clone(&output_slot);
let output_ready = Arc::clone(&output_ready);
async move {
let sink = loop {
let notified = output_ready.notified();
if let Some(sink) = output_slot.lock().take() {
break sink;
}
notified.await;
};
let mut bytes = vec![0_u8; 64 * 1024];
loop {
match stream.read(&mut bytes).await {
Ok(0) => break,
Ok(count) => {
if sink.send(bytes[..count].to_vec()).is_err() {
break;
}
}
Err(_) => break,
}
}
drop(sink);
}
})
.map_err(|error| format!("bind job data-plane output: {error}"))?;
Ok(Self {
result_sink,
result_ready,
connector,
output_path,
actor_plane,
host_endpoint_json,
})
@ -177,43 +122,17 @@ impl JobDataPlanePort for EmbeddedJobDataPlane {
fn configure(
&self,
job_id: u64,
result_peer: &str,
_result_peer: &str,
) -> Result<BTreeMap<String, String>, String> {
let assignment = serde_json::from_str::<EmbeddedDataPlaneAssignment>(result_peer)
.map_err(|error| format!("parse job data-plane assignment: {error}"))?;
let sink = self.connector.connect(
assignment.result_endpoint,
INFERENCE_RESULTS_EDGE_ID,
DATA_PLANE_CONNECT_DEADLINE,
)?;
*self.result_sink.lock() = Some(sink);
self.result_ready.notify_one();
self.actor_plane.configure_run(job_id.to_string())?;
let mut env = self.actor_plane.handoff_env(&self.host_endpoint_json);
env.insert(
"SWACTOR_DATA_PLANE_OUTPUT".to_owned(),
self.output_path.to_string_lossy().into_owned(),
);
Ok(env)
Ok(self.actor_plane.handoff_env(&self.host_endpoint_json))
}
fn session_ended(&self, _job_id: u64) {
self.result_sink.lock().take();
self.actor_plane.close();
}
}
fn remove_stale_socket(path: &Path) -> Result<(), String> {
match std::fs::remove_file(path) {
Ok(()) => Ok(()),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(error) => Err(format!(
"remove stale job data-plane socket {}: {error}",
path.display()
)),
}
}
/// Out-of-band identity one side publishes so the other can route to it.
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct NodeIdentity {
@ -221,6 +140,17 @@ pub struct NodeIdentity {
pub actor_hex: String,
}
struct OrchestratorResultSink {
bytes: Arc<Mutex<Vec<u8>>>,
}
impl StreamConsumer for OrchestratorResultSink {
fn consume(&self, bytes: &[u8]) -> Result<(), String> {
self.bytes.lock().extend_from_slice(bytes);
Ok(())
}
}
pub(crate) struct JobOrchestratorSession {
_engine: Engine,
driver: IrohDriver,
@ -229,6 +159,10 @@ pub(crate) struct JobOrchestratorSession {
orch: ActorAddress,
identity: NodeIdentity,
landing: PathBuf,
_namespace_authority: DataNamespaceAuthority,
_result_plane: Arc<ActorJobDataPlane>,
_result_completion: ActorCompletion<Result<(), DataPlaneError>>,
_result_bytes: Arc<Mutex<Vec<u8>>>,
}
type JobComposition = (Engine, IrohDriver, DistributionRuntimeStack);
@ -611,6 +545,47 @@ fn start_orchestrator_mode(
Some(relay_mode) => build_composition_with_relay(relay_mode)?,
None => build_composition()?,
};
let namespace_authority = DataNamespaceAuthority::start(
&stack,
&driver,
landing.join(".swactor-data-namespace.json"),
)?;
let namespace = install_namespace_client(&stack, &driver)?;
let result_capability = JobCapability::new(ActorAddress::new_random().0);
let result_transport: Arc<dyn data_plane::stream_transport::StreamTransport> =
driver.stream_transport();
let result_plane = Arc::new(ActorJobDataPlane::new(
&stack.runtime,
ActorJobDataPlaneConfig {
arena_bytes: JOB_ARENA_BYTES,
arena_generation: 2,
session_generation: 2,
capability: result_capability,
job_context: JobContext {
run_id: "unconfigured".to_owned(),
read_prefixes: vec![DataPath::parse("/runs").expect("static run prefix")],
write_prefixes: Vec::new(),
},
namespace: Some(namespace.client),
transfer_receiver: None,
source_sender: None,
source_publisher: Some(namespace.source_publisher),
route_registrar: None,
stream_transport: Some(result_transport),
},
)?);
result_plane.configure_run("0".to_owned())?;
let result_client = result_plane.attach_local()?;
let result_bytes = Arc::new(Mutex::new(Vec::new()));
let result_consumer: Arc<dyn StreamConsumer> = Arc::new(OrchestratorResultSink {
bytes: Arc::clone(&result_bytes),
});
let result_completion = result_client
.collect_stream(
DataPath::parse("/runs/0/results/inference").expect("static result path"),
result_consumer,
)
.map_err(|error| format!("register inference result sink: {error}"))?;
let done = stack
.runtime
.new_inbox::<JobDone>()
@ -630,6 +605,10 @@ fn start_orchestrator_mode(
orch,
identity,
landing,
_namespace_authority: namespace_authority,
_result_plane: result_plane,
_result_completion: result_completion,
_result_bytes: result_bytes,
})
}

View file

@ -1983,14 +1983,12 @@ fn run() -> Result<(), String> {
let data_plane = EmbeddedJobDataPlane::start(
&stack,
crate::job_deploy::EmbeddedJobDataPlaneConfig {
engine: engine.handle(),
connector: driver.edge_connector(),
root: &workdir,
host_endpoint: driver.endpoint_addr(),
namespace: namespace.client,
transfer_receiver,
source_sender,
source_publisher: namespace.source_publisher,
stream_transport: driver.stream_transport(),
},
)?;
let job_route_registrar = Arc::new(MyelinChildRouteRegistrar::new(

View file

@ -7,6 +7,7 @@
//! only through these closures.
use iroh::EndpointAddr;
use iroh_driver::telemetry_transport::PullCollectorConfig;
use iroh_driver::{IrohDriver, PullCollectorHandle, TelemetryQuicHeader, spawn_pull_collector};
use parking_lot::Mutex;
use std::collections::{BTreeMap, BTreeSet};
@ -82,12 +83,14 @@ impl FrameCollector {
flow_id[8..].copy_from_slice(&node_id.to_le_bytes());
let collector = spawn_pull_collector(
engine,
endpoint,
peer,
flow_id,
Vec::new(),
SubscriptionRequest::all(),
Arc::clone(&self.pull_fanout),
PullCollectorConfig {
endpoint,
peer,
flow_id,
token: Vec::new(),
request: SubscriptionRequest::all(),
fanout: Arc::clone(&self.pull_fanout),
},
self.pull_header_tx.clone(),
);
if let Some(previous) = self

View file

@ -43,6 +43,15 @@ pub(crate) struct OrchTelemetry {
descriptor: StreamDescriptor,
}
pub(crate) struct BootstrapEmission<'a> {
pub(crate) dashboard: Option<&'a DashboardSupport>,
pub(crate) channel: &'a str,
pub(crate) run_id: u64,
pub(crate) node_id: u64,
pub(crate) phase: &'a str,
pub(crate) status: &'a str,
pub(crate) detail: Value,
}
impl OrchTelemetry {
pub(crate) fn new(run_id: u64, frame_log: Option<&Path>) -> Result<Self, String> {
let stream = StreamId::new(NodeId::new("myelin-orchestrator"), Lifetime(run_id));
@ -110,6 +119,10 @@ impl OrchTelemetry {
id
}
pub(crate) fn producer(&self) -> TelemetryProducer {
self.producer.clone()
}
pub(crate) fn emit_event(
&mut self,
dashboard: Option<&DashboardSupport>,
@ -140,27 +153,27 @@ impl OrchTelemetry {
status: &str,
detail: Value,
) {
self.emit_bootstrap_to_channel(
self.emit_bootstrap_to_channel(BootstrapEmission {
dashboard,
MYELIN_ORCH_BOOTSTRAP,
channel: MYELIN_ORCH_BOOTSTRAP,
run_id,
node_id,
phase,
status,
detail,
);
});
}
pub(crate) fn emit_bootstrap_to_channel(
&mut self,
dashboard: Option<&DashboardSupport>,
channel: &str,
run_id: u64,
node_id: u64,
phase: &str,
status: &str,
detail: Value,
) {
pub(crate) fn emit_bootstrap_to_channel(&mut self, emission: BootstrapEmission<'_>) {
let BootstrapEmission {
dashboard,
channel,
run_id,
node_id,
phase,
status,
detail,
} = emission;
let benchmark = benchmark::stamp("myelin-orchestrator");
let payload = serde_json::to_vec(&json!({
"schema_version": benchmark["schema_version"].clone(),

View file

@ -10,7 +10,7 @@ use crate::provisioning::{
};
pub(crate) fn node_stream_id(run_id: u64, node_id: u64) -> StreamId {
StreamId::new(NodeId::new(&node_id.to_string()), Lifetime(run_id))
StreamId::new(NodeId::new(node_id.to_string()), Lifetime(run_id))
}
#[derive(Clone)]

View file

@ -538,7 +538,7 @@ async fn submit_node_job(
ManualControlMsg::Query { reply_to }
}) {
Ok(response_rx) => response_rx,
Err(response) => return response,
Err(response) => return *response,
};
let model = match response_rx.await {
Ok(ManualControlReply::Status(model)) => model,
@ -689,7 +689,7 @@ async fn request_reply(
) -> Response {
let response_rx = match begin_request_reply(state, timeout, build) {
Ok(response_rx) => response_rx,
Err(response) => return response,
Err(response) => return *response,
};
match response_rx.await {
@ -718,7 +718,7 @@ fn begin_request_reply(
state: &ControlHttpState,
timeout: Duration,
build: impl FnOnce(ActorAddress) -> ManualControlMsg,
) -> Result<tokio::sync::oneshot::Receiver<ManualControlReply>, Response> {
) -> Result<tokio::sync::oneshot::Receiver<ManualControlReply>, Box<Response>> {
let (response_tx, response_rx) = tokio::sync::oneshot::channel();
let response_tx = Arc::new(Mutex::new(Some(response_tx)));
let reply_to = state
@ -730,26 +730,30 @@ fn begin_request_reply(
timeout,
})
.map_err(|error| {
(
StatusCode::SERVICE_UNAVAILABLE,
Json(ErrorResponse {
error: format!("create control reply observer: {error}"),
}),
Box::new(
(
StatusCode::SERVICE_UNAVAILABLE,
Json(ErrorResponse {
error: format!("create control reply observer: {error}"),
}),
)
.into_response(),
)
.into_response()
})?;
if let Err(error) = state
.runtime
.send_to(state.orchestrator, OrchestratorMsg::Manual(build(reply_to)))
{
let _ = state.runtime.stop_actor(reply_to);
return Err((
StatusCode::SERVICE_UNAVAILABLE,
Json(ErrorResponse {
error: format!("orchestrator control actor unavailable: {error}"),
}),
)
.into_response());
return Err(Box::new(
(
StatusCode::SERVICE_UNAVAILABLE,
Json(ErrorResponse {
error: format!("orchestrator control actor unavailable: {error}"),
}),
)
.into_response(),
));
}
Ok(response_rx)
}
@ -998,7 +1002,7 @@ mod properties {
status,
})
};
let reply = |result: Result<_, Response>| match result {
let reply = |result: Result<_, Box<Response>>| match result {
Ok(receiver) => PendingHttpObservation::Reply {
index,
action: action.clone(),
@ -1191,9 +1195,10 @@ mod properties {
HttpAction::from_raw(kind, command_slot, value)
})
.collect::<Vec<_>>();
let mut config = RuntimeConfig::default();
config.worker_count = 1;
let parts = RuntimeParts::new(config);
let parts = RuntimeParts::new(RuntimeConfig {
worker_count: 1,
..RuntimeConfig::default()
});
let runtime = parts.runtime().clone();
let backend = SteppingBackend::new();
let engine =
@ -1294,9 +1299,10 @@ mod properties {
fn generated_duplicate_control_replies_deliver_first_once_and_remove_observer(
replies in prop::collection::vec(any::<u8>(), 0..=16)
) {
let mut config = RuntimeConfig::default();
config.worker_count = 1;
let parts = RuntimeParts::new(config);
let parts = RuntimeParts::new(RuntimeConfig {
worker_count: 1,
..RuntimeConfig::default()
});
let runtime = parts.runtime().clone();
let backend = SteppingBackend::new();
let engine =
@ -1380,9 +1386,10 @@ mod properties {
#[test]
fn reply_observer_disappearance_returns_a_bounded_terminal_http_response() {
let mut config = RuntimeConfig::default();
config.worker_count = 1;
let parts = RuntimeParts::new(config);
let parts = RuntimeParts::new(RuntimeConfig {
worker_count: 1,
..RuntimeConfig::default()
});
let runtime = parts.runtime().clone();
let backend = SteppingBackend::new();
let engine =

View file

@ -1,14 +1,50 @@
(() => {
const CONTROL_ID = 'myelin-fleet-control';
const CONFIRM_ID = 'myelin-confirm-dialog';
const CONFIRM_STYLE_ID = 'myelin-confirm-dialog-style';
const selectedJobs = new Map();
async function syncControl() {
const nodeView = document.querySelector('.node-view[data-node]');
if (!nodeView) return;
const rawNodeId = nodeView.getAttribute('data-node') || '';
if (!/^\d+$/.test(rawNodeId)) return;
const logicalNodeId = Number(rawNodeId);
function confirmKill(logicalNodeId) {
let dialog = document.getElementById(CONFIRM_ID);
if (!dialog) {
const style = document.createElement('style');
style.id = CONFIRM_STYLE_ID;
style.textContent = `
.myelin-confirm { width:min(440px,calc(100vw - 32px));padding:0;color:var(--text);background:var(--panel);border:1px solid var(--bad);border-radius:var(--r);box-shadow:0 18px 60px rgba(0,0,0,.55) }
.myelin-confirm::backdrop { background:rgba(0,6,12,.78) }
.myelin-confirm form { display:grid;gap:14px;padding:18px }
.myelin-confirm h2,.myelin-confirm p { margin:0 }
.myelin-confirm h2 { color:var(--bad) }
.myelin-confirm-actions { display:flex;justify-content:flex-end;gap:8px }
.myelin-confirm button { padding:6px 12px;background:transparent;color:var(--text);border:1px solid var(--border);border-radius:var(--r);cursor:pointer;font:600 13px var(--mono) }
.myelin-confirm button[value="confirm"] { color:var(--danger-ink);background:var(--danger-fill);border-color:var(--danger-border) }
`;
document.head.append(style);
dialog = document.createElement('dialog');
dialog.id = CONFIRM_ID;
dialog.className = 'myelin-confirm';
dialog.setAttribute('aria-labelledby', 'myelin-confirm-title');
dialog.setAttribute('aria-describedby', 'myelin-confirm-message');
dialog.innerHTML = `<form method="dialog">
<h2 id="myelin-confirm-title">Terminate managed node?</h2>
<p id="myelin-confirm-message"></p>
<div class="myelin-confirm-actions">
<button value="cancel" autofocus>Cancel</button>
<button value="confirm">Terminate node</button>
</div>
</form>`;
document.body.append(dialog);
}
dialog.querySelector('#myelin-confirm-message').textContent =
`Terminate managed node ${logicalNodeId}? Vast.ai contracts are destroyed and billing stops.`;
dialog.returnValue = 'cancel';
return new Promise(resolve => {
dialog.addEventListener('close', () => resolve(dialog.returnValue === 'confirm'), { once: true });
dialog.showModal();
});
}
async function syncControl() {
let model;
try {
const response = await fetch('/api/control/status', { cache: 'no-store' });
@ -18,6 +54,17 @@
} catch (_) {
return;
}
window.dispatchEvent(new CustomEvent('dashboard-hardware-source', {
detail: {
source: model?.provider?.provisioning_mode === 'mock' ? 'orchestrator' : 'node',
},
}));
const nodeView = document.querySelector('.node-view[data-node]');
if (!nodeView) return;
const rawNodeId = nodeView.getAttribute('data-node') || '';
if (!/^\d+$/.test(rawNodeId)) return;
const logicalNodeId = Number(rawNodeId);
const node = model?.nodes?.find(candidate => candidate.logical_node_id === logicalNodeId);
if (!node) return;
@ -130,7 +177,7 @@
};
killButton.onclick = async () => {
if (!window.confirm(`Kill managed node ${logicalNodeId}? Vast.ai contracts are destroyed and billing stops.`)) return;
if (!await confirmKill(logicalNodeId)) return;
killButton.disabled = true;
message.textContent = 'Submitting kill…';
const commandId = `fleet-kill-${globalThis.crypto?.randomUUID?.() || Date.now()}`;

View file

@ -3441,8 +3441,8 @@ mod tests {
proptest! {
#![proptest_config(ProptestConfig {
cases: 128,
max_shrink_iters: 2_000,
cases: 16,
max_shrink_iters: 256,
..ProptestConfig::default()
})]

View file

@ -92,6 +92,19 @@
.sort-button[data-direction="asc"]::before { top: 3px; border-bottom: 5px solid currentColor; }
.sort-button[data-direction="desc"]::before { bottom: 3px; border-top: 5px solid currentColor; }
.sort-button:hover:not(:disabled), .sort-button[aria-pressed="true"] { color: var(--cyan); border-color: var(--cyan); background: var(--panel-hover); }
.offers-heading { display: flex; align-items: center; justify-content: space-between; gap: 12px; flex-wrap: wrap; margin-bottom: 12px; }
.offers-heading h2 { margin: 0; }
.offer-actions { display: flex; align-items: center; gap: 10px; flex-wrap: wrap; }
.offer-selection { margin: 0; font: 12px var(--mono); }
dialog.confirm-dialog {
width: min(440px, calc(100vw - 32px)); padding: 0; color: var(--text);
background: var(--panel); border: 1px solid var(--amber); border-radius: var(--r);
box-shadow: 0 18px 60px rgba(0, 0, 0, .55);
}
dialog.confirm-dialog::backdrop { background: rgba(0, 6, 12, .78); }
.confirm-form { display: grid; gap: 14px; padding: 18px; }
.confirm-form h2, .confirm-form p { margin: 0; }
.confirm-actions { display: flex; justify-content: flex-end; gap: 8px; }
@media (max-width: 680px) { #search-form { grid-template-columns: 1fr; } }
@media (prefers-reduced-motion: reduce) { * { transition-duration: .01ms !important; } }
</style>
@ -144,12 +157,27 @@
</section>
</div>
<section id="vast-offers-card" class="card" style="margin-top:1rem">
<h2>Offers</h2><p id="offer-status" class="muted"></p><p id="offer-error" class="error"></p>
<div class="offers-heading">
<h2>Offers</h2>
<div class="offer-actions">
<p class="offer-selection">Selected: <code id="selection">none (0/8)</code></p>
<button id="provision-button" class="primary" disabled>Provision selected offers</button>
</div>
</div>
<p id="offer-status" class="muted"></p><p id="offer-error" class="error"></p>
<div class="offers-scroll"><table id="offers-table"><thead><tr><th>Select</th><th>Offer<span class="sort-controls"><button type="button" class="sort-button" data-sort="offer_id" data-direction="asc" title="Sort offer ascending" aria-label="Sort offer ascending">↑</button><button type="button" class="sort-button" data-sort="offer_id" data-direction="desc" title="Sort offer descending" aria-label="Sort offer descending">↓</button></span></th><th>Host<span class="sort-controls"><button type="button" class="sort-button" data-sort="host_id" data-direction="asc" title="Sort host ascending" aria-label="Sort host ascending">↑</button><button type="button" class="sort-button" data-sort="host_id" data-direction="desc" title="Sort host descending" aria-label="Sort host descending">↓</button></span></th><th>GPU<span class="sort-controls"><button type="button" class="sort-button" data-sort="gpu_model" data-direction="asc" title="Sort GPU ascending" aria-label="Sort GPU ascending">↑</button><button type="button" class="sort-button" data-sort="gpu_model" data-direction="desc" title="Sort GPU descending" aria-label="Sort GPU descending">↓</button></span></th><th>RAM<span class="sort-controls"><button type="button" class="sort-button" data-sort="gpu_ram_mb" data-direction="asc" title="Sort RAM ascending" aria-label="Sort RAM ascending">↑</button><button type="button" class="sort-button" data-sort="gpu_ram_mb" data-direction="desc" title="Sort RAM descending" aria-label="Sort RAM descending">↓</button></span></th><th>CC<span class="sort-controls"><button type="button" class="sort-button" data-sort="compute_cap" data-direction="asc" title="Sort compute capability ascending" aria-label="Sort compute capability ascending">↑</button><button type="button" class="sort-button" data-sort="compute_cap" data-direction="desc" title="Sort compute capability descending" aria-label="Sort compute capability descending">↓</button></span></th><th>Verified<span class="sort-controls"><button type="button" class="sort-button" data-sort="verification" data-direction="asc" title="Sort verification ascending" aria-label="Sort verification ascending">↑</button><button type="button" class="sort-button" data-sort="verification" data-direction="desc" title="Sort verification descending" aria-label="Sort verification descending">↓</button></span></th><th>Reliability<span class="sort-controls"><button type="button" class="sort-button" data-sort="reliability" data-direction="asc" title="Sort reliability ascending" aria-label="Sort reliability ascending">↑</button><button type="button" class="sort-button" data-sort="reliability" data-direction="desc" title="Sort reliability descending" aria-label="Sort reliability descending">↓</button></span></th><th>Down<span class="sort-controls"><button type="button" class="sort-button" data-sort="download_mbps" data-direction="asc" title="Sort download speed ascending" aria-label="Sort download speed ascending">↑</button><button type="button" class="sort-button" data-sort="download_mbps" data-direction="desc" title="Sort download speed descending" aria-label="Sort download speed descending">↓</button></span></th><th>Up<span class="sort-controls"><button type="button" class="sort-button" data-sort="upload_mbps" data-direction="asc" title="Sort upload speed ascending" aria-label="Sort upload speed ascending">↑</button><button type="button" class="sort-button" data-sort="upload_mbps" data-direction="desc" title="Sort upload speed descending" aria-label="Sort upload speed descending">↓</button></span></th><th>Location<span class="sort-controls"><button type="button" class="sort-button" data-sort="location" data-direction="asc" title="Sort location ascending" aria-label="Sort location ascending">↑</button><button type="button" class="sort-button" data-sort="location" data-direction="desc" title="Sort location descending" aria-label="Sort location descending">↓</button></span></th><th>$/hr<span class="sort-controls"><button type="button" class="sort-button" data-sort="hourly_price" data-direction="asc" title="Sort price ascending" aria-label="Sort price ascending">↑</button><button type="button" class="sort-button" data-sort="hourly_price" data-direction="desc" title="Sort price descending" aria-label="Sort price descending">↓</button></span></th></tr></thead><tbody id="offers"></tbody></table></div>
<p>Selected exact offer IDs: <code id="selection">none (0/8)</code></p>
<button id="provision-button" class="primary" disabled>Provision selected offers</button>
</section>
</main>
<dialog id="confirm-dialog" class="confirm-dialog" aria-labelledby="confirm-title" aria-describedby="confirm-message">
<form method="dialog" class="confirm-form">
<h2 id="confirm-title">Confirm action</h2>
<p id="confirm-message"></p>
<div class="confirm-actions">
<button value="cancel" autofocus>Cancel</button>
<button id="confirm-submit" class="primary" value="confirm">Confirm</button>
</div>
</form>
</dialog>
<script>
'use strict';
const $=id=>document.getElementById(id); const MAX_SELECTED_OFFERS=8; const PAGE_STATE_KEY='myelin.provision.page.v1'; const SEARCH_FIELDS=['gpu_model','min_gpu_ram_mb','min_compute_cap','min_reliability','min_download_mbps','min_upload_mbps','max_hourly_price','blacklist_hosts','require_verified']; let selected=new Set(); let provisioningMode='real'; let currentOffers=[]; let offerSorts=[]; let sortDoubleClickStartedActive=null;
@ -158,6 +186,7 @@ const number=(form,name)=>{const raw=value(form,name);return raw===null?null:Num
const escapeHtml=raw=>String(raw??'').replace(/[&<>"']/g,char=>({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[char]));
async function jsonFetch(url,options={}){const response=await fetch(url,{...options,headers:{'content-type':'application/json',...(options.headers||{})}});const text=await response.text();let body=null;try{body=text?JSON.parse(text):null}catch{body={error:text}}if(!response.ok)throw new Error(body?.error||`${response.status} ${response.statusText}`);return body}
function commandId(prefix){return `${prefix}-${crypto.randomUUID()}`}
function confirmAction(title,message,confirmLabel){const dialog=$('confirm-dialog');$('confirm-title').textContent=title;$('confirm-message').textContent=message;$('confirm-submit').textContent=confirmLabel;dialog.returnValue='cancel';return new Promise(resolve=>{dialog.addEventListener('close',()=>resolve(dialog.returnValue==='confirm'),{once:true});dialog.showModal()})}
function formSnapshot(form,names){return Object.fromEntries(names.map(name=>{const field=form.elements[name];return [name,field.type==='checkbox'?field.checked:field.value]}))}
function restoreForm(form,state){if(!state)return;Object.entries(state).forEach(([name,saved])=>{const field=form.elements[name];if(!field)return;if(field.type==='checkbox')field.checked=Boolean(saved);else field.value=String(saved??'')})}
function persistPageState(){try{sessionStorage.setItem(PAGE_STATE_KEY,JSON.stringify({config:formSnapshot($('config-form'),['ssh_identity','bootstrap_command']),search:formSnapshot($('search-form'),SEARCH_FIELDS),local:formSnapshot($('local-provision-form'),['count']),offers:currentOffers,selected:[...selected],sorts:offerSorts}))}catch{}}
@ -165,13 +194,13 @@ function restorePageState(){try{const saved=JSON.parse(sessionStorage.getItem(PA
function updateSelectionUi(){const count=selected.size;$('selection').textContent=count?`${[...selected].join(', ')} (${count}/${MAX_SELECTED_OFFERS})`:`none (0/${MAX_SELECTED_OFFERS})`;$('provision-button').disabled=count===0;$('provision-button').textContent=count===0?(provisioningMode==='mock'?'Simulate selected offers':'Provision selected offers'):`${provisioningMode==='mock'?'Simulate':'Provision'} ${count} selected offer${count===1?'':'s'}`}
async function refresh(){try{const reply=await jsonFetch('/api/control/status');const model=reply.Status;const provider=model.provider.name;provisioningMode=model.provider.provisioning_mode||'real';const vast=provider==='vastai';const simulated=vast&&provisioningMode==='mock';const local=provider==='process'||provider==='docker';$('provider-config-card').hidden=!vast;$('vast-search-card').hidden=!vast;$('vast-offers-card').hidden=!vast;$('local-provision-card').hidden=!local;$('simulation-banner').hidden=!simulated;document.querySelectorAll('[data-real-vastai]').forEach(element=>element.hidden=simulated);$('local-provider-name').textContent=provider;$('readiness').textContent=`${provider} · ${provisioningMode} · ${model.provider.kind}`;$('readiness').className=model.provider.kind==='ready'?'status ok':'status';$('readiness-error').textContent=model.provider.error||'';$('search-button').disabled=!vast||model.provider.kind!=='ready';$('local-provision-button').disabled=!local||model.provider.kind!=='ready';updateSelectionUi()}catch(error){$('readiness-error').textContent=error.message}}
$('config-form').addEventListener('submit',async event=>{event.preventDefault();const form=event.currentTarget;try{await jsonFetch('/api/control/provider',{method:'POST',body:JSON.stringify({api_key:value(form,'api_key'),ssh_identity:value(form,'ssh_identity'),bootstrap_command:value(form,'bootstrap_command')})});form.elements.api_key.value='';persistPageState();await refresh()}catch(error){$('readiness-error').textContent=error.message}});
$('local-provision-form').addEventListener('submit',async event=>{event.preventDefault();const form=event.currentTarget;const count=number(form,'count')||1;const provider=$('local-provider-name').textContent;if(!confirm(`Provision ${count} ${provider} node${count===1?'':'s'}?`))return;try{await jsonFetch('/api/control/provision',{method:'POST',body:JSON.stringify({command_id:commandId('provision'),count,selected_offer_ids:[],image:$('runtime-image').value.trim()})});$('local-provision-error').textContent='';await refresh()}catch(error){$('local-provision-error').textContent=error.message}});
$('local-provision-form').addEventListener('submit',async event=>{event.preventDefault();const form=event.currentTarget;const count=number(form,'count')||1;const provider=$('local-provider-name').textContent;if(!await confirmAction('Provision nodes?',`Provision ${count} ${provider} node${count===1?'':'s'}?`,'Provision'))return;try{await jsonFetch('/api/control/provision',{method:'POST',body:JSON.stringify({command_id:commandId('provision'),count,selected_offer_ids:[],image:$('runtime-image').value.trim()})});$('local-provision-error').textContent='';await refresh()}catch(error){$('local-provision-error').textContent=error.message}});
$('search-form').addEventListener('submit',async event=>{event.preventDefault();const form=event.currentTarget;const blacklist=(value(form,'blacklist_hosts')||'').split(',').map(v=>Number(v.trim())).filter(Number.isFinite);$('offer-status').textContent='Searching…';$('offer-error').textContent='';try{const reply=await jsonFetch('/api/control/offers',{method:'POST',body:JSON.stringify({gpu_model:value(form,'gpu_model'),min_gpu_ram_mb:number(form,'min_gpu_ram_mb'),min_compute_cap:number(form,'min_compute_cap'),min_reliability:number(form,'min_reliability'),require_verified:form.elements.require_verified.checked,min_download_mbps:number(form,'min_download_mbps'),min_upload_mbps:number(form,'min_upload_mbps'),max_hourly_price:number(form,'max_hourly_price'),blacklist_hosts:blacklist,count:null})});currentOffers=reply.Offers;selected.clear();renderOffers();updateSelectionUi();$('offer-status').textContent=currentOffers.length===0?'No offers match this search.':`${currentOffers.length} matching offer${currentOffers.length===1?'':'s'}`;persistPageState()}catch(error){$('offer-status').textContent='';$('offer-error').textContent=error.message}});
function compareOfferValues(left,right){const leftMissing=left===null||left===undefined||left==='';const rightMissing=right===null||right===undefined||right==='';if(leftMissing||rightMissing)return leftMissing===rightMissing?0:leftMissing?1:-1;if(typeof left==='number'&&typeof right==='number')return left-right;return String(left).localeCompare(String(right),undefined,{numeric:true,sensitivity:'base'})}
function renderOffers(){const offers=[...currentOffers];if(offerSorts.length)offers.sort((left,right)=>{for(const sort of offerSorts){const compared=compareOfferValues(left[sort.key],right[sort.key]);if(compared!==0)return compared*(sort.direction==='asc'?1:-1)}return Number(left.offer_id)-Number(right.offer_id)});$('offers').replaceChildren(...offers.map(offer=>{const tr=document.createElement('tr');tr.innerHTML=`<td><input type="checkbox" data-offer="${Number(offer.offer_id)}" style="width:auto"></td><td>${escapeHtml(offer.offer_id)}</td><td>${escapeHtml(offer.host_id)}</td><td>${escapeHtml(offer.gpu_model)}</td><td>${escapeHtml(offer.gpu_ram_mb)}</td><td>${escapeHtml(offer.compute_cap)}</td><td>${escapeHtml(offer.verification)}</td><td>${escapeHtml(offer.reliability)}</td><td>${escapeHtml(offer.download_mbps)}</td><td>${escapeHtml(offer.upload_mbps)}</td><td>${escapeHtml(offer.location)}</td><td>${escapeHtml(Number(offer.hourly_price).toFixed(3))}</td>`;tr.querySelector('[data-offer]').checked=selected.has(Number(offer.offer_id));return tr}));document.querySelectorAll('.sort-button').forEach(button=>button.setAttribute('aria-pressed',String(offerSorts.some(sort=>sort.key===button.dataset.sort&&sort.direction===button.dataset.direction))))}
$('offers-table').addEventListener('click',event=>{const button=event.target.closest('.sort-button');if(!button)return;const id=`${button.dataset.sort}:${button.dataset.direction}`;const activeIndex=offerSorts.findIndex(sort=>sort.key===button.dataset.sort&&sort.direction===button.dataset.direction);if(event.detail===1)sortDoubleClickStartedActive=activeIndex>=0?id:null;if(event.detail===2&&sortDoubleClickStartedActive===id){offerSorts.splice(activeIndex,1);sortDoubleClickStartedActive=null}else if(activeIndex<0){const columnIndex=offerSorts.findIndex(sort=>sort.key===button.dataset.sort);const next={key:button.dataset.sort,direction:button.dataset.direction};if(columnIndex>=0)offerSorts.splice(columnIndex,1,next);else offerSorts.push(next)}renderOffers();persistPageState()});
$('offers').addEventListener('change',event=>{const id=Number(event.target.dataset.offer);if(!Number.isFinite(id))return;if(event.target.checked){if(selected.size>=MAX_SELECTED_OFFERS){event.target.checked=false;$('offer-error').textContent=`Select at most ${MAX_SELECTED_OFFERS} offers.`;return}selected.add(id)}else selected.delete(id);$('offer-error').textContent='';updateSelectionUi();persistPageState()});
$('provision-button').addEventListener('click',async()=>{try{const ids=[...selected];if(ids.length===0)return;const question=provisioningMode==='mock'?`Simulate provisioning exact Vast.ai offer IDs ${ids.join(', ')}? No rental will be created.`:`Provision exact Vast.ai offer IDs ${ids.join(', ')}? This can incur charges.`;if(!confirm(question))return;await jsonFetch('/api/control/provision',{method:'POST',body:JSON.stringify({command_id:commandId('provision'),count:ids.length,selected_offer_ids:ids,image:$('runtime-image').value.trim()})});const provisioned=new Set(ids);currentOffers=currentOffers.filter(offer=>!provisioned.has(Number(offer.offer_id)));selected.clear();renderOffers();updateSelectionUi();persistPageState();await refresh()}catch(error){$('offer-error').textContent=error.message}});
$('provision-button').addEventListener('click',async()=>{try{const ids=[...selected];if(ids.length===0)return;const simulated=provisioningMode==='mock';const question=simulated?`Simulate provisioning exact Vast.ai offer IDs ${ids.join(', ')}? No rental will be created.`:`Provision exact Vast.ai offer IDs ${ids.join(', ')}? This can incur charges.`;if(!await confirmAction(simulated?'Simulate provisioning?':'Provision selected offers?',question,simulated?'Simulate':'Provision'))return;await jsonFetch('/api/control/provision',{method:'POST',body:JSON.stringify({command_id:commandId('provision'),count:ids.length,selected_offer_ids:ids,image:$('runtime-image').value.trim()})});const provisioned=new Set(ids);currentOffers=currentOffers.filter(offer=>!provisioned.has(Number(offer.offer_id)));selected.clear();renderOffers();updateSelectionUi();persistPageState();await refresh()}catch(error){$('offer-error').textContent=error.message}});
$('config-form').addEventListener('input',persistPageState);$('search-form').addEventListener('input',persistPageState);$('local-provision-form').addEventListener('input',persistPageState);restorePageState();refresh();setInterval(refresh,1000);
</script>
</body></html>

View file

@ -99,6 +99,7 @@ impl DataNode {
source_sender: Some(Arc::clone(&self.sender)),
source_publisher: Some(Arc::clone(&self.namespace.source_publisher)),
route_registrar: None,
stream_transport: None,
})
.expect("host session"),
)

View file

@ -1,14 +1,26 @@
#![cfg(target_os = "linux")]
use std::collections::BTreeSet;
use std::sync::Arc;
use std::time::Duration;
use data_plane::bootstrap::{
ENV_ARENA_FD, ENV_DATA_PLANE_ACTOR, ENV_DATA_PLANE_ENDPOINT, ENV_JOB_CAPABILITY,
};
use data_plane::data_plane::StreamConsumer;
use data_plane::namespace::{
DataDirectoryActor, NamespaceClient, NamespaceClientActor, NamespaceDiscovery,
};
use data_plane::path::{DataPath, JobContext};
use data_plane::protocol::JobCapability;
use data_plane::source::BlobSourcePublisher;
use data_plane::stream_transport::{LocalStreamTransport, StreamTransport};
use futures_lite::future;
use parking_lot::Mutex;
use swactor::actor::ActorAddress;
use swactor::config::RuntimeConfig;
use swactor::runtime::RuntimeParts;
use swactor_engine::{Engine, TokioBackend, TokioConfig};
use crate::job_data_plane::ActorJobDataPlane;
@ -39,6 +51,7 @@ fn plane() -> ActorJobDataPlane {
source_sender: None,
source_publisher: None,
route_registrar: None,
stream_transport: None,
},
)
.expect("actor data-plane")
@ -69,3 +82,121 @@ fn handoff_contains_one_descriptor_and_private_actor_metadata() {
assert!(flags >= 0);
assert_eq!(flags & libc::FD_CLOEXEC, 0);
}
struct StaticDiscovery(ActorAddress);
impl NamespaceDiscovery for StaticDiscovery {
fn current_directory(&self) -> Option<ActorAddress> {
Some(self.0)
}
}
struct LocalPublisher;
impl BlobSourcePublisher for LocalPublisher {
fn publish_source(&self, _source: ActorAddress) -> Result<(), String> {
Ok(())
}
}
struct BytesConsumer(Arc<Mutex<Vec<u8>>>);
impl StreamConsumer for BytesConsumer {
fn consume(&self, bytes: &[u8]) -> Result<(), String> {
self.0.lock().extend_from_slice(bytes);
Ok(())
}
}
#[test]
fn canonical_inference_result_uses_native_stream_end_to_end() {
let state_root = std::env::temp_dir().join(format!(
"myelin-stream-test-{}",
ActorAddress::new_random().to_full_hex()
));
let parts = RuntimeParts::new(RuntimeConfig::default());
let runtime = parts.runtime().clone();
let engine = Engine::new(
parts,
TokioBackend::new(TokioConfig::default()).expect("tokio backend"),
)
.expect("engine");
let directory = runtime
.spawn(
DataDirectoryActor::recover(state_root.join("namespace.json"), |_recovery, _length| {
Err(data_plane::namespace::NamespaceError::SourceRecovery(
"no recovered blob sources".to_owned(),
))
})
.expect("directory"),
)
.expect("spawn directory");
let proxy = runtime
.spawn(NamespaceClientActor::new(
engine.handle(),
runtime.create_sender(),
Arc::new(StaticDiscovery(directory)),
Duration::from_millis(5),
))
.expect("namespace proxy");
let namespace = NamespaceClient::new(runtime.clone(), proxy);
let transport: Arc<dyn StreamTransport> = Arc::new(LocalStreamTransport::new());
let publisher: Arc<dyn BlobSourcePublisher> = Arc::new(LocalPublisher);
let make_plane = |read: bool| {
ActorJobDataPlane::new(
&runtime,
crate::job_data_plane::ActorJobDataPlaneConfig {
arena_bytes: 1 << 20,
arena_generation: if read { 21 } else { 22 },
session_generation: if read { 31 } else { 32 },
capability: CAPABILITY,
job_context: JobContext {
run_id: "0".to_owned(),
read_prefixes: if read {
vec![path("/runs/0/results")]
} else {
Vec::new()
},
write_prefixes: if read {
Vec::new()
} else {
vec![path("/runs/0/results")]
},
},
namespace: Some(namespace.clone()),
transfer_receiver: None,
source_sender: None,
source_publisher: Some(Arc::clone(&publisher)),
route_registrar: None,
stream_transport: Some(Arc::clone(&transport)),
},
)
.expect("actor plane")
};
let reader_plane = make_plane(true);
let writer_plane = make_plane(false);
let reader = reader_plane.attach_local().expect("reader attachment");
let writer = writer_plane.attach_local().expect("writer attachment");
let logical = path("/runs/0/results/inference");
let observed = Arc::new(Mutex::new(Vec::new()));
let consumer: Arc<dyn StreamConsumer> = Arc::new(BytesConsumer(Arc::clone(&observed)));
let completed = reader
.collect_stream(logical.clone(), consumer)
.expect("register result sink");
future::block_on(async {
let mut writer = writer.write_stream(&logical).await.expect("open writer");
writer
.write(br#"{"device":"CUDA:0","output":[2.75,-8.75]}"#)
.await
.expect("write result");
writer.close().await.expect("close writer");
});
completed.wait().expect("result sink completed");
assert_eq!(
&*observed.lock(),
br#"{"device":"CUDA:0","output":[2.75,-8.75]}"#
);
let _ = std::fs::remove_dir_all(state_root);
}

View file

@ -1,305 +0,0 @@
#!/usr/bin/env python3
"""Tinygrad-backed device bridge probe for Rust Myelin bridge tests.
Line-delimited JSON control only. Payload bytes live in the arena file whose
path Rust passes during initialize.
"""
from __future__ import annotations
import json
import os
import struct
import sys
from dataclasses import dataclass
from typing import Any
try:
from tinygrad import Tensor # type: ignore
except Exception as exc: # pragma: no cover - exercised from Rust process tests
print(
json.dumps(
{
"type": "worker_fatal",
"reason": "tinygrad_unavailable",
"message": str(exc),
}
),
flush=True,
)
raise SystemExit(2)
@dataclass
class DeviceObject:
dtype: str
shape: str
extent: int
values: list[int]
tensor: Any
@dataclass
class PendingCopy:
kind: str
copy_id: int
handle_id: int
host_offset: int
device_offset: int
length: int
arena_path: str | None = None
arena_bytes = 0
generation = 0
objects: dict[int, DeviceObject] = {}
pending_copies: dict[int, PendingCopy] = {}
fail_next: str | None = None
def emit(obj: dict[str, Any]) -> None:
print(json.dumps(obj, separators=(",", ":")), flush=True)
def backend_error(reason: str) -> None:
emit({"type": "backend_error", "reason": reason})
def fatal(reason: str, message: str) -> None:
emit({"type": "worker_fatal", "reason": reason, "message": message})
def require_arena() -> str:
if arena_path is None:
raise RuntimeError("arena is not initialized")
return arena_path
def consume_failure(reason: str) -> bool:
global fail_next
if fail_next == reason:
fail_next = None
backend_error(reason)
return True
return False
def check_u32_range(offset: int, length: int) -> None:
if offset < 0 or length < 0 or offset % 4 != 0 or length % 4 != 0:
raise ValueError("u32 ranges must be non-negative and 4-byte aligned")
def tensor_from_values(values: list[int]) -> Any:
try:
return Tensor(values, dtype="uint32").realize()
except Exception:
return Tensor(values, dtype="int32").realize()
def allocate_tensor(dtype: str, extent: int) -> tuple[list[int], Any]:
if dtype == "u32":
if extent % 4 != 0:
raise ValueError("u32 extent must be 4-byte aligned")
values = [0] * (extent // 4)
return values, tensor_from_values(values)
if dtype == "f16":
values = [0] * (extent // 2)
return values, Tensor(values, dtype="float16").realize()
raise ValueError(f"unsupported dtype {dtype!r}")
def update_tensor(obj: DeviceObject) -> None:
if obj.dtype == "u32":
obj.tensor = tensor_from_values(obj.values)
elif obj.dtype == "f16":
obj.tensor = Tensor(obj.values, dtype="float16").realize()
else:
raise ValueError(f"unsupported dtype {obj.dtype!r}")
def materialized_values(obj: DeviceObject) -> list[int]:
return [int(v) for v in obj.tensor.tolist()]
def read_arena(offset: int, length: int) -> bytes:
path = require_arena()
with open(path, "rb", buffering=0) as f:
f.seek(offset)
data = f.read(length)
if len(data) != length:
raise EOFError("short arena read")
return data
def write_arena(offset: int, data: bytes) -> None:
path = require_arena()
with open(path, "r+b", buffering=0) as f:
f.seek(offset)
f.write(data)
f.flush()
def perform_host_to_device(copy: PendingCopy) -> None:
obj = objects[copy.handle_id]
if obj.dtype != "u32":
raise ValueError("payload copy is implemented for u32 test objects only")
check_u32_range(copy.device_offset, copy.length)
payload = read_arena(copy.host_offset, copy.length)
words = list(struct.unpack("<" + "I" * (copy.length // 4), payload))
start = copy.device_offset // 4
end = start + len(words)
if end > len(obj.values):
raise ValueError("device range out of bounds")
obj.values[start:end] = words
update_tensor(obj)
def perform_device_to_host(copy: PendingCopy) -> None:
obj = objects[copy.handle_id]
if obj.dtype != "u32":
raise ValueError("payload copy is implemented for u32 test objects only")
check_u32_range(copy.device_offset, copy.length)
values = materialized_values(obj)
start = copy.device_offset // 4
end = start + (copy.length // 4)
if end > len(values):
raise ValueError("device range out of bounds")
payload = struct.pack("<" + "I" * (end - start), *values[start:end])
write_arena(copy.host_offset, payload)
def perform_copy(copy: PendingCopy) -> None:
if copy.kind == "host_to_device":
perform_host_to_device(copy)
elif copy.kind == "device_to_host":
perform_device_to_host(copy)
else:
raise ValueError(f"unknown copy kind {copy.kind!r}")
def handle(req: dict[str, Any]) -> bool:
global arena_path, arena_bytes, generation, fail_next
typ = req.get("type")
if typ == "initialize":
arena_path = str(req["arena_path"])
arena_bytes = int(req["arena_bytes"])
generation = int(req["generation"])
with open(arena_path, "r+b", buffering=0) as f:
f.truncate(arena_bytes)
emit({"type": "worker_ready", "generation": generation})
return True
if typ == "alloc":
if consume_failure("allocation_failed"):
return True
handle_id = int(req["handle_id"])
dtype = str(req["dtype"])
shape = str(req["shape"])
extent = int(req["extent"])
values, tensor = allocate_tensor(dtype, extent)
objects[handle_id] = DeviceObject(dtype=dtype, shape=shape, extent=extent, values=values, tensor=tensor)
emit({"type": "allocated", "handle_id": handle_id})
return True
if typ in ("host_to_device", "device_to_host"):
if consume_failure("copy_failed"):
return True
copy = PendingCopy(
kind=typ,
copy_id=int(req["copy_id"]),
handle_id=int(req["handle_id"]),
host_offset=int(req["host_offset"]),
device_offset=int(req["device_offset"]),
length=int(req["len"]),
)
if copy.handle_id not in objects:
backend_error("copy_failed")
return True
if bool(req.get("defer", False)):
pending_copies[copy.copy_id] = copy
emit({"type": "copy_started", "copy_id": copy.copy_id})
else:
perform_copy(copy)
emit({"type": "copy_completed", "copy_id": copy.copy_id})
return True
if typ == "complete_copy":
copy_id = int(req["copy_id"])
copy = pending_copies.pop(copy_id, None)
if copy is None:
backend_error("copy_failed")
return True
perform_copy(copy)
emit({"type": "copy_completed", "copy_id": copy_id})
return True
if typ == "wrap_for_tinygrad":
if consume_failure("invalid_view"):
return True
handle_id = int(req["handle_id"])
obj = objects.get(handle_id)
if obj is None:
backend_error("invalid_view")
return True
dtype = str(req["dtype"])
shape = str(req["shape"])
if obj.dtype != dtype or obj.shape != shape:
backend_error("invalid_view")
return True
# Force materialization at view time so success depends on live tensor state.
_ = obj.tensor.tolist()
emit({"type": "view", "handle_id": handle_id, "dtype": dtype, "shape": shape})
return True
if typ == "free":
handle_id = int(req["handle_id"])
if handle_id not in objects:
backend_error("invalid_view")
return True
del objects[handle_id]
emit({"type": "freed", "handle_id": handle_id})
return True
if typ == "restart":
generation = int(req["generation"])
objects.clear()
pending_copies.clear()
fail_next = None
emit({"type": "worker_ready", "generation": generation})
return True
if typ == "fail_next":
fail_next = str(req["failure"])
emit({"type": "ok"})
return True
if typ == "shutdown":
emit({"type": "worker_stopped"})
return False
fatal("protocol_error", f"unknown command type {typ!r}")
return False
def main() -> int:
for line in sys.stdin:
line = line.strip()
if not line:
continue
try:
req = json.loads(line)
if not isinstance(req, dict):
raise ValueError("request must be an object")
if not handle(req):
return 0
except SystemExit:
raise
except Exception as exc:
fatal("backend_exception", str(exc))
return 2
return 0
if __name__ == "__main__":
raise SystemExit(main())

File diff suppressed because it is too large Load diff

View file

@ -8,7 +8,7 @@ version = "0.1.0"
requires-python = ">=3.9"
[dependency-groups]
dev = ["jupyter", "ipykernel", "pytest"]
dev = ["jupyter", "ipykernel", "maturin>=1.7,<2", "pytest", "pytest-timeout"]
[tool.uv]
cache-keys = [

View file

@ -2,12 +2,15 @@ use std::collections::HashMap;
use std::ffi::{CString, c_int, c_void};
use std::os::fd::{FromRawFd, OwnedFd, RawFd};
use std::ptr;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex, RwLock};
use std::time::Duration;
use data_plane::blob::{Blob, BlobView, ContentDigest, WritableArenaView};
use data_plane::bootstrap as dp_bootstrap;
use data_plane::data_plane::{BlobWriter, DataPlane, DataPlaneBootstrap, parse_actor_address};
use data_plane::data_plane::{
BlobWriter, DataPlane, DataPlaneBootstrap, StreamReader, StreamWriter, parse_actor_address,
};
use data_plane::path::DataPath;
use data_plane::protocol::{
BlobFailure, DataPlaneError, JobCapability, register_data_plane_codecs,
@ -23,24 +26,22 @@ use parking_lot::Mutex as ParkingMutex;
use pyo3::exceptions::{PyBufferError, PyPermissionError, PyRuntimeError};
use pyo3::ffi;
use pyo3::prelude::*;
use pyo3::types::{PyAny, PyModule};
use pyo3::types::{PyAny, PyBytes, PyModule};
use swactor::actor::{ActorAddress, ActorInterface, Ctx};
use swactor::config::RuntimeConfig;
use swactor::runtime::{Runtime, RuntimeParts};
use swactor_engine::{ActorCompletion, Engine, EngineHandle, TokioBackend, TokioConfig};
use swactor_transport::{CodecRegistry, CodecRemoteSink, TransportRouter};
use tokio::io::{AsyncWriteExt, BufWriter};
use tokio::net::UnixStream;
const ROUTE_POLL: Duration = Duration::from_millis(5);
const ROUTE_DEADLINE: Duration = Duration::from_secs(5);
const LEGACY_OUTPUT_ENV: &str = "SWACTOR_DATA_PLANE_OUTPUT";
pyo3::create_exception!(swactor, SwactorError, pyo3::exceptions::PyException);
pyo3::create_exception!(swactor, BootstrapError, SwactorError);
pyo3::create_exception!(swactor, DataPathError, SwactorError);
pyo3::create_exception!(swactor, BlobError, SwactorError);
pyo3::create_exception!(swactor, SessionError, SwactorError);
pyo3::create_exception!(swactor, StreamError, SwactorError);
fn bootstrap_error(message: impl Into<String>) -> PyErr {
PyErr::new::<BootstrapError, _>(message.into())
@ -56,6 +57,11 @@ fn data_plane_error(error: DataPlaneError) -> PyErr {
PyErr::new::<DataPathError, _>(format!("data path not found: {path}"))
}
DataPlaneError::Blob(reason) => PyErr::new::<BlobError, _>(format!("{reason:?}")),
DataPlaneError::WrongEntryType { .. }
| DataPlaneError::PathReplaced(_)
| DataPlaneError::PeerLost
| DataPlaneError::StreamFault(_)
| DataPlaneError::StreamClosed => PyErr::new::<StreamError, _>(error.to_string()),
DataPlaneError::Attachment(reason) => {
PyErr::new::<SessionError, _>(format!("attachment failed: {reason:?}"))
}
@ -229,7 +235,6 @@ fn build_child_routing(
pub struct PyDataPlane {
inner: Arc<DataPlane>,
_routing: Arc<JobRouting>,
legacy_output: Option<String>,
}
#[pymethods]
@ -257,20 +262,34 @@ impl PyDataPlane {
})
}
fn read_stream(&self, _path: String) -> PyResult<()> {
Err(PyRuntimeError::new_err(
"actor-driven stream reads are not installed",
))
fn read_stream<'py>(&self, py: Python<'py>, path: String) -> PyResult<Bound<'py, PyAny>> {
let path = DataPath::parse(path)
.map_err(|error| PyErr::new::<DataPathError, _>(error.to_string()))?;
let data_plane = self.inner.clone();
pyo3_async_runtimes::tokio::future_into_py(py, async move {
let reader = data_plane
.read_stream(&path)
.await
.map_err(data_plane_error)?;
Python::with_gil(|py| {
Py::new(
py,
PyStreamReader {
reader: Arc::new(tokio::sync::Mutex::new(reader)),
},
)
})
})
}
fn write_stream(&self, path: String) -> PyResult<PyStreamContext> {
DataPath::parse(path).map_err(|error| PyErr::new::<DataPathError, _>(error.to_string()))?;
let socket = self.legacy_output.clone().ok_or_else(|| {
PyRuntimeError::new_err("temporary deployment stream bridge is not configured")
})?;
let path = DataPath::parse(path)
.map_err(|error| PyErr::new::<DataPathError, _>(error.to_string()))?;
Ok(PyStreamContext {
socket,
data_plane: self.inner.clone(),
path,
stream: Arc::new(tokio::sync::Mutex::new(None)),
entered: Arc::new(AtomicBool::new(false)),
})
}
}
@ -618,9 +637,25 @@ unsafe fn release_buffer_format(view: *mut ffi::Py_buffer) {
}
}
#[pyclass(name = "StreamReader")]
pub struct PyStreamReader {
reader: Arc<tokio::sync::Mutex<StreamReader>>,
}
#[pymethods]
impl PyStreamReader {
fn read<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
let reader = self.reader.clone();
pyo3_async_runtimes::tokio::future_into_py(py, async move {
let bytes = reader.lock().await.read().await.map_err(data_plane_error)?;
Python::with_gil(|py| Ok(bytes.map(|bytes| PyBytes::new(py, &bytes).unbind())))
})
}
}
#[pyclass(name = "StreamWriter")]
pub struct PyStreamWriter {
stream: Arc<tokio::sync::Mutex<Option<BufWriter<UnixStream>>>>,
stream: Arc<tokio::sync::Mutex<Option<StreamWriter>>>,
}
#[pymethods]
@ -629,56 +664,67 @@ impl PyStreamWriter {
let stream = self.stream.clone();
pyo3_async_runtimes::tokio::future_into_py(py, async move {
let mut stream = stream.lock().await;
let stream = stream
let writer = stream
.as_mut()
.ok_or_else(|| PyRuntimeError::new_err("stream writer is closed"))?;
stream
.write_all(&bytes)
.await
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
Ok(())
writer.write(&bytes).await.map_err(data_plane_error)
})
}
}
#[pyclass(name = "_StreamWriteContext")]
pub struct PyStreamContext {
socket: String,
stream: Arc<tokio::sync::Mutex<Option<BufWriter<UnixStream>>>>,
data_plane: Arc<DataPlane>,
path: DataPath,
stream: Arc<tokio::sync::Mutex<Option<StreamWriter>>>,
entered: Arc<AtomicBool>,
}
#[pymethods]
impl PyStreamContext {
fn __aenter__<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
let socket = self.socket.clone();
if self.entered.swap(true, Ordering::AcqRel) {
return Err(PyRuntimeError::new_err(
"stream write context cannot be entered twice",
));
}
let data_plane = self.data_plane.clone();
let path = self.path.clone();
let state = self.stream.clone();
let entered = self.entered.clone();
pyo3_async_runtimes::tokio::future_into_py(py, async move {
let stream = UnixStream::connect(&socket).await.map_err(|error| {
PyRuntimeError::new_err(format!("connect output stream: {error}"))
})?;
*state.lock().await = Some(BufWriter::new(stream));
Python::with_gil(|py| Py::new(py, PyStreamWriter { stream: state }))
match data_plane.write_stream(&path).await {
Ok(writer) => {
*state.lock().await = Some(writer);
Python::with_gil(|py| Py::new(py, PyStreamWriter { stream: state }))
}
Err(error) => {
entered.store(false, Ordering::Release);
Err(data_plane_error(error))
}
}
})
}
fn __aexit__<'py>(
&self,
py: Python<'py>,
_exception_type: &Bound<'_, PyAny>,
exception_type: &Bound<'_, PyAny>,
_exception: &Bound<'_, PyAny>,
_traceback: &Bound<'_, PyAny>,
) -> PyResult<Bound<'py, PyAny>> {
let clean = exception_type.is_none();
let state = self.stream.clone();
pyo3_async_runtimes::tokio::future_into_py(py, async move {
if let Some(mut stream) = state.lock().await.take() {
stream
.flush()
.await
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
stream
.shutdown()
.await
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
let mut writer = state
.lock()
.await
.take()
.ok_or_else(|| PyRuntimeError::new_err("stream write context is not active"))?;
if clean {
writer.close().await.map_err(data_plane_error)?;
} else {
writer.abort().map_err(data_plane_error)?;
}
Ok(false)
})
@ -736,7 +782,6 @@ fn run(py: Python<'_>, main: Bound<'_, PyAny>) -> PyResult<()> {
PyDataPlane {
inner: Arc::new(bootstrap.data_plane),
_routing: Arc::new(routing),
legacy_output: std::env::var(LEGACY_OUTPUT_ENV).ok(),
},
)?;
let context = Py::new(py, PyContext { data })?;
@ -848,6 +893,19 @@ impl data_plane::source::BlobSourcePublisher for DebugSourceRegistrar {
}
}
#[cfg(all(debug_assertions, target_os = "linux"))]
struct DebugStreamCaptureConsumer {
bytes: Arc<ParkingMutex<Vec<u8>>>,
}
#[cfg(all(debug_assertions, target_os = "linux"))]
impl data_plane::data_plane::StreamConsumer for DebugStreamCaptureConsumer {
fn consume(&self, bytes: &[u8]) -> Result<(), String> {
self.bytes.lock().extend_from_slice(bytes);
Ok(())
}
}
#[cfg(all(debug_assertions, target_os = "linux"))]
#[pyclass(name = "_TestDataPlaneHost")]
struct PyTestDataPlaneHost {
@ -855,6 +913,7 @@ struct PyTestDataPlaneHost {
_engine: Engine,
handoff: data_plane::bootstrap::JobHandoff,
namespace_root: std::path::PathBuf,
stream_data_plane: Arc<DataPlane>,
}
#[cfg(all(debug_assertions, target_os = "linux"))]
@ -879,6 +938,57 @@ impl PyTestDataPlaneHost {
fn arena_fd(&self) -> RawFd {
std::os::fd::AsRawFd::as_raw_fd(&self.handoff.arena_fd)
}
fn read_stream<'py>(&self, py: Python<'py>, path: String) -> PyResult<Bound<'py, PyAny>> {
let path = DataPath::parse(path)
.map_err(|error| PyErr::new::<DataPathError, _>(error.to_string()))?;
let data_plane = self.stream_data_plane.clone();
pyo3_async_runtimes::tokio::future_into_py(py, async move {
let reader = data_plane
.read_stream(&path)
.await
.map_err(data_plane_error)?;
Python::with_gil(|py| {
Py::new(
py,
PyStreamReader {
reader: Arc::new(tokio::sync::Mutex::new(reader)),
},
)
})
})
}
fn capture_stream(&self, path: String) -> PyResult<PyTestStreamCapture> {
let path = DataPath::parse(path)
.map_err(|error| PyErr::new::<DataPathError, _>(error.to_string()))?;
let bytes = Arc::new(ParkingMutex::new(Vec::new()));
let consumer: Arc<dyn data_plane::data_plane::StreamConsumer> =
Arc::new(DebugStreamCaptureConsumer {
bytes: Arc::clone(&bytes),
});
let completion = self
.stream_data_plane
.collect_stream(path, consumer)
.map_err(data_plane_error)?;
Ok(PyTestStreamCapture { completion, bytes })
}
}
#[cfg(all(debug_assertions, target_os = "linux"))]
#[pyclass(name = "_TestStreamCapture")]
struct PyTestStreamCapture {
completion: ActorCompletion<Result<(), DataPlaneError>>,
bytes: Arc<ParkingMutex<Vec<u8>>>,
}
#[cfg(all(debug_assertions, target_os = "linux"))]
#[pymethods]
impl PyTestStreamCapture {
fn result<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyBytes>> {
self.completion.wait().map_err(data_plane_error)?;
Ok(PyBytes::new(py, &self.bytes.lock()))
}
}
#[cfg(all(debug_assertions, target_os = "linux"))]
@ -976,6 +1086,8 @@ fn _test_data_plane_host() -> PyResult<PyTestDataPlaneHost> {
))
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
let namespace = data_plane::namespace::NamespaceClient::new(runtime.clone(), namespace_proxy);
let stream_transport: Arc<dyn data_plane::stream_transport::StreamTransport> =
Arc::new(data_plane::stream_transport::LocalStreamTransport::new());
let host_session = runtime
.spawn(
data_plane::host::HostDataPlaneSessionActor::new(
@ -998,11 +1110,12 @@ fn _test_data_plane_host() -> PyResult<PyTestDataPlaneHost> {
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?,
],
},
namespace: Some(namespace),
namespace: Some(namespace.clone()),
transfer_receiver: Some(Arc::new(DebugBlobReceiver)),
source_sender: Some(source_sender),
source_publisher: Some(source_publisher),
source_sender: Some(Arc::clone(&source_sender)),
source_publisher: Some(Arc::clone(&source_publisher)),
route_registrar: Some(registrar),
stream_transport: Some(Arc::clone(&stream_transport)),
},
)
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?,
@ -1010,6 +1123,59 @@ fn _test_data_plane_host() -> PyResult<PyTestDataPlaneHost> {
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
data_plane::host::install_session_env(&mut handoff, host_session, capability);
let mut sink_arena = data_plane::arena::ArenaManager::boot(data_plane::arena::ArenaConfig {
node_id: data_plane::arena::NodeId(2),
reservation_ceiling: 1 << 20,
base_alignment: 64,
})
.map_err(|error| PyRuntimeError::new_err(format!("debug sink arena: {error:?}")))?;
let sink_handoff = data_plane::bootstrap::write_bootstrap(
&mut sink_arena,
data_plane::bootstrap::BootstrapSpec {
arena_generation: 2,
alignment: 64,
},
)
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
let sink_session = runtime
.spawn(
data_plane::host::HostDataPlaneSessionActor::new(
data_plane::host::HostDataPlaneConfig {
runtime: runtime.clone(),
arena: sink_arena,
arena_generation: 2,
session_generation: 2,
capability,
job_context: data_plane::path::JobContext {
run_id: "test-run".to_owned(),
read_prefixes: vec![
DataPath::parse("/runs/test-run/results")
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?,
],
write_prefixes: Vec::new(),
},
namespace: Some(namespace),
transfer_receiver: None,
source_sender: None,
source_publisher: None,
route_registrar: None,
stream_transport: Some(stream_transport),
},
)
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?,
)
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
let stream_data_plane = Arc::new(
future::block_on(DataPlaneBootstrap::attach(
sink_handoff.arena_fd,
runtime.clone(),
sink_session,
capability,
))
.map_err(data_plane_error)?
.data_plane,
);
driver.enable_actor_bridge(iroh_driver::ActorBridgeConfig {
runtime: runtime.clone(),
codec: codecs,
@ -1031,6 +1197,7 @@ fn _test_data_plane_host() -> PyResult<PyTestDataPlaneHost> {
_engine: engine,
handoff,
namespace_root,
stream_data_plane,
})
}
@ -1040,14 +1207,17 @@ pub fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
module.add("DataPathError", module.py().get_type::<DataPathError>())?;
module.add("BlobError", module.py().get_type::<BlobError>())?;
module.add("SessionError", module.py().get_type::<SessionError>())?;
module.add("StreamError", module.py().get_type::<StreamError>())?;
module.add_class::<PyDataPlane>()?;
module.add_class::<PyBlob>()?;
module.add_class::<PyBlobView>()?;
module.add_class::<PyStreamWriter>()?;
module.add_class::<PyStreamReader>()?;
module.add_class::<PyContext>()?;
module.add_function(wrap_pyfunction!(run, module)?)?;
#[cfg(all(debug_assertions, target_os = "linux"))]
{
module.add_class::<PyTestStreamCapture>()?;
module.add_class::<PyTestDataPlaneHost>()?;
module.add_function(wrap_pyfunction!(_test_data_plane_host, module)?)?;
}

View file

@ -2,16 +2,15 @@
from __future__ import annotations
import asyncio
import ctypes
import json
import os
import struct
import subprocess
import importlib
import socket
import sys
import tempfile
import threading
from pathlib import Path
import pytest
@ -215,39 +214,29 @@ def test_missing_path_and_authorization_are_typed(monkeypatch, host):
swactor.run(main)
def test_temporary_output_stream_bridge_remains_available(monkeypatch, host, tmp_path):
def test_native_stream_round_trip_has_ordered_eof(monkeypatch, host):
install_host_env(monkeypatch, host)
socket_path = tmp_path / "output.sock"
listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
listener.bind(str(socket_path))
listener.listen(1)
monkeypatch.setenv("SWACTOR_DATA_PLANE_OUTPUT", str(socket_path))
received = []
def receive():
connection, _ = listener.accept()
with connection:
chunks = []
while chunk := connection.recv(4096):
chunks.append(chunk)
received.append(b"".join(chunks))
receiver = threading.Thread(target=receive)
receiver.start()
async def main(ctx):
async def receive():
reader = await ctx.data.read_stream(
"/runs/self/results/predictions"
)
while (chunk := await reader.read()) is not None:
received.append(chunk)
receiver = asyncio.create_task(receive())
async with ctx.data.write_stream(
"/runs/self/results/predictions"
) as stream:
await stream.write(b"temporary-result")
await stream.write(b"native-")
await stream.write(b"result")
await receiver
try:
swactor.run(main)
receiver.join(timeout=2)
assert not receiver.is_alive()
assert received == [b"temporary-result"]
finally:
listener.close()
swactor.run(main)
assert b"".join(received) == b"native-result"
assert "SWACTOR_DATA_PLANE_OUTPUT" not in host.env()
def test_invalid_capability_prevents_main(monkeypatch, host):
@ -350,34 +339,18 @@ def test_real_exec_attachment_and_blob_mapping(host):
not Path("/dev/nvidia0").exists(),
reason="CUDA device is unavailable",
)
def test_real_exec_tinygrad_cuda_scenario(host, tmp_path):
socket_path = tmp_path / "cuda-output.sock"
listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
listener.bind(str(socket_path))
listener.listen(1)
received = []
def receive():
connection, _ = listener.accept()
with connection:
chunks = []
while chunk := connection.recv(4096):
chunks.append(chunk)
received.append(b"".join(chunks))
receiver = threading.Thread(target=receive)
receiver.start()
def test_real_exec_tinygrad_cuda_scenario(host):
capture = host.capture_stream("/runs/self/results/inference")
env = {
key: value
for key, value in os.environ.items()
if key not in (*BOOTSTRAP_ENV, *OLD_WAKE_ENV)
if key not in (*BOOTSTRAP_ENV, *OLD_WAKE_ENV, "SWACTOR_DATA_PLANE_OUTPUT")
}
env.update(dict(host.env()))
env.update(
{
"CUDA_PTX": "1",
"DEV": "CUDA",
"SWACTOR_DATA_PLANE_OUTPUT": str(socket_path),
}
)
script = (
@ -387,21 +360,16 @@ def test_real_exec_tinygrad_cuda_scenario(host, tmp_path):
/ "jobs"
/ "tiny_linear_inference.py"
)
try:
result = subprocess.run(
[sys.executable, str(script)],
env=env,
pass_fds=(host.arena_fd(),),
text=True,
capture_output=True,
timeout=45,
check=False,
)
receiver.join(timeout=2)
assert result.returncode == 0, result.stderr
assert not receiver.is_alive()
payload = json.loads(received[0])
assert payload["device"].startswith("CUDA")
assert payload["output"] == pytest.approx([2.75, -8.75])
finally:
listener.close()
result = subprocess.run(
[sys.executable, str(script)],
env=env,
pass_fds=(host.arena_fd(),),
text=True,
capture_output=True,
timeout=45,
check=False,
)
assert result.returncode == 0, result.stderr
payload = json.loads(capture.result())
assert payload["device"].startswith("CUDA")
assert payload["output"] == pytest.approx([2.75, -8.75])

View file

@ -1282,6 +1282,30 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/af/33/ee4519fa02ed11a94aef9559552f3b17bb863f2ecfe1a35dc7f548cde231/matplotlib_inline-0.2.1-py3-none-any.whl", hash = "sha256:d56ce5156ba6085e00a9d54fead6ed29a9c47e215cd1bba2e976ef39f5710a76", size = 9516, upload-time = "2025-10-23T09:00:20.675Z" },
]
[[package]]
name = "maturin"
version = "1.14.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "tomli", marker = "python_full_version < '3.11'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/e7/b3/addd877f871fb1860d46d3a4f206ecb10b946c85846805e6367631926fd3/maturin-1.14.1.tar.gz", hash = "sha256:9d6577a62cd08e0ceba7a0db06fb098e0c9b1b3429bad747a4f3a18215a1b3df", size = 369637, upload-time = "2026-06-19T05:19:49.774Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/f4/f0/97c5a5bd9c71653a066c0976a484eaaae50b9369557838a4176b7b0bdaa5/maturin-1.14.1-py3-none-linux_armv6l.whl", hash = "sha256:522292398945442cdafa9daeb2271b2340fbde57027b818f923f88eab04174f8", size = 10207496, upload-time = "2026-06-19T05:19:09.321Z" },
{ url = "https://files.pythonhosted.org/packages/fe/83/294bca639b0e052f1e2f65199b3db258780c7d4e31408b934c9c974a1379/maturin-1.14.1-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:ffe5ad71f21d1e6603c4dd75f7fee34adf5ed5ebcebb692886549888ebb329ed", size = 19680113, upload-time = "2026-06-19T05:19:13.43Z" },
{ url = "https://files.pythonhosted.org/packages/43/b6/79c881410a3b1c187f7eb3d407aecae646c6a4433d630d72200359015e83/maturin-1.14.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:f3306078070c1508fd715b9116070cbcaff5959024272a9f1e6f5cb29768b86c", size = 10169205, upload-time = "2026-06-19T05:19:16.615Z" },
{ url = "https://files.pythonhosted.org/packages/93/9d/44b6f26dcb7f7a04c5501ac2dbb6ca1490150682baa525ca5860504f9eab/maturin-1.14.1-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:cd457cd88961156e26379e1155bd287cc0ec1c8b2f1582b0660fb31b87c8842d", size = 10188098, upload-time = "2026-06-19T05:19:19.736Z" },
{ url = "https://files.pythonhosted.org/packages/1a/bd/9c0d5d6983905ce2c9edaa073a7e89355a9cf7f396988e05d32f1c37785d/maturin-1.14.1-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:dfc54ae32e6fcb18302193ab9a30b0b25eefffba994ae13238974805533ef75e", size = 10627576, upload-time = "2026-06-19T05:19:22.713Z" },
{ url = "https://files.pythonhosted.org/packages/e5/33/b096412bd6a7cb399652b260666f901adf88a687181a6dbd6a3f89f0a94e/maturin-1.14.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:a131d912b5267e640bc96d70f4914e10590aed64082ec9abacba7cea52004224", size = 10085181, upload-time = "2026-06-19T05:19:25.69Z" },
{ url = "https://files.pythonhosted.org/packages/56/8d/08c3bf469c38a23c9e6c877e338193001eb604d010fedc08341974e38528/maturin-1.14.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:be18fc568fb76884c0205456336892a75105ec398e6b667cd777c6268bd06d69", size = 10026363, upload-time = "2026-06-19T05:19:28.904Z" },
{ url = "https://files.pythonhosted.org/packages/3a/a4/c4d1a92839f8745ab4aab988a7db884a79d6d710bd3b286fcf9316dece1a/maturin-1.14.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:994a0c8ba3ad8a92b3a9ee1b02645d200d610216b15cff5102b0fe65e8e08666", size = 13321347, upload-time = "2026-06-19T05:19:32.411Z" },
{ url = "https://files.pythonhosted.org/packages/b3/fa/170f04624d03fd07d2a8b1b67de83a127af93aef9eaa425839553347297b/maturin-1.14.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:be80866363e605d137991b491a741a84cde9ae350183c4c85f49690ca9aaaa65", size = 10877609, upload-time = "2026-06-19T05:19:35.448Z" },
{ url = "https://files.pythonhosted.org/packages/61/ad/1ae2e1d0ded282bf2c55ac13f0811d87deb425e200ae64a15785675dede9/maturin-1.14.1-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:5282dffd4b539d2be245f4e5b1a5ab6bc1033b58f4a4872f5833f9d43c954aa4", size = 10417316, upload-time = "2026-06-19T05:19:38.28Z" },
{ url = "https://files.pythonhosted.org/packages/fb/27/bf677183920718da49cd7982d6a3ffc440aad8919329f571d189f81b7bdf/maturin-1.14.1-py3-none-win32.whl", hash = "sha256:1a04de0a20188f95c721b5702eed18140bdcccb28c386797093eca3f62f4d4e0", size = 8931293, upload-time = "2026-06-19T05:19:41.183Z" },
{ url = "https://files.pythonhosted.org/packages/63/4b/585adeb9167b08d3cdff0032a938b0e72655c92003df4f52c3f696a1bcc2/maturin-1.14.1-py3-none-win_amd64.whl", hash = "sha256:3c9f94640ecc4895e94abaf834a0684430032c865b2748a36c12461fd9252fdd", size = 10314067, upload-time = "2026-06-19T05:19:44.389Z" },
{ url = "https://files.pythonhosted.org/packages/51/d4/dac8c0720ae246be1700afb6fbdbbea20fe35b13f6570b2f70faa005df77/maturin-1.14.1-py3-none-win_arm64.whl", hash = "sha256:15cea8fcb3ba47dd636f50092bb34baea8b04ac777392f23e6bf8a9a61efb894", size = 9718943, upload-time = "2026-06-19T05:19:47.49Z" },
]
[[package]]
name = "mistune"
version = "3.2.0"
@ -1644,6 +1668,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" },
]
[[package]]
name = "pytest-timeout"
version = "2.4.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
{ name = "pytest", version = "9.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/ac/82/4c9ecabab13363e72d880f2fb504c5f750433b2b6f16e99f4ec21ada284c/pytest_timeout-2.4.0.tar.gz", hash = "sha256:7e68e90b01f9eff71332b25001f85c75495fc4e3a836701876183c4bcfd0540a", size = 17973, upload-time = "2025-05-05T19:44:34.99Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl", hash = "sha256:c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2", size = 14382, upload-time = "2025-05-05T19:44:33.502Z" },
]
[[package]]
name = "python-dateutil"
version = "2.9.0.post0"
@ -2313,8 +2350,10 @@ dev = [
{ name = "ipykernel", version = "6.31.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
{ name = "ipykernel", version = "7.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
{ name = "jupyter" },
{ name = "maturin" },
{ name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
{ name = "pytest", version = "9.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
{ name = "pytest-timeout" },
]
[package.metadata]
@ -2323,7 +2362,9 @@ dev = [
dev = [
{ name = "ipykernel" },
{ name = "jupyter" },
{ name = "maturin", specifier = ">=1.7,<2" },
{ name = "pytest" },
{ name = "pytest-timeout" },
]
[[package]]

View file

@ -11,3 +11,6 @@ crate-type = ["cdylib"]
swactor = { path = "../../..", default-features = false, features = ["wasm", "std"] }
swactor-engine = { path = "../../engine", default-features = false }
wasm-bindgen = "0.2"
[lints]
workspace = true

View file

@ -103,6 +103,12 @@ pub struct WasmRuntime {
backend: SteppingBackend,
}
impl Default for WasmRuntime {
fn default() -> Self {
Self::new()
}
}
#[wasm_bindgen]
impl WasmRuntime {
#[wasm_bindgen(constructor)]

View file

@ -23,3 +23,6 @@ demo-control = []
[dev-dependencies]
proptest = "1"
tower = { version = "0.5", features = ["util"] }
[lints]
workspace = true

View file

@ -187,9 +187,10 @@ mod properties {
concurrent in any::<bool>(),
destination_disappears in any::<bool>(),
) {
let mut config = RuntimeConfig::default();
config.worker_count = 1;
let parts = RuntimeParts::new(config);
let parts = RuntimeParts::new(RuntimeConfig {
worker_count: 1,
..RuntimeConfig::default()
});
let runtime = parts.runtime().clone();
let backend = SteppingBackend::new();
let _engine =

View file

@ -236,7 +236,9 @@ struct NodeCard {
errors: Vec<String>,
cpu: Option<CpuSnapshot>,
gpu: Option<GpuSnapshot>,
memory: Option<telemetry::hardware::memory::HostMemorySample>,
net: Option<crate::hardware_view::NetSnapshot>,
storage: Option<telemetry::hardware::storage::HostStorageSample>,
process: Option<crate::hardware_view::ProcessSnapshot>,
history: Vec<HardwareHistorySnapshot>,
actor_summary: ActorSummarySnapshot,
@ -329,15 +331,9 @@ fn stream_key(stream: &StreamEvent) -> String {
fn node_card(node: &FusedNode, now: Instant) -> NodeCard {
let summary = node.hardware.summary();
let totals = node.actors.totals();
let mut roster: Vec<RosterRow> = node.actors.actors.values().map(roster_row).collect();
// Busiest actors first; ties fall back to address for stable rendering.
roster.sort_by(|left, right| {
right
.msg_per_sec
.partial_cmp(&left.msg_per_sec)
.map_or(std::cmp::Ordering::Equal, |order| order)
.then_with(|| left.address.cmp(&right.address))
});
// Address-keyed map order is the stable default. Volatile telemetry must
// not move a row out from under the pointer; the page offers explicit sorts.
let roster: Vec<RosterRow> = node.actors.actors.values().map(roster_row).collect();
NodeCard {
stream: StreamKeySnapshot {
key: stream_key(&node.stream),
@ -352,7 +348,9 @@ fn node_card(node: &FusedNode, now: Instant) -> NodeCard {
errors: node.hardware.errors(),
cpu: node.hardware.cpu.as_ref().map(CpuSnapshot::from),
gpu: node.hardware.gpu.as_ref().map(GpuSnapshot::from),
memory: node.hardware.memory.clone(),
net: node.hardware.net.clone(),
storage: node.hardware.storage.clone(),
process: node.hardware.process.clone(),
history: node
.hardware
@ -362,11 +360,16 @@ fn node_card(node: &FusedNode, now: Instant) -> NodeCard {
ms_ago: duration_ms(now.duration_since(sample.at)),
sample_unix_ms: sample.sample_unix_ms,
cpu_total_percent: sample.cpu_total_percent,
cpu_cores_percent: sample.cpu_cores_percent.clone(),
gpu_max_percent: sample.gpu_max_percent,
gpu_memory_used_mib: sample.gpu_memory_used_mib,
gpu_memory_total_mib: sample.gpu_memory_total_mib,
net_rx_bps: sample.net_rx_bps,
net_tx_bps: sample.net_tx_bps,
memory_used_percent: sample.memory_used_percent,
memory_pressure_some_avg10: sample.memory_pressure_some_avg10,
storage_used_percent: sample.storage_used_percent,
io_pressure_some_avg10: sample.io_pressure_some_avg10,
})
.collect(),
actor_summary: ActorSummarySnapshot {
@ -667,6 +670,191 @@ mod tests {
assert_eq!(still_live.first(), Some(&"orch"));
}
#[test]
fn hardware_channels_fold_into_fleet_snapshot() {
let view = ControlPlaneView::default();
let stream = StreamId::new(NodeId::new("worker"), Lifetime(1));
let cpu = json!({
"schema":"host.cpu.v1",
"seq":1,
"sample_unix_ms":1_000,
"query_elapsed_ms":1,
"host":{
"logical_cpus":8,
"total_percent":42.5,
"idle_percent":57.5,
"iowait_percent":0.0,
"steal_percent":0.0,
"load1":1.0,
"load5":0.5,
"load15":0.25
},
"cores":[
{"index":0,"total_percent":25.0,"idle_percent":75.0,"iowait_percent":0.0,"steal_percent":0.0},
{"index":1,"total_percent":60.0,"idle_percent":40.0,"iowait_percent":0.0,"steal_percent":0.0}
],
"processes":[],
"error":null
});
let gpu = json!({
"schema":"host.gpu.v1",
"seq":1,
"sample_unix_ms":1_000,
"query_elapsed_ms":2,
"gpus":[{
"index":0,
"uuid":"gpu-0",
"name":"test gpu",
"memory_used_mib":512,
"memory_total_mib":4096,
"utilization_gpu_percent":71,
"utilization_memory_percent":12,
"temperature_c":55,
"power_draw_w":25.0
}],
"processes":[],
"error":null
});
let memory = json!({
"schema":"host.memory.v1",
"seq":1,
"sample_unix_ms":2_000,
"query_elapsed_ms":1,
"total_bytes":16_000,
"available_bytes":4_000,
"used_bytes":12_000,
"cached_bytes":2_000,
"swap_total_bytes":8_000,
"swap_used_bytes":1_000,
"pressure":{
"some_avg10":1.25,
"some_avg60":0.75,
"some_avg300":0.5,
"some_total_us":100,
"full_avg10":0.1,
"full_avg60":0.05,
"full_avg300":0.01,
"full_total_us":10
},
"error":null
});
let net_sample = |seq, sample_unix_ms, rx_bytes, tx_bytes| {
json!({
"schema":"host.net.v1",
"seq":seq,
"sample_unix_ms":sample_unix_ms,
"interfaces":[{
"name":"eth0",
"rx_bytes":rx_bytes,
"tx_bytes":tx_bytes,
"rx_packets":10,
"tx_packets":10,
"rx_errors":0,
"tx_errors":0,
"rx_dropped":0,
"tx_dropped":0
}],
"error":null
})
};
let storage = json!({
"schema":"host.storage.v1",
"seq":1,
"sample_unix_ms":2_000,
"query_elapsed_ms":1,
"filesystems":[{
"mount":"/",
"total_bytes":100_000,
"used_bytes":80_000,
"available_bytes":20_000,
"used_percent":80.0
}],
"pressure":{
"some_avg10":2.5,
"some_avg60":1.5,
"some_avg300":0.5,
"some_total_us":200,
"full_avg10":0.2,
"full_avg60":0.1,
"full_avg300":0.05,
"full_total_us":20
},
"error":null
});
for (position, channel, payload) in [
(0, "host.cpu", cpu),
(1, "host.gpu", gpu),
(2, "host.memory", memory),
(3, "host.net", net_sample(0, 1_000, 1_000, 2_000)),
(4, "host.net", net_sample(1, 2_000, 2_000, 3_500)),
(5, "host.storage", storage),
] {
ingest_json(
&view,
&stream,
position,
channel,
serde_json::to_vec(&payload).expect("hardware payload"),
);
}
let snapshot = view.snapshot_json();
let node = &snapshot["live"][0];
assert_eq!(node["cpu"]["host"]["total_percent"], json!(42.5));
assert_eq!(node["gpu"]["gpus"][0]["utilization_gpu_percent"], json!(71));
assert_eq!(node["cpu"]["cores"][1]["total_percent"], json!(60.0));
assert_eq!(node["memory"]["used_bytes"], json!(12_000));
assert_eq!(node["net"]["interfaces"][0]["rx_bps"], json!(1_000.0));
assert_eq!(node["net"]["interfaces"][0]["tx_bps"], json!(1_500.0));
assert_eq!(
node["storage"]["filesystems"][0]["used_percent"],
json!(80.0)
);
assert_eq!(node["history"][0]["cpu_cores_percent"], json!([25.0, 60.0]));
assert_eq!(
node["history"][0]["memory_pressure_some_avg10"],
json!(1.25)
);
assert_eq!(node["history"][0]["io_pressure_some_avg10"], json!(2.5));
assert_eq!(node["last_sample_unix_ms"], json!(2_000));
assert!(node["errors"].as_array().expect("errors").is_empty());
}
#[test]
fn roster_default_order_does_not_follow_volatile_throughput() {
let view = ControlPlaneView::default();
let stream = StreamId::new(NodeId::new("node"), Lifetime(1));
ingest_json(
&view,
&stream,
0,
"runtime.actors",
actors_payload(
0,
json!([
{ "address": "zz", "messages_processed": 100 },
{ "address": "aa", "messages_processed": 1 }
]),
),
);
{
let mut state = view.state.write();
let actors = &mut state.streams.get_mut("node#1").expect("node").actors.actors;
actors.get_mut("zz").expect("zz actor").msg_per_sec = 10_000.0;
actors.get_mut("aa").expect("aa actor").msg_per_sec = 1.0;
}
let snapshot = view.snapshot_json();
let addresses: Vec<&str> = snapshot["live"][0]["roster"]
.as_array()
.expect("roster")
.iter()
.map(|actor| actor["address"].as_str().expect("address"))
.collect();
assert_eq!(addresses, vec!["aa", "zz"]);
}
#[test]
fn newer_life_generation_evicts_superseded_stream() {
let view = ControlPlaneView::default();
@ -701,7 +889,7 @@ mod tests {
fn stale_pool_is_hard_capped() {
let view = ControlPlaneView::default();
for index in 0..(STALE_POOL_CAP as u64 + 5) {
let stream = StreamId::new(NodeId::new(&format!("old-{index}")), Lifetime(1));
let stream = StreamId::new(NodeId::new(format!("old-{index}")), Lifetime(1));
ingest_json(
&view,
&stream,

View file

@ -87,6 +87,12 @@
table { width: 100%; border-collapse: collapse; font-family: var(--mono); font-size: 12px; }
th, td { text-align: left; padding: 7px 8px; border-bottom: 1px solid var(--divider); font-variant-numeric: tabular-nums; }
th { color: var(--amber); font-weight: 600; font-size: 10px; text-transform: uppercase; letter-spacing: .08em; }
.roster-sort-controls { display: inline-flex; gap: 2px; margin-left: 4px; vertical-align: middle; }
.roster-sort-button { position: relative; width: 14px; height: 14px; padding: 0; color: var(--muted); background: transparent; border: 1px solid transparent; border-radius: var(--r); font-size: 0; line-height: 0; cursor: pointer; }
.roster-sort-button::before { content: ""; position: absolute; left: 3px; width: 0; height: 0; border-left: 3px solid transparent; border-right: 3px solid transparent; }
.roster-sort-button[data-direction="asc"]::before { top: 3px; border-bottom: 5px solid currentColor; }
.roster-sort-button[data-direction="desc"]::before { bottom: 3px; border-top: 5px solid currentColor; }
.roster-sort-button:hover, .roster-sort-button[aria-pressed="true"] { color: var(--cyan); border-color: var(--cyan); background: var(--panel-hover); }
tbody tr[data-addr] { cursor: pointer; transition: background var(--t); }
tbody tr[data-addr]:hover { background: var(--row-hover); }
tbody tr[data-selected="true"] { background: var(--selected); box-shadow: inset 2px 0 0 var(--selected-edge); }
@ -102,6 +108,33 @@
.empty { padding: 24px; border: 1px dashed var(--border); border-radius: var(--r); color: var(--muted); font-family: var(--mono); font-size: 12px; }
.notice { font-size: 12px; color: var(--muted); margin: 6px 0; }
canvas { width: 100%; height: 64px; background: var(--inset); border: 1px solid var(--border); border-radius: var(--r); }
.hardware-source { margin-bottom: 8px; }
.hardware-grid { display: grid; grid-template-columns: repeat(12, minmax(0, 1fr)); gap: 8px; }
.hw-card { min-width: 0; padding: 10px; background: var(--inset); border: 1px solid var(--divider); border-radius: var(--r); }
.hw-card.cpu { grid-column: span 7; }
.hw-card.memory { grid-column: span 5; }
.hw-card.storage, .hw-card.gpu, .hw-card.network { grid-column: span 4; }
.hw-card-head { display: flex; align-items: baseline; justify-content: space-between; gap: 8px; margin-bottom: 7px; }
.hw-card-title { color: var(--muted); font: 700 10px/1.2 var(--mono); text-transform: uppercase; letter-spacing: .09em; }
.hw-card-value { color: var(--text); font: 700 16px/1 var(--mono); font-variant-numeric: tabular-nums; }
.hw-card-subtitle { margin-top: 5px; color: var(--muted); font: 11px/1.35 var(--mono); }
.metric-pair { display: flex; justify-content: space-between; gap: 8px; margin-top: 6px; color: var(--muted); font: 11px/1.3 var(--mono); }
.metric-pair strong { color: var(--text); font-weight: 600; }
.pressure-value.ok { color: var(--ok); } .pressure-value.warn { color: var(--amber); } .pressure-value.bad { color: var(--bad); }
canvas.thread-graph { height: 72px; background: var(--bg); border-color: var(--divider); image-rendering: pixelated; }
.thread-strip { display: grid; grid-template-columns: repeat(auto-fit, minmax(26px, 1fr)); gap: 3px; margin-top: 5px; }
.thread-cell { min-width: 0; height: 20px; display: grid; place-items: center; border: 1px solid var(--divider); border-radius: var(--r); color: var(--text); font: 9px/1 var(--mono); font-variant-numeric: tabular-nums; }
.network-list { display: grid; gap: 6px; }
.network-row { display: grid; grid-template-columns: minmax(0, 1fr) auto; gap: 8px; font: 11px/1.25 var(--mono); }
.network-row .interface { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; color: var(--muted); }
.hw-empty { color: var(--muted); font: 11px/1.35 var(--mono); }
@media (max-width: 900px) {
.hw-card.cpu { grid-column: span 12; }
.hw-card.memory, .hw-card.storage, .hw-card.gpu, .hw-card.network { grid-column: span 6; }
}
@media (max-width: 600px) {
.hw-card.cpu, .hw-card.memory, .hw-card.storage, .hw-card.gpu, .hw-card.network { grid-column: span 12; }
}
.err { color: var(--bad); }
@media (prefers-reduced-motion: reduce) { * { transition-duration: 0.01ms !important; animation: none !important; } }
</style>
@ -134,6 +167,8 @@ const params = new URLSearchParams(window.location.search);
let selectedStream = params.get('stream') || null;
let selectedActor = params.get('actor') || null;
let rosterFilter = '';
let rosterSort = { key: 'address', direction: 'asc' };
let hardwareSource = 'node';
let lastSnapshot = null;
let detailTimer = null;
// A1: last-rendered HTML per region. A poll that yields identical markup
@ -144,6 +179,14 @@ let lastRosterHtml = null;
let lastMachineHtml = null;
let lastDossierHtml = null;
window.addEventListener('dashboard-hardware-source', event => {
const next = event.detail?.source === 'orchestrator' ? 'orchestrator' : 'node';
if (next === hardwareSource) return;
hardwareSource = next;
lastMachineHtml = null;
if (lastSnapshot) render();
});
// Receipt ages (dossier) render as empty spans carrying an absolute epoch —
// stable across polls — and a 1 Hz pass rewrites their textContent. All other
// ticking text (card seen/stats) is written per-poll by updateCardTexts.
@ -173,6 +216,17 @@ function fmtRate(value) {
if (n >= 1e3) return (n / 1e3).toFixed(1) + ' k';
return n.toFixed(n < 10 ? 1 : 0);
}
function fmtBytes(value) {
const n = Number(value);
if (!Number.isFinite(n)) return '—';
const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
let scaled = Math.max(0, n), unit = 0;
while (scaled >= 1024 && unit < units.length - 1) {
scaled /= 1024;
unit += 1;
}
return scaled.toFixed(scaled < 10 && unit > 0 ? 1 : 0) + ' ' + units[unit];
}
function typeShort(name) {
if (!name) return '—';
const parts = String(name).split('::');
@ -291,6 +345,14 @@ function updateCardTexts(nodes) {
}
}
function hardwareNodeFor(node, live, stale) {
if (hardwareSource !== 'orchestrator' || node.stream.origin === 'orchestrator') {
return { node, mirrored: false };
}
const orchestrator = live.concat(stale).find(candidate => candidate.stream.origin === 'orchestrator');
return orchestrator ? { node: orchestrator, mirrored: true } : { node, mirrored: false };
}
function renderNode(page, node, live, stale) {
const summary = node.actor_summary || {};
const active = document.activeElement;
@ -333,7 +395,8 @@ function renderNode(page, node, live, stale) {
if (actorCount) actorCount.textContent = fmt(summary.actors);
const seen = page.querySelector('h2 [data-seen]');
if (seen) seen.textContent = node.live ? 'live' : 'stale ' + ago(node.last_seen_ms_ago) + ' ago';
const machine = machineDetail(node);
const machineSource = hardwareNodeFor(node, live, stale);
const machine = machineDetail(machineSource.node, machineSource.mirrored);
const machineSlot = document.getElementById('machine-detail');
if (machineSlot && machine !== lastMachineHtml) {
machineSlot.innerHTML = machine;
@ -347,67 +410,192 @@ function renderNode(page, node, live, stale) {
}
}
function machineDetail(node) {
const cpu = node.cpu, gpu = node.gpu, net = node.net;
const rows = [];
const cpuTotal = cpu && cpu.host ? cpu.host.total_percent : null;
rows.push(`<div class="bar-row"><span class="name">CPU</span>${bar(cpuTotal)}<span class="num">${cpuTotal == null ? '—' : fmt(cpuTotal, 1) + '%'}</span></div>`);
if (gpu && gpu.gpus) {
gpu.gpus.forEach((device, index) => {
const util = device.utilization_gpu_percent;
const used = device.memory_used_mib || 0, total = device.memory_total_mib || 0;
rows.push(`<div class="bar-row"><span class="name">GPU ${fmt(index)}</span>${bar(util)}<span class="num">${util == null ? '—' : fmt(util) + '%'}</span></div>`);
if (total) rows.push(`<div class="bar-row"><span class="name">&nbsp;</span>${bar(used, total)}<span class="num">${fmt(used)}/${fmt(total)} MiB</span></div>`);
});
}
if (net && net.interfaces) {
net.interfaces.slice(0, 4).forEach(nic => {
rows.push(`<div class="bar-row"><span class="name" title="${esc(nic.name)}">${esc(nic.name.slice(0, 12))}</span><span class="num" style="width:auto;flex:1">${fmtRate(nic.rx_bps)}↓ ${fmtRate(nic.tx_bps)}↑</span></div>`);
});
}
if (node.history && node.history.length) drawMachineHistory(node.history);
const errors = (node.errors || []).map(e => `<div class="notice err">${esc(e)}</div>`).join('');
return `<div class="bars">${rows.join('')}</div><canvas id="machine-spark" width="600" height="64"></canvas>${errors}`;
function percentOf(used, total) {
const numerator = Number(used), denominator = Number(total);
return Number.isFinite(numerator) && Number.isFinite(denominator) && denominator > 0
? numerator * 100 / denominator
: null;
}
function drawMachineHistory(history) {
function pressureTone(value, warn = 1, bad = 5) {
const pressure = Number(value);
if (!Number.isFinite(pressure)) return '';
return pressure >= bad ? 'bad' : pressure >= warn ? 'warn' : 'ok';
}
function machineDetail(node, mirrored = false) {
const cpu = node.cpu, gpu = node.gpu, memory = node.memory, net = node.net, storage = node.storage;
const cpuHost = cpu?.host;
const cpuTotal = cpuHost?.total_percent;
const cores = [...(cpu?.cores || [])].sort((left, right) => left.index - right.index);
const threadHeight = Math.min(92, Math.max(42, cores.length * 4));
drawThreadGraph(node.history || [], cores.map(core => core.total_percent));
const threadCells = cores.map(core => {
const value = core.total_percent;
const load = value == null ? 0 : Math.max(0, Math.min(100, Number(value)));
const label = value == null ? '—' : fmt(value);
return `<span class="thread-cell" title="Thread ${fmt(core.index)} · ${value == null ? 'no sample' : fmt(value, 1) + '%'}" style="background:linear-gradient(to top,var(--ok) 0%,var(--ok) ${load}%,var(--bg) ${load}%,var(--bg) 100%)">${label}</span>`;
}).join('');
const load = [cpuHost?.load1, cpuHost?.load5, cpuHost?.load15].map(value => fmt(value, 2)).join(' / ');
const cpuLabel = cpuTotal == null ? '—' : fmt(cpuTotal, 1) + '%';
const memoryPercent = percentOf(memory?.used_bytes, memory?.total_bytes);
const memoryPressure = memory?.pressure?.some_avg10;
const swapPercent = percentOf(memory?.swap_used_bytes, memory?.swap_total_bytes);
const memoryLabel = memoryPercent == null ? '—' : fmt(memoryPercent, 1) + '%';
const filesystem = storage?.filesystems?.[0];
const storagePercent = filesystem?.used_percent;
const ioPressure = storage?.pressure?.some_avg10;
const storageLabel = storagePercent == null ? '—' : fmt(storagePercent, 1) + '%';
const gpuDevices = gpu?.gpus || [];
const gpuMax = gpuDevices.reduce((maximum, device) => {
const value = Number(device.utilization_gpu_percent);
return Number.isFinite(value) ? Math.max(maximum, value) : maximum;
}, 0);
const gpuRows = gpuDevices.map((device, index) => {
const utilization = device.utilization_gpu_percent;
const memoryPercent = percentOf(device.memory_used_mib, device.memory_total_mib);
return `<div class="metric-pair"><span>GPU ${fmt(index)}</span><strong>${utilization == null ? '—' : fmt(utilization) + '%'}</strong></div>
${bar(utilization)}
<div class="metric-pair"><span>Memory</span><strong>${device.memory_total_mib ? fmt(device.memory_used_mib) + ' / ' + fmt(device.memory_total_mib) + ' MiB' : '—'}</strong></div>
${device.memory_total_mib ? bar(memoryPercent) : ''}`;
}).join('');
const networkRows = (net?.interfaces || []).slice(0, 4).map(nic => `
<div class="network-row">
<span class="interface" title="${esc(nic.name)}">${esc(nic.name)}</span>
<span>${fmtRate(nic.rx_bps)}↓&nbsp; ${fmtRate(nic.tx_bps)}↑</span>
</div>`).join('');
const source = mirrored ? '<div class="hardware-source muted">Mock node · local orchestrator hardware</div>' : '';
const errors = (node.errors || []).map(error => `<div class="notice err">${esc(error)}</div>`).join('');
return `${source}<div class="hardware-grid">
<section class="hw-card cpu">
<div class="hw-card-head"><span class="hw-card-title">CPU · ${fmt(cores.length)} threads</span><span class="hw-card-value">${cpuLabel}</span></div>
<canvas id="thread-graph" class="thread-graph" width="560" height="${threadHeight}" style="height:${threadHeight}px" role="img" aria-label="Per-thread CPU utilization history"></canvas>
<div class="thread-strip">${threadCells || '<span class="hw-empty">Waiting for per-thread samples</span>'}</div>
<div class="hw-card-subtitle">Load 1 / 5 / 15 min&nbsp; ${load}</div>
</section>
<section class="hw-card memory">
<div class="hw-card-head"><span class="hw-card-title">Memory</span><span class="hw-card-value">${memoryLabel}</span></div>
${bar(memoryPercent)}
<div class="metric-pair"><span>Used</span><strong>${fmtBytes(memory?.used_bytes)} / ${fmtBytes(memory?.total_bytes)}</strong></div>
<div class="metric-pair"><span>Available</span><strong>${fmtBytes(memory?.available_bytes)}</strong></div>
<div class="metric-pair"><span>Swap</span><strong>${swapPercent == null ? '—' : fmt(swapPercent, 1) + '%'}</strong></div>
<div class="metric-pair"><span>PSI some · 10s</span><strong class="pressure-value ${pressureTone(memoryPressure)}">${memoryPressure == null ? '—' : fmt(memoryPressure, 2) + '%'}</strong></div>
</section>
<section class="hw-card storage">
<div class="hw-card-head"><span class="hw-card-title">Storage · ${esc(filesystem?.mount || '/')}</span><span class="hw-card-value">${storageLabel}</span></div>
${bar(storagePercent)}
<div class="metric-pair"><span>Used</span><strong>${fmtBytes(filesystem?.used_bytes)} / ${fmtBytes(filesystem?.total_bytes)}</strong></div>
<div class="metric-pair"><span>Available</span><strong>${fmtBytes(filesystem?.available_bytes)}</strong></div>
<div class="metric-pair"><span>I/O PSI · 10s</span><strong class="pressure-value ${pressureTone(ioPressure)}">${ioPressure == null ? '—' : fmt(ioPressure, 2) + '%'}</strong></div>
</section>
<section class="hw-card gpu">
<div class="hw-card-head"><span class="hw-card-title">GPU · ${fmt(gpuDevices.length)} devices</span><span class="hw-card-value">${gpuDevices.length ? fmt(gpuMax) + '%' : '—'}</span></div>
${gpuRows || '<div class="hw-empty">No GPU telemetry</div>'}
</section>
<section class="hw-card network">
<div class="hw-card-head"><span class="hw-card-title">Network</span><span class="hw-card-value">${fmt(net?.interfaces?.length || 0)}</span></div>
<div class="network-list">${networkRows || '<div class="hw-empty">No interface telemetry</div>'}</div>
</section>
</div>${errors}`;
}
function drawThreadGraph(history, currentCores) {
requestAnimationFrame(() => {
const canvas = document.getElementById('machine-spark');
const canvas = document.getElementById('thread-graph');
if (!canvas) return;
const ratio = Math.max(1, window.devicePixelRatio || 1);
const rect = canvas.getBoundingClientRect();
const width = Math.max(1, Math.round(rect.width));
const height = Math.max(1, Math.round(rect.height));
const pixelWidth = Math.round(width * ratio), pixelHeight = Math.round(height * ratio);
if (canvas.width !== pixelWidth || canvas.height !== pixelHeight) {
canvas.width = pixelWidth;
canvas.height = pixelHeight;
}
const ctx = canvas.getContext('2d');
ctx.clearRect(0, 0, canvas.width, canvas.height);
const points = history.map(h => ({ cpu: h.cpu_total_percent, gpu: h.gpu_max_percent }));
drawLine(ctx, points.map(p => p.gpu), '#fbbf24');
drawLine(ctx, points.map(p => p.cpu), '#34d399');
ctx.setTransform(ratio, 0, 0, ratio, 0, 0);
const styles = getComputedStyle(document.documentElement);
const background = styles.getPropertyValue('--bg').trim() || '#00060c';
const divider = styles.getPropertyValue('--divider').trim() || '#0d2c4a';
const foreground = styles.getPropertyValue('--ok').trim() || '#00d400';
ctx.globalAlpha = 1;
ctx.fillStyle = background;
ctx.fillRect(0, 0, width, height);
const threadCount = Math.max(currentCores.length, ...history.map(point => point.cpu_cores_percent?.length || 0));
if (!threadCount) return;
const rowHeight = height / threadCount;
ctx.fillStyle = divider;
for (let row = 1; row < threadCount; row += 1) {
ctx.fillRect(0, Math.floor(row * rowHeight), width, 1);
}
const slot = 2;
const points = history
.map(point => point.cpu_cores_percent || [])
.slice(-Math.floor(width / slot));
if (!points.length) points.push(currentCores);
ctx.fillStyle = foreground;
points.forEach((values, pointIndex) => {
const x = width - (points.length - pointIndex) * slot;
values.forEach((value, threadIndex) => {
if (value == null || !Number.isFinite(Number(value))) return;
const utilization = Math.max(0, Math.min(100, Number(value)));
ctx.globalAlpha = 0.10 + utilization * 0.009;
const y = Math.ceil(threadIndex * rowHeight);
ctx.fillRect(x, y, slot, Math.max(1, Math.floor(rowHeight) - 1));
});
});
ctx.globalAlpha = 1;
});
}
function drawLine(ctx, values, color) {
const valid = values.filter(v => v != null);
if (valid.length < 2) return;
const max = Math.max(100, ...valid);
const step = ctx.canvas.width / (values.length - 1 || 1);
ctx.strokeStyle = color;
ctx.lineWidth = 1.5;
ctx.beginPath();
let started = false;
values.forEach((value, index) => {
if (value == null) return;
const x = index * step;
const y = ctx.canvas.height - (value / max) * (ctx.canvas.height - 6) - 3;
if (!started) { ctx.moveTo(x, y); started = true; } else ctx.lineTo(x, y);
});
ctx.stroke();
function rosterSortControls(key, label) {
return `<span class="roster-sort-controls">
<button type="button" class="roster-sort-button" data-sort="${key}" data-direction="asc" aria-label="Sort ${label} ascending" title="Sort ${label} ascending" aria-pressed="${rosterSort.key === key && rosterSort.direction === 'asc'}">↑</button>
<button type="button" class="roster-sort-button" data-sort="${key}" data-direction="desc" aria-label="Sort ${label} descending" title="Sort ${label} descending" aria-pressed="${rosterSort.key === key && rosterSort.direction === 'desc'}">↓</button>
</span>`;
}
function compareRosterValues(left, right) {
const leftMissing = left == null || left === '';
const rightMissing = right == null || right === '';
if (leftMissing || rightMissing) return leftMissing === rightMissing ? 0 : leftMissing ? 1 : -1;
if (typeof left === 'number' && typeof right === 'number') return left - right;
if (typeof left === 'boolean' && typeof right === 'boolean') return Number(left) - Number(right);
return String(left).localeCompare(String(right), undefined, { numeric: true, sensitivity: 'base' });
}
function isLegacyHardwareSampler(actor) {
const actorType = String(actor.actor_type || '');
return (actorType.includes('BlockingSamplerActor') && actorType.includes('telemetry::hardware'))
|| /Host(?:Cpu|Gpu|Net)SamplerActor/.test(actorType);
}
function renderRoster(node) {
const wrap = document.getElementById('roster-wrap');
if (!wrap) return;
let rows = node.roster || [];
let rows = [...(node.roster || [])];
if (hardwareSource === 'orchestrator') {
// Mock containers may come from an older local image. Current samplers are
// engine tasks; do not leak legacy sampler actors into the simulated roster.
rows = rows.filter(actor => !isLegacyHardwareSampler(actor));
const actorCount = document.querySelector('[data-actor-count]');
if (actorCount) actorCount.textContent = fmt(rows.length);
}
if (rosterFilter) {
const q = rosterFilter.toLowerCase();
rows = rows.filter(a => `${a.name || ''} ${a.actor_type || ''} ${a.address} ${a.worker_id ?? ''}`.toLowerCase().includes(q));
}
rows.sort((left, right) => {
const compared = compareRosterValues(left[rosterSort.key], right[rosterSort.key]);
return (rosterSort.direction === 'asc' ? compared : -compared)
|| String(left.address).localeCompare(String(right.address));
});
const capped = rows.slice(0, ROSTER_RENDER_CAP);
const notice = rows.length > capped.length
? `<div class="notice">showing ${fmt(capped.length)} of ${fmt(rows.length)} — refine the filter to see more</div>`
@ -417,7 +605,7 @@ function renderRoster(node) {
html = '<div class="empty">No actors on this stream (or none match the filter).</div>';
} else {
html = notice + `<table>
<thead><tr><th>Actor</th><th>Type</th><th>State</th><th>Mailbox</th><th>Msg/s</th><th>Processed</th><th>Worker</th><th>Last message</th></tr></thead>
<thead><tr><th>Actor${rosterSortControls('address', 'actor')}</th><th>Type${rosterSortControls('actor_type', 'type')}</th><th>State${rosterSortControls('poisoned', 'state')}</th><th>Mailbox${rosterSortControls('mailbox_depth', 'mailbox')}</th><th>Msg/s${rosterSortControls('msg_per_sec', 'throughput')}</th><th>Processed${rosterSortControls('messages_processed', 'processed count')}</th><th>Worker${rosterSortControls('worker_id', 'worker')}</th><th>Last message${rosterSortControls('last_msg_type', 'last message')}</th></tr></thead>
<tbody>${capped.map(a => `<tr data-addr="${esc(a.address)}" tabindex="0" ${a.address === selectedActor ? 'data-selected="true"' : ''}>
<td class="mono">${esc(shortAddr(a.address))}${a.name ? `<br><span class="muted">${esc(a.name)}</span>` : ''}</td>
<td><span class="muted" title="${esc(a.actor_type || '')}">${typeShort(a.actor_type)}</span></td>
@ -552,6 +740,12 @@ pageEl.addEventListener('click', e => {
render();
return;
}
const sortButton = e.target.closest('.roster-sort-button');
if (sortButton) {
rosterSort = { key: sortButton.dataset.sort, direction: sortButton.dataset.direction };
render();
return;
}
const rosterRow = e.target.closest('tr[data-addr]');
if (rosterRow) {
selectedActor = rosterRow.getAttribute('data-addr');

View file

@ -8,7 +8,9 @@ use telemetry::hardware::cpu::{
use telemetry::hardware::gpu::{
GpuDeviceSample, GpuProcessSample, HOST_GPU_CHANNEL, HostGpuSample,
};
use telemetry::hardware::memory::{HOST_MEMORY_CHANNEL, HostMemorySample};
use telemetry::hardware::net::{HOST_NET_CHANNEL, HostNetSample, NetInterfaceSample};
use telemetry::hardware::storage::{HOST_STORAGE_CHANNEL, HostStorageSample};
use serde::Serialize;
@ -27,7 +29,9 @@ pub(crate) struct NodeHardwareState {
decode_errors: BTreeMap<&'static str, String>,
pub(crate) cpu: Option<HostCpuSample>,
pub(crate) gpu: Option<HostGpuSample>,
pub(crate) memory: Option<HostMemorySample>,
pub(crate) net: Option<NetSnapshot>,
pub(crate) storage: Option<HostStorageSample>,
pub(crate) process: Option<ProcessSnapshot>,
pub(crate) history: VecDeque<HardwareHistoryState>,
}
@ -39,7 +43,9 @@ impl NodeHardwareState {
decode_errors: BTreeMap::new(),
cpu: None,
gpu: None,
memory: None,
net: None,
storage: None,
process: None,
history: VecDeque::with_capacity(HISTORY_CAP),
}
@ -64,6 +70,14 @@ impl NodeHardwareState {
}
Err(error) => self.store_decode_error(HOST_GPU_CHANNEL, error),
},
HOST_MEMORY_CHANNEL => match HostMemorySample::decode(payload) {
Ok(sample) => {
self.memory = Some(sample);
self.decode_errors.remove(HOST_MEMORY_CHANNEL);
self.update_history(now);
}
Err(error) => self.store_decode_error(HOST_MEMORY_CHANNEL, error),
},
HOST_NET_CHANNEL => match HostNetSample::decode(payload) {
Ok(sample) => {
self.net = Some(NetSnapshot::from_sample(sample, self.net.as_ref()));
@ -72,6 +86,14 @@ impl NodeHardwareState {
}
Err(error) => self.store_decode_error(HOST_NET_CHANNEL, error),
},
HOST_STORAGE_CHANNEL => match HostStorageSample::decode(payload) {
Ok(sample) => {
self.storage = Some(sample);
self.decode_errors.remove(HOST_STORAGE_CHANNEL);
self.update_history(now);
}
Err(error) => self.store_decode_error(HOST_STORAGE_CHANNEL, error),
},
_ => {
if channel.starts_with("proc.") && channel.ends_with(".lifecycle") {
self.process = decode_process_snapshot(payload);
@ -99,6 +121,35 @@ impl NodeHardwareState {
.as_ref()
.and_then(|sample| sample.host.as_ref())
.and_then(|host| host.total_percent);
let cpu_cores_percent = self
.cpu
.as_ref()
.map(|sample| {
sample
.cores
.iter()
.map(|core| core.total_percent)
.collect::<Vec<_>>()
})
.unwrap_or_default();
let memory_used_percent = self.memory.as_ref().and_then(|sample| {
Some(sample.used_bytes? as f64 * 100.0 / sample.total_bytes?.max(1) as f64)
});
let memory_pressure_some_avg10 = self
.memory
.as_ref()
.and_then(|sample| sample.pressure.as_ref())
.map(|pressure| pressure.some_avg10);
let storage_used_percent = self
.storage
.as_ref()
.and_then(|sample| sample.filesystems.first())
.and_then(|filesystem| filesystem.used_percent);
let io_pressure_some_avg10 = self
.storage
.as_ref()
.and_then(|sample| sample.pressure.as_ref())
.map(|pressure| pressure.some_avg10);
let mut gpu_max_percent = None;
let mut gpu_memory_used_mib = 0_u64;
@ -128,17 +179,24 @@ impl NodeHardwareState {
sample_unix_ms: [
self.cpu.as_ref().map(|sample| sample.sample_unix_ms),
self.gpu.as_ref().map(|sample| sample.sample_unix_ms),
self.memory.as_ref().map(|sample| sample.sample_unix_ms),
self.net.as_ref().map(|sample| sample.sample_unix_ms),
self.storage.as_ref().map(|sample| sample.sample_unix_ms),
]
.into_iter()
.flatten()
.max(),
cpu_total_percent,
cpu_cores_percent,
gpu_max_percent,
gpu_memory_used_mib,
gpu_memory_total_mib,
net_rx_bps,
net_tx_bps,
memory_used_percent,
memory_pressure_some_avg10,
storage_used_percent,
io_pressure_some_avg10,
}
}
@ -149,11 +207,16 @@ impl NodeHardwareState {
{
last.sample_unix_ms = summary.sample_unix_ms;
last.cpu_total_percent = summary.cpu_total_percent;
last.cpu_cores_percent = summary.cpu_cores_percent.clone();
last.gpu_max_percent = summary.gpu_max_percent;
last.gpu_memory_used_mib = summary.gpu_memory_used_mib;
last.gpu_memory_total_mib = summary.gpu_memory_total_mib;
last.net_rx_bps = summary.net_rx_bps;
last.net_tx_bps = summary.net_tx_bps;
last.memory_used_percent = summary.memory_used_percent;
last.memory_pressure_some_avg10 = summary.memory_pressure_some_avg10;
last.storage_used_percent = summary.storage_used_percent;
last.io_pressure_some_avg10 = summary.io_pressure_some_avg10;
return;
}
@ -164,11 +227,16 @@ impl NodeHardwareState {
at: now,
sample_unix_ms: summary.sample_unix_ms,
cpu_total_percent: summary.cpu_total_percent,
cpu_cores_percent: summary.cpu_cores_percent,
gpu_max_percent: summary.gpu_max_percent,
gpu_memory_used_mib: summary.gpu_memory_used_mib,
gpu_memory_total_mib: summary.gpu_memory_total_mib,
net_rx_bps: summary.net_rx_bps,
net_tx_bps: summary.net_tx_bps,
memory_used_percent: summary.memory_used_percent,
memory_pressure_some_avg10: summary.memory_pressure_some_avg10,
storage_used_percent: summary.storage_used_percent,
io_pressure_some_avg10: summary.io_pressure_some_avg10,
});
}
@ -180,9 +248,23 @@ impl NodeHardwareState {
if let Some(error) = self.gpu.as_ref().and_then(|sample| sample.error.as_ref()) {
errors.push(format!("{HOST_GPU_CHANNEL}: {error}"));
}
if let Some(error) = self
.memory
.as_ref()
.and_then(|sample| sample.error.as_ref())
{
errors.push(format!("{HOST_MEMORY_CHANNEL}: {error}"));
}
if let Some(error) = self.net.as_ref().and_then(|sample| sample.error.as_ref()) {
errors.push(format!("{HOST_NET_CHANNEL}: {error}"));
}
if let Some(error) = self
.storage
.as_ref()
.and_then(|sample| sample.error.as_ref())
{
errors.push(format!("{HOST_STORAGE_CHANNEL}: {error}"));
}
errors
}
}
@ -308,26 +390,36 @@ impl NetInterfaceSnapshot {
}
}
#[derive(Clone, Copy)]
#[derive(Clone)]
pub(crate) struct HardwareSummary {
pub(crate) sample_unix_ms: Option<u64>,
pub(crate) cpu_total_percent: Option<f64>,
pub(crate) cpu_cores_percent: Vec<Option<f64>>,
pub(crate) gpu_max_percent: Option<u64>,
pub(crate) gpu_memory_used_mib: u64,
pub(crate) gpu_memory_total_mib: u64,
pub(crate) net_rx_bps: f64,
pub(crate) net_tx_bps: f64,
pub(crate) memory_used_percent: Option<f64>,
pub(crate) memory_pressure_some_avg10: Option<f64>,
pub(crate) storage_used_percent: Option<f64>,
pub(crate) io_pressure_some_avg10: Option<f64>,
}
pub(crate) struct HardwareHistoryState {
pub(crate) at: Instant,
pub(crate) sample_unix_ms: Option<u64>,
pub(crate) cpu_total_percent: Option<f64>,
pub(crate) cpu_cores_percent: Vec<Option<f64>>,
pub(crate) gpu_max_percent: Option<u64>,
pub(crate) gpu_memory_used_mib: u64,
pub(crate) gpu_memory_total_mib: u64,
pub(crate) net_rx_bps: f64,
pub(crate) net_tx_bps: f64,
pub(crate) memory_used_percent: Option<f64>,
pub(crate) memory_pressure_some_avg10: Option<f64>,
pub(crate) storage_used_percent: Option<f64>,
pub(crate) io_pressure_some_avg10: Option<f64>,
}
#[derive(Serialize)]
@ -383,11 +475,16 @@ pub(crate) struct HardwareHistorySnapshot {
pub(crate) ms_ago: u64,
pub(crate) sample_unix_ms: Option<u64>,
pub(crate) cpu_total_percent: Option<f64>,
pub(crate) cpu_cores_percent: Vec<Option<f64>>,
pub(crate) gpu_max_percent: Option<u64>,
pub(crate) gpu_memory_used_mib: u64,
pub(crate) gpu_memory_total_mib: u64,
pub(crate) net_rx_bps: f64,
pub(crate) net_tx_bps: f64,
pub(crate) memory_used_percent: Option<f64>,
pub(crate) memory_pressure_some_avg10: Option<f64>,
pub(crate) storage_used_percent: Option<f64>,
pub(crate) io_pressure_some_avg10: Option<f64>,
}
pub(crate) fn duration_ms(duration: Duration) -> u64 {

View file

@ -469,7 +469,7 @@ mod tests {
}
fn is_valid_for_route(&self) -> bool {
self.payload % 5 == 0
self.payload.is_multiple_of(5)
}
}

View file

@ -20,6 +20,7 @@
//! out. Receipts are bounded and interval-spaced so noisy actors cannot flood
//! the page.
use std::cmp::Reverse;
use std::collections::{BTreeMap, VecDeque};
use std::time::{Duration, Instant};
@ -172,8 +173,10 @@ impl RuntimeState {
}
pub(crate) fn totals(&self) -> Totals {
let mut totals = Totals::default();
totals.actors = self.actors.len().min(u32::MAX as usize) as u32;
let mut totals = Totals {
actors: self.actors.len().min(u32::MAX as usize) as u32,
..Totals::default()
};
for actor in self.actors.values() {
totals.mailbox_depth = totals.mailbox_depth.saturating_add(actor.mailbox_depth);
totals.msg_per_sec += actor.msg_per_sec;
@ -449,7 +452,7 @@ fn parse_message_type_counts(value: Option<&Value>) -> Option<Vec<(String, u64)>
.iter()
.filter_map(|(name, count)| value_to_u64(count).map(|count| (name.clone(), count)))
.collect();
out.sort_by(|a, b| b.1.cmp(&a.1));
out.sort_by_key(|&(_, count)| Reverse(count));
return Some(out);
}
None

View file

@ -14,6 +14,7 @@ swactor = { path = "../..", features = ["serde", "transport"] }
swactor-transport = { path = "../transport" }
swactor-engine = { path = "../engine" }
parking_lot = "0.12"
[dev-dependencies]
parking_lot = "0.12"

View file

@ -46,6 +46,7 @@
//! - P11 (binding slice) fast path performs no syscalls;
//! - P12 one copy per side per byte.
use serde::{Deserialize, Serialize};
use std::ptr::NonNull;
use std::sync::atomic::{AtomicU64, Ordering};
@ -85,7 +86,7 @@ pub struct ByteRingSpec {
}
/// A located, installed ring: what `attach` needs to find it again.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct RingHandle {
/// Lease start (the header lives here).
pub offset: u64,
@ -93,9 +94,11 @@ pub struct RingHandle {
pub capacity: u64,
/// Ring generation.
pub generation: u64,
/// Arena lease identity used for deterministic release.
pub lease_id: u64,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum Role {
Producer,
Consumer,
@ -112,15 +115,139 @@ pub struct Reservation {
}
/// Record framing layered on the byte stream (property P9).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum RecordKind {
Data,
Eof,
Fault,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct RecordMeta {
pub kind: RecordKind,
pub len: u64,
}
/// Borrowed committed record. Dropping the view releases the complete framed
/// record back to the producer.
pub struct PinnedRecord<'a> {
endpoint: &'a mut Endpoint,
kind: RecordKind,
payload_start: u64,
payload_len: u64,
record_start: u64,
record_len: u64,
generation: u64,
released: bool,
}
impl std::fmt::Debug for PinnedRecord<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("PinnedRecord")
.field("kind", &self.kind)
.field("payload_len", &self.payload_len)
.field("record_start", &self.record_start)
.finish_non_exhaustive()
}
}
impl PinnedRecord<'_> {
pub fn kind(&self) -> RecordKind {
self.kind
}
pub fn len(&self) -> usize {
self.payload_len as usize
}
pub fn is_empty(&self) -> bool {
self.payload_len == 0
}
pub fn spans(&self) -> (&[u8], &[u8]) {
self.endpoint
.read_spans(self.payload_start, self.payload_len)
}
pub fn release(mut self) -> Result<(), FlowError> {
self.release_inner()?;
self.released = true;
Ok(())
}
fn release_inner(&mut self) -> Result<(), FlowError> {
self.endpoint.check("release_record", Role::Consumer)?;
let generation = self.endpoint.field_u64(OFF_GENERATION);
if generation != self.generation {
return Err(FlowError::StaleReservation {
reservation: self.generation,
ring: generation,
});
}
let consume = self.endpoint.consume_cursor();
if consume != self.record_start {
return Err(FlowError::PinnedRecordMoved {
expected: self.record_start,
found: consume,
});
}
self.endpoint
.atomic(OFF_CONSUME)
.store(self.record_start + self.record_len, Ordering::Release);
Ok(())
}
}
impl Drop for PinnedRecord<'_> {
fn drop(&mut self) {
if !self.released && self.release_inner().is_ok() {
self.released = true;
}
}
}
/// Producer reservation for a complete framed record. Payload bytes are
/// written directly into the ring and remain invisible until `commit`.
pub struct WritableRecord<'a> {
endpoint: &'a mut Endpoint,
reservation: Option<Reservation>,
payload_start: u64,
payload_len: u64,
}
impl std::fmt::Debug for WritableRecord<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("WritableRecord")
.field("payload_len", &self.payload_len)
.finish_non_exhaustive()
}
}
impl WritableRecord<'_> {
pub fn len(&self) -> usize {
self.payload_len as usize
}
pub fn is_empty(&self) -> bool {
self.payload_len == 0
}
pub fn spans_mut(&mut self) -> (&mut [u8], &mut [u8]) {
self.endpoint
.write_spans(self.payload_start, self.payload_len)
}
pub fn commit(mut self) -> Result<(), FlowError> {
let reservation = self
.reservation
.take()
.expect("writable record commits at most once");
self.endpoint.commit(reservation)
}
}
impl RecordKind {
fn to_byte(self) -> u8 {
pub fn to_byte(self) -> u8 {
match self {
Self::Data => 1,
Self::Eof => 2,
@ -128,7 +255,7 @@ impl RecordKind {
}
}
fn from_byte(byte: u8) -> Option<Self> {
pub fn from_byte(byte: u8) -> Option<Self> {
match byte {
1 => Some(Self::Data),
2 => Some(Self::Eof),
@ -194,6 +321,7 @@ pub enum AttachError {
pub enum RecordError {
LengthExceedsCapacity { len: u64, capacity: u64 },
InvalidKind(u8),
LengthExceedsWire { len: u64 },
}
#[derive(Clone, Debug, PartialEq, Eq)]
@ -218,6 +346,10 @@ pub enum FlowError {
},
/// The header stopped validating mid-protocol (property P5).
Corrupt(HeaderError),
PinnedRecordMoved {
expected: u64,
found: u64,
},
BadRecord(RecordError),
Io,
}
@ -230,6 +362,59 @@ pub struct Endpoint {
role: Role,
}
#[derive(Clone, Copy)]
pub struct RingProbe {
header: NonNull<u8>,
info: RingHandle,
}
unsafe impl Send for RingProbe {}
unsafe impl Sync for RingProbe {}
impl RingProbe {
fn atomic(&self, off: u64) -> &AtomicU64 {
// SAFETY: probes are created only from a successfully attached,
// aligned endpoint and remain bounded by that ring's live lease.
unsafe { &*(self.header.as_ptr().add(off as usize) as *const AtomicU64) }
}
pub fn positions(&self) -> Result<(u64, u64), FlowError> {
let generation = self.atomic(OFF_GENERATION).load(Ordering::Relaxed);
if generation != self.info.generation {
return Err(FlowError::StaleReservation {
reservation: self.info.generation,
ring: generation,
});
}
let commit = self.atomic(OFF_COMMIT).load(Ordering::Acquire);
let consume = self.atomic(OFF_CONSUME).load(Ordering::Acquire);
if commit < consume {
return Err(FlowError::Corrupt(HeaderError::CommitBelowConsume {
commit,
consume,
}));
}
if commit - consume > self.info.capacity {
return Err(FlowError::Corrupt(HeaderError::ReadableExceedsCapacity {
commit,
consume,
capacity: self.info.capacity,
}));
}
Ok((commit, consume))
}
pub fn has_data(&self) -> bool {
self.positions()
.is_ok_and(|(commit, consume)| commit > consume)
}
pub fn has_capacity(&self) -> bool {
self.positions()
.is_ok_and(|(commit, consume)| commit - consume < self.info.capacity)
}
}
// SAFETY: an endpoint touches only its role-owned cursor field and the data
// region under the single-writer protocol (properties P3/P4); the raw
// pointer is never dereferenced outside `[header, header + DATA_OFFSET +
@ -295,6 +480,7 @@ pub fn install(arena: &mut ArenaManager, spec: ByteRingSpec) -> Result<RingHandl
offset: lease.layout.start_offset,
capacity: spec.capacity,
generation: spec.generation,
lease_id: lease.ring_id.0,
})
}
@ -339,6 +525,38 @@ pub fn attach(
Ok(endpoint)
}
/// Attach through a child/host process-local mapping of the same arena.
pub fn attach_mapped(
arena: &crate::mapped_arena::MappedArena,
handle: RingHandle,
role: Role,
) -> Result<Endpoint, AttachError> {
let total = DATA_OFFSET
.checked_add(handle.capacity)
.ok_or(AttachError::OutOfBounds {
end: u64::MAX,
arena_len: arena.len() as u64,
})?;
let range =
arena
.checked_range(handle.offset, total)
.map_err(|_| AttachError::OutOfBounds {
end: handle.offset.saturating_add(total),
arena_len: arena.len() as u64,
})?;
let endpoint = Endpoint {
header: arena.ptr_at(range.start),
info: handle,
role,
};
endpoint.validate_fixed().map_err(AttachError::Header)?;
endpoint
.validate_generation(handle.generation)
.map_err(AttachError::Header)?;
endpoint.validate_cursors().map_err(AttachError::Header)?;
Ok(endpoint)
}
fn lease_ring(
arena: &mut ArenaManager,
request_id: u64,
@ -360,6 +578,32 @@ fn lease_ring(
}
impl Endpoint {
pub fn capacity(&self) -> u64 {
self.info.capacity
}
pub fn role(&self) -> Role {
self.role
}
/// Current published producer and consumer positions. This role-neutral
/// observation is used only to wait for an already-committed clean close
/// to enter the downstream bounded transport.
pub fn positions(&self) -> Result<(u64, u64), FlowError> {
self.validate_fixed().map_err(FlowError::Corrupt)?;
self.validate_generation(self.info.generation)
.map_err(FlowError::Corrupt)?;
self.validate_cursors().map_err(FlowError::Corrupt)?;
Ok((self.commit_cursor(), self.consume_cursor()))
}
pub fn probe(&self) -> RingProbe {
RingProbe {
header: self.header,
info: self.info,
}
}
// ─── field access ────────────────────────────────────────────────────────
/// SAFETY: callers keep `self` alive; the pointer is bounds-checked at
@ -490,6 +734,36 @@ impl Endpoint {
}
}
fn read_spans(&self, stream_pos: u64, len: u64) -> (&[u8], &[u8]) {
let capacity = self.info.capacity as usize;
let start = (stream_pos % self.info.capacity) as usize;
let len = len as usize;
let first = len.min(capacity - start);
// SAFETY: `stream_pos` and `len` describe a validated committed
// record no larger than the ring. The split ranges do not overlap.
unsafe {
(
std::slice::from_raw_parts(self.data_ptr().add(start), first),
std::slice::from_raw_parts(self.data_ptr(), len - first),
)
}
}
fn write_spans(&mut self, stream_pos: u64, len: u64) -> (&mut [u8], &mut [u8]) {
let capacity = self.info.capacity as usize;
let start = (stream_pos % self.info.capacity) as usize;
let len = len as usize;
let first = len.min(capacity - start);
// SAFETY: the producer exclusively owns this uncommitted reservation;
// it is no larger than the ring and the split ranges do not overlap.
unsafe {
(
std::slice::from_raw_parts_mut(self.data_ptr().add(start), first),
std::slice::from_raw_parts_mut(self.data_ptr(), len - first),
)
}
}
fn copy_out(&self, stream_pos: u64, len: u64) -> Vec<u8> {
let capacity = self.info.capacity as usize;
let start = (stream_pos % self.info.capacity) as usize;
@ -610,6 +884,34 @@ impl Endpoint {
// ─── records (P9) ────────────────────────────────────────────────────────
/// Reserve one complete record for direct producer-side payload writes.
pub fn reserve_record(
&mut self,
kind: RecordKind,
len: u64,
) -> Result<WritableRecord<'_>, FlowError> {
if len > u64::from(u32::MAX) {
return Err(FlowError::BadRecord(RecordError::LengthExceedsWire { len }));
}
if RECORD_HEADER_LEN + len > self.info.capacity {
return Err(FlowError::BadRecord(RecordError::LengthExceedsCapacity {
len,
capacity: self.info.capacity,
}));
}
let reservation = self.reserve(RECORD_HEADER_LEN + len)?;
let mut prefix = [0_u8; RECORD_HEADER_LEN as usize];
prefix[0] = kind.to_byte();
prefix[1..5].copy_from_slice(&(len as u32).to_le_bytes());
self.copy_into(reservation.start, &prefix);
let payload_start = reservation.start + RECORD_HEADER_LEN;
Ok(WritableRecord {
endpoint: self,
reservation: Some(reservation),
payload_start,
payload_len: len,
})
}
/// Frame and commit one record in a single step.
pub fn send_record(&mut self, kind: RecordKind, bytes: &[u8]) -> Result<(), FlowError> {
let len = bytes.len() as u64;
@ -628,6 +930,72 @@ impl Endpoint {
self.commit(reservation)
}
/// Inspect the next complete committed record without pinning or consuming
/// it. Used by local direct transfer to reserve destination capacity before
/// borrowing the source payload.
pub fn next_record_meta(&self) -> Result<Option<RecordMeta>, FlowError> {
self.check("next_record_meta", Role::Consumer)?;
self.validate_generation(self.info.generation)
.map_err(FlowError::Corrupt)?;
let (commit, consume) = (self.commit_cursor(), self.consume_cursor());
let readable = commit - consume;
if readable < RECORD_HEADER_LEN {
return Ok(None);
}
let prefix = self.copy_out(consume, RECORD_HEADER_LEN);
let kind = RecordKind::from_byte(prefix[0])
.ok_or(FlowError::BadRecord(RecordError::InvalidKind(prefix[0])))?;
let len = u64::from(u32::from_le_bytes(prefix[1..5].try_into().unwrap()));
if RECORD_HEADER_LEN + len > self.info.capacity {
return Err(FlowError::BadRecord(RecordError::LengthExceedsCapacity {
len,
capacity: self.info.capacity,
}));
}
if readable < RECORD_HEADER_LEN + len {
return Ok(None);
}
Ok(Some(RecordMeta { kind, len }))
}
/// Borrow one complete committed record without copying its payload.
/// The record remains pinned until the returned view is released or
/// dropped.
pub fn peek_record(&mut self) -> Result<Option<PinnedRecord<'_>>, FlowError> {
self.check("peek_record", Role::Consumer)?;
self.validate_generation(self.info.generation)
.map_err(FlowError::Corrupt)?;
let (commit, consume) = (self.commit_cursor(), self.consume_cursor());
let readable = commit - consume;
if readable < RECORD_HEADER_LEN {
return Ok(None);
}
let prefix = self.copy_out(consume, RECORD_HEADER_LEN);
let kind = RecordKind::from_byte(prefix[0])
.ok_or(FlowError::BadRecord(RecordError::InvalidKind(prefix[0])))?;
let len = u64::from(u32::from_le_bytes(prefix[1..5].try_into().unwrap()));
if RECORD_HEADER_LEN + len > self.info.capacity {
return Err(FlowError::BadRecord(RecordError::LengthExceedsCapacity {
len,
capacity: self.info.capacity,
}));
}
let record_len = RECORD_HEADER_LEN + len;
if readable < record_len {
return Ok(None);
}
let generation = self.info.generation;
Ok(Some(PinnedRecord {
endpoint: self,
kind,
payload_start: consume + RECORD_HEADER_LEN,
payload_len: len,
record_start: consume,
record_len,
generation,
released: false,
}))
}
/// Receive one complete record; `Ok(None)` when nothing (or only a
/// torn, uncommitted prefix) is readable.
pub fn recv_record(&mut self) -> Result<Option<(RecordKind, Vec<u8>)>, FlowError> {

File diff suppressed because it is too large Load diff

View file

@ -1,6 +1,6 @@
//! Host-side session, binding, and arena-allocation actors.
use std::collections::HashSet;
use std::collections::{HashMap, HashSet};
use std::os::fd::{FromRawFd, OwnedFd};
use std::sync::Arc;
@ -20,18 +20,27 @@ use crate::blob_transfer::{
BlobTransferEvent, BlobTransferId, BlobTransferOffer, BlobTransferReceiver, BlobTransferSender,
};
use crate::bootstrap::JobHandoff;
use crate::byte_ring::{self, ByteRingSpec, RingHandle, Role};
use crate::mapped_arena::MappedArena;
use crate::namespace::{
BlobBinding as NamespaceBlobBinding, DataDirectoryOut, NamespaceClient, NamespaceClientIn,
NamespaceError, NamespaceRequest, OperationId, SourceRecovery,
NamespaceError, NamespaceRequest, OperationId, SourceRecovery, StreamIncarnation, StreamMatch,
StreamRole,
};
use crate::path::{DataPath, JobContext};
use crate::protocol::{
AttachmentFailure, ChildSessionIn, DataOperation, DataPlaneError, HostSessionIn, JobCapability,
AttachmentFailure, ChildSessionIn, DataOperation, DataPlaneError, HostSessionIn, HostStreamIn,
JobCapability,
};
use crate::source::{BlobSourceIn, BlobSourcePublisher, BlobSourceRetirement, FileBlobSourceActor};
use crate::stream_transport::{
StreamPeerDescriptor, StreamSinkRequest, StreamSourceRequest, StreamTransport,
StreamTransportEvent, StreamTransportNotifier,
};
const BLOB_ALIGNMENT: u64 = 64;
const FIRST_BLOB_REQUEST_ID: u64 = 2;
const STREAM_RING_CAPACITY: u64 = 256 * 1024;
pub trait HostRouteRegistrar: Send + Sync + 'static {
fn register_child(
@ -53,6 +62,7 @@ pub struct HostDataPlaneConfig {
pub source_sender: Option<Arc<dyn BlobTransferSender>>,
pub source_publisher: Option<Arc<dyn BlobSourcePublisher>>,
pub route_registrar: Option<Arc<dyn HostRouteRegistrar>>,
pub stream_transport: Option<Arc<dyn StreamTransport>>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@ -75,9 +85,12 @@ pub struct HostDataPlaneSessionActor {
source_sender: Option<Arc<dyn BlobTransferSender>>,
source_publisher: Option<Arc<dyn BlobSourcePublisher>>,
route_registrar: Option<Arc<dyn HostRouteRegistrar>>,
stream_arena: Option<Arc<MappedArena>>,
stream_transport: Option<Arc<dyn StreamTransport>>,
child_session: Option<ActorAddress>,
allocator: Option<ActorAddress>,
active_bindings: HashSet<ActorAddress>,
stream_bindings: HashMap<ActorAddress, ActorAddress>,
state: HostSessionState,
}
@ -92,6 +105,21 @@ impl HostDataPlaneSessionActor {
.job_context
.validate()
.map_err(|error| DataPlaneError::InvalidPath(error.to_string()))?;
let stream_arena = if config.stream_transport.is_some() {
let fd = unsafe { libc::dup(config.arena.arena_fd()) };
if fd < 0 {
return Err(DataPlaneError::SessionFailed(format!(
"duplicate arena backing for streams: {}",
std::io::Error::last_os_error()
)));
}
let owned = unsafe { OwnedFd::from_raw_fd(fd) };
let (mapped, _) = MappedArena::map(owned)
.map_err(|error| DataPlaneError::SessionFailed(error.to_string()))?;
Some(Arc::new(mapped))
} else {
None
};
Ok(Self {
arena: Some(config.arena),
arena_generation: config.arena_generation,
@ -104,9 +132,12 @@ impl HostDataPlaneSessionActor {
source_sender: config.source_sender,
source_publisher: config.source_publisher,
route_registrar: config.route_registrar,
stream_arena,
stream_transport: config.stream_transport,
child_session: None,
allocator: None,
active_bindings: HashSet::new(),
stream_bindings: HashMap::new(),
state: HostSessionState::AwaitingAttachment,
})
}
@ -159,7 +190,10 @@ impl HostDataPlaneSessionActor {
}
fn maybe_finish_close(&mut self) {
if self.state == HostSessionState::Closing && self.active_bindings.is_empty() {
if self.state == HostSessionState::Closing
&& self.active_bindings.is_empty()
&& self.stream_bindings.is_empty()
{
self.state = HostSessionState::Closed;
}
}
@ -329,6 +363,133 @@ impl ActorInterface for HostDataPlaneSessionActor {
),
}
}
ref message @ (HostSessionIn::OpenReadStream {
ref path,
child_session,
operation,
replace,
}
| HostSessionIn::OpenWriteStream {
ref path,
child_session,
operation,
replace,
}) => {
let role = match message {
HostSessionIn::OpenReadStream { .. } => StreamRole::Sink,
HostSessionIn::OpenWriteStream { .. } => StreamRole::Source,
_ => unreachable!(),
};
let data_operation = match role {
StreamRole::Source => DataOperation::WriteStream,
StreamRole::Sink => DataOperation::ReadStream,
};
let resolved = match self.validate_open(child_session, path, data_operation) {
Ok(path) => path,
Err(error) => {
self.send_open_failure(ctx, child_session, operation, error);
return;
}
};
let (Some(namespace), Some(arena), Some(transport)) = (
self.namespace.clone(),
self.stream_arena.clone(),
self.stream_transport.clone(),
) else {
self.send_open_failure(
ctx,
child_session,
operation,
DataPlaneError::SessionFailed(
"stream namespace or transport service is unavailable".to_owned(),
),
);
return;
};
let local_descriptor = match transport.descriptor() {
Ok(descriptor) => descriptor,
Err(reason) => {
self.send_open_failure(
ctx,
child_session,
operation,
DataPlaneError::StreamFault(reason),
);
return;
}
};
let binding = HostStreamBindingActor {
runtime: self.runtime.clone(),
host_session: ctx.self_addr(),
allocator: self.allocator.expect("allocator started"),
child_session,
operation,
path: resolved,
replace,
role,
namespace,
arena,
transport,
local_descriptor,
ring: None,
matched: None,
peer_descriptor: None,
transport_installed: false,
transport_ready: false,
transport_quiesced: false,
opened: false,
terminal: None,
data_waiters: Vec::new(),
capacity_waiters: Vec::new(),
close_waiters: Vec::new(),
release_started: false,
};
match ctx.spawn(binding) {
Ok(binding) => {
if let Some(publisher) = &self.source_publisher
&& let Err(error) = publisher.publish_source(binding)
{
let _ = ctx.stop_actor(binding);
self.send_open_failure(
ctx,
child_session,
operation,
DataPlaneError::SessionFailed(format!(
"publish stream endpoint: {error}"
)),
);
return;
}
self.stream_bindings.insert(operation, binding);
}
Err(error) => self.send_open_failure(
ctx,
child_session,
operation,
DataPlaneError::SessionFailed(error.to_string()),
),
}
}
HostSessionIn::CancelStream { operation } => {
if let Some(binding) = self.stream_bindings.get(&operation).copied() {
let _ = ctx.send(
binding,
HostStreamIn::Close {
clean: false,
reply_to: None,
},
);
}
}
HostSessionIn::StreamControl { binding, message } => {
if self
.stream_bindings
.values()
.any(|stream_binding| *stream_binding == binding)
{
let _ = ctx.send(binding, message);
}
}
HostSessionIn::ReleaseBlob {
binding,
lease_id,
@ -391,6 +552,8 @@ impl ActorInterface for HostDataPlaneSessionActor {
}
HostSessionIn::BindingDone { binding } | HostSessionIn::BindingDetached { binding } => {
self.active_bindings.remove(&binding);
self.stream_bindings
.retain(|_, stream_binding| *stream_binding != binding);
self.maybe_finish_close();
}
HostSessionIn::ConfigureRun { run_id, reply_to } => {
@ -422,6 +585,15 @@ impl ActorInterface for HostDataPlaneSessionActor {
for binding in self.active_bindings.iter().copied() {
let _ = ctx.send(binding, HostBindingIn::SessionClosed);
}
for binding in self.stream_bindings.values().copied() {
let _ = ctx.send(
binding,
HostStreamIn::Close {
clean: false,
reply_to: None,
},
);
}
self.maybe_finish_close();
}
}
@ -465,6 +637,14 @@ enum ArenaAllocatorIn {
transfer: ActorAddress,
kind: AllocationKind,
},
AllocateStream {
binding: ActorAddress,
capacity: u64,
},
ReleaseStream {
binding: ActorAddress,
ring: RingHandle,
},
ValidateSealed {
binding: ActorAddress,
lease: BlobLease,
@ -664,6 +844,65 @@ impl ArenaAllocatorActor {
ctx.spawn(source)
.map_err(|error| DataPlaneError::SessionFailed(error.to_string()))
}
fn allocate_stream(&mut self, capacity: u64) -> Result<RingHandle, DataPlaneError> {
let request_id = self.next_request_id;
self.next_request_id = self.next_request_id.checked_add(1).ok_or_else(|| {
DataPlaneError::SessionFailed("stream request id exhausted".to_owned())
})?;
let generation = self.next_generation;
self.next_generation = self
.next_generation
.checked_add(1)
.filter(|next| *next != 0)
.ok_or_else(|| {
DataPlaneError::SessionFailed("stream generation exhausted".to_owned())
})?;
byte_ring::install(
&mut self.arena,
ByteRingSpec {
capacity,
generation,
alignment: BLOB_ALIGNMENT,
request_id,
},
)
.map_err(|error| match error {
byte_ring::InstallError::LeaseRejected(_) | byte_ring::InstallError::LeaseQueued => {
DataPlaneError::ArenaExhausted
}
other => {
DataPlaneError::SessionFailed(format!("stream ring installation failed: {other:?}"))
}
})
}
fn release_stream(&mut self, ring: RingHandle) -> Result<(), DataPlaneError> {
let ring_id = RingId(ring.lease_id);
let Some(allocation) = self.arena.lookup_lease(ring_id) else {
return Err(DataPlaneError::StreamFault(
"stream arena lease is no longer live".to_owned(),
));
};
if allocation.layout.start_offset != ring.offset
|| allocation.layout.data_bytes != ring.capacity
{
return Err(DataPlaneError::StreamFault(
"stream arena lease does not match ring handle".to_owned(),
));
}
let events = self.arena.request(ArenaRequest::ReleaseRing {
ring_id,
proof: QuiescenceProof::verified(),
});
if matches!(events.as_slice(), [ArenaEvent::RingReleased { .. }]) {
Ok(())
} else {
Err(DataPlaneError::SessionFailed(
"arena rejected stream release".to_owned(),
))
}
}
}
impl ActorInterface for ArenaAllocatorActor {
@ -699,6 +938,14 @@ impl ActorInterface for ArenaAllocatorActor {
let _ = ctx.send(transfer, BlobTransferEvent::AllocatorFailed(error.into()));
}
}
ArenaAllocatorIn::AllocateStream { binding, capacity } => {
let result = self.allocate_stream(capacity);
let _ = ctx.send(binding, HostStreamIn::Allocated(result));
}
ArenaAllocatorIn::ReleaseStream { binding, ring } => {
let result = self.release_stream(ring);
let _ = ctx.send(binding, HostStreamIn::ReleaseComplete(result));
}
ArenaAllocatorIn::SealTransfer {
transfer,
lease,
@ -1056,7 +1303,6 @@ impl ActorInterface for NamespacePublishActor {
Err(error) => HostBindingIn::PublicationRejected(namespace_error(error)),
};
let _ = ctx.send(self.binding, response);
ctx.stop_self();
}
}
@ -1118,9 +1364,557 @@ impl ActorInterface for NamespaceUnpublishActor {
}
}
struct NamespaceStreamOpenActor {
proxy: ActorAddress,
parent: ActorAddress,
path: DataPath,
role: StreamRole,
replace: bool,
descriptor: Vec<u8>,
operation_id: OperationId,
completed: bool,
}
impl ActorInterface for NamespaceStreamOpenActor {
type Incoming = DataDirectoryOut;
type Response = ();
fn on_start(&mut self, ctx: &Ctx<'_>) {
let _ = ctx.send(
self.proxy,
NamespaceClientIn::Request {
request: NamespaceRequest::OpenStream {
path: self.path.clone(),
role: self.role,
endpoint: self.parent,
descriptor: self.descriptor.clone(),
replace: self.replace,
operation_id: self.operation_id,
},
reply_to: ctx.self_addr(),
},
);
}
fn handle(&mut self, ctx: &Ctx<'_>, message: DataDirectoryOut) {
let result = match message {
DataDirectoryOut::StreamOpened { result, .. } => result,
other => Err(NamespaceError::Protocol(format!(
"expected stream-open reply, received {other:?}"
))),
};
self.completed = true;
let _ = ctx.send(self.parent, HostStreamIn::NamespaceMatched(result));
ctx.stop_self();
}
fn on_stop(&mut self, ctx: &Ctx<'_>) {
if !self.completed {
let _ = ctx.send(
self.proxy,
NamespaceClientIn::Cancel {
reply_to: ctx.self_addr(),
},
);
}
}
}
struct NamespaceStreamCloseActor {
proxy: ActorAddress,
path: DataPath,
incarnation: StreamIncarnation,
}
impl ActorInterface for NamespaceStreamCloseActor {
type Incoming = DataDirectoryOut;
type Response = ();
fn on_start(&mut self, ctx: &Ctx<'_>) {
let _ = ctx.send(
self.proxy,
NamespaceClientIn::Request {
request: NamespaceRequest::CloseStream {
path: self.path.clone(),
incarnation: self.incarnation,
},
reply_to: ctx.self_addr(),
},
);
}
fn handle(&mut self, ctx: &Ctx<'_>, _message: DataDirectoryOut) {
ctx.stop_self();
}
}
struct RuntimeStreamNotifier {
runtime: Runtime,
target: ActorAddress,
}
impl StreamTransportNotifier for RuntimeStreamNotifier {
fn notify(&self, event: StreamTransportEvent) {
let _ = self
.runtime
.send_to(self.target, HostStreamIn::Transport(event));
}
}
struct HostStreamBindingActor {
runtime: Runtime,
host_session: ActorAddress,
allocator: ActorAddress,
child_session: ActorAddress,
operation: ActorAddress,
path: DataPath,
replace: bool,
role: StreamRole,
namespace: NamespaceClient,
arena: Arc<MappedArena>,
transport: Arc<dyn StreamTransport>,
local_descriptor: StreamPeerDescriptor,
ring: Option<RingHandle>,
matched: Option<StreamMatch>,
peer_descriptor: Option<StreamPeerDescriptor>,
transport_installed: bool,
transport_ready: bool,
opened: bool,
transport_quiesced: bool,
terminal: Option<DataPlaneError>,
data_waiters: Vec<ActorAddress>,
capacity_waiters: Vec<ActorAddress>,
close_waiters: Vec<ActorAddress>,
release_started: bool,
}
impl HostStreamBindingActor {
fn operation_id(address: ActorAddress) -> OperationId {
let mut bytes = [0_u8; 16];
bytes.copy_from_slice(&address.0[..16]);
OperationId::from_u128(u128::from_le_bytes(bytes))
}
fn fail_open(&mut self, ctx: &Ctx<'_>, error: DataPlaneError) {
if !self.opened {
let _ = ctx.send(
self.child_session,
ChildSessionIn::OperationFailed {
operation: self.operation,
error: error.clone(),
},
);
}
self.begin_terminal(ctx, error, true);
}
fn try_install_transport(&mut self, ctx: &Ctx<'_>) {
if self.transport_installed || self.terminal.is_some() {
return;
}
let (Some(ring), Some(matched)) = (self.ring, self.matched.clone()) else {
return;
};
let endpoint_role = match self.role {
StreamRole::Source => Role::Consumer,
StreamRole::Sink => Role::Producer,
};
let endpoint = match byte_ring::attach_mapped(&self.arena, ring, endpoint_role) {
Ok(endpoint) => endpoint,
Err(error) => {
self.fail_open(
ctx,
DataPlaneError::StreamFault(format!("attach host stream ring: {error:?}")),
);
return;
}
};
let notifier: Arc<dyn StreamTransportNotifier> = Arc::new(RuntimeStreamNotifier {
runtime: self.runtime.clone(),
target: ctx.self_addr(),
});
let install = match self.role {
StreamRole::Source => {
let Some(peer) = self.peer_descriptor.clone() else {
return;
};
self.transport.install_source(StreamSourceRequest {
incarnation: matched.incarnation,
peer,
endpoint,
notifier,
})
}
StreamRole::Sink => self.transport.install_sink(StreamSinkRequest {
incarnation: matched.incarnation,
endpoint,
notifier,
}),
};
match install {
Ok(()) => {
self.transport_installed = true;
if self.role == StreamRole::Sink && matched.sink_descriptor.is_empty() {
let _ = ctx.send(
matched.source,
HostStreamIn::PeerOffer {
incarnation: matched.incarnation,
descriptor: self.local_descriptor.clone(),
},
);
}
}
Err(reason) => self.fail_open(ctx, DataPlaneError::StreamFault(reason)),
}
}
fn open_if_ready(&mut self, ctx: &Ctx<'_>) {
if self.opened || !self.transport_ready || self.terminal.is_some() {
return;
}
let Some(ring) = self.ring else {
return;
};
self.opened = true;
let _ = ctx.send(
self.child_session,
ChildSessionIn::StreamOpened {
operation: self.operation,
host_binding: ctx.self_addr(),
ring,
role: match self.role {
StreamRole::Source => Role::Producer,
StreamRole::Sink => Role::Consumer,
},
},
);
}
fn wake_waiters(
ctx: &Ctx<'_>,
child_session: ActorAddress,
waiters: &mut Vec<ActorAddress>,
result: Result<(), DataPlaneError>,
) {
for reply_to in waiters.drain(..) {
let _ = ctx.send(
child_session,
ChildSessionIn::StreamWake {
reply_to,
result: result.clone(),
},
);
}
}
fn begin_terminal(&mut self, ctx: &Ctx<'_>, error: DataPlaneError, notify_peer: bool) {
if self.terminal.is_some() {
return;
}
self.terminal = Some(error.clone());
Self::wake_waiters(
ctx,
self.child_session,
&mut self.data_waiters,
Err(error.clone()),
);
Self::wake_waiters(
ctx,
self.child_session,
&mut self.capacity_waiters,
Err(error.clone()),
);
if let Some(matched) = &self.matched {
if notify_peer {
let peer = match self.role {
StreamRole::Source => matched.sink,
StreamRole::Sink => matched.source,
};
let peer_error = if matches!(error, DataPlaneError::StreamClosed) {
DataPlaneError::StreamClosed
} else {
DataPlaneError::PeerLost
};
let _ = ctx.send(
peer,
HostStreamIn::PeerTerminated {
incarnation: matched.incarnation,
error: peer_error,
},
);
}
let _ = ctx.spawn(NamespaceStreamCloseActor {
proxy: self.namespace.proxy(),
path: self.path.clone(),
incarnation: matched.incarnation,
});
if self.transport_installed {
self.transport.terminate(matched.incarnation);
if !self.transport_quiesced {
return;
}
}
}
self.release_ring(ctx);
}
fn complete_release(&mut self, ctx: &Ctx<'_>, result: Result<(), DataPlaneError>) {
for reply_to in self.close_waiters.drain(..) {
let _ = ctx.send(
self.child_session,
ChildSessionIn::StreamWake {
reply_to,
result: result.clone(),
},
);
}
let _ = ctx.send(
self.host_session,
HostSessionIn::BindingDone {
binding: ctx.self_addr(),
},
);
ctx.stop_self();
}
fn release_ring(&mut self, ctx: &Ctx<'_>) {
if self.release_started {
return;
}
self.release_started = true;
if let Some(ring) = self.ring {
let _ = ctx.send(
self.allocator,
ArenaAllocatorIn::ReleaseStream {
binding: ctx.self_addr(),
ring,
},
);
} else {
self.complete_release(ctx, Ok(()));
}
}
}
impl ActorInterface for HostStreamBindingActor {
type Incoming = HostStreamIn;
type Response = ();
fn on_start(&mut self, ctx: &Ctx<'_>) {
let _ = ctx.send(
self.allocator,
ArenaAllocatorIn::AllocateStream {
binding: ctx.self_addr(),
capacity: STREAM_RING_CAPACITY,
},
);
let _ = ctx.spawn(NamespaceStreamOpenActor {
proxy: self.namespace.proxy(),
parent: ctx.self_addr(),
path: self.path.clone(),
role: self.role,
descriptor: self.local_descriptor.0.clone(),
replace: self.replace,
operation_id: Self::operation_id(ctx.self_addr()),
completed: false,
});
}
fn handle(&mut self, ctx: &Ctx<'_>, message: HostStreamIn) {
match message {
HostStreamIn::NamespaceMatched(result) => match result {
Ok(matched) => {
let expected = match self.role {
StreamRole::Source => matched.source,
StreamRole::Sink => matched.sink,
};
if expected != ctx.self_addr() {
self.fail_open(
ctx,
DataPlaneError::StreamFault(
"namespace matched the wrong stream endpoint".to_owned(),
),
);
return;
}
if self.role == StreamRole::Source && !matched.sink_descriptor.is_empty() {
self.peer_descriptor =
Some(StreamPeerDescriptor(matched.sink_descriptor.clone()));
}
self.matched = Some(matched);
self.try_install_transport(ctx);
}
Err(error) => self.fail_open(ctx, namespace_error(error)),
},
HostStreamIn::Allocated(result) => match result {
Ok(ring) => {
self.ring = Some(ring);
self.try_install_transport(ctx);
}
Err(error) => self.fail_open(ctx, error),
},
HostStreamIn::PeerOffer {
incarnation,
descriptor,
} => {
if self.role != StreamRole::Source {
return;
}
if self
.matched
.as_ref()
.is_some_and(|matched| matched.incarnation == incarnation)
{
self.peer_descriptor = Some(descriptor);
self.try_install_transport(ctx);
}
}
HostStreamIn::Transport(StreamTransportEvent::Ready) => {
self.transport_ready = true;
self.open_if_ready(ctx);
}
HostStreamIn::Transport(StreamTransportEvent::DataAvailable) => {
Self::wake_waiters(ctx, self.child_session, &mut self.data_waiters, Ok(()));
}
HostStreamIn::Transport(StreamTransportEvent::CapacityAvailable) => {
Self::wake_waiters(ctx, self.child_session, &mut self.capacity_waiters, Ok(()));
}
HostStreamIn::Transport(StreamTransportEvent::Fault(reason)) => {
self.fail_open(ctx, DataPlaneError::StreamFault(reason));
}
HostStreamIn::Transport(StreamTransportEvent::Quiesced) => {
self.transport_quiesced = true;
if self.terminal.is_some() {
self.release_ring(ctx);
}
}
HostStreamIn::DataAvailable => {
if let Some(matched) = &self.matched {
self.transport.source_progress(matched.incarnation);
}
}
HostStreamIn::CapacityAvailable => {
if let Some(matched) = &self.matched {
self.transport.sink_progress(matched.incarnation);
}
}
HostStreamIn::WaitData { reply_to } => {
let result = self
.terminal
.as_ref()
.map(|error| Err(error.clone()))
.or_else(|| {
self.matched
.as_ref()
.filter(|matched| self.transport.sink_has_data(matched.incarnation))
.map(|_| Ok(()))
});
if let Some(result) = result {
let _ = ctx.send(
self.child_session,
ChildSessionIn::StreamWake { reply_to, result },
);
} else {
self.data_waiters.push(reply_to);
}
}
HostStreamIn::WaitCapacity { reply_to } => {
let result = self
.terminal
.as_ref()
.map(|error| Err(error.clone()))
.or_else(|| {
self.matched
.as_ref()
.filter(|matched| {
self.transport.source_has_capacity(matched.incarnation)
})
.map(|_| Ok(()))
});
if let Some(result) = result {
let _ = ctx.send(
self.child_session,
ChildSessionIn::StreamWake { reply_to, result },
);
} else {
self.capacity_waiters.push(reply_to);
}
}
HostStreamIn::Close { clean, reply_to } => {
if let Some(reply_to) = reply_to {
self.close_waiters.push(reply_to);
}
let error = if clean {
DataPlaneError::StreamClosed
} else {
DataPlaneError::OperationCancelled
};
self.fail_open(ctx, error);
}
HostStreamIn::PeerTerminated { incarnation, error } => {
if self
.matched
.as_ref()
.is_some_and(|matched| matched.incarnation == incarnation)
{
if !self.opened {
let _ = ctx.send(
self.child_session,
ChildSessionIn::OperationFailed {
operation: self.operation,
error: error.clone(),
},
);
}
self.begin_terminal(ctx, error, false);
}
}
HostStreamIn::ReleaseComplete(result) => {
if let Err(error) = &result
&& !self.opened
{
let _ = ctx.send(
self.child_session,
ChildSessionIn::OperationFailed {
operation: self.operation,
error: error.clone(),
},
);
}
self.complete_release(ctx, result);
}
}
}
fn on_stop(&mut self, _ctx: &Ctx<'_>) {
if let Some(matched) = &self.matched {
self.transport.terminate(matched.incarnation);
}
}
}
fn namespace_error(error: NamespaceError) -> DataPlaneError {
match error {
NamespaceError::PathNotFound(path) => DataPlaneError::PathNotFound(path),
NamespaceError::WrongEntryType {
path,
expected,
found,
} => DataPlaneError::WrongEntryType {
path,
expected,
found,
},
NamespaceError::PathReplaced(path) => DataPlaneError::PathReplaced(path),
NamespaceError::DuplicateStreamRole { path, role } => {
DataPlaneError::SessionFailed(format!("stream path {path} already has a {role:?}"))
}
NamespaceError::StaleIncarnation { path, incarnation } => {
DataPlaneError::SessionFailed(format!(
"stream path {path} no longer names incarnation {}:{}",
incarnation.authority_epoch, incarnation.revision
))
}
NamespaceError::SourceRecovery(reason) => DataPlaneError::SourceFailure(reason),
NamespaceError::DirectoryUnavailable(reason)
| NamespaceError::Storage(reason)
@ -1345,6 +2139,7 @@ impl HostBlobBindingActor {
fn finish_without_lease(&mut self, ctx: &Ctx<'_>, outcome: ReleaseOutcome) {
self.state = HostBindingState::Released;
let read_released = matches!(outcome, ReleaseOutcome::ReadReleased);
if let Some(auxiliary) = self.auxiliary.take() {
let _ = ctx.stop_actor(auxiliary);
}
@ -1354,6 +2149,9 @@ impl HostBlobBindingActor {
ChildSessionIn::WriteAborted { operation },
);
}
if read_released {
let _ = ctx.send(self.child_session, ChildSessionIn::BlobReleased);
}
let _ = ctx.send(
self.host_session,
HostSessionIn::BindingDone {

View file

@ -25,3 +25,4 @@ pub mod path;
pub mod protocol;
pub mod ring;
pub mod source;
pub mod stream_transport;

View file

@ -33,9 +33,51 @@ pub struct BlobBinding {
pub revision: u64,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum EntryKind {
Blob,
Stream,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum StreamRole {
Source,
Sink,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct StreamIncarnation {
pub authority_epoch: u64,
pub revision: u64,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct StreamMatch {
pub incarnation: StreamIncarnation,
pub source: ActorAddress,
pub source_descriptor: Vec<u8>,
pub sink_descriptor: Vec<u8>,
pub sink: ActorAddress,
pub revision: u64,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum NamespaceError {
PathNotFound(DataPath),
WrongEntryType {
path: DataPath,
expected: EntryKind,
found: EntryKind,
},
PathReplaced(DataPath),
DuplicateStreamRole {
path: DataPath,
role: StreamRole,
},
StaleIncarnation {
path: DataPath,
incarnation: StreamIncarnation,
},
OperationConflict(OperationId),
Storage(String),
SourceRecovery(String),
@ -47,6 +89,25 @@ impl fmt::Display for NamespaceError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::PathNotFound(path) => write!(f, "data path not found: {path}"),
Self::WrongEntryType {
path,
expected,
found,
} => write!(
f,
"data path {path} has entry kind {found:?}, expected {expected:?}"
),
Self::PathReplaced(path) => {
write!(f, "pending data path was replaced: {path}")
}
Self::DuplicateStreamRole { path, role } => {
write!(f, "stream path {path} already has a {role:?}")
}
Self::StaleIncarnation { path, incarnation } => write!(
f,
"stream path {path} no longer names incarnation {}:{}",
incarnation.authority_epoch, incarnation.revision
),
Self::OperationConflict(operation) => write!(
f,
"namespace operation ID {:02x?} was reused for a different request",
@ -92,6 +153,26 @@ pub enum DataDirectoryIn {
operation_id: OperationId,
reply_to: ActorAddress,
},
OpenStream {
request_id: DirectoryRequestId,
path: DataPath,
role: StreamRole,
descriptor: Vec<u8>,
endpoint: ActorAddress,
replace: bool,
operation_id: OperationId,
reply_to: ActorAddress,
},
CancelStream {
path: DataPath,
operation_id: OperationId,
},
CloseStream {
request_id: DirectoryRequestId,
path: DataPath,
incarnation: StreamIncarnation,
reply_to: ActorAddress,
},
}
impl NetworkMessage for DataDirectoryIn {
@ -117,14 +198,25 @@ pub enum DataDirectoryOut {
authority_epoch: u64,
result: Result<MutationReceipt, NamespaceError>,
},
StreamOpened {
request_id: DirectoryRequestId,
authority_epoch: u64,
result: Result<StreamMatch, NamespaceError>,
},
StreamClosed {
request_id: DirectoryRequestId,
authority_epoch: u64,
result: Result<(), NamespaceError>,
},
}
impl DataDirectoryOut {
pub fn request_id(&self) -> DirectoryRequestId {
match self {
Self::Registered { request_id, .. }
| Self::Resolved { request_id, .. }
| Self::Unregistered { request_id, .. } => *request_id,
| Self::Unregistered { request_id, .. }
| Self::StreamOpened { request_id, .. }
| Self::StreamClosed { request_id, .. } => *request_id,
}
}
}
@ -145,8 +237,19 @@ pub enum NamespaceRequest {
path: DataPath,
operation_id: OperationId,
},
OpenStream {
path: DataPath,
role: StreamRole,
endpoint: ActorAddress,
descriptor: Vec<u8>,
replace: bool,
operation_id: OperationId,
},
CloseStream {
path: DataPath,
incarnation: StreamIncarnation,
},
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum NamespaceClientIn {
Request {
@ -177,9 +280,42 @@ enum RuntimeSource {
Unavailable(String),
}
struct PendingStream {
role: StreamRole,
endpoint: ActorAddress,
operation_id: OperationId,
request_id: DirectoryRequestId,
descriptor: Vec<u8>,
reply_to: ActorAddress,
revision: u64,
}
struct StreamOpenRequest {
request_id: DirectoryRequestId,
path: DataPath,
role: StreamRole,
endpoint: ActorAddress,
replace: bool,
descriptor: Vec<u8>,
operation_id: OperationId,
reply_to: ActorAddress,
}
struct ActiveStream {
binding: StreamMatch,
source_operation: OperationId,
sink_operation: OperationId,
}
enum RuntimeStream {
Pending(PendingStream),
Active(ActiveStream),
}
pub struct DataDirectoryActor {
store: NamespaceStore,
sources: BTreeMap<DataPath, RuntimeSource>,
streams: BTreeMap<DataPath, RuntimeStream>,
authority_epoch: u64,
}
@ -202,6 +338,7 @@ impl DataDirectoryActor {
Ok(Self {
store,
sources,
streams: BTreeMap::new(),
authority_epoch,
})
}
@ -232,6 +369,257 @@ impl DataDirectoryActor {
})
}
fn bind_stream(
&mut self,
path: DataPath,
operation_id: OperationId,
retired: Option<ActorAddress>,
) -> Result<MutationReceipt, NamespaceError> {
let request = MutationRequest::BindStream { path: path.clone() };
if let Some(replayed) = self.replay(operation_id, &request) {
return replayed;
}
let revision = self.store.snapshot().next_revision;
let next_revision = revision
.checked_add(1)
.filter(|revision| *revision != 0)
.ok_or(NamespaceStoreError::RevisionExhausted)?;
let receipt = MutationReceipt { revision };
let mut next = self.store.snapshot().clone();
next.next_revision = next_revision;
next.bindings.remove(&path);
if let Some(retired) = retired
&& !next.retirements.contains(&retired)
{
next.retirements.push(retired);
}
next.operations.insert(
operation_id,
PersistedOperation {
request,
result: PersistedMutationResult::Committed(receipt),
},
);
self.store.commit(next)?;
self.sources.remove(&path);
Ok(receipt)
}
fn send_stream_result(
&self,
ctx: &Ctx<'_>,
request_id: DirectoryRequestId,
reply_to: ActorAddress,
result: Result<StreamMatch, NamespaceError>,
) {
let _ = ctx.send(
reply_to,
NamespaceClientIn::DirectoryReply(DataDirectoryOut::StreamOpened {
request_id,
authority_epoch: self.authority_epoch,
result,
}),
);
}
fn displace_stream(&mut self, ctx: &Ctx<'_>, path: &DataPath) {
if let Some(RuntimeStream::Pending(pending)) = self.streams.remove(path) {
self.send_stream_result(
ctx,
pending.request_id,
pending.reply_to,
Err(NamespaceError::PathReplaced(path.clone())),
);
}
}
fn open_stream(&mut self, ctx: &Ctx<'_>, request: StreamOpenRequest) {
let StreamOpenRequest {
request_id,
path,
role,
endpoint,
replace,
descriptor,
operation_id,
reply_to,
} = request;
if let Some(RuntimeStream::Pending(pending)) = self.streams.get_mut(&path)
&& pending.operation_id == operation_id
{
if pending.role != role || pending.endpoint != endpoint {
self.send_stream_result(
ctx,
request_id,
reply_to,
Err(NamespaceError::OperationConflict(operation_id)),
);
} else {
pending.request_id = request_id;
pending.reply_to = reply_to;
}
return;
}
if let Some(RuntimeStream::Active(active)) = self.streams.get(&path)
&& (active.source_operation == operation_id || active.sink_operation == operation_id)
{
self.send_stream_result(ctx, request_id, reply_to, Ok(active.binding.clone()));
return;
}
let compatible_pending = matches!(
self.streams.get(&path),
Some(RuntimeStream::Pending(pending)) if pending.role != role
);
if replace && self.streams.contains_key(&path) && !compatible_pending {
self.displace_stream(ctx, &path);
}
if let Some(RuntimeStream::Pending(pending)) = self.streams.remove(&path) {
if pending.role == role {
self.streams
.insert(path.clone(), RuntimeStream::Pending(pending));
self.send_stream_result(
ctx,
request_id,
reply_to,
Err(NamespaceError::DuplicateStreamRole { path, role }),
);
return;
}
let (
source,
sink,
source_descriptor,
sink_descriptor,
source_operation,
sink_operation,
) = match role {
StreamRole::Source => (
endpoint,
pending.endpoint,
descriptor,
pending.descriptor,
operation_id,
pending.operation_id,
),
StreamRole::Sink => (
pending.endpoint,
endpoint,
pending.descriptor,
descriptor,
pending.operation_id,
operation_id,
),
};
let binding = StreamMatch {
incarnation: StreamIncarnation {
authority_epoch: self.authority_epoch,
revision: pending.revision,
},
source,
source_descriptor,
sink_descriptor,
sink,
revision: pending.revision,
};
self.send_stream_result(
ctx,
pending.request_id,
pending.reply_to,
Ok(binding.clone()),
);
self.send_stream_result(ctx, request_id, reply_to, Ok(binding.clone()));
self.streams.insert(
path,
RuntimeStream::Active(ActiveStream {
binding,
source_operation,
sink_operation,
}),
);
return;
}
if self.streams.contains_key(&path) {
self.send_stream_result(
ctx,
request_id,
reply_to,
Err(NamespaceError::DuplicateStreamRole { path, role }),
);
return;
}
if self.store.snapshot().bindings.contains_key(&path) && !replace {
self.send_stream_result(
ctx,
request_id,
reply_to,
Err(NamespaceError::WrongEntryType {
path,
expected: EntryKind::Stream,
found: EntryKind::Blob,
}),
);
return;
}
let retired = self.sources.get(&path).and_then(|source| match source {
RuntimeSource::Available(actor) => Some(*actor),
RuntimeSource::Unavailable(_) => None,
});
match self.bind_stream(path.clone(), operation_id, retired) {
Ok(receipt) => {
if let Some(retired) = retired {
let _ = ctx.send(retired, BlobSourceIn::Retire);
}
self.streams.insert(
path,
RuntimeStream::Pending(PendingStream {
role,
descriptor,
endpoint,
operation_id,
request_id,
reply_to,
revision: receipt.revision,
}),
);
}
Err(error) => self.send_stream_result(ctx, request_id, reply_to, Err(error)),
}
}
fn cancel_stream(&mut self, path: &DataPath, operation_id: OperationId) {
let should_remove = matches!(
self.streams.get(path),
Some(RuntimeStream::Pending(pending)) if pending.operation_id == operation_id
);
if should_remove {
self.streams.remove(path);
}
}
fn close_stream(
&mut self,
path: &DataPath,
incarnation: StreamIncarnation,
) -> Result<(), NamespaceError> {
let matches = matches!(
self.streams.get(path),
Some(RuntimeStream::Active(active)) if active.binding.incarnation == incarnation
);
if matches {
self.streams.remove(path);
Ok(())
} else {
Err(NamespaceError::StaleIncarnation {
path: path.clone(),
incarnation,
})
}
}
fn register(
&mut self,
path: DataPath,
@ -283,6 +671,13 @@ impl DataDirectoryActor {
}
fn resolve(&self, path: &DataPath) -> Result<BlobBinding, NamespaceError> {
if self.streams.contains_key(path) {
return Err(NamespaceError::WrongEntryType {
path: path.clone(),
expected: EntryKind::Blob,
found: EntryKind::Stream,
});
}
let persisted = self
.store
.snapshot()
@ -376,6 +771,7 @@ impl ActorInterface for DataDirectoryActor {
operation_id,
reply_to,
} => {
let logical = path.clone();
let replayed = self.store.snapshot().operations.contains_key(&operation_id);
let retired = (!replayed)
.then(|| self.sources.get(&path))
@ -390,6 +786,9 @@ impl ActorInterface for DataDirectoryActor {
{
let _ = ctx.send(retired, BlobSourceIn::Retire);
}
if result.is_ok() {
self.displace_stream(ctx, &logical);
}
let _ = ctx.send(
reply_to,
NamespaceClientIn::DirectoryReply(DataDirectoryOut::Registered {
@ -420,6 +819,21 @@ impl ActorInterface for DataDirectoryActor {
operation_id,
reply_to,
} => {
if self.streams.contains_key(&path) {
let _ = ctx.send(
reply_to,
NamespaceClientIn::DirectoryReply(DataDirectoryOut::Unregistered {
request_id,
authority_epoch: self.authority_epoch,
result: Err(NamespaceError::WrongEntryType {
path,
expected: EntryKind::Blob,
found: EntryKind::Stream,
}),
}),
);
return;
}
let replayed = self.store.snapshot().operations.contains_key(&operation_id);
let retired = (!replayed)
.then(|| self.sources.get(&path))
@ -443,6 +857,47 @@ impl ActorInterface for DataDirectoryActor {
}),
);
}
DataDirectoryIn::OpenStream {
request_id,
path,
role,
endpoint,
descriptor,
replace,
operation_id,
reply_to,
} => self.open_stream(
ctx,
StreamOpenRequest {
request_id,
path,
role,
endpoint,
replace,
descriptor,
operation_id,
reply_to,
},
),
DataDirectoryIn::CancelStream { path, operation_id } => {
self.cancel_stream(&path, operation_id);
}
DataDirectoryIn::CloseStream {
request_id,
path,
incarnation,
reply_to,
} => {
let result = self.close_stream(&path, incarnation);
let _ = ctx.send(
reply_to,
NamespaceClientIn::DirectoryReply(DataDirectoryOut::StreamClosed {
request_id,
authority_epoch: self.authority_epoch,
result,
}),
);
}
}
}
}
@ -513,6 +968,29 @@ impl NamespaceClientActor {
operation_id: *operation_id,
reply_to: ctx.self_addr(),
},
NamespaceRequest::OpenStream {
path,
role,
endpoint,
descriptor,
replace,
operation_id,
} => DataDirectoryIn::OpenStream {
request_id,
path: path.clone(),
role: *role,
endpoint: *endpoint,
descriptor: descriptor.clone(),
replace: *replace,
operation_id: *operation_id,
reply_to: ctx.self_addr(),
},
NamespaceRequest::CloseStream { path, incarnation } => DataDirectoryIn::CloseStream {
request_id,
path: path.clone(),
incarnation: *incarnation,
reply_to: ctx.self_addr(),
},
};
let _ = ctx.send(directory, message);
}
@ -558,6 +1036,25 @@ impl ActorInterface for NamespaceClientActor {
.insert(request_id, PendingRequest { request, reply_to });
}
NamespaceClientIn::Cancel { reply_to } => {
let cancelled: Vec<(DataPath, OperationId)> = self
.pending
.values()
.filter(|pending| pending.reply_to == reply_to)
.filter_map(|pending| match &pending.request {
NamespaceRequest::OpenStream {
path, operation_id, ..
} => Some((path.clone(), *operation_id)),
_ => None,
})
.collect();
if let Some(directory) = self.discovery.current_directory() {
for (path, operation_id) in cancelled {
let _ = ctx.send(
directory,
DataDirectoryIn::CancelStream { path, operation_id },
);
}
}
self.pending
.retain(|_, pending| pending.reply_to != reply_to);
}
@ -692,6 +1189,92 @@ impl NamespaceClient {
))),
}
}
async fn open_stream_inner(
&self,
path: DataPath,
role: StreamRole,
endpoint: ActorAddress,
replace: bool,
operation_id: OperationId,
) -> Result<StreamMatch, NamespaceError> {
match self
.request(NamespaceRequest::OpenStream {
path,
role,
endpoint,
replace,
descriptor: Vec::new(),
operation_id,
})
.await?
{
DataDirectoryOut::StreamOpened { result, .. } => result,
other => Err(NamespaceError::Protocol(format!(
"expected stream-open reply, received {other:?}"
))),
}
}
pub async fn open_stream(
&self,
path: DataPath,
role: StreamRole,
endpoint: ActorAddress,
operation_id: OperationId,
) -> Result<StreamMatch, NamespaceError> {
self.open_stream_inner(path, role, endpoint, false, operation_id)
.await
}
pub async fn replace_with_stream(
&self,
path: DataPath,
role: StreamRole,
endpoint: ActorAddress,
operation_id: OperationId,
) -> Result<StreamMatch, NamespaceError> {
self.open_stream_inner(path, role, endpoint, true, operation_id)
.await
}
pub async fn close_stream(
&self,
path: DataPath,
incarnation: StreamIncarnation,
) -> Result<(), NamespaceError> {
match self
.request(NamespaceRequest::CloseStream { path, incarnation })
.await?
{
DataDirectoryOut::StreamClosed { result, .. } => result,
other => Err(NamespaceError::Protocol(format!(
"expected stream-close reply, received {other:?}"
))),
}
}
}
struct DirectoryStreamCancellation {
runtime: Runtime,
directory: ActorAddress,
path: DataPath,
operation_id: OperationId,
armed: bool,
}
impl Drop for DirectoryStreamCancellation {
fn drop(&mut self) {
if self.armed {
let _ = self.runtime.send_to(
self.directory,
DataDirectoryIn::CancelStream {
path: self.path.clone(),
operation_id: self.operation_id,
},
);
}
}
}
#[derive(Clone)]
@ -814,6 +1397,100 @@ impl DirectoryClient {
))),
}
}
async fn open_stream_inner(
&self,
path: DataPath,
role: StreamRole,
endpoint: ActorAddress,
replace: bool,
operation_id: OperationId,
) -> Result<StreamMatch, NamespaceError> {
let inbox = self
.runtime
.new_inbox::<NamespaceClientIn>()
.map_err(|error| NamespaceError::DirectoryUnavailable(error.to_string()))?;
let mut cancellation = DirectoryStreamCancellation {
runtime: self.runtime.clone(),
directory: self.directory,
path: path.clone(),
operation_id,
armed: true,
};
self.runtime
.send_to(
self.directory,
DataDirectoryIn::OpenStream {
request_id: self.request_id(),
path,
role,
endpoint,
replace,
operation_id,
descriptor: Vec::new(),
reply_to: *inbox.addr(),
},
)
.map_err(|error| NamespaceError::DirectoryUnavailable(error.to_string()))?;
let result = match self.receive(&inbox).await? {
DataDirectoryOut::StreamOpened { result, .. } => result,
other => Err(NamespaceError::Protocol(format!(
"expected stream-open reply, received {other:?}"
))),
};
cancellation.armed = false;
result
}
pub async fn open_stream(
&self,
path: DataPath,
role: StreamRole,
endpoint: ActorAddress,
operation_id: OperationId,
) -> Result<StreamMatch, NamespaceError> {
self.open_stream_inner(path, role, endpoint, false, operation_id)
.await
}
pub async fn replace_with_stream(
&self,
path: DataPath,
role: StreamRole,
endpoint: ActorAddress,
operation_id: OperationId,
) -> Result<StreamMatch, NamespaceError> {
self.open_stream_inner(path, role, endpoint, true, operation_id)
.await
}
pub async fn close_stream(
&self,
path: DataPath,
incarnation: StreamIncarnation,
) -> Result<(), NamespaceError> {
let inbox = self
.runtime
.new_inbox::<NamespaceClientIn>()
.map_err(|error| NamespaceError::DirectoryUnavailable(error.to_string()))?;
self.runtime
.send_to(
self.directory,
DataDirectoryIn::CloseStream {
request_id: self.request_id(),
path,
incarnation,
reply_to: *inbox.addr(),
},
)
.map_err(|error| NamespaceError::DirectoryUnavailable(error.to_string()))?;
match self.receive(&inbox).await? {
DataDirectoryOut::StreamClosed { result, .. } => result,
other => Err(NamespaceError::Protocol(format!(
"expected stream-close reply, received {other:?}"
))),
}
}
}
pub fn register_namespace_codecs(registry: &mut CodecRegistry) {

View file

@ -72,6 +72,9 @@ pub enum MutationRequest {
length: u64,
recovery: SourceRecovery,
},
BindStream {
path: DataPath,
},
Unregister {
path: DataPath,
},

View file

@ -7,8 +7,11 @@ use swactor::actor::ActorAddress;
use swactor_transport::{CodecRegistry, JsonCodec, NetworkMessage};
use crate::blob::{BlobError, BlobLease, BlobMetadata};
use crate::byte_ring::{RingHandle, Role};
use crate::ids::BlobLeaseId;
use crate::namespace::{EntryKind, NamespaceError, StreamIncarnation, StreamMatch};
use crate::path::DataPath;
use crate::stream_transport::{StreamPeerDescriptor, StreamTransportEvent};
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct JobCapability([u8; 32]);
@ -108,7 +111,15 @@ pub enum DataPlaneError {
ArenaExhausted,
Blob(BlobFailure),
OperationCancelled,
StreamsDeferred,
WrongEntryType {
path: DataPath,
expected: EntryKind,
found: EntryKind,
},
PathReplaced(DataPath),
PeerLost,
StreamFault(String),
StreamClosed,
}
impl fmt::Display for DataPlaneError {
@ -127,7 +138,18 @@ impl fmt::Display for DataPlaneError {
Self::ArenaExhausted => f.write_str("data-plane arena is exhausted"),
Self::Blob(reason) => write!(f, "blob lease failure: {reason:?}"),
Self::OperationCancelled => f.write_str("data-plane operation was cancelled"),
Self::StreamsDeferred => f.write_str("actor-driven streams are not installed"),
Self::WrongEntryType {
path,
expected,
found,
} => write!(
f,
"data path {path} has entry kind {found:?}, expected {expected:?}"
),
Self::PathReplaced(path) => write!(f, "data path was replaced: {path}"),
Self::PeerLost => f.write_str("stream peer was lost"),
Self::StreamFault(reason) => write!(f, "stream fault: {reason}"),
Self::StreamClosed => f.write_str("stream is closed"),
}
}
}
@ -162,6 +184,25 @@ pub enum HostSessionIn {
child_session: ActorAddress,
operation: ActorAddress,
},
OpenReadStream {
path: DataPath,
child_session: ActorAddress,
operation: ActorAddress,
replace: bool,
},
OpenWriteStream {
path: DataPath,
child_session: ActorAddress,
operation: ActorAddress,
replace: bool,
},
CancelStream {
operation: ActorAddress,
},
StreamControl {
binding: ActorAddress,
message: HostStreamIn,
},
ReleaseBlob {
binding: ActorAddress,
lease_id: BlobLeaseId,
@ -224,6 +265,33 @@ pub enum ChildSessionIn {
length: u64,
reply_to: ActorAddress,
},
OpenReadStream {
path: DataPath,
reply_to: ActorAddress,
replace: bool,
},
OpenWriteStream {
path: DataPath,
reply_to: ActorAddress,
replace: bool,
},
CancelStream {
reply_to: ActorAddress,
},
StreamWake {
reply_to: ActorAddress,
result: Result<(), DataPlaneError>,
},
StreamControl {
binding: ActorAddress,
message: HostStreamIn,
},
ReleaseBlob {
binding: ActorAddress,
lease_id: BlobLeaseId,
generation: u64,
},
BlobReleased,
BlobOpened {
operation: ActorAddress,
host_binding: ActorAddress,
@ -236,6 +304,12 @@ pub enum ChildSessionIn {
lease: BlobLease,
metadata: BlobMetadata,
},
StreamOpened {
operation: ActorAddress,
host_binding: ActorAddress,
ring: RingHandle,
role: Role,
},
OperationFailed {
operation: ActorAddress,
error: DataPlaneError,
@ -258,9 +332,44 @@ impl NetworkMessage for ChildSessionIn {
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum HostStreamIn {
NamespaceMatched(Result<StreamMatch, NamespaceError>),
Allocated(Result<RingHandle, DataPlaneError>),
PeerOffer {
incarnation: StreamIncarnation,
descriptor: StreamPeerDescriptor,
},
Transport(StreamTransportEvent),
DataAvailable,
CapacityAvailable,
WaitData {
reply_to: ActorAddress,
},
WaitCapacity {
reply_to: ActorAddress,
},
Close {
clean: bool,
reply_to: Option<ActorAddress>,
},
PeerTerminated {
incarnation: StreamIncarnation,
error: DataPlaneError,
},
ReleaseComplete(Result<(), DataPlaneError>),
}
impl NetworkMessage for HostStreamIn {
fn type_tag() -> &'static str {
"data-plane.host-stream.v1"
}
}
pub fn register_data_plane_codecs(registry: &mut CodecRegistry) {
registry.register::<HostSessionIn, _>(JsonCodec::default());
registry.register::<ChildSessionIn, _>(JsonCodec::default());
registry.register::<HostStreamIn, _>(JsonCodec::default());
crate::namespace::register_namespace_codecs(registry);
crate::blob_transfer::register_blob_transfer_codecs(registry);
crate::source::register_blob_source_codecs(registry);

View file

@ -0,0 +1,311 @@
//! Replaceable bounded transport port for SPSC stream incarnations.
use std::collections::BTreeMap;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use parking_lot::Mutex;
use serde::{Deserialize, Serialize};
use crate::byte_ring::{Endpoint, FlowError};
use crate::namespace::StreamIncarnation;
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct StreamPeerDescriptor(pub Vec<u8>);
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum StreamTransportEvent {
Ready,
DataAvailable,
CapacityAvailable,
Quiesced,
Fault(String),
}
pub trait StreamTransportNotifier: Send + Sync + 'static {
fn notify(&self, event: StreamTransportEvent);
}
pub struct StreamSourceRequest {
pub incarnation: StreamIncarnation,
pub peer: StreamPeerDescriptor,
pub endpoint: Endpoint,
pub notifier: Arc<dyn StreamTransportNotifier>,
}
pub struct StreamSinkRequest {
pub incarnation: StreamIncarnation,
pub endpoint: Endpoint,
pub notifier: Arc<dyn StreamTransportNotifier>,
}
/// Transport effect boundary. Namespace, terminal, reconnection, and lease
/// policy stay in the data-plane actors that invoke this port.
pub trait StreamTransport: Send + Sync + 'static {
fn descriptor(&self) -> Result<StreamPeerDescriptor, String>;
fn install_source(&self, request: StreamSourceRequest) -> Result<(), String>;
fn install_sink(&self, request: StreamSinkRequest) -> Result<(), String>;
fn source_progress(&self, incarnation: StreamIncarnation);
fn sink_progress(&self, incarnation: StreamIncarnation);
fn source_has_capacity(&self, _incarnation: StreamIncarnation) -> bool {
false
}
fn sink_has_data(&self, _incarnation: StreamIncarnation) -> bool {
false
}
fn terminate(&self, incarnation: StreamIncarnation);
}
struct LocalSource {
endpoint: Endpoint,
notifier: Arc<dyn StreamTransportNotifier>,
}
struct LocalSink {
endpoint: Endpoint,
notifier: Arc<dyn StreamTransportNotifier>,
}
#[derive(Default)]
struct LocalTransfer {
source: Option<LocalSource>,
sink: Option<LocalSink>,
ready: bool,
}
#[derive(Default)]
struct LocalState {
transfers: BTreeMap<StreamIncarnation, LocalTransfer>,
}
static NEXT_LOCAL_TRANSPORT: AtomicU64 = AtomicU64::new(1);
/// Direct in-process adapter. Host composition shares one instance between
/// sessions on a node. Payload bytes copy once from the source ring spans to
/// the destination ring spans; no payload-sized staging allocation exists.
pub struct LocalStreamTransport {
id: u64,
state: Mutex<LocalState>,
}
impl Default for LocalStreamTransport {
fn default() -> Self {
Self::new()
}
}
impl LocalStreamTransport {
pub fn new() -> Self {
Self {
id: NEXT_LOCAL_TRANSPORT.fetch_add(1, Ordering::Relaxed),
state: Mutex::new(LocalState::default()),
}
}
fn drive(&self, incarnation: StreamIncarnation) {
let mut notifications = Vec::new();
let mut fault = None;
{
let mut state = self.state.lock();
let Some(transfer) = state.transfers.get_mut(&incarnation) else {
return;
};
let (Some(source), Some(sink)) = (&mut transfer.source, &mut transfer.sink) else {
return;
};
if !transfer.ready {
transfer.ready = true;
notifications.push((Arc::clone(&source.notifier), StreamTransportEvent::Ready));
notifications.push((Arc::clone(&sink.notifier), StreamTransportEvent::Ready));
}
let mut moved = false;
loop {
let meta = match source.endpoint.next_record_meta() {
Ok(Some(meta)) => meta,
Ok(None) => break,
Err(error) => {
fault = Some(format_flow_error(error));
break;
}
};
let mut destination = match sink.endpoint.reserve_record(meta.kind, meta.len) {
Ok(destination) => destination,
Err(FlowError::InsufficientSpace { .. }) => break,
Err(error) => {
fault = Some(format_flow_error(error));
break;
}
};
let source_view = match source.endpoint.peek_record() {
Ok(Some(view)) => view,
Ok(None) => {
fault = Some("source record disappeared after inspection".to_owned());
break;
}
Err(error) => {
fault = Some(format_flow_error(error));
break;
}
};
let (source_first, source_second) = source_view.spans();
let (destination_first, destination_second) = destination.spans_mut();
copy_spans(
source_first,
source_second,
destination_first,
destination_second,
);
if let Err(error) = destination.commit() {
fault = Some(format_flow_error(error));
break;
}
if let Err(error) = source_view.release() {
fault = Some(format_flow_error(error));
break;
}
moved = true;
}
if moved {
notifications.push((
Arc::clone(&source.notifier),
StreamTransportEvent::CapacityAvailable,
));
notifications.push((
Arc::clone(&sink.notifier),
StreamTransportEvent::DataAvailable,
));
}
if let Some(reason) = &fault {
notifications.push((
Arc::clone(&source.notifier),
StreamTransportEvent::Fault(reason.clone()),
));
notifications.push((
Arc::clone(&sink.notifier),
StreamTransportEvent::Fault(reason.clone()),
));
}
}
for (notifier, event) in notifications {
notifier.notify(event);
}
if fault.is_some() {
self.state.lock().transfers.remove(&incarnation);
}
}
}
impl StreamTransport for LocalStreamTransport {
fn descriptor(&self) -> Result<StreamPeerDescriptor, String> {
Ok(StreamPeerDescriptor(self.id.to_le_bytes().to_vec()))
}
fn install_source(&self, request: StreamSourceRequest) -> Result<(), String> {
if request.peer != self.descriptor()? {
return Err("local stream peer belongs to a different transport instance".to_owned());
}
let incarnation = request.incarnation;
let mut state = self.state.lock();
let transfer = state.transfers.entry(incarnation).or_default();
if transfer.source.is_some() {
return Err("stream source is already installed".to_owned());
}
transfer.source = Some(LocalSource {
endpoint: request.endpoint,
notifier: request.notifier,
});
drop(state);
self.drive(incarnation);
Ok(())
}
fn install_sink(&self, request: StreamSinkRequest) -> Result<(), String> {
let incarnation = request.incarnation;
let mut state = self.state.lock();
let transfer = state.transfers.entry(incarnation).or_default();
if transfer.sink.is_some() {
return Err("stream sink is already installed".to_owned());
}
transfer.sink = Some(LocalSink {
endpoint: request.endpoint,
notifier: request.notifier,
});
drop(state);
self.drive(incarnation);
Ok(())
}
fn source_progress(&self, incarnation: StreamIncarnation) {
self.drive(incarnation);
}
fn sink_progress(&self, incarnation: StreamIncarnation) {
self.drive(incarnation);
}
fn source_has_capacity(&self, incarnation: StreamIncarnation) -> bool {
self.state
.lock()
.transfers
.get(&incarnation)
.and_then(|transfer| transfer.source.as_ref())
.is_some_and(|source| source.endpoint.probe().has_capacity())
}
fn sink_has_data(&self, incarnation: StreamIncarnation) -> bool {
self.state
.lock()
.transfers
.get(&incarnation)
.and_then(|transfer| transfer.sink.as_ref())
.is_some_and(|sink| sink.endpoint.probe().has_data())
}
fn terminate(&self, incarnation: StreamIncarnation) {
let transfer = self.state.lock().transfers.remove(&incarnation);
if let Some(transfer) = transfer {
if let Some(source) = transfer.source {
source.notifier.notify(StreamTransportEvent::Quiesced);
}
if let Some(sink) = transfer.sink {
sink.notifier.notify(StreamTransportEvent::Quiesced);
}
}
}
}
fn copy_spans(
source_first: &[u8],
source_second: &[u8],
destination_first: &mut [u8],
destination_second: &mut [u8],
) {
debug_assert_eq!(
source_first.len() + source_second.len(),
destination_first.len() + destination_second.len()
);
let sources = [source_first, source_second];
let mut source_index = 0;
let mut source_offset = 0;
for destination in [destination_first, destination_second] {
let mut destination_offset = 0;
while destination_offset < destination.len() {
while source_index < sources.len() && source_offset == sources[source_index].len() {
source_index += 1;
source_offset = 0;
}
let source = sources[source_index];
let take = (destination.len() - destination_offset).min(source.len() - source_offset);
destination[destination_offset..destination_offset + take]
.copy_from_slice(&source[source_offset..source_offset + take]);
destination_offset += take;
source_offset += take;
}
}
}
fn format_flow_error(error: FlowError) -> String {
format!("byte ring transport fault: {error:?}")
}

View file

@ -24,6 +24,7 @@ const CAPABILITY: JobCapability = JobCapability::new([9; 32]);
const ARENA_GENERATION: u64 = 17;
const SESSION_GENERATION: u64 = 29;
const WEIGHTS: &[u8] = b"0123456789abcdefghijklmn";
static STREAM_TEST_LOCK: parking_lot::Mutex<()> = parking_lot::Mutex::new(());
struct DirectRuntimeSink {
destination: Runtime,
@ -167,7 +168,54 @@ impl data_plane::source::BlobSourcePublisher for NoopSourceRegistrar {
}
}
struct RejectingStreamTransport;
impl data_plane::stream_transport::StreamTransport for RejectingStreamTransport {
fn descriptor(&self) -> Result<data_plane::stream_transport::StreamPeerDescriptor, String> {
Ok(data_plane::stream_transport::StreamPeerDescriptor(vec![1]))
}
fn install_source(
&self,
_request: data_plane::stream_transport::StreamSourceRequest,
) -> Result<(), String> {
Err("injected source transport failure".to_owned())
}
fn install_sink(
&self,
_request: data_plane::stream_transport::StreamSinkRequest,
) -> Result<(), String> {
Err("injected sink transport failure".to_owned())
}
fn source_progress(&self, _incarnation: data_plane::namespace::StreamIncarnation) {}
fn sink_progress(&self, _incarnation: data_plane::namespace::StreamIncarnation) {}
fn terminate(&self, _incarnation: data_plane::namespace::StreamIncarnation) {}
}
struct CollectBytes(Arc<parking_lot::Mutex<Vec<u8>>>);
impl data_plane::data_plane::StreamConsumer for CollectBytes {
fn consume(&self, bytes: &[u8]) -> Result<(), String> {
self.0.lock().extend_from_slice(bytes);
Ok(())
}
}
fn harness(arena_bytes: u64) -> Harness {
harness_with_transport(
arena_bytes,
Arc::new(data_plane::stream_transport::LocalStreamTransport::new()),
)
}
fn harness_with_transport(
arena_bytes: u64,
stream_transport: Arc<dyn data_plane::stream_transport::StreamTransport>,
) -> Harness {
let temp = TempState::new();
let mut arena = ArenaManager::boot(ArenaConfig {
node_id: NodeId(1),
@ -194,12 +242,20 @@ fn harness(arena_bytes: u64) -> Harness {
}));
let host_engine = Engine::new(
host_parts,
TokioBackend::new(TokioConfig::default()).expect("host backend"),
TokioBackend::new(TokioConfig {
worker_threads: 1,
..TokioConfig::default()
})
.expect("host backend"),
)
.expect("host engine");
let child_engine = Engine::new(
child_parts,
TokioBackend::new(TokioConfig::default()).expect("child backend"),
TokioBackend::new(TokioConfig {
worker_threads: 1,
..TokioConfig::default()
})
.expect("child backend"),
)
.expect("child engine");
@ -269,6 +325,7 @@ fn harness(arena_bytes: u64) -> Harness {
source_sender: Some(sender),
source_publisher: Some(Arc::new(NoopSourceRegistrar)),
route_registrar: None,
stream_transport: Some(stream_transport),
})
.expect("host session config"),
)
@ -314,7 +371,15 @@ fn attachment_without_a_host_reply_fails_on_actor_deadline() {
.unwrap();
let (parts, runtime) = runtime_parts();
runtime.set_remote_sink(Arc::new(BlackHoleSink));
let engine = Engine::new(parts, TokioBackend::new(TokioConfig::default()).unwrap()).unwrap();
let engine = Engine::new(
parts,
TokioBackend::new(TokioConfig {
worker_threads: 1,
..TokioConfig::default()
})
.unwrap(),
)
.unwrap();
let (mapped, resolved) = DataPlaneBootstrap::map_arena(handoff.arena_fd).unwrap();
let result = future::block_on(DataPlaneBootstrap::attach_mapped_with_deadline(
mapped,
@ -618,3 +683,194 @@ fn write_blob_seals_once_and_abort_publishes_nothing() {
Err(DataPlaneError::PathNotFound(_))
));
}
#[test]
fn stream_endpoints_open_only_after_match_and_deliver_eof_in_order() {
let _stream_test = STREAM_TEST_LOCK.lock();
let harness = harness(2 << 20);
let data_plane = harness.bootstrap.data_plane.clone();
let logical = path("/runs/self/results/inference");
future::block_on(async {
let mut reader_open = Box::pin(data_plane.read_stream(&logical));
assert!(
future::poll_once(reader_open.as_mut()).await.is_none(),
"reader open waits for its source"
);
let mut writer_open = Box::pin(data_plane.write_stream(&logical));
let mut writer = writer_open.as_mut().await.expect("writer opens");
let mut reader = reader_open.await.expect("reader opens");
writer.write(b"first").await.expect("write first");
writer.write(b"second").await.expect("write second");
assert_eq!(
reader.read().await.expect("read first"),
Some(b"first".to_vec())
);
assert_eq!(
reader.read().await.expect("read second"),
Some(b"second".to_vec())
);
writer.close().await.expect("clean writer close");
assert!(matches!(
writer.write(b"late").await,
Err(DataPlaneError::StreamClosed)
));
assert_eq!(reader.read().await.expect("read eof"), None);
assert_eq!(reader.read().await.expect("sticky eof"), None);
});
}
#[test]
fn stream_writer_suspends_until_reader_releases_bounded_capacity() {
let _stream_test = STREAM_TEST_LOCK.lock();
let harness = harness(2 << 20);
let data_plane = harness.bootstrap.data_plane.clone();
let logical = path("/runs/self/results/backpressure");
future::block_on(async {
let mut reader_open = Box::pin(data_plane.read_stream(&logical));
assert!(future::poll_once(reader_open.as_mut()).await.is_none());
let mut writer = data_plane
.write_stream(&logical)
.await
.expect("writer opens");
let mut reader = reader_open.await.expect("reader opens");
let capacity = writer.capacity() as usize;
let payload: Vec<u8> = (0..(capacity * 2 + 97))
.map(|index| (index % 251) as u8)
.collect();
let mut writing = Box::pin(writer.write(&payload));
assert!(
future::poll_once(writing.as_mut()).await.is_none(),
"bounded source and destination rings must eventually suspend the writer"
);
let mut observed = reader
.read()
.await
.expect("read releases destination capacity")
.expect("first data");
writing.await.expect("writer resumes");
while observed.len() < payload.len() {
observed.extend(
reader
.read()
.await
.expect("read remaining")
.expect("remaining data"),
);
}
assert_eq!(observed, payload);
writer.close().await.expect("close");
assert_eq!(reader.read().await.expect("eof"), None);
});
}
#[test]
fn transport_startup_failure_faults_both_pending_opens() {
let _stream_test = STREAM_TEST_LOCK.lock();
let harness = harness_with_transport(2 << 20, Arc::new(RejectingStreamTransport));
let data_plane = harness.bootstrap.data_plane.clone();
let logical = path("/runs/self/results/faulted");
future::block_on(async {
let mut reader_open = Box::pin(data_plane.read_stream(&logical));
assert!(future::poll_once(reader_open.as_mut()).await.is_none());
let writer_error = match data_plane.write_stream(&logical).await {
Ok(_) => panic!("writer must not open when transport setup fails"),
Err(error) => error,
};
let reader_error = match reader_open.await {
Ok(_) => panic!("reader must not open when transport setup fails"),
Err(error) => error,
};
assert!(matches!(
writer_error,
DataPlaneError::PeerLost | DataPlaneError::StreamFault(_)
));
assert!(matches!(
reader_error,
DataPlaneError::PeerLost | DataPlaneError::StreamFault(_)
));
});
}
#[test]
fn peer_replacement_requires_and_supports_a_fresh_incarnation() {
let _stream_test = STREAM_TEST_LOCK.lock();
let harness = harness(2 << 20);
let data_plane = harness.bootstrap.data_plane.clone();
let logical = path("/runs/self/results/failover");
future::block_on(async {
let mut first_reader_open = Box::pin(data_plane.read_stream(&logical));
assert!(
future::poll_once(first_reader_open.as_mut())
.await
.is_none()
);
let mut first_writer = data_plane
.write_stream(&logical)
.await
.expect("first writer");
let mut first_reader = first_reader_open.await.expect("first reader");
first_writer.write(b"old").await.expect("old write");
assert_eq!(
first_reader.read().await.expect("old read"),
Some(b"old".to_vec())
);
first_writer.abort().expect("abort first incarnation");
assert!(matches!(
first_reader.read().await,
Err(DataPlaneError::PeerLost)
));
let mut replacement_writer_open = Box::pin(data_plane.write_stream_replacing(&logical));
assert!(
future::poll_once(replacement_writer_open.as_mut())
.await
.is_none(),
"replacement writer waits for an explicit new reader"
);
let mut replacement_reader = data_plane
.read_stream(&logical)
.await
.expect("replacement reader");
let mut replacement_writer = replacement_writer_open.await.expect("replacement writer");
replacement_writer.write(b"new").await.expect("new write");
assert_eq!(
replacement_reader.read().await.expect("new read"),
Some(b"new".to_vec())
);
replacement_writer.close().await.expect("new close");
assert_eq!(replacement_reader.read().await.expect("new eof"), None);
});
}
#[test]
fn actor_stream_consumer_registers_before_writer_and_collects_to_eof() {
let _stream_test = STREAM_TEST_LOCK.lock();
let harness = harness(2 << 20);
let data_plane = harness.bootstrap.data_plane.clone();
let logical = path("/runs/self/results/collector");
let observed = Arc::new(parking_lot::Mutex::new(Vec::new()));
let consumer: Arc<dyn data_plane::data_plane::StreamConsumer> =
Arc::new(CollectBytes(Arc::clone(&observed)));
let completion = data_plane
.collect_stream(logical.clone(), consumer)
.expect("spawn collector");
future::block_on(async {
let mut writer = data_plane
.write_stream(&logical)
.await
.expect("writer matches collector");
writer.write(b"actor-").await.expect("first write");
writer.write(b"consumer").await.expect("second write");
writer.close().await.expect("close");
});
completion.wait().expect("collector completes");
assert_eq!(&*observed.lock(), b"actor-consumer");
}

View file

@ -470,3 +470,90 @@ fn operations_revalidate_cursors_and_never_panic() {
})
);
}
#[test]
fn pinned_record_blocks_capacity_until_release() {
let (arena, handle) = installed(32, 1);
let mut producer = attach(&arena, handle, Role::Producer).expect("producer");
let mut consumer = attach(&arena, handle, Role::Consumer).expect("consumer");
producer
.send_record(RecordKind::Data, &[7; 20])
.expect("send");
let view = consumer
.peek_record()
.expect("peek")
.expect("record must be visible");
assert_eq!(view.kind(), RecordKind::Data);
assert_eq!(view.len(), 20);
assert!(view.spans().1.is_empty());
assert_eq!(
producer.reserve(8).unwrap_err(),
FlowError::InsufficientSpace {
requested: 8,
free: 7,
}
);
drop(view);
assert_eq!(producer.reserve(8).expect("capacity released").len, 8);
}
#[test]
fn pinned_record_exposes_wrapped_payload_as_two_spans() {
let (arena, handle) = installed(32, 1);
let mut producer = attach(&arena, handle, Role::Producer).expect("producer");
let mut consumer = attach(&arena, handle, Role::Consumer).expect("consumer");
producer
.send_record(RecordKind::Data, &[1; 18])
.expect("first");
assert_eq!(
consumer.recv_record().expect("consume first"),
Some((RecordKind::Data, vec![1; 18]))
);
let expected: Vec<u8> = (0..15).collect();
producer
.send_record(RecordKind::Data, &expected)
.expect("wrapped record");
let view = consumer
.peek_record()
.expect("peek")
.expect("wrapped record visible");
let (first, second) = view.spans();
assert!(!first.is_empty());
assert!(!second.is_empty());
let observed: Vec<u8> = first.iter().chain(second).copied().collect();
assert_eq!(observed, expected);
}
#[test]
fn writable_record_is_invisible_until_commit() {
let (arena, handle) = installed(64, 1);
let mut producer = attach(&arena, handle, Role::Producer).expect("producer");
let mut consumer = attach(&arena, handle, Role::Consumer).expect("consumer");
let mut reservation = producer
.reserve_record(RecordKind::Data, 17)
.expect("reserve record");
let (first, second) = reservation.spans_mut();
for (index, byte) in first.iter_mut().chain(second).enumerate() {
*byte = index as u8;
}
assert!(consumer.peek_record().expect("peek uncommitted").is_none());
reservation.commit().expect("commit");
let view = consumer
.peek_record()
.expect("peek")
.expect("committed record");
let observed: Vec<u8> = view
.spans()
.0
.iter()
.chain(view.spans().1)
.copied()
.collect();
assert_eq!(observed, (0..17).collect::<Vec<_>>());
}

View file

@ -5,8 +5,8 @@ use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;
use data_plane::namespace::{
DataDirectoryActor, DirectoryClient, NamespaceClient, NamespaceClientActor, NamespaceClientIn,
NamespaceDiscovery, NamespaceError, OperationId, SourceRecovery,
DataDirectoryActor, DirectoryClient, EntryKind, NamespaceClient, NamespaceClientActor,
NamespaceClientIn, NamespaceDiscovery, NamespaceError, OperationId, SourceRecovery, StreamRole,
};
use data_plane::path::DataPath;
use futures_lite::future;
@ -80,7 +80,11 @@ fn spawn_directory(store: &Path) -> DirectoryHarness {
let directory = runtime.spawn(actor).expect("spawn directory actor");
let engine = Engine::new(
parts,
TokioBackend::new(TokioConfig::default()).expect("tokio backend"),
TokioBackend::new(TokioConfig {
worker_threads: 1,
..TokioConfig::default()
})
.expect("tokio backend"),
)
.expect("directory engine");
DirectoryHarness {
@ -168,6 +172,222 @@ fn namespace_mutations_are_linearizable_and_durable() {
));
}
#[test]
fn stream_rendezvous_is_symmetric_and_incarnations_are_isolated() {
let state = TempState::new("stream-rendezvous");
let directory = spawn_directory(&state.store());
let logical = path("/runs/7/results");
future::block_on(async {
let mut source_open = Box::pin(directory.client.open_stream(
logical.clone(),
StreamRole::Source,
source(10),
OperationId::from_u128(10),
));
assert!(future::poll_once(source_open.as_mut()).await.is_none());
let sink_match = directory
.client
.open_stream(
logical.clone(),
StreamRole::Sink,
source(11),
OperationId::from_u128(11),
)
.await
.expect("sink matches source");
let source_match = source_open.await.expect("source matches sink");
assert_eq!(source_match, sink_match);
assert_eq!(source_match.source, source(10));
assert_eq!(source_match.sink, source(11));
directory
.client
.close_stream(logical.clone(), source_match.incarnation)
.await
.expect("close first incarnation");
let mut sink_open = Box::pin(directory.client.open_stream(
logical.clone(),
StreamRole::Sink,
source(12),
OperationId::from_u128(12),
));
assert!(future::poll_once(sink_open.as_mut()).await.is_none());
let second_source = directory
.client
.open_stream(
logical,
StreamRole::Source,
source(13),
OperationId::from_u128(13),
)
.await
.expect("source matches waiting sink");
let second_sink = sink_open.await.expect("sink matches source");
assert_eq!(second_source, second_sink);
assert_ne!(source_match.incarnation, second_source.incarnation);
});
}
#[test]
fn typed_paths_require_explicit_rebinding() {
let state = TempState::new("typed-path");
let directory = spawn_directory(&state.store());
let logical = path("/typed/value");
let blob_source = source(20);
future::block_on(async {
directory
.client
.register(
logical.clone(),
blob_source,
4,
recovery(blob_source),
OperationId::from_u128(20),
)
.await
.expect("register blob");
assert!(matches!(
directory
.client
.open_stream(
logical.clone(),
StreamRole::Source,
source(21),
OperationId::from_u128(21),
)
.await,
Err(NamespaceError::WrongEntryType {
expected: EntryKind::Stream,
found: EntryKind::Blob,
..
})
));
let mut source_open = Box::pin(directory.client.replace_with_stream(
logical.clone(),
StreamRole::Source,
source(22),
OperationId::from_u128(22),
));
assert!(future::poll_once(source_open.as_mut()).await.is_none());
assert!(matches!(
directory.client.resolve(logical.clone()).await,
Err(NamespaceError::WrongEntryType {
expected: EntryKind::Blob,
found: EntryKind::Stream,
..
})
));
let sink_match = directory
.client
.open_stream(
logical,
StreamRole::Sink,
source(23),
OperationId::from_u128(23),
)
.await
.expect("match rebound stream");
assert_eq!(
source_open.await.expect("rebound source matched"),
sink_match
);
});
}
#[test]
fn replacing_waiting_stream_displaces_old_open() {
let state = TempState::new("stream-displacement");
let directory = spawn_directory(&state.store());
let logical = path("/replace/waiting");
future::block_on(async {
let mut old_open = Box::pin(directory.client.open_stream(
logical.clone(),
StreamRole::Source,
source(30),
OperationId::from_u128(30),
));
assert!(future::poll_once(old_open.as_mut()).await.is_none());
let mut replacement = Box::pin(directory.client.replace_with_stream(
logical.clone(),
StreamRole::Source,
source(31),
OperationId::from_u128(31),
));
assert!(future::poll_once(replacement.as_mut()).await.is_none());
assert!(matches!(
old_open.await,
Err(NamespaceError::PathReplaced(found)) if found == logical
));
let sink_match = directory
.client
.open_stream(
logical,
StreamRole::Sink,
source(32),
OperationId::from_u128(32),
)
.await
.expect("sink matches replacement");
assert_eq!(
replacement.await.expect("replacement source matched"),
sink_match
);
assert_eq!(sink_match.source, source(31));
});
}
#[test]
fn duplicate_stream_role_fails_without_replacing_the_waiter() {
let state = TempState::new("duplicate-stream-role");
let directory = spawn_directory(&state.store());
let logical = path("/duplicate/source");
future::block_on(async {
let mut first = Box::pin(directory.client.open_stream(
logical.clone(),
StreamRole::Source,
source(51),
OperationId::from_u128(51),
));
assert!(future::poll_once(first.as_mut()).await.is_none());
assert!(matches!(
directory
.client
.open_stream(
logical.clone(),
StreamRole::Source,
source(52),
OperationId::from_u128(52),
)
.await,
Err(NamespaceError::DuplicateStreamRole {
role: StreamRole::Source,
..
})
));
let matched = directory
.client
.open_stream(
logical,
StreamRole::Sink,
source(53),
OperationId::from_u128(53),
)
.await
.expect("sink matches original source");
assert_eq!(matched.source, source(51));
assert_eq!(first.await.expect("original source survives"), matched);
});
}
#[test]
fn committed_mutation_retry_has_at_most_once_effect() {
let state = TempState::new("idempotent");
@ -369,9 +589,15 @@ struct ModelBinding {
revision: u64,
}
#[derive(Clone, Debug)]
enum TypedModelEntry {
Blob(ModelBinding),
Stream(data_plane::namespace::StreamMatch),
}
proptest! {
#![proptest_config(ProptestConfig {
cases: 16,
cases: 8,
max_shrink_iters: 128,
..ProptestConfig::default()
})]
@ -434,9 +660,98 @@ proptest! {
let observed = future::block_on(directory.client.resolve(path.clone()))
.expect("all model bindings remain resolvable");
prop_assert_eq!(observed.source, expected.source);
prop_assert_eq!(observed.length, expected.length);
prop_assert_eq!(observed.revision, expected.revision);
}
}
}
#[test]
fn typed_binding_action_strings_match_reference_model(actions in prop::collection::vec(any::<u8>(), 1..64)) {
let state = TempState::new("typed-stateful");
let directory = spawn_directory(&state.store());
let paths = [path("/state/a"), path("/state/b"), path("/state/c")];
let mut model = BTreeMap::<DataPath, TypedModelEntry>::new();
let mut next_operation = 10_000_u128;
for (step, action) in actions.into_iter().enumerate() {
let logical = paths[usize::from(action) % paths.len()].clone();
match action % 4 {
0 => {
let actor = source(action.wrapping_add(step as u8).wrapping_add(1));
let length = u64::from(action) + 1;
let receipt = future::block_on(directory.client.register(
logical.clone(),
actor,
length,
recovery(actor),
OperationId::from_u128(next_operation),
)).expect("blob rebind");
next_operation += 1;
model.insert(logical, TypedModelEntry::Blob(ModelBinding {
source: actor,
length,
revision: receipt.revision,
}));
}
1 => {
let source_actor = source(action.wrapping_add(41));
let sink_actor = source(action.wrapping_add(97));
let mut source_open = Box::pin(directory.client.replace_with_stream(
logical.clone(),
StreamRole::Source,
source_actor,
OperationId::from_u128(next_operation),
));
next_operation += 1;
prop_assert!(future::block_on(future::poll_once(source_open.as_mut())).is_none());
let sink_match = future::block_on(directory.client.open_stream(
logical.clone(),
StreamRole::Sink,
sink_actor,
OperationId::from_u128(next_operation),
)).expect("sink match");
next_operation += 1;
let source_match = future::block_on(source_open).expect("source match");
prop_assert_eq!(&source_match, &sink_match);
model.insert(logical, TypedModelEntry::Stream(sink_match));
}
2 => {
if let Some(TypedModelEntry::Stream(binding)) = model.get(&logical) {
future::block_on(directory.client.close_stream(
logical.clone(),
binding.incarnation,
)).expect("close current stream");
model.remove(&logical);
}
}
_ => {
let observed = future::block_on(directory.client.resolve(logical.clone()));
match model.get(&logical) {
Some(TypedModelEntry::Blob(expected)) => {
let observed = observed.expect("blob resolves");
prop_assert_eq!(observed.source, expected.source);
prop_assert_eq!(observed.length, expected.length);
prop_assert_eq!(observed.revision, expected.revision);
}
Some(TypedModelEntry::Stream(_)) => {
let wrong_type = matches!(
observed,
Err(NamespaceError::WrongEntryType {
expected: EntryKind::Blob,
found: EntryKind::Stream,
..
})
);
prop_assert!(wrong_type, "blob lookup must reject a stream binding");
}
None => prop_assert!(matches!(
observed,
Err(NamespaceError::PathNotFound(found)) if found == logical
)),
}
}
}
}
}
}

View file

@ -202,12 +202,20 @@ fn host_read_resolves_file_source_and_seals_final_arena_lease() {
}));
let host_engine = Engine::new(
host_parts,
TokioBackend::new(TokioConfig::default()).unwrap(),
TokioBackend::new(TokioConfig {
worker_threads: 1,
..TokioConfig::default()
})
.unwrap(),
)
.unwrap();
let child_engine = Engine::new(
child_parts,
TokioBackend::new(TokioConfig::default()).unwrap(),
TokioBackend::new(TokioConfig {
worker_threads: 1,
..TokioConfig::default()
})
.unwrap(),
)
.unwrap();
@ -264,6 +272,7 @@ fn host_read_resolves_file_source_and_seals_final_arena_lease() {
source_sender: Some(Arc::clone(&sender)),
source_publisher: Some(Arc::clone(&source_publisher)),
route_registrar: None,
stream_transport: None,
})
.unwrap(),
)
@ -296,6 +305,7 @@ fn host_read_resolves_file_source_and_seals_final_arena_lease() {
source_sender: Some(Arc::clone(&sender)),
source_publisher: Some(source_publisher),
route_registrar: None,
stream_transport: None,
})
.unwrap(),
)

View file

@ -0,0 +1,260 @@
#![cfg(target_os = "linux")]
use std::sync::Arc;
use data_plane::arena::{ArenaConfig, ArenaManager, NodeId};
use data_plane::byte_ring::{ByteRingSpec, RecordKind, Role, attach, install};
use data_plane::namespace::StreamIncarnation;
use data_plane::stream_transport::{
LocalStreamTransport, StreamSinkRequest, StreamSourceRequest, StreamTransport,
StreamTransportEvent, StreamTransportNotifier,
};
use parking_lot::Mutex;
use proptest::prelude::*;
#[derive(Default)]
struct Events(Mutex<Vec<StreamTransportEvent>>);
impl StreamTransportNotifier for Events {
fn notify(&self, event: StreamTransportEvent) {
self.0.lock().push(event);
}
}
fn arena(node: u64) -> ArenaManager {
ArenaManager::boot(ArenaConfig {
node_id: NodeId(node),
reservation_ceiling: 1 << 20,
base_alignment: 64,
})
.expect("arena")
}
#[test]
fn local_transport_preserves_order_backpressure_and_eof() {
let mut source_arena = arena(1);
let source_handle = install(
&mut source_arena,
ByteRingSpec {
capacity: 32,
generation: 1,
alignment: 64,
request_id: 1,
},
)
.expect("source ring");
let mut destination_arena = arena(2);
let destination_handle = install(
&mut destination_arena,
ByteRingSpec {
capacity: 16,
generation: 2,
alignment: 64,
request_id: 2,
},
)
.expect("destination ring");
let mut writer = attach(&source_arena, source_handle, Role::Producer).expect("writer");
let source_pump = attach(&source_arena, source_handle, Role::Consumer).expect("source pump");
let destination_pump =
attach(&destination_arena, destination_handle, Role::Producer).expect("destination pump");
let mut reader =
attach(&destination_arena, destination_handle, Role::Consumer).expect("reader");
let transport = LocalStreamTransport::new();
let incarnation = StreamIncarnation {
authority_epoch: 7,
revision: 11,
};
let source_events = Arc::new(Events::default());
let sink_events = Arc::new(Events::default());
transport
.install_sink(StreamSinkRequest {
incarnation,
endpoint: destination_pump,
notifier: sink_events.clone(),
})
.expect("install sink");
transport
.install_source(StreamSourceRequest {
incarnation,
peer: transport.descriptor().expect("descriptor"),
endpoint: source_pump,
notifier: source_events.clone(),
})
.expect("install source");
assert!(
source_events
.0
.lock()
.contains(&StreamTransportEvent::Ready)
);
assert!(sink_events.0.lock().contains(&StreamTransportEvent::Ready));
writer
.send_record(RecordKind::Data, b"first")
.expect("write first");
transport.source_progress(incarnation);
writer
.send_record(RecordKind::Data, b"next!")
.expect("write second");
transport.source_progress(incarnation);
assert_eq!(
reader.recv_record().expect("read first"),
Some((RecordKind::Data, b"first".to_vec()))
);
assert_eq!(reader.recv_record().expect("second remains upstream"), None);
transport.sink_progress(incarnation);
assert_eq!(
reader.recv_record().expect("read second"),
Some((RecordKind::Data, b"next!".to_vec()))
);
writer.send_record(RecordKind::Eof, b"").expect("write eof");
transport.source_progress(incarnation);
assert_eq!(
reader.recv_record().expect("read eof"),
Some((RecordKind::Eof, Vec::new()))
);
}
#[test]
fn local_transport_rejects_a_descriptor_from_another_backend() {
let first = LocalStreamTransport::new();
let second = LocalStreamTransport::new();
assert_ne!(
first.descriptor().expect("first descriptor"),
second.descriptor().expect("second descriptor")
);
}
proptest! {
#![proptest_config(ProptestConfig {
cases: 32,
max_shrink_iters: 256,
..ProptestConfig::default()
})]
#[test]
fn randomized_payloads_preserve_exact_bytes(
payload in prop::collection::vec(any::<u8>(), 0..4096),
raw_capacity in 16_u8..128,
chunk_seeds in prop::collection::vec(1_u8..=255, 1..32),
) {
let capacity = u64::from(raw_capacity);
let mut source_arena = arena(21);
let source_handle = install(
&mut source_arena,
ByteRingSpec {
capacity,
generation: 1,
alignment: 64,
request_id: 1,
},
).expect("source ring");
let mut destination_arena = arena(22);
let destination_handle = install(
&mut destination_arena,
ByteRingSpec {
capacity,
generation: 2,
alignment: 64,
request_id: 2,
},
).expect("destination ring");
let mut writer = attach(&source_arena, source_handle, Role::Producer).expect("writer");
let source_pump = attach(&source_arena, source_handle, Role::Consumer).expect("source pump");
let destination_pump =
attach(&destination_arena, destination_handle, Role::Producer).expect("destination pump");
let mut reader =
attach(&destination_arena, destination_handle, Role::Consumer).expect("reader");
let transport = LocalStreamTransport::new();
let incarnation = StreamIncarnation {
authority_epoch: 3,
revision: 9,
};
transport.install_sink(StreamSinkRequest {
incarnation,
endpoint: destination_pump,
notifier: Arc::new(Events::default()),
}).expect("sink");
transport.install_source(StreamSourceRequest {
incarnation,
peer: transport.descriptor().expect("descriptor"),
endpoint: source_pump,
notifier: Arc::new(Events::default()),
}).expect("source");
let max_chunk = capacity as usize - 5;
let mut offset = 0;
let mut seed_index = 0;
let mut observed = Vec::new();
while offset < payload.len() {
let chunk_len = usize::from(chunk_seeds[seed_index % chunk_seeds.len()])
.min(max_chunk)
.min(payload.len() - offset);
seed_index += 1;
loop {
match writer.send_record(
RecordKind::Data,
&payload[offset..offset + chunk_len],
) {
Ok(()) => break,
Err(data_plane::byte_ring::FlowError::InsufficientSpace { .. }) => {
if let Some((RecordKind::Data, bytes)) =
reader.recv_record().expect("drain destination")
{
observed.extend(bytes);
transport.sink_progress(incarnation);
} else {
transport.source_progress(incarnation);
}
}
Err(error) => panic!("unexpected source error: {error:?}"),
}
}
offset += chunk_len;
transport.source_progress(incarnation);
}
loop {
match writer.send_record(RecordKind::Eof, &[]) {
Ok(()) => break,
Err(data_plane::byte_ring::FlowError::InsufficientSpace { .. }) => {
if let Some((RecordKind::Data, bytes)) =
reader.recv_record().expect("drain for eof")
{
observed.extend(bytes);
transport.sink_progress(incarnation);
} else {
transport.source_progress(incarnation);
}
}
Err(error) => panic!("unexpected eof error: {error:?}"),
}
}
transport.source_progress(incarnation);
let mut steps = 0;
loop {
steps += 1;
prop_assert!(steps < payload.len() + 1024, "transfer made no bounded progress");
match reader.recv_record().expect("receive") {
Some((RecordKind::Data, bytes)) => {
observed.extend(bytes);
transport.sink_progress(incarnation);
}
Some((RecordKind::Eof, _)) => break,
Some((RecordKind::Fault, bytes)) => {
return Err(TestCaseError::fail(format!(
"unexpected fault record: {bytes:?}"
)));
}
None => transport.source_progress(incarnation),
}
}
prop_assert_eq!(observed, payload);
}
}

View file

@ -15,3 +15,6 @@ tokio = { workspace = true, optional = true }
[dev-dependencies]
proptest = "1"
[lints]
workspace = true

View file

@ -18,8 +18,7 @@ use swactor::runtime::{ExternalSender, Runtime, RuntimeParts};
pub struct Engine {
/// Retained so the engine owns the runtime handle it drives for its full
/// lifetime. Core workers are moved into substrate tasks at construction.
#[allow(dead_code)]
runtime: Runtime,
_runtime: Runtime,
backend: Arc<dyn ExecutionBackend>,
}
@ -41,7 +40,10 @@ impl Engine {
// no Tokio feature gate — so core progression does not silently
// disappear when an alternate backend is used (ENGINE_SPEC.md).
crate::core_driver::install(workers, &backend);
Ok(Engine { runtime, backend })
Ok(Engine {
_runtime: runtime,
backend,
})
}
/// Return a clonable handle for scheduling engine work.

View file

@ -3,8 +3,6 @@
//! Imports only public `swactor` APIs and exposes no private engine state. See
//! `ENGINE_SPEC.md`.
#![allow(dead_code)]
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::time::{Duration, Instant};
@ -23,9 +21,10 @@ pub fn default_runtime_parts() -> (RuntimeParts, Runtime) {
runtime_parts(RuntimeConfig::default())
}
pub fn runtime_parts_with_workers(worker_count: usize) -> (RuntimeParts, Runtime) {
let mut config = RuntimeConfig::default();
config.worker_count = worker_count;
runtime_parts(config)
runtime_parts(RuntimeConfig {
worker_count,
..RuntimeConfig::default()
})
}
pub fn default_parts() -> RuntimeParts {

View file

@ -10,7 +10,7 @@
//! These tests exercise the native Tokio backend specifically; the
//! non-Tokio portability proof lives in `engine_unit.rs`.
mod common;
pub mod common;
use common::*;
use std::sync::Arc;

View file

@ -5,7 +5,7 @@
//! These tests exercise internal logic directly and use the [`SteppingBackend`]
//! to prove substrate independence without Tokio (ENGINE_SPEC.md).
mod common;
pub mod common;
use common::*;
use std::sync::Arc;
@ -1318,8 +1318,7 @@ fn engine_instant_is_ordered() {
/// A tasks-only probe backend that shares a sentinel `Arc<()>` so the test can
/// observe exactly when the engine's strong backend reference is released.
struct SentinelBackend {
#[allow(dead_code)]
sentinel: Arc<()>,
_sentinel: Arc<()>,
}
impl ExecutionBackend for SentinelBackend {
@ -1347,7 +1346,7 @@ fn dropping_engine_releases_backend_even_with_live_handles() {
let engine = Engine::new(
parts,
SentinelBackend {
sentinel: sentinel.clone(),
_sentinel: sentinel.clone(),
},
)
.expect("tasks capability present");

View file

@ -33,6 +33,7 @@ use distribution::types::NodeId;
use crate::edge_transport::spawn_edge_send_pump as spawn_edge_sender_task;
use crate::edge_transport::{EDGE_ALPN, EdgeSendHandle, spawn_edge_recv_pump};
use crate::stream_transport::{IrohStreamTransport, STREAM_ALPN};
use crate::telemetry_transport::{
TELEMETRY_ALPN, TelemetryQuicHeader, TelemetryQuicRead, read_events_from_stream,
spawn_subscription_writer,
@ -327,6 +328,7 @@ pub struct IrohDriver {
/// iroh work is scheduled through this handle; it never exposes the raw
/// Tokio runtime (ENGINE_SPEC.md §7).
engine: EngineHandle,
stream_transport: Arc<IrohStreamTransport>,
conns: Arc<Mutex<ConnCache>>,
peer_auth: Option<Arc<Mutex<PeerAllowList>>>,
/// Collects connections from background join tasks.
@ -450,7 +452,7 @@ impl IrohDriver {
let secret_key = config.secret_key;
let (endpoint_tx, endpoint_rx) = std::sync::mpsc::channel::<Result<Endpoint, String>>();
engine.spawn(async move {
let mut all_alpns = vec![ALPN.to_vec()];
let mut all_alpns = vec![ALPN.to_vec(), STREAM_ALPN.to_vec()];
all_alpns.extend(additional_alpns);
let mut builder = Endpoint::builder(iroh::endpoint::presets::Minimal)
.relay_mode(effective_relay_mode)
@ -481,6 +483,7 @@ impl IrohDriver {
format!("engine endpoint-bind task dropped: {e}").into()
})?
.map_err(|e| -> Box<dyn std::error::Error> { e.into() })?;
let stream_transport = Arc::new(IrohStreamTransport::new(engine.clone(), endpoint.clone()));
let relay_url = endpoint
.addr()
.relay_urls()
@ -504,6 +507,7 @@ impl IrohDriver {
let peer_auth = config.peer_auth.clone();
let swim_buf = Arc::clone(&accepted_conns);
let other_buf = Arc::clone(&other_accepted_conns);
let accepted_streams = Arc::clone(&stream_transport);
engine.spawn(async move {
while let Some(incoming) = ep.accept().await {
if let Ok(conn) = incoming.await {
@ -522,6 +526,8 @@ impl IrohDriver {
let negotiated_alpn = conn.alpn().to_vec();
if negotiated_alpn == ALPN {
swim_buf.lock().push((node_id, conn));
} else if negotiated_alpn == STREAM_ALPN {
accepted_streams.accept_connection(conn);
} else {
other_buf.lock().push((node_id, negotiated_alpn, conn));
}
@ -534,6 +540,7 @@ impl IrohDriver {
keypair,
endpoint,
engine,
stream_transport,
conns: Arc::new(Mutex::new(ConnCache {
connections: HashMap::new(),
next_generation: 1,
@ -561,6 +568,10 @@ impl IrohDriver {
self.endpoint.clone()
}
pub fn stream_transport(&self) -> Arc<IrohStreamTransport> {
Arc::clone(&self.stream_transport)
}
/// Drain accepted connections whose negotiated ALPN exactly matches `alpn`.
pub fn drain_accepted_for_alpn(&self, alpn: &[u8]) -> Vec<(NodeId, Connection)> {
let mut pending = self.other_accepted_conns.lock();

View file

@ -13,6 +13,7 @@ pub mod blob_transfer;
pub mod edge_transport;
pub mod endpoint_advertisement;
pub mod iroh_driver;
pub mod stream_transport;
pub mod telemetry_transport;
pub use blob_transfer::{IrohBlobTransferReceiver, IrohBlobTransferSender};
@ -25,6 +26,7 @@ pub use iroh_driver::{
};
pub use edge_transport::{EDGE_ALPN, EdgeSendHandle};
pub use stream_transport::{IrohStreamTransport, STREAM_ALPN};
pub use telemetry_transport::{
PullCollectorConfig, PullCollectorHandle, TELEMETRY_ALPN, TelemetryQuicHeader,

View file

@ -0,0 +1,431 @@
//! Iroh-specific implementation of the data-plane SPSC stream transport port.
use std::collections::BTreeMap;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use data_plane::byte_ring::{Endpoint as RingEndpoint, FlowError, RecordKind, RingProbe};
use data_plane::namespace::StreamIncarnation;
use data_plane::stream_transport::{
StreamPeerDescriptor, StreamSinkRequest, StreamSourceRequest, StreamTransport,
StreamTransportEvent, StreamTransportNotifier,
};
use iroh::endpoint::{Connection, RecvStream, SendStream};
use iroh::{Endpoint as IrohEndpoint, EndpointAddr};
use parking_lot::Mutex;
use swactor_engine::EngineHandle;
use tokio::io::AsyncWriteExt;
use tokio::sync::mpsc;
pub const STREAM_ALPN: &[u8] = b"swactor/data-plane-spsc/1";
const PREAMBLE_LEN: usize = 16;
const RECORD_HEADER_LEN: usize = 5;
#[derive(Clone)]
struct TaskControl {
wake: mpsc::Sender<()>,
progress_pending: Arc<AtomicBool>,
cancelled: Arc<AtomicBool>,
}
impl TaskControl {
fn pair() -> (Self, mpsc::Receiver<()>) {
let (wake, receiver) = mpsc::channel(1);
(
Self {
wake,
progress_pending: Arc::new(AtomicBool::new(false)),
cancelled: Arc::new(AtomicBool::new(false)),
},
receiver,
)
}
fn progress(&self) {
if !self.progress_pending.swap(true, Ordering::AcqRel) {
let _ = self.wake.try_send(());
}
}
fn cancel(&self) {
self.cancelled.store(true, Ordering::Release);
self.progress();
}
async fn wait(&self, receiver: &mut mpsc::Receiver<()>) -> bool {
if self.cancelled.load(Ordering::Acquire) {
return false;
}
if receiver.recv().await.is_none() {
return false;
}
self.progress_pending.store(false, Ordering::Release);
!self.cancelled.load(Ordering::Acquire)
}
}
struct PendingSink {
endpoint: RingEndpoint,
notifier: Arc<dyn StreamTransportNotifier>,
}
#[derive(Default)]
struct TransportState {
pending_sinks: BTreeMap<StreamIncarnation, PendingSink>,
controls: BTreeMap<StreamIncarnation, Vec<TaskControl>>,
source_probes: BTreeMap<StreamIncarnation, RingProbe>,
sink_probes: BTreeMap<StreamIncarnation, RingProbe>,
}
struct Inner {
engine: EngineHandle,
endpoint: IrohEndpoint,
state: Mutex<TransportState>,
}
/// Cloneable adapter capability installed into `HostDataPlaneConfig`.
#[derive(Clone)]
pub struct IrohStreamTransport {
inner: Arc<Inner>,
}
impl IrohStreamTransport {
pub fn new(engine: EngineHandle, endpoint: IrohEndpoint) -> Self {
Self {
inner: Arc::new(Inner {
engine,
endpoint,
state: Mutex::new(TransportState::default()),
}),
}
}
pub(crate) fn accept_connection(&self, connection: Connection) {
let transport = self.clone();
self.inner.engine.spawn(async move {
while let Ok(mut recv) = connection.accept_uni().await {
let mut preamble = [0_u8; PREAMBLE_LEN];
if recv.read_exact(&mut preamble).await.is_err() {
continue;
}
let incarnation = decode_incarnation(preamble);
let pending = transport
.inner
.state
.lock()
.pending_sinks
.remove(&incarnation);
let Some(pending) = pending else {
continue;
};
let (control, receiver) = TaskControl::pair();
transport
.inner
.state
.lock()
.controls
.entry(incarnation)
.or_default()
.push(control.clone());
transport.inner.engine.spawn(run_sink(
incarnation,
recv,
pending.endpoint,
pending.notifier,
control,
receiver,
));
}
});
}
fn register_control(&self, incarnation: StreamIncarnation, control: TaskControl) {
self.inner
.state
.lock()
.controls
.entry(incarnation)
.or_default()
.push(control);
}
fn progress_controls(&self, incarnation: StreamIncarnation) {
if let Some(controls) = self.inner.state.lock().controls.get(&incarnation) {
for control in controls {
control.progress();
}
}
}
}
impl StreamTransport for IrohStreamTransport {
fn descriptor(&self) -> Result<StreamPeerDescriptor, String> {
serde_json::to_vec(&self.inner.endpoint.addr())
.map(StreamPeerDescriptor)
.map_err(|error| format!("encode iroh stream endpoint: {error}"))
}
fn install_source(&self, request: StreamSourceRequest) -> Result<(), String> {
let peer: EndpointAddr = serde_json::from_slice(&request.peer.0)
.map_err(|error| format!("decode iroh stream endpoint: {error}"))?;
let (control, receiver) = TaskControl::pair();
self.register_control(request.incarnation, control.clone());
let endpoint = self.inner.endpoint.clone();
let probe = request.endpoint.probe();
self.inner
.state
.lock()
.source_probes
.insert(request.incarnation, probe);
self.inner.engine.spawn(run_source(
request.incarnation,
endpoint,
peer,
request.endpoint,
request.notifier,
control,
receiver,
));
Ok(())
}
fn install_sink(&self, request: StreamSinkRequest) -> Result<(), String> {
let mut state = self.inner.state.lock();
if state.pending_sinks.contains_key(&request.incarnation) {
return Err("iroh stream sink is already installed".to_owned());
}
let probe = request.endpoint.probe();
state.sink_probes.insert(request.incarnation, probe);
state.pending_sinks.insert(
request.incarnation,
PendingSink {
endpoint: request.endpoint,
notifier: request.notifier,
},
);
Ok(())
}
fn source_progress(&self, incarnation: StreamIncarnation) {
self.progress_controls(incarnation);
}
fn sink_progress(&self, incarnation: StreamIncarnation) {
self.progress_controls(incarnation);
}
fn source_has_capacity(&self, incarnation: StreamIncarnation) -> bool {
self.inner
.state
.lock()
.source_probes
.get(&incarnation)
.is_some_and(RingProbe::has_capacity)
}
fn sink_has_data(&self, incarnation: StreamIncarnation) -> bool {
self.inner
.state
.lock()
.sink_probes
.get(&incarnation)
.is_some_and(RingProbe::has_data)
}
fn terminate(&self, incarnation: StreamIncarnation) {
let (pending, controls) = {
let mut state = self.inner.state.lock();
let pending = state.pending_sinks.remove(&incarnation);
let controls = state.controls.remove(&incarnation).unwrap_or_default();
state.source_probes.remove(&incarnation);
state.sink_probes.remove(&incarnation);
(pending, controls)
};
if let Some(pending) = pending {
pending.notifier.notify(StreamTransportEvent::Quiesced);
}
for control in controls {
control.cancel();
}
}
}
async fn run_source(
incarnation: StreamIncarnation,
endpoint: IrohEndpoint,
peer: EndpointAddr,
mut source: RingEndpoint,
notifier: Arc<dyn StreamTransportNotifier>,
control: TaskControl,
mut receiver: mpsc::Receiver<()>,
) {
let result = async {
let connection = endpoint
.connect(peer, STREAM_ALPN)
.await
.map_err(|error| format!("connect stream incarnation: {error}"))?;
let mut send = connection
.open_uni()
.await
.map_err(|error| format!("open stream incarnation: {error}"))?;
send.write_all(&encode_incarnation(incarnation))
.await
.map_err(|error| format!("write stream preamble: {error}"))?;
send.flush()
.await
.map_err(|error| format!("flush stream preamble: {error}"))?;
notifier.notify(StreamTransportEvent::Ready);
loop {
if control.cancelled.load(Ordering::Acquire) {
return Ok(());
}
let mut moved = false;
while let Some(meta) = source
.next_record_meta()
.map_err(|error| format!("inspect source ring: {error:?}"))?
{
let view = source
.peek_record()
.map_err(|error| format!("pin source ring: {error:?}"))?
.ok_or_else(|| "source record disappeared after inspection".to_owned())?;
write_record(&mut send, meta.kind, view.spans())
.await
.map_err(|error| format!("write stream record: {error}"))?;
view.release()
.map_err(|error| format!("release source ring: {error:?}"))?;
notifier.notify(StreamTransportEvent::CapacityAvailable);
moved = true;
if matches!(meta.kind, RecordKind::Eof | RecordKind::Fault) {
send.finish()
.map_err(|error| format!("finish stream incarnation: {error}"))?;
match send
.stopped()
.await
.map_err(|error| format!("await stream finish: {error}"))?
{
Some(code) => {
return Err(format!("peer stopped stream incarnation: {code}"));
}
None => return Ok(()),
}
}
}
if !moved && !control.wait(&mut receiver).await {
return Ok(());
}
}
}
.await;
if let Err(reason) = result {
notifier.notify(StreamTransportEvent::Fault(reason));
}
notifier.notify(StreamTransportEvent::Quiesced);
}
async fn write_record(
send: &mut SendStream,
kind: RecordKind,
spans: (&[u8], &[u8]),
) -> Result<(), String> {
let len = spans.0.len() + spans.1.len();
let len = u32::try_from(len).map_err(|_| "stream record exceeds u32 framing".to_owned())?;
let mut header = [0_u8; RECORD_HEADER_LEN];
header[0] = kind.to_byte();
header[1..].copy_from_slice(&len.to_le_bytes());
send.write_all(&header)
.await
.map_err(|error| error.to_string())?;
if !spans.0.is_empty() {
send.write_all(spans.0)
.await
.map_err(|error| error.to_string())?;
}
if !spans.1.is_empty() {
send.write_all(spans.1)
.await
.map_err(|error| error.to_string())?;
}
send.flush().await.map_err(|error| error.to_string())
}
async fn run_sink(
_incarnation: StreamIncarnation,
mut recv: RecvStream,
mut sink: RingEndpoint,
notifier: Arc<dyn StreamTransportNotifier>,
control: TaskControl,
mut receiver: mpsc::Receiver<()>,
) {
notifier.notify(StreamTransportEvent::Ready);
let result = async {
loop {
if control.cancelled.load(Ordering::Acquire) {
return Ok(());
}
let mut header = [0_u8; RECORD_HEADER_LEN];
recv.read_exact(&mut header)
.await
.map_err(|error| format!("read stream record header: {error}"))?;
let kind = RecordKind::from_byte(header[0])
.ok_or_else(|| format!("invalid stream record kind {}", header[0]))?;
let len = u64::from(u32::from_le_bytes(header[1..].try_into().unwrap()));
let mut reservation = loop {
match sink.reserve_record(kind, len) {
Ok(reservation) => break reservation,
Err(FlowError::InsufficientSpace { .. }) => {
if !control.wait(&mut receiver).await {
return Ok(());
}
}
Err(error) => return Err(format!("reserve sink ring: {error:?}")),
}
};
let (first, second) = reservation.spans_mut();
if !first.is_empty() {
recv.read_exact(first)
.await
.map_err(|error| format!("read first stream span: {error}"))?;
}
if !second.is_empty() {
recv.read_exact(second)
.await
.map_err(|error| format!("read second stream span: {error}"))?;
}
reservation
.commit()
.map_err(|error| format!("commit sink ring: {error:?}"))?;
notifier.notify(StreamTransportEvent::DataAvailable);
if matches!(kind, RecordKind::Eof | RecordKind::Fault) {
let mut trailing = [0_u8; 1];
match recv
.read(&mut trailing)
.await
.map_err(|error| format!("read stream finish: {error}"))?
{
None | Some(0) => return Ok(()),
Some(_) => return Err("bytes followed terminal stream record".to_owned()),
}
}
}
}
.await;
if let Err(reason) = result {
notifier.notify(StreamTransportEvent::Fault(reason));
}
notifier.notify(StreamTransportEvent::Quiesced);
}
fn encode_incarnation(incarnation: StreamIncarnation) -> [u8; PREAMBLE_LEN] {
let mut encoded = [0_u8; PREAMBLE_LEN];
encoded[..8].copy_from_slice(&incarnation.authority_epoch.to_le_bytes());
encoded[8..].copy_from_slice(&incarnation.revision.to_le_bytes());
encoded
}
fn decode_incarnation(encoded: [u8; PREAMBLE_LEN]) -> StreamIncarnation {
StreamIncarnation {
authority_epoch: u64::from_le_bytes(encoded[..8].try_into().unwrap()),
revision: u64::from_le_bytes(encoded[8..].try_into().unwrap()),
}
}

View file

@ -0,0 +1,137 @@
pub mod common;
use std::sync::Arc;
use std::sync::mpsc::{self, Receiver, Sender};
use std::time::Duration;
use data_plane::arena::{ArenaConfig, ArenaManager, NodeId};
use data_plane::byte_ring::{ByteRingSpec, RecordKind, Role, attach, install};
use data_plane::namespace::StreamIncarnation;
use data_plane::stream_transport::{
StreamSinkRequest, StreamSourceRequest, StreamTransport, StreamTransportEvent,
StreamTransportNotifier,
};
use common::iroh::make_driver;
struct ChannelNotifier(Sender<StreamTransportEvent>);
impl StreamTransportNotifier for ChannelNotifier {
fn notify(&self, event: StreamTransportEvent) {
let _ = self.0.send(event);
}
}
fn arena(node: u64) -> ArenaManager {
ArenaManager::boot(ArenaConfig {
node_id: NodeId(node),
reservation_ceiling: 1 << 20,
base_alignment: 64,
})
.expect("arena")
}
fn recv_until(receiver: &Receiver<StreamTransportEvent>, expected: StreamTransportEvent) {
loop {
let event = receiver
.recv_timeout(Duration::from_secs(10))
.expect("transport event deadline");
if event == expected {
return;
}
if let StreamTransportEvent::Fault(reason) = event {
panic!("unexpected stream fault: {reason}");
}
}
}
#[test]
fn iroh_adapter_satisfies_ordering_and_terminal_contract() {
let source_node = make_driver();
let sink_node = make_driver();
let source_transport = source_node.driver.stream_transport();
let sink_transport = sink_node.driver.stream_transport();
let mut source_arena = arena(1);
let source_handle = install(
&mut source_arena,
ByteRingSpec {
capacity: 128,
generation: 1,
alignment: 64,
request_id: 1,
},
)
.expect("source ring");
let mut sink_arena = arena(2);
let sink_handle = install(
&mut sink_arena,
ByteRingSpec {
capacity: 128,
generation: 2,
alignment: 64,
request_id: 2,
},
)
.expect("sink ring");
let mut writer = attach(&source_arena, source_handle, Role::Producer).expect("writer");
let source_pump = attach(&source_arena, source_handle, Role::Consumer).expect("source pump");
let sink_pump = attach(&sink_arena, sink_handle, Role::Producer).expect("sink pump");
let mut reader = attach(&sink_arena, sink_handle, Role::Consumer).expect("reader");
let incarnation = StreamIncarnation {
authority_epoch: 17,
revision: 23,
};
let (source_tx, source_rx) = mpsc::channel();
let (sink_tx, sink_rx) = mpsc::channel();
sink_transport
.install_sink(StreamSinkRequest {
incarnation,
endpoint: sink_pump,
notifier: Arc::new(ChannelNotifier(sink_tx)),
})
.expect("install sink");
source_transport
.install_source(StreamSourceRequest {
incarnation,
peer: sink_transport.descriptor().expect("sink descriptor"),
endpoint: source_pump,
notifier: Arc::new(ChannelNotifier(source_tx)),
})
.expect("install source");
recv_until(&source_rx, StreamTransportEvent::Ready);
recv_until(&sink_rx, StreamTransportEvent::Ready);
writer
.send_record(RecordKind::Data, b"one")
.expect("write one");
writer
.send_record(RecordKind::Data, b"two")
.expect("write two");
source_transport.source_progress(incarnation);
recv_until(&sink_rx, StreamTransportEvent::DataAvailable);
recv_until(&sink_rx, StreamTransportEvent::DataAvailable);
assert_eq!(
reader.recv_record().expect("read one"),
Some((RecordKind::Data, b"one".to_vec()))
);
assert_eq!(
reader.recv_record().expect("read two"),
Some((RecordKind::Data, b"two".to_vec()))
);
sink_transport.sink_progress(incarnation);
writer.send_record(RecordKind::Eof, &[]).expect("write eof");
source_transport.source_progress(incarnation);
recv_until(&sink_rx, StreamTransportEvent::DataAvailable);
assert_eq!(
reader.recv_record().expect("read eof"),
Some((RecordKind::Eof, Vec::new()))
);
recv_until(&source_rx, StreamTransportEvent::Quiesced);
recv_until(&sink_rx, StreamTransportEvent::Quiesced);
source_transport.terminate(incarnation);
sink_transport.terminate(incarnation);
}

View file

@ -7,6 +7,8 @@ license = "AGPL-3.0-only"
[dependencies]
swactor = { path = "../..", features = ["serde", "transport"] }
swactor-transport = { path = "../transport" }
swactor-engine = { path = "../engine" }
futures-channel = "0.3"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
iroh = "0.98"
@ -14,3 +16,6 @@ crossbeam-channel = "0.5"
[target.'cfg(target_os = "linux")'.dependencies]
libc = "0.2"
[lints]
workspace = true

View file

@ -17,6 +17,7 @@ pub trait FrameSink: Send {
/// Ship one positioned frame for `stream`. Best-effort: a sink may drop.
fn ship(&mut self, stream: &StreamId, frame: &Frame);
}
pub type ProcessChannelRouter = dyn Fn(&str, bool) -> ChannelId + Send + Sync;
/// Static identity a node needs to build its mux.
pub struct EmitterConfig {
@ -27,7 +28,7 @@ pub struct EmitterConfig {
struct MuxProcObserver {
mux: Arc<Mux>,
channel_for: Arc<dyn Fn(&str, bool) -> ChannelId + Send + Sync>,
channel_for: Arc<ProcessChannelRouter>,
}
impl ProcessOutputObserver for MuxProcObserver {

View file

@ -16,6 +16,7 @@ use serde::{Deserialize, Serialize};
use swactor::process_observer::ProcessOutputObserver;
use swactor::stats::{ActorSnapshot, StatsHook};
use crate::emit::ProcessChannelRouter;
use crate::frame::{
ChannelContent, ChannelDescriptor, ChannelFilter, ChannelId, ChannelRef, FrameDelivery,
SourceFilter, StreamDescriptor, StreamId, StreamOrigin, SubscriptionRequest, TelemetryEvent,
@ -721,7 +722,7 @@ fn register_channel(
/// Legacy/custom process-output observer adapter that submits stdout/stderr chunks as frames.
pub struct TelemetryProcessObserver {
producer: TelemetryProducer,
channel_for: Arc<dyn Fn(&str, bool) -> ChannelId + Send + Sync>,
channel_for: Arc<ProcessChannelRouter>,
}
impl ProcessOutputObserver for TelemetryProcessObserver {

View file

@ -324,10 +324,9 @@ fn parse_cpu_snapshot(raw: &str) -> Option<CpuSnapshot> {
} else if let Some(index) = label
.strip_prefix("cpu")
.and_then(|suffix| suffix.parse::<u32>().ok())
&& let Some(times) = parse_cpu_times(line)
{
if let Some(times) = parse_cpu_times(line) {
cores.push(CpuCoreTimes { index, times });
}
cores.push(CpuCoreTimes { index, times });
}
}

View file

@ -0,0 +1,151 @@
use std::collections::BTreeMap;
use std::fs;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use serde::{Deserialize, Serialize};
use super::pressure::{self, PressureSample};
use crate::record::Record;
pub const HOST_MEMORY_CHANNEL: &str = "host.memory";
pub const MEMORY_SAMPLE_INTERVAL: Duration = Duration::from_secs(1);
const SCHEMA: &str = "host.memory.v1";
const KIB: u64 = 1024;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct HostMemorySample {
pub schema: String,
pub seq: u64,
pub sample_unix_ms: u64,
pub query_elapsed_ms: Option<u64>,
pub total_bytes: Option<u64>,
pub available_bytes: Option<u64>,
pub used_bytes: Option<u64>,
pub cached_bytes: Option<u64>,
pub swap_total_bytes: Option<u64>,
pub swap_used_bytes: Option<u64>,
pub pressure: Option<PressureSample>,
pub error: Option<String>,
}
impl Record for HostMemorySample {
const CHANNEL: &'static str = HOST_MEMORY_CHANNEL;
}
pub fn sample(seq: u64) -> HostMemorySample {
let started = Instant::now();
let sample_unix_ms = unix_ms_now();
let memory = match fs::read_to_string("/proc/meminfo") {
Ok(raw) => match parse_meminfo(&raw) {
Some(memory) => memory,
None => return HostMemorySample::error(seq, "parse /proc/meminfo"),
},
Err(error) => return HostMemorySample::error(seq, format!("read /proc/meminfo: {error}")),
};
HostMemorySample {
schema: SCHEMA.to_owned(),
seq,
sample_unix_ms,
query_elapsed_ms: Some(elapsed_ms(started)),
total_bytes: Some(memory.total_bytes),
available_bytes: Some(memory.available_bytes),
used_bytes: Some(memory.total_bytes.saturating_sub(memory.available_bytes)),
cached_bytes: Some(memory.cached_bytes),
swap_total_bytes: Some(memory.swap_total_bytes),
swap_used_bytes: Some(
memory
.swap_total_bytes
.saturating_sub(memory.swap_free_bytes),
),
pressure: pressure::read("memory").ok(),
error: None,
}
}
impl HostMemorySample {
fn error(seq: u64, error: impl Into<String>) -> Self {
Self {
schema: SCHEMA.to_owned(),
seq,
sample_unix_ms: unix_ms_now(),
query_elapsed_ms: None,
total_bytes: None,
available_bytes: None,
used_bytes: None,
cached_bytes: None,
swap_total_bytes: None,
swap_used_bytes: None,
pressure: pressure::read("memory").ok(),
error: Some(error.into()),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct MemoryCounters {
total_bytes: u64,
available_bytes: u64,
cached_bytes: u64,
swap_total_bytes: u64,
swap_free_bytes: u64,
}
fn parse_meminfo(raw: &str) -> Option<MemoryCounters> {
let mut values = BTreeMap::new();
for line in raw.lines() {
let (name, rest) = line.split_once(':')?;
let value_kib = rest.split_whitespace().next()?.parse::<u64>().ok()?;
values.insert(name, value_kib.saturating_mul(KIB));
}
let cached_bytes = values
.get("Cached")
.copied()
.unwrap_or(0)
.saturating_add(values.get("SReclaimable").copied().unwrap_or(0));
Some(MemoryCounters {
total_bytes: *values.get("MemTotal")?,
available_bytes: *values.get("MemAvailable")?,
cached_bytes,
swap_total_bytes: values.get("SwapTotal").copied().unwrap_or(0),
swap_free_bytes: values.get("SwapFree").copied().unwrap_or(0),
})
}
fn unix_ms_now() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis()
.try_into()
.unwrap_or(u64::MAX)
}
fn elapsed_ms(started: Instant) -> u64 {
started.elapsed().as_millis().try_into().unwrap_or(u64::MAX)
}
#[cfg(test)]
mod tests {
use super::{parse_meminfo, sample};
#[test]
fn derives_used_cached_and_swap_memory() {
let counters = parse_meminfo(
"MemTotal: 1000 kB\nMemAvailable: 400 kB\nCached: 100 kB\nSReclaimable: 20 kB\nSwapTotal: 200 kB\nSwapFree: 150 kB\n",
)
.expect("memory counters");
assert_eq!(counters.total_bytes, 1_024_000);
assert_eq!(counters.available_bytes, 409_600);
assert_eq!(counters.cached_bytes, 122_880);
assert_eq!(counters.swap_total_bytes - counters.swap_free_bytes, 51_200);
}
#[test]
fn samples_live_memory() {
let sample = sample(7);
assert_eq!(sample.seq, 7);
assert!(sample.total_bytes.is_some_and(|total| total > 0));
assert!(sample.error.is_none());
}
}

View file

@ -1,3 +1,103 @@
pub mod cpu;
pub mod gpu;
pub mod memory;
pub mod net;
pub mod pressure;
pub mod storage;
use std::time::Duration;
use swactor_engine::EngineHandle;
/// Run a stateful blocking sampler on the engine without creating an actor.
///
/// Sampling never overlaps: the next interval is armed only after the previous
/// blocking sample has returned and `observed` has consumed its result.
pub fn spawn_blocking_sampler<State, Sample, Started, Observed>(
engine: EngineHandle,
period: Duration,
state: State,
sample: fn(State, u64) -> (State, Sample),
started: Started,
mut observed: Observed,
) where
State: Send + 'static,
Sample: Send + 'static,
Started: FnOnce() + Send + 'static,
Observed: FnMut(u64, Sample) + Send + 'static,
{
let task_engine = engine.clone();
engine.spawn(async move {
started();
let blocking_work = task_engine.blocking_work_sender();
let mut interval = task_engine.interval(period);
let mut state = state;
let mut seq = 0_u64;
loop {
(&mut interval).await;
let current_state = state;
let (sample_tx, sample_rx) = futures_channel::oneshot::channel();
let work = Box::new(move || {
let _ = sample_tx.send(sample(current_state, seq));
});
if blocking_work.submit(work).is_err() {
return;
}
let Ok((next_state, result)) = sample_rx.await else {
return;
};
state = next_state;
observed(seq, result);
seq = seq.saturating_add(1);
}
});
}
#[cfg(test)]
mod tests {
use std::sync::mpsc;
use std::time::Duration;
use swactor::config::RuntimeConfig;
use swactor::runtime::RuntimeParts;
use swactor_engine::{Engine, TokioBackend, TokioConfig};
use super::spawn_blocking_sampler;
#[test]
fn blocking_sampler_runs_sequentially_with_monotonic_sequences() {
let engine = Engine::new(
RuntimeParts::new(RuntimeConfig::default()),
TokioBackend::new(TokioConfig::default()).expect("Tokio backend"),
)
.expect("engine");
let (observed_tx, observed_rx) = mpsc::channel();
spawn_blocking_sampler(
engine.handle(),
Duration::from_millis(1),
0_u64,
|state, seq| (state + 1, (seq, state)),
|| {},
move |seq, result| {
observed_tx
.send((seq, result))
.expect("observation receiver")
},
);
assert_eq!(
observed_rx
.recv_timeout(Duration::from_secs(1))
.expect("first sample"),
(0, (0, 0)),
);
assert_eq!(
observed_rx
.recv_timeout(Duration::from_secs(1))
.expect("second sample"),
(1, (1, 1)),
);
}
}

View file

@ -0,0 +1,95 @@
use std::fs;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct PressureSample {
pub some_avg10: f64,
pub some_avg60: f64,
pub some_avg300: f64,
pub some_total_us: u64,
pub full_avg10: Option<f64>,
pub full_avg60: Option<f64>,
pub full_avg300: Option<f64>,
pub full_total_us: Option<u64>,
}
pub(crate) fn read(resource: &str) -> Result<PressureSample, String> {
let path = format!("/proc/pressure/{resource}");
let raw = fs::read_to_string(&path).map_err(|error| format!("read {path}: {error}"))?;
parse(&raw).ok_or_else(|| format!("parse {path}"))
}
fn parse(raw: &str) -> Option<PressureSample> {
let some = parse_row(raw.lines().find(|line| line.starts_with("some "))?)?;
let full = raw
.lines()
.find(|line| line.starts_with("full "))
.and_then(parse_row);
Some(PressureSample {
some_avg10: some.avg10,
some_avg60: some.avg60,
some_avg300: some.avg300,
some_total_us: some.total_us,
full_avg10: full.as_ref().map(|row| row.avg10),
full_avg60: full.as_ref().map(|row| row.avg60),
full_avg300: full.as_ref().map(|row| row.avg300),
full_total_us: full.map(|row| row.total_us),
})
}
#[derive(Debug, Clone, Copy)]
struct PressureRow {
avg10: f64,
avg60: f64,
avg300: f64,
total_us: u64,
}
fn parse_row(line: &str) -> Option<PressureRow> {
let mut avg10 = None;
let mut avg60 = None;
let mut avg300 = None;
let mut total_us = None;
for field in line.split_whitespace().skip(1) {
let (name, value) = field.split_once('=')?;
match name {
"avg10" => avg10 = value.parse().ok(),
"avg60" => avg60 = value.parse().ok(),
"avg300" => avg300 = value.parse().ok(),
"total" => total_us = value.parse().ok(),
_ => {}
}
}
Some(PressureRow {
avg10: avg10?,
avg60: avg60?,
avg300: avg300?,
total_us: total_us?,
})
}
#[cfg(test)]
mod tests {
use super::parse;
#[test]
fn parses_some_and_full_pressure_rows() {
let sample = parse(
"some avg10=1.25 avg60=2.50 avg300=3.75 total=1234\nfull avg10=0.10 avg60=0.20 avg300=0.30 total=42\n",
)
.expect("pressure sample");
assert_eq!(sample.some_avg10, 1.25);
assert_eq!(sample.some_total_us, 1234);
assert_eq!(sample.full_avg10, Some(0.10));
assert_eq!(sample.full_total_us, Some(42));
}
#[test]
fn accepts_cpu_pressure_without_full_row() {
let sample =
parse("some avg10=0.00 avg60=0.01 avg300=0.02 total=99\n").expect("pressure sample");
assert_eq!(sample.some_total_us, 99);
assert_eq!(sample.full_avg10, None);
}
}

View file

@ -0,0 +1,123 @@
use std::ffi::CString;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use serde::{Deserialize, Serialize};
use super::pressure::{self, PressureSample};
use crate::record::Record;
pub const HOST_STORAGE_CHANNEL: &str = "host.storage";
pub const STORAGE_SAMPLE_INTERVAL: Duration = Duration::from_secs(1);
const SCHEMA: &str = "host.storage.v1";
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct HostStorageSample {
pub schema: String,
pub seq: u64,
pub sample_unix_ms: u64,
pub query_elapsed_ms: Option<u64>,
pub filesystems: Vec<FilesystemSample>,
pub pressure: Option<PressureSample>,
pub error: Option<String>,
}
impl Record for HostStorageSample {
const CHANNEL: &'static str = HOST_STORAGE_CHANNEL;
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct FilesystemSample {
pub mount: String,
pub total_bytes: u64,
pub used_bytes: u64,
pub available_bytes: u64,
pub used_percent: Option<f64>,
}
pub fn sample(seq: u64) -> HostStorageSample {
let started = Instant::now();
match read_filesystem("/") {
Ok(filesystem) => HostStorageSample {
schema: SCHEMA.to_owned(),
seq,
sample_unix_ms: unix_ms_now(),
query_elapsed_ms: Some(elapsed_ms(started)),
filesystems: vec![filesystem],
pressure: pressure::read("io").ok(),
error: None,
},
Err(error) => HostStorageSample {
schema: SCHEMA.to_owned(),
seq,
sample_unix_ms: unix_ms_now(),
query_elapsed_ms: Some(elapsed_ms(started)),
filesystems: Vec::new(),
pressure: pressure::read("io").ok(),
error: Some(error),
},
}
}
#[cfg(target_os = "linux")]
fn read_filesystem(mount: &str) -> Result<FilesystemSample, String> {
let path = CString::new(mount).map_err(|error| format!("filesystem path: {error}"))?;
let mut stats = std::mem::MaybeUninit::<libc::statvfs>::uninit();
// SAFETY: `path` is a live NUL-terminated string and `stats` points to writable storage.
let result = unsafe { libc::statvfs(path.as_ptr(), stats.as_mut_ptr()) };
if result != 0 {
return Err(format!(
"statvfs {mount}: {}",
std::io::Error::last_os_error()
));
}
// SAFETY: successful `statvfs` initialized the output structure.
let stats = unsafe { stats.assume_init() };
let fragment_size = stats.f_frsize;
let total_bytes = stats.f_blocks.saturating_mul(fragment_size);
let free_bytes = stats.f_bfree.saturating_mul(fragment_size);
let available_bytes = stats.f_bavail.saturating_mul(fragment_size);
let used_bytes = total_bytes.saturating_sub(free_bytes);
let used_percent = (total_bytes > 0).then_some(used_bytes as f64 * 100.0 / total_bytes as f64);
Ok(FilesystemSample {
mount: mount.to_owned(),
total_bytes,
used_bytes,
available_bytes,
used_percent,
})
}
#[cfg(not(target_os = "linux"))]
fn read_filesystem(mount: &str) -> Result<FilesystemSample, String> {
Err(format!("filesystem sampling unsupported for {mount}"))
}
fn unix_ms_now() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis()
.try_into()
.unwrap_or(u64::MAX)
}
fn elapsed_ms(started: Instant) -> u64 {
started.elapsed().as_millis().try_into().unwrap_or(u64::MAX)
}
#[cfg(test)]
mod tests {
use super::sample;
#[test]
fn samples_root_filesystem_capacity() {
let sample = sample(9);
assert_eq!(sample.seq, 9);
let root = sample.filesystems.first().expect("root filesystem");
assert_eq!(root.mount, "/");
assert!(root.total_bytes > 0);
assert!(root.used_bytes <= root.total_bytes);
assert!(sample.error.is_none());
}
}

View file

@ -4,7 +4,7 @@
//! payloads into a bounded queue first, then assigns a single monotonic position
//! sequence while draining accepted payloads.
use crossbeam_channel::{Receiver, Sender, TryRecvError, TrySendError, bounded};
use crossbeam_channel::{Receiver, Sender, TrySendError, bounded};
use std::sync::atomic::{AtomicU64, Ordering};
use crate::frame::{ChannelId, Frame, Position, StreamId};
@ -26,7 +26,7 @@ pub struct Mux {
impl Mux {
/// Create a mux for `stream` with a bounded outgoing queue.
pub fn new(stream: StreamId, capacity: usize) -> Self {
let capacity = capacity.max(1).min(1_048_576);
let capacity = capacity.clamp(1, 1_048_576);
let (tx, rx) = bounded(capacity);
Mux {
stream,
@ -61,20 +61,15 @@ impl Mux {
/// Pull all currently queued frames in mux queue order.
pub fn drain(&self) -> Vec<Frame> {
let mut frames = Vec::new();
loop {
match self.rx.try_recv() {
Ok(pending) => {
// Position is consumed only after a pending frame has left
// the queue; failed submit never reaches this point.
let position = Position(self.next.fetch_add(1, Ordering::Relaxed));
frames.push(Frame {
channel: pending.channel,
position,
payload: pending.payload,
});
}
Err(TryRecvError::Empty) | Err(TryRecvError::Disconnected) => break,
}
while let Ok(pending) = self.rx.try_recv() {
// Position is consumed only after a pending frame has left
// the queue; failed submit never reaches this point.
let position = Position(self.next.fetch_add(1, Ordering::Relaxed));
frames.push(Frame {
channel: pending.channel,
position,
payload: pending.payload,
});
}
frames
}

View file

@ -27,17 +27,6 @@ impl Record for ResourceSample {
const CHANNEL: &'static str = "host.resource";
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
struct MembershipTransition {
peer: String,
from: String,
to: String,
}
impl Record for MembershipTransition {
const CHANNEL: &'static str = "membership";
}
fn stream() -> StreamId {
StreamId::new(NodeId::new("node-alpha"), Lifetime(1))
}

View file

@ -10,7 +10,7 @@ use telemetry::ingest::Consumer;
use telemetry::mux::Mux;
use telemetry::transport::Delivery;
use telemetry::wire::{decode_delivery, encode_delivery};
use telemetry::{ChannelId, Lifetime, NodeId, Position, Record, StreamId};
use telemetry::{ChannelId, Lifetime, NodeId, Record, StreamId};
const RESOURCE_CHANNEL: ChannelId = ChannelId(1);
const LOG_CHANNEL: ChannelId = ChannelId(2);

View file

@ -14,3 +14,6 @@ ed25519-dalek = { version = "2", features = ["std", "rand_core", "serde"] }
rand_core = { version = "0.6", features = ["getrandom"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
[lints]
workspace = true

View file

@ -51,7 +51,7 @@ pub fn hex_encode(bytes: &[u8]) -> String {
/// Hex-decode a string into bytes. Returns `None` on invalid input.
pub fn hex_decode(hex: &str) -> Option<Vec<u8>> {
if hex.len() % 2 != 0 {
if !hex.len().is_multiple_of(2) {
return None;
}
let mut bytes = Vec::with_capacity(hex.len() / 2);

View file

@ -19,7 +19,7 @@ pub fn hex_encode(bytes: &[u8]) -> String {
/// Hex-decode a string into bytes. Returns `None` on invalid input.
pub fn hex_decode(hex: &str) -> Option<Vec<u8>> {
if hex.len() % 2 != 0 {
if !hex.len().is_multiple_of(2) {
return None;
}
let mut out = Vec::with_capacity(hex.len() / 2);

View file

@ -352,7 +352,9 @@ fn round_trip_across_two_runtimes() {
// Verify all 3 replies
for i in 0..3u32 {
let pong = inbox_a.try_recv().expect(&format!("missing pong #{i}"));
let pong = inbox_a
.try_recv()
.unwrap_or_else(|| panic!("missing pong #{i}"));
assert_eq!(pong, Pong { value: i * 10 + 1 });
}
assert!(inbox_a.try_recv().is_none(), "no extra messages");

View file

@ -611,7 +611,6 @@ pub(crate) struct ResumeSignal;
/// Minimal core interface: send, spawn, stop, and extension access.
/// Registry methods (naming, monitoring, groups) and timer scheduling
/// are provided by extension traits in `swactor-std`.
#[allow(private_interfaces)]
pub trait ContextInner {
fn send_any(&self, addr: ActorAddress, msg: Box<dyn Any + Send>) -> Result<(), Error>;
fn spawn_any(&self, request: SpawnRequest);

View file

@ -118,6 +118,10 @@ impl<T: Message> Admin<T> {
}
pub(crate) type AdminBoxedReply = Box<dyn Any + Send>;
pub(crate) type AdminGetState =
Box<dyn FnOnce(ActorAddress, &dyn AnyActor, ActorTypeMetadata) -> AdminBoxedReply + Send>;
pub(crate) type AdminReplaceState =
Box<dyn FnOnce(&mut dyn AnyActor, ActorTypeMetadata) -> AdminResult<OperationResult> + Send>;
pub(crate) enum AdminCommand {
ListActors {
@ -130,17 +134,13 @@ pub(crate) enum AdminCommand {
GetActorState {
actor: ActorAddress,
reply_to: ActorAddress,
get: Box<
dyn FnOnce(ActorAddress, &dyn AnyActor, ActorTypeMetadata) -> AdminBoxedReply + Send,
>,
get: AdminGetState,
not_found: Box<dyn FnOnce(ActorAddress) -> AdminBoxedReply + Send>,
},
ReplaceActorState {
actor: ActorAddress,
reply_to: ActorAddress,
replace: Box<
dyn FnOnce(&mut dyn AnyActor, ActorTypeMetadata) -> AdminResult<OperationResult> + Send,
>,
replace: AdminReplaceState,
},
StopActor {
actor: ActorAddress,

View file

@ -1,6 +1,7 @@
use crate::Instant;
use std::any::Any;
use std::cell::RefCell;
use std::cmp::Reverse;
use std::collections::{HashMap, VecDeque};
use std::sync::Arc;
use std::sync::atomic::Ordering;
@ -124,7 +125,7 @@ impl Worker {
|| self
.worker_ext
.as_ref()
.map_or(false, |e| e.has_pending_work())
.is_some_and(|extension| extension.has_pending_work())
}
/// Run one synchronous worker pass. Returns `true` if any work was done.
@ -928,7 +929,7 @@ impl ActorPool {
let snap_depth = slot.mailbox.len();
let mut snap_type_counts: Vec<(&'static str, u64)> =
slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect();
snap_type_counts.sort_by(|a, b| b.1.cmp(&a.1));
snap_type_counts.sort_by_key(|&(_, count)| Reverse(count));
let ctx = Ctx::new(
wctx,
@ -1132,7 +1133,7 @@ impl ActorPool {
if is_on_stop_eligible(slot.stopping, slot.poisoned) {
let mut type_counts: Vec<(&'static str, u64)> =
slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect();
type_counts.sort_by(|a, b| b.1.cmp(&a.1));
type_counts.sort_by_key(|&(_, count)| Reverse(count));
let ctx = Ctx::new(
inner,
addr,
@ -1159,7 +1160,7 @@ impl ActorPool {
out.extend(self.actors.iter().map(|(&addr, slot)| {
let mut type_counts: Vec<(&'static str, u64)> =
slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect();
type_counts.sort_by(|a, b| b.1.cmp(&a.1));
type_counts.sort_by_key(|&(_, count)| Reverse(count));
let metadata = slot.actor.metadata();
ActorSnapshot {
address: addr,

View file

@ -3,7 +3,7 @@
//! Covers: spawning, on_start, lifecycle decision paths, parent-child delegation,
//! graceful stop, panic isolation, dead actor cleanup, and watching (ActorExited).
mod common;
pub mod common;
use common::*;
use parking_lot::Mutex;

View file

@ -1,7 +1,5 @@
// Shared types and helpers for runtime test files.
#![allow(dead_code, unused_imports)]
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};

View file

@ -64,10 +64,6 @@ impl RuntimeExtension for SeamExtension {
&self,
dead: &[(ActorAddress, StopReason, Option<ExitValue>)],
) -> Vec<(ActorAddress, Box<dyn Any + Send>)> {
let _ = dead
.iter()
.map(|(_, reason, value)| (reason, value))
.count();
let Some(report_to) = *self.state.death_report_to.lock() else {
return Vec::new();
};

205
tests/lint_policy.rs Normal file
View file

@ -0,0 +1,205 @@
use std::collections::BTreeSet;
use std::fs;
use std::path::{Path, PathBuf};
use syn::visit::Visit;
use syn::{Attribute, Meta};
const SOURCE_ROOTS: &[&str] = &["src", "crates", "apps", "xtask", "tools", "tests"];
const FORBIDDEN_ATTRIBUTES: &[&str] = &["allow", "expect"];
#[derive(Default)]
struct SuppressionVisitor {
found: bool,
}
impl<'ast> Visit<'ast> for SuppressionVisitor {
fn visit_attribute(&mut self, attribute: &'ast Attribute) {
if FORBIDDEN_ATTRIBUTES
.iter()
.any(|name| attribute.path().is_ident(name))
|| matches!(
&attribute.meta,
Meta::List(list)
if list.path.is_ident("cfg_attr") && tokens_contain_suppression(&list.tokens)
)
{
self.found = true;
}
syn::visit::visit_attribute(self, attribute);
}
}
#[test]
fn workspace_sources_do_not_suppress_lints() {
let workspace = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let mut rust_sources = Vec::new();
for root in SOURCE_ROOTS {
collect_rust_sources(&workspace.join(root), &mut rust_sources);
}
rust_sources.sort();
let mut violations = Vec::new();
for source in rust_sources {
let text = fs::read_to_string(&source)
.unwrap_or_else(|error| panic!("read {}: {error}", source.display()));
let syntax = syn::parse_file(&text)
.unwrap_or_else(|error| panic!("parse {}: {error}", source.display()));
let mut visitor = SuppressionVisitor::default();
visitor.visit_file(&syntax);
if visitor.found {
violations.push(
source
.strip_prefix(&workspace)
.unwrap_or(&source)
.display()
.to_string(),
);
}
}
assert!(
violations.is_empty(),
"lint suppression attributes are forbidden; fix the warning instead:\n{}",
violations.join("\n")
);
}
#[test]
fn suppression_visitor_detects_direct_and_conditional_attributes() {
for source in [
"#[allow(dead_code)] fn hidden() {}",
"#![expect(unused_imports)]",
"#[cfg_attr(test, allow(clippy::too_many_arguments))] fn hidden() {}",
"#![cfg_attr(feature = \"strict\", expect(dead_code))]",
] {
let syntax = syn::parse_file(source).expect("valid suppression probe");
let mut visitor = SuppressionVisitor::default();
visitor.visit_file(&syntax);
assert!(visitor.found, "suppression escaped detection: {source}");
}
let syntax = syn::parse_file("#[derive(Clone)] struct Clean;").unwrap();
let mut visitor = SuppressionVisitor::default();
visitor.visit_file(&syntax);
assert!(!visitor.found);
}
#[test]
fn source_tree_test_modules_are_wired() {
let workspace = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let mut test_directories = Vec::new();
for root in SOURCE_ROOTS {
collect_source_test_directories(&workspace.join(root), &mut test_directories);
}
test_directories.sort();
let mut unwired = Vec::new();
for directory in test_directories {
let module_path = directory.join("mod.rs");
let module_source = fs::read_to_string(&module_path)
.unwrap_or_else(|error| panic!("read {}: {error}", module_path.display()));
let module = syn::parse_file(&module_source)
.unwrap_or_else(|error| panic!("parse {}: {error}", module_path.display()));
let declared = module
.items
.iter()
.filter_map(|item| match item {
syn::Item::Mod(item) => Some(item.ident.to_string()),
_ => None,
})
.collect::<BTreeSet<_>>();
for entry in fs::read_dir(&directory)
.unwrap_or_else(|error| panic!("read directory {}: {error}", directory.display()))
{
let path = entry.expect("read test-module entry").path();
if path.extension().is_some_and(|extension| extension == "rs")
&& path.file_stem().is_some_and(|stem| stem != "mod")
{
let module_name = path
.file_stem()
.expect("test module stem")
.to_string_lossy()
.to_string();
if !declared.contains(&module_name) {
unwired.push(
path.strip_prefix(&workspace)
.unwrap_or(&path)
.display()
.to_string(),
);
}
}
}
}
assert!(
unwired.is_empty(),
"source-tree test modules must be declared by their adjacent mod.rs:\n{}",
unwired.join("\n")
);
}
fn collect_source_test_directories(directory: &Path, directories: &mut Vec<PathBuf>) {
if !directory.exists() {
return;
}
if directory.file_name().is_some_and(|name| name == "tests")
&& directory
.parent()
.and_then(Path::file_name)
.is_some_and(|name| name == "src")
{
directories.push(directory.to_path_buf());
return;
}
let entries = fs::read_dir(directory)
.unwrap_or_else(|error| panic!("read directory {}: {error}", directory.display()));
for entry in entries {
let path = entry.expect("read source-tree entry").path();
if path.is_dir() {
collect_source_test_directories(&path, directories);
}
}
}
fn collect_rust_sources(directory: &Path, sources: &mut Vec<PathBuf>) {
if !directory.exists() {
return;
}
let entries = fs::read_dir(directory)
.unwrap_or_else(|error| panic!("read directory {}: {error}", directory.display()));
for entry in entries {
let entry = entry.unwrap_or_else(|error| panic!("read directory entry: {error}"));
let path = entry.path();
if path.is_dir() {
collect_rust_sources(&path, sources);
} else if path.extension().is_some_and(|extension| extension == "rs") {
sources.push(path);
}
}
}
fn tokens_contain_suppression(tokens: &proc_macro2::TokenStream) -> bool {
let mut token_trees = tokens.clone().into_iter().peekable();
while let Some(token) = token_trees.next() {
match token {
proc_macro2::TokenTree::Ident(identifier)
if FORBIDDEN_ATTRIBUTES.iter().any(|name| identifier == *name)
&& matches!(
token_trees.peek(),
Some(proc_macro2::TokenTree::Group(group))
if group.delimiter() == proc_macro2::Delimiter::Parenthesis
) =>
{
return true;
}
proc_macro2::TokenTree::Group(group) if tokens_contain_suppression(&group.stream()) => {
return true;
}
_ => {}
}
}
false
}

View file

@ -3,7 +3,7 @@
//! Covers: routing correctness at scale, send-from-within-handler patterns,
//! address error handling, and fairness/budgets.
mod common;
pub mod common;
use common::*;
use std::sync::Arc;

View file

@ -4,7 +4,7 @@
//! `docs/specs/drafts/MULTICORE_SPEC.md`. They observe behavior through public
//! APIs only — never inspecting source layout.
mod common;
pub mod common;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
@ -157,9 +157,10 @@ fn worker_of(stats: &swactor::stats::RuntimeStats, addr: ActorAddress) -> usize
}
fn config_with(workers: usize) -> RuntimeConfig {
let mut c = RuntimeConfig::default();
c.worker_count = workers;
c
RuntimeConfig {
worker_count: workers,
..RuntimeConfig::default()
}
}
// ─── Phase 1: single-thread host advances every worker once ─────────────────

View file

@ -3,7 +3,7 @@
//! Uses proptest for randomized testing and proptest-state-machine for
//! stateful property testing with automatic shrinking of failing sequences.
mod common;
pub mod common;
use common::plain_host;
use std::collections::HashMap;

View file

@ -1,6 +1,6 @@
//! Runtime Admin API tests — inventory, typed actor state, lifecycle control, and scheduling.
mod common;
pub mod common;
use common::*;
use std::collections::HashSet;

View file

@ -3,7 +3,7 @@
//! Covers: high-volume delivery, panic isolation under load, and sustained
//! throughput with no message loss. All tests are tick-driven (single worker).
mod common;
pub mod common;
use common::*;
use std::sync::Arc;

View file

@ -3,7 +3,7 @@
//! Tests only the std APIs used by production crates: runtime naming,
//! runtime groups, actor-side watch, actor-side group join, and extension install.
mod common;
pub mod common;
use common::*;
#[derive(Clone)]

View file

@ -7,3 +7,6 @@ publish = false
[lib]
path = "src/lib.rs"
[lints]
workspace = true

View file

@ -10,6 +10,12 @@ dependencies = [
"tokio",
]
[[package]]
name = "atomic-waker"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
[[package]]
name = "bitflags"
version = "2.13.1"
@ -182,6 +188,7 @@ dependencies = [
name = "swactor"
version = "0.1.0"
dependencies = [
"atomic-waker",
"crossbeam-queue",
"crossbeam-utils",
"getrandom",

View file

@ -1,4 +1,3 @@
#![allow(clippy::disallowed_methods, dead_code, unused_must_use)]
use std::thread::sleep as renamed_sleep;
use std::time::Duration;

View file

@ -10,6 +10,12 @@ dependencies = [
"swactor-engine",
]
[[package]]
name = "atomic-waker"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
[[package]]
name = "bitflags"
version = "2.13.1"
@ -182,6 +188,7 @@ dependencies = [
name = "swactor"
version = "0.1.0"
dependencies = [
"atomic-waker",
"crossbeam-queue",
"crossbeam-utils",
"getrandom",

View file

@ -18,3 +18,6 @@ urlencoding = "2"
[dev-dependencies]
wiremock = "0.6"
[lints]
workspace = true

View file

@ -48,50 +48,53 @@ fn env_optional_positive_f64(name: &str) -> Option<f64> {
impl SelectionPolicy {
/// Build selection policy from the historical `PP_*` environment knobs.
pub fn from_env() -> Self {
let mut policy = Self::default();
policy.min_gpu_ram_mb = env_positive_u64(ENV_GPU_MIN_RAM_MB);
if let Some(min_compute_cap) = env_positive_u64(ENV_MIN_COMPUTE_CAP) {
policy.min_compute_cap = Some(min_compute_cap);
}
policy.min_down_mbps = env_nonnegative_f64(ENV_MIN_INET_DOWN_MBPS, 100.0);
policy.min_reliability = std::env::var(ENV_MIN_RELIABILITY)
.ok()
.and_then(|s| s.trim().parse::<f64>().ok())
.filter(|&v| (0.0..=1.0).contains(&v))
.unwrap_or(0.95);
policy.require_verified = truthy_env(ENV_REQUIRE_VERIFIED);
policy.min_up_mbps = env_optional_positive_f64(ENV_MIN_INET_UP_MBPS);
policy.max_dph_total = env_optional_positive_f64(ENV_MAX_DPH_TOTAL);
policy.drop_cheap_frac = std::env::var(ENV_DROP_CHEAP_FRAC)
.ok()
.and_then(|s| s.trim().parse::<f64>().ok())
.filter(|v| v.is_finite())
.map(|v| v.clamp(0.0, 0.99))
.unwrap_or(0.30);
policy.image_size_gb = env_optional_positive_f64(ENV_IMAGE_SIZE_GB);
let defaults = Self::default();
let mut blacklist_hosts = defaults.blacklist_hosts;
if let Ok(raw) = std::env::var(ENV_BLACKLIST_HOSTS) {
policy
.blacklist_hosts
.extend(raw.split(',').filter_map(|s| s.trim().parse::<u64>().ok()));
blacklist_hosts.extend(
raw.split(',')
.filter_map(|value| value.trim().parse::<u64>().ok()),
);
}
Self {
gpu_name: defaults.gpu_name,
min_gpu_ram_mb: env_positive_u64(ENV_GPU_MIN_RAM_MB),
min_compute_cap: env_positive_u64(ENV_MIN_COMPUTE_CAP).or(defaults.min_compute_cap),
min_down_mbps: env_nonnegative_f64(ENV_MIN_INET_DOWN_MBPS, 100.0),
min_reliability: std::env::var(ENV_MIN_RELIABILITY)
.ok()
.and_then(|value| value.trim().parse::<f64>().ok())
.filter(|&value| (0.0..=1.0).contains(&value))
.unwrap_or(0.95),
require_verified: truthy_env(ENV_REQUIRE_VERIFIED),
min_up_mbps: env_optional_positive_f64(ENV_MIN_INET_UP_MBPS),
max_dph_total: env_optional_positive_f64(ENV_MAX_DPH_TOTAL),
blacklist_hosts,
drop_cheap_frac: std::env::var(ENV_DROP_CHEAP_FRAC)
.ok()
.and_then(|value| value.trim().parse::<f64>().ok())
.filter(|value| value.is_finite())
.map(|value| value.clamp(0.0, 0.99))
.unwrap_or(0.30),
image_size_gb: env_optional_positive_f64(ENV_IMAGE_SIZE_GB),
}
policy
}
}
impl LifecyclePolicy {
/// Build lifecycle policy from environment, using caller-provided poll cadence.
pub fn from_env(poll_interval: Duration) -> Self {
let mut policy = Self::default();
policy.lease_pace = Duration::from_millis(
std::env::var(ENV_LEASE_PACE_MS)
.ok()
.and_then(|s| s.trim().parse::<u64>().ok())
.unwrap_or(600),
);
policy.poll_interval = poll_interval;
if let Some(state_timeout_secs) = env_positive_u64(ENV_STATE_TIMEOUT_SECS) {
policy.state_timeout = Duration::from_secs(state_timeout_secs);
Self {
lease_pace: Duration::from_millis(
std::env::var(ENV_LEASE_PACE_MS)
.ok()
.and_then(|value| value.trim().parse::<u64>().ok())
.unwrap_or(600),
),
poll_interval,
state_timeout: env_positive_u64(ENV_STATE_TIMEOUT_SECS)
.map(Duration::from_secs)
.unwrap_or_else(|| Self::default().state_timeout),
}
policy
}
}

View file

@ -9,9 +9,9 @@ pub(crate) fn reachable_offers(offers: Vec<Offer>, policy: &SelectionPolicy) ->
.filter(|o| {
o.geolocation
.as_deref()
.map_or(false, |g| !g.to_uppercase().contains("CN"))
.is_some_and(|geolocation| !geolocation.to_uppercase().contains("CN"))
})
.filter(|o| o.host_id.map_or(true, |h| !blacklist.contains(&h)))
.filter(|offer| offer.host_id.is_none_or(|host| !blacklist.contains(&host)))
.filter(|o| o.verification.as_deref() != Some("deverified"))
.filter(|o| policy.max_dph_total.is_none_or(|max| o.dph_total <= max))
.filter(|o| {

View file

@ -83,14 +83,14 @@ fn next_eligible_offer<'a>(
failed_host_ids: &HashSet<u64>,
preferred_offer_id: Option<u64>,
) -> Option<&'a Offer> {
if let Some(offer_id) = preferred_offer_id {
if let Some(offer) = pool.iter().find(|o| o.id == offer_id) {
if !tried_offer_ids.contains(&offer.id)
&& offer.host_id.is_none_or(|h| !failed_host_ids.contains(&h))
{
return Some(offer);
}
}
if let Some(offer_id) = preferred_offer_id
&& let Some(offer) = pool.iter().find(|offer| offer.id == offer_id)
&& !tried_offer_ids.contains(&offer.id)
&& offer
.host_id
.is_none_or(|host| !failed_host_ids.contains(&host))
{
return Some(offer);
}
pool.iter().find(|o| {
@ -108,18 +108,32 @@ fn env_for_index(req: &ProvisionRequest, index: u32) -> BTreeMap<String, String>
env
}
async fn provision_one(
client: &reqwest::Client,
base_url: &str,
api_key: &str,
req: &ProvisionRequest,
pool: &[Offer],
struct ProvisionOneRequest<'a> {
client: &'a reqwest::Client,
base_url: &'a str,
api_key: &'a str,
request: &'a ProvisionRequest,
pool: &'a [Offer],
index: u32,
tried_offer_ids: &mut Vec<u64>,
used_host_ids: &mut HashSet<u64>,
failed_host_ids: &mut HashSet<u64>,
tried_offer_ids: &'a mut Vec<u64>,
used_host_ids: &'a mut HashSet<u64>,
failed_host_ids: &'a mut HashSet<u64>,
preferred_offer_id: Option<u64>,
) -> Result<ProvisionedInstance, String> {
}
async fn provision_one(request: ProvisionOneRequest<'_>) -> Result<ProvisionedInstance, String> {
let ProvisionOneRequest {
client,
base_url,
api_key,
request: req,
pool,
index,
tried_offer_ids,
used_host_ids,
failed_host_ids,
preferred_offer_id,
} = request;
let mut attempt = 1_u64;
loop {
let offer = match next_eligible_offer(
@ -230,20 +244,21 @@ pub async fn provision_fleet(
let mut used_host_ids = HashSet::new();
let mut failed_host_ids = HashSet::new();
for index in 0..req.count {
match provision_one(
match provision_one(ProvisionOneRequest {
client,
base_url,
api_key,
&req,
&pool,
request: &req,
pool: &pool,
index,
&mut tried_offer_ids,
&mut used_host_ids,
&mut failed_host_ids,
req.preferred_offer_id
tried_offer_ids: &mut tried_offer_ids,
used_host_ids: &mut used_host_ids,
failed_host_ids: &mut failed_host_ids,
preferred_offer_id: req
.preferred_offer_id
.filter(|_| req.count == 1)
.or_else(|| first_wave.get(index as usize).map(|offer| offer.id)),
)
})
.await
{
Ok(info) => created.push(info),
@ -279,18 +294,18 @@ pub async fn provision_fleet(
);
}
eprintln!("lease_chain: replacing index {index}");
match provision_one(
match provision_one(ProvisionOneRequest {
client,
base_url,
api_key,
&req,
&pool,
request: &req,
pool: &pool,
index,
&mut tried_offer_ids,
&mut used_host_ids,
&mut failed_host_ids,
None,
)
tried_offer_ids: &mut tried_offer_ids,
used_host_ids: &mut used_host_ids,
failed_host_ids: &mut failed_host_ids,
preferred_offer_id: None,
})
.await
{
Ok(info) => created[idx] = info,

View file

@ -151,10 +151,10 @@ pub fn plan_distinct_host_first_wave(
if selected.len() == target {
break;
}
if let Some(host_id) = offer.host_id {
if !selected_hosts.insert(host_id) {
continue;
}
if let Some(host_id) = offer.host_id
&& !selected_hosts.insert(host_id)
{
continue;
}
selected_ids.insert(offer.id);
selected.push(offer.clone());

View file

@ -25,3 +25,6 @@ libc = "0.2"
[dev-dependencies]
proptest = "1"
[lints]
workspace = true

View file

@ -107,9 +107,10 @@ mod properties {
codes in prop::collection::vec(any::<u8>(), 0..=32),
registration in 0_usize..=32,
) {
let mut config = RuntimeConfig::default();
config.worker_count = 1;
let parts = RuntimeParts::new(config);
let parts = RuntimeParts::new(RuntimeConfig {
worker_count: 1,
..RuntimeConfig::default()
});
let runtime = parts.runtime().clone();
let backend = SteppingBackend::new();
let _engine =

View file

@ -503,7 +503,6 @@ pub struct NodeEdgeAgent {
}
impl NodeEdgeAgent {
#[allow(clippy::too_many_arguments)]
pub fn new(
attempt: u64,
logical_node: String,
@ -756,7 +755,7 @@ impl swactor::actor::ActorInterface for EdgeAckRelay {
if let Some(addr) = self.supervisor.get() {
let _ = self
.sender
.send_to(addr.clone(), crate::demo::feed::SupervisorMsg::EdgeAck(ack));
.send_to(*addr, crate::demo::feed::SupervisorMsg::EdgeAck(ack));
}
}
}

View file

@ -215,25 +215,43 @@ pub struct SupervisorActor {
shutdown: Option<ShutdownState>,
}
pub struct SupervisorConfig {
pub driver: ClusterDriver,
pub executor: IdempotentEffectExecutor<DemoBackend, EngineSpawner>,
pub manager: NodeManager,
pub driver_handle: std::sync::Arc<crate::demo::DemoDriverHandle>,
pub telemetry: SupervisorTelemetry,
pub dashboard: dashboard::DashboardHandle,
pub sender: swactor::runtime::ExternalSender,
pub registry: provisioning::BootstrapRegistry,
pub collector: std::sync::Arc<dyn provisioning::NodeTelemetryCollector>,
pub engine: EngineHandle,
pub remote_sub: telemetry::TelemetrySubscription,
pub initial_slots: Vec<String>,
pub run_id: RunId,
pub launch: crate::demo::LaunchStyle,
pub edge_actor: ActorAddress,
}
impl SupervisorActor {
#[allow(clippy::too_many_arguments)]
pub fn new(
driver: ClusterDriver,
executor: IdempotentEffectExecutor<DemoBackend, EngineSpawner>,
manager: NodeManager,
driver_handle: std::sync::Arc<crate::demo::DemoDriverHandle>,
mut telemetry: SupervisorTelemetry,
dashboard: dashboard::DashboardHandle,
sender: swactor::runtime::ExternalSender,
registry: provisioning::BootstrapRegistry,
collector: std::sync::Arc<dyn provisioning::NodeTelemetryCollector>,
engine: EngineHandle,
remote_sub: telemetry::TelemetrySubscription,
initial_slots: Vec<String>,
run_id: RunId,
launch: crate::demo::LaunchStyle,
edge_actor: ActorAddress,
) -> Self {
pub fn new(config: SupervisorConfig) -> Self {
let SupervisorConfig {
driver,
executor,
manager,
driver_handle,
mut telemetry,
dashboard,
sender,
registry,
collector,
engine,
remote_sub,
initial_slots,
run_id,
launch,
edge_actor,
} = config;
let events_channel = telemetry.register("prov.reconciler.events");
let snapshot_channel = telemetry.register("prov.reconciler.snapshot");
Self {
@ -814,7 +832,7 @@ impl SupervisorActor {
/// cards and the control view stay live between lifecycle transitions.
fn emit_node_status(&mut self, now: SystemTime) {
self.status_tick = self.status_tick.wrapping_add(1);
if self.status_tick % 4 != 0 {
if !self.status_tick.is_multiple_of(4) {
return; // 250ms ticks → heartbeat every second
}
let attempts: Vec<u64> = self.nodes.keys().copied().collect();
@ -968,7 +986,6 @@ impl SupervisorActor {
}
}
#[allow(clippy::too_many_arguments)]
fn publish_frame(
dashboard: &dashboard::DashboardHandle,
stream: &telemetry::frame::StreamId,
@ -1489,7 +1506,7 @@ mod properties {
fn control_command(kind: u8, attempt: u64, index: usize) -> dashboard::control::ControlCommand {
let command_id = format!("generated-command-{index}");
if kind % 2 == 0 {
if kind.is_multiple_of(2) {
dashboard::control::ControlCommand::Kill {
command_id,
node: format!("node-{attempt}"),
@ -1568,9 +1585,10 @@ mod properties {
fn generated_supervisor_transitions_are_once_only_nonblocking_and_clean(
actions in supervisor_actions(),
) {
let mut config = RuntimeConfig::default();
config.worker_count = 1;
let parts = RuntimeParts::new(config);
let parts = RuntimeParts::new(RuntimeConfig {
worker_count: 1,
..RuntimeConfig::default()
});
let runtime = parts.runtime().clone();
let sender = runtime.create_sender();
let backend = SteppingBackend::new();
@ -1644,25 +1662,25 @@ mod properties {
.spawn(AnnounceActor::new(manager.clone(), sender.clone()))
.expect("spawn generated announce actor");
let supervisor = runtime
.spawn(SupervisorActor::new(
.spawn(SupervisorActor::new(SupervisorConfig {
driver,
executor,
manager.clone(),
manager: manager.clone(),
driver_handle,
telemetry,
dashboard,
sender.clone(),
sender: sender.clone(),
registry,
Arc::new(evidence.clone()),
engine.handle(),
collector: Arc::new(evidence.clone()),
engine: engine.handle(),
remote_sub,
Vec::new(),
RunId(1),
crate::demo::LaunchStyle::Process {
initial_slots: Vec::new(),
run_id: RunId(1),
launch: crate::demo::LaunchStyle::Process {
exe: "generated-demo-node".into(),
},
*edge_inbox.addr(),
))
edge_actor: *edge_inbox.addr(),
}))
.expect("spawn generated supervisor actor");
drive(&backend, 8);

View file

@ -50,8 +50,8 @@ use provisioning::reconciler::ClusterDriver;
use provisioning::{ClusterShape, RunId};
use feed::{
EngineSpawner, SupervisorActor, SupervisorMsg, SupervisorTelemetry, demo_retry_policy,
initial_slots,
EngineSpawner, SupervisorActor, SupervisorConfig, SupervisorMsg, SupervisorTelemetry,
demo_retry_policy, initial_slots,
};
use provider::{DemoBackend, DemoProvider, NodeManager};
@ -68,9 +68,10 @@ pub const DEFAULT_NODES: u64 = 3;
pub(super) fn shared_test_driver() -> Arc<IrohDriver> {
thread_local! {
static DRIVER: (Arc<IrohDriver>, Engine) = {
let mut config = RuntimeConfig::default();
config.worker_count = 1;
let parts = RuntimeParts::new(config);
let parts = RuntimeParts::new(RuntimeConfig {
worker_count: 1,
..RuntimeConfig::default()
});
let backend = TokioBackend::new(TokioConfig::default())
.expect("create shared demo test backend");
let engine =
@ -99,10 +100,10 @@ pub(super) fn shared_test_driver() -> Arc<IrohDriver> {
/// replaced by a rebuild while this process runs, so fall back through
/// argv[0] and PATH.
fn resolve_exe() -> std::path::PathBuf {
if let Ok(path) = std::env::current_exe() {
if !path.to_string_lossy().ends_with(" (deleted)") {
return path;
}
if let Ok(path) = std::env::current_exe()
&& !path.to_string_lossy().ends_with(" (deleted)")
{
return path;
}
if let Some(arg0) = std::env::args_os().next() {
let candidate = std::path::PathBuf::from(&arg0);
@ -189,12 +190,14 @@ impl provisioning::NodeTelemetryCollector for DemoTelemetryCollector {
};
iroh_driver::spawn_pull_collector_to_actor(
&self.engine,
self.endpoint.clone(),
addr,
flow_id,
Vec::new(),
telemetry::SubscriptionRequest::all(),
Arc::clone(&self.fanout),
iroh_driver::PullCollectorConfig {
endpoint: self.endpoint.clone(),
peer: addr,
flow_id,
token: Vec::new(),
request: telemetry::SubscriptionRequest::all(),
fanout: Arc::clone(&self.fanout),
},
self.sender.clone(),
header_actor,
);
@ -347,15 +350,15 @@ fn run_supervisor(args: &[String]) -> Result<(), String> {
let route_view: RouteView =
Arc::new(std::sync::RwLock::new(std::collections::HashMap::new()));
let outbox: Outbox = Arc::new(std::sync::Mutex::new(Vec::new()));
driver.enable_actor_bridge(
runtime.clone(),
Arc::new(codec),
driver.enable_actor_bridge(iroh_driver::ActorBridgeConfig {
runtime: runtime.clone(),
codec: Arc::new(codec),
routes,
announce,
swim: announce,
relay_mirror,
route_view,
outbox,
);
});
driver.install_actor_bridge_pump(Duration::from_millis(250));
}
@ -458,23 +461,23 @@ fn run_supervisor(args: &[String]) -> Result<(), String> {
sender.clone(),
supervisor_slot.clone(),
)?;
let supervisor = SupervisorActor::new(
cluster_driver,
let supervisor = SupervisorActor::new(SupervisorConfig {
driver: cluster_driver,
executor,
manager.clone(),
manager: manager.clone(),
driver_handle,
telemetry,
dashboard.clone(),
sender.clone(),
bootstrap_registry,
dashboard: dashboard.clone(),
sender: sender.clone(),
registry: bootstrap_registry,
collector,
engine.handle(),
engine: engine.handle(),
remote_sub,
slots,
RunId(1),
launch.clone(),
initial_slots: slots,
run_id: RunId(1),
launch: launch.clone(),
edge_actor,
);
});
// Control plane: dashboard → supervisor.
control::install(&runtime, supervisor_slot.clone());
@ -483,7 +486,7 @@ fn run_supervisor(args: &[String]) -> Result<(), String> {
.spawn(supervisor)
.map_err(|e| format!("spawn supervisor actor: {e}"))?;
supervisor_slot
.set(supervisor_addr.clone())
.set(supervisor_addr)
.expect("supervisor address slot set once");
manager.set_spawn_actor(sender.clone(), supervisor_slot.clone());
@ -650,9 +653,10 @@ mod properties {
let pid = child.id() as i32;
let stdout = child.stdout.take().expect("capture demo supervisor stdout");
let mut config = RuntimeConfig::default();
config.worker_count = 1;
let parts = RuntimeParts::new(config);
let parts = RuntimeParts::new(RuntimeConfig {
worker_count: 1,
..RuntimeConfig::default()
});
let runtime = parts.runtime().clone();
let engine = Engine::new(
parts,

View file

@ -179,15 +179,15 @@ pub fn run_node_role(supervisor_addr_json: &str, attempt: u64) -> Result<(), Str
let route_view: RouteView =
Arc::new(std::sync::RwLock::new(std::collections::HashMap::new()));
let outbox: Outbox = Arc::new(std::sync::Mutex::new(Vec::new()));
driver.enable_actor_bridge(
runtime.clone(),
Arc::new(codec),
driver.enable_actor_bridge(iroh_driver::ActorBridgeConfig {
runtime: runtime.clone(),
codec: Arc::new(codec),
routes,
edge_agent,
swim: edge_agent,
relay_mirror,
route_view,
outbox,
);
});
driver.install_actor_bridge_pump(Duration::from_millis(250));
}
let driver = Arc::new(driver);
@ -199,7 +199,7 @@ pub fn run_node_role(supervisor_addr_json: &str, attempt: u64) -> Result<(), Str
// Join, then announce identity + advertised address to the supervisor's
// bootstrap actor over the control plane (readiness + telemetry dial).
driver.join(&[supervisor_addr.clone()]);
driver.join(std::slice::from_ref(&supervisor_addr));
let addr_json =
serde_json::to_string(&driver.endpoint_addr()).map_err(|e| format!("addr: {e}"))?;
@ -546,9 +546,10 @@ mod properties {
actions in node_actions(),
attempt in any::<u64>(),
) {
let mut config = RuntimeConfig::default();
config.worker_count = 1;
let parts = RuntimeParts::new(config);
let parts = RuntimeParts::new(RuntimeConfig {
worker_count: 1,
..RuntimeConfig::default()
});
let runtime = parts.runtime().clone();
let backend = SteppingBackend::new();
let engine =

Some files were not shown because too many files have changed in this diff Show more