docs(vastai): split remote readiness from prioritized local blockers

Commit the E2E plan and two linked checkpoint handoffs. Describe the frozen-checkpoint remote diagnostic track, working deployment/runtime/campaign paths, evidence limits, and the prerequisites for later paid VastAI execution.

Prioritize complete workflow timing, remaining safety verification, and fresh identity-bound qualification on the local track. Remove the internal workload/cleanup budget split as a standalone blocker without changing runtime limits or paid admission guards. Replace the stale active-stream failure status with newer Gate A/campaign evidence and the 186 passing focused review tests.

Validate relative handoff links. Historical target artifacts remain outside Git and must be preserved separately; no remote or paid execution was performed.

.cargo/config.toml

@@ -1,3 +1,6 @@
+[unstable]
+profile-hint-mostly-unused = true
+
 [build]
 rustc-workspace-wrapper = "tools/actor-control-flow-lint/rustc-wrapper.py"
 

Cargo.toml

@@ -10,6 +10,7 @@ members = [
     "crates/distribution",
     "crates/iroh-driver",
     "crates/dashboard",
+    "crates/myelin-control-contract",
     "apps/myelin",
     "xtask",
     "tools/vastai",
@@ -37,6 +38,11 @@ exclude = ["crates/bindings/wasm-crypto", "examples"]
 # Without this, members declared tokio with different features, so building
 # one member vs another (or `run` vs `test`) recompiled tokio each time.
 tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros", "process", "io-util", "sync", "time", "net", "signal"] }
+iroh = { version = "1.1", default-features = false, features = ["portmapper", "tls-ring"] }
+iroh-base = { version = "1.1", default-features = false, features = ["key"] }
+iroh-relay = { version = "1.1", default-features = false, features = ["test-utils"] }
+reqwest = { version = "0.13", default-features = false, features = ["json", "rustls-no-provider"] }
+rustls = { version = "0.23", default-features = false, features = ["ring", "std"] }
 
 [package]
 name = "swactor"
@@ -49,6 +55,48 @@ autobenches = false
 debug = true
 strip = false
 
+# Development builds omit debug information; the diagnostic profile restores
+# full debugger support without changing release artifacts.
+[profile.dev]
+debug = 0
+
+[profile.dev.package.iroh]
+hint-mostly-unused = true
+
+[profile.dev.package."noq-proto"]
+hint-mostly-unused = true
+
+[profile.dev.package.reqwest]
+hint-mostly-unused = true
+
+[profile.dev.package.rustls]
+hint-mostly-unused = true
+
+[profile.dev.package.tokio]
+hint-mostly-unused = true
+
+[profile.test]
+debug = 0
+
+[profile.test.package.iroh]
+hint-mostly-unused = true
+
+[profile.test.package."noq-proto"]
+hint-mostly-unused = true
+
+[profile.test.package.reqwest]
+hint-mostly-unused = true
+
+[profile.test.package.rustls]
+hint-mostly-unused = true
+
+[profile.test.package.tokio]
+hint-mostly-unused = true
+
+[profile.diagnostic]
+inherits = "dev"
+debug = 2
+
 [lib]
 crate-type = ["rlib"]
 

apps/myelin/Cargo.toml

@@ -18,6 +18,7 @@ provisioning = { path = "../../crates/provisioning" }
 dashboard = { path = "../../crates/dashboard" }
 futures-lite = "2"
 serde_json = "1"
+myelin-control-contract = { path = "../../crates/myelin-control-contract" }
 serde = { version = "1", features = ["derive"] }
 swactor = { path = "../..", features = ["serde", "transport"] }
 swactor-engine = { path = "../../crates/engine" }
@@ -26,12 +27,13 @@ swactor-process = { path = "../../crates/process" }
 swactor-process-context = { path = "../../crates/process-context" }
 distribution = { path = "../../crates/distribution" }
 iroh-driver = { path = "../../crates/iroh-driver" }
-iroh = "0.98"
+iroh.workspace = true
 tokio.workspace = true
 swactor-vastai = { path = "../../tools/vastai" }
 parking_lot = "0.12"
 blake3 = "1"
-toml = "0.8"
+sha2 = "0.10"
+toml = "1.1"
 ureq = "2"
 axum = "0.8"
 

apps/myelin/node-image/Dockerfile.base

@@ -9,6 +9,8 @@ RUN apt-get update && \
     apt-get install -y --no-install-recommends \
         ca-certificates \
         procps \
+        python3 \
+        python3-pip \
         openssh-server && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* \

apps/myelin/node-image/e2e_python_launcher.py

@@ -1,57 +1,48 @@
 #!/usr/bin/env python3
-import argparse
 import base64
 import os
-import runpy
 import sys
-import signal
-import tempfile
 import zlib
 
 
 def main() -> None:
-    parser = argparse.ArgumentParser()
-    source = parser.add_mutually_exclusive_group(required=True)
-    source.add_argument("--file")
-    source.add_argument("--source-base64")
-    source.add_argument("--source-env")
-    source.add_argument("--source-env-zlib")
-    args = parser.parse_args()
-
-    if args.file is not None:
-        runpy.run_path(args.file, run_name="__main__")
-        return
-
-    encoded = args.source_base64
-    if args.source_env is not None:
-        encoded = os.environ.get(args.source_env)
+    if len(sys.argv) != 3:
+        raise RuntimeError("expected exactly one generated program source option")
+    option, value = sys.argv[1:]
+    if option == "--file":
+        with open(value, "rb") as source:
+            program = source.read()
+        filename = value
+    elif option == "--source-base64":
+        program = base64.b64decode(value, validate=True)
+        filename = "<myelin-e2e-generated>"
+    elif option in {"--source-env", "--source-env-zlib"}:
+        encoded = os.environ.get(value)
         if encoded is None:
             raise RuntimeError(
-                f"generated program environment variable is missing: {args.source_env}"
+                f"generated program environment variable is missing: {value}"
             )
-    if args.source_env_zlib is not None:
-        encoded = os.environ.get(args.source_env_zlib)
-        if encoded is None:
-            raise RuntimeError(
-                "compressed generated program environment variable is missing: "
-                f"{args.source_env_zlib}"
-            )
-    if encoded is None:
-        raise RuntimeError("generated program source is missing")
+        program = base64.b64decode(encoded, validate=True)
+        if option == "--source-env-zlib":
+            program = zlib.decompress(program)
+        filename = "<myelin-e2e-generated>"
+    else:
+        raise RuntimeError(f"unsupported generated program source option: {option}")
 
-    program = base64.b64decode(encoded, validate=True)
-    if args.source_env_zlib is not None:
-        program = zlib.decompress(program)
     if os.environ.get("MYELIN_E2E_HOLD_BEFORE_BOOTSTRAP") == "1":
+        import signal
+
         signal.pause()
-    with tempfile.NamedTemporaryFile(prefix="myelin-e2e-", suffix=".py", delete=False) as handle:
-        handle.write(program)
-        path = handle.name
-    try:
-        sys.argv = [path]
-        runpy.run_path(path, run_name="__main__")
-    finally:
-        os.unlink(path)
+    sys.argv = [filename]
+    namespace = {
+        "__name__": "__main__",
+        "__file__": filename,
+        "__package__": None,
+        "__loader__": None,
+        "__spec__": None,
+        "__cached__": None,
+    }
+    exec(compile(program, filename, "exec"), namespace)
 
 
 if __name__ == "__main__":

apps/myelin/proptest-regressions/orchestration/provider_adapters/vastai/mod.txt

@@ -7,3 +7,4 @@
 cc 1feb8f8e1d264d84e655f5b3e9d3f489450acfc29129f97df9982e3b636f76f6 # shrinks to terminal_kind = 0, extra_polls = 0
 cc f8e47b839c0043733e02d2549e2b87ed4655b29e5492c7418afd30e56dd5a118 # shrinks to retrying = false, actions = []
 cc b621752b491766cbc484a8c9836abdf2a19220c9c551d4f2ff8fd781bfb38d6e # shrinks to run_id = 1, node_id = 1, contract_id = 1
+cc 519f735486975dd09f9499bde9c46e0e6780471389897e7255f844f4751e404d # shrinks to status_index = 0, empty = false

apps/myelin/src/data_namespace.rs

@@ -1,21 +1,25 @@
 use std::path::Path;
-use std::sync::Arc;
+use std::sync::{Arc, OnceLock};
 use std::time::Duration;
 
 use data_plane::blob_transfer::BlobTransferSender;
 use data_plane::control::{DataNamespaceService, DataPlaneControl};
-use data_plane::namespace::{NamespaceClient, NamespaceClientActor, NamespaceDiscovery};
+use data_plane::namespace::{
+    NamespaceClient, NamespaceClientActor, NamespaceClientIn, NamespaceDiscovery,
+};
 use data_plane::source::BlobSourcePublisher;
-use distribution::directory_actor::DirectoryIn;
+use distribution::directory_actor::{DirectoryClaims, DirectoryIn};
 use distribution::registry_actor::{RegistryIn, RegistryView};
-use iroh_driver::{ActorRegistrar, IrohBlobTransferSender, IrohDriver};
+use distribution::transport_bridge::RouteView;
+use iroh_driver::{ActorRegistrar, ConnectionWatch, IrohBlobTransferSender, IrohDriver};
 use swactor::actor::{ActorAddress, ActorInterface, Ctx};
 use swactor::runtime::Runtime;
 
 use crate::orchestration::distribution_stack::DistributionRuntimeStack;
 
 pub(crate) const DATA_DIRECTORY_SERVICE: &str = "swactor.data-directory";
-const NAMESPACE_RETRY: Duration = Duration::from_millis(100);
+const NAMESPACE_RETRY: Duration = Duration::from_secs(1);
+const RECOVERED_SERVICE_TIMESTAMP_BASE: u64 = 1_u64 << 63;
 
 #[derive(Clone)]
 struct LocalActorPublisher {
@@ -25,53 +29,114 @@ struct LocalActorPublisher {
 }
 
 impl LocalActorPublisher {
-    fn publish(&self, actor: ActorAddress) -> Result<(), String> {
-        let claim = self.registrar.register_actor(actor, 1);
+    fn publish(&self, actor: ActorAddress, generation: u64) -> Result<[u8; 32], String> {
+        let claim = self.registrar.register_actor(actor, generation);
         self.runtime
             .send_to(self.distribution_directory, DirectoryIn::Register(claim))
-            .map_err(|error| error.to_string())
+            .map_err(|error| error.to_string())?;
+        Ok(self.registrar.node_id_bytes())
     }
 }
 
 impl BlobSourcePublisher for LocalActorPublisher {
-    fn publish_source(&self, source: ActorAddress) -> Result<(), String> {
-        self.publish(source)
+    fn publish_source(&self, source: ActorAddress) -> Result<[u8; 32], String> {
+        self.publish(source, 1)
     }
 }
 struct RegistryNamespaceDiscovery {
     view: RegistryView,
+    claims: DirectoryClaims,
+    routes: RouteView,
+    connection_watch: OnceLock<ConnectionWatch>,
+}
+
+impl RegistryNamespaceDiscovery {
+    fn current_authority(&self) -> Option<(ActorAddress, u64)> {
+        let view = self.view.read().expect("registry view poisoned");
+        let binding = view
+            .entries
+            .iter()
+            .find(|entry| entry.name == DATA_DIRECTORY_SERVICE && !entry.tombstone)?;
+        let (host, generation) = self.claims.location(&binding.actor_addr)?;
+        if host != binding.node_id
+            || self
+                .routes
+                .read()
+                .expect("route view poisoned")
+                .get(&binding.actor_addr)
+                != Some(&host)
+        {
+            return None;
+        }
+        // Registry timestamps are Lamport ordering, not durable epochs: a
+        // repeated publication or tombstone takeover advances them. The
+        // authority signs its unchanged epoch into the actor-location claim.
+        let epoch = generation
+            .checked_sub(RECOVERED_SERVICE_TIMESTAMP_BASE)
+            .filter(|epoch| *epoch != 0)?;
+        Some((binding.actor_addr, epoch))
+    }
 }
 
 impl NamespaceDiscovery for RegistryNamespaceDiscovery {
     fn current_directory(&self) -> Option<ActorAddress> {
-        self.view
-            .read()
-            .expect("registry view poisoned")
-            .entries
-            .iter()
-            .find(|entry| entry.name == DATA_DIRECTORY_SERVICE && !entry.tombstone)
-            .map(|entry| entry.actor_addr)
+        self.current_authority().map(|(directory, _)| directory)
+    }
+
+    fn accepts_authority_epoch(&self, epoch: u64) -> bool {
+        self.current_authority()
+            .is_some_and(|(_, expected)| epoch == expected)
     }
 }
 
-#[derive(Clone)]
-enum NamespaceServicePublisherIn {
-    Tick,
-}
+/// How many publisher ticks (1 s each) a recovery re-bind keeps pushing the
+/// fresh directory binding directly to unacknowledged persisted workers.
+/// The ceiling is unchanged: successful peers retire on their exact ACK,
+/// while missing peers retain the full dial/recovery fallback budget.
+const RECOVERY_REBIND_TICKS: u32 = 120;
 
 struct NamespaceServicePublisher {
     registry: ActorAddress,
     view: RegistryView,
     directory: ActorAddress,
+    timestamp: u64,
+    /// Directed recovery re-bind state: persisted peers still to re-bind and
+    /// the remaining bounded tick budget.
+    rebind: Option<(Vec<swactor_transport::NodeId>, u32)>,
 }
 
 impl ActorInterface for NamespaceServicePublisher {
-    type Incoming = NamespaceServicePublisherIn;
+    type Incoming = RegistryIn;
     type Response = ();
 
-    fn handle(&mut self, ctx: &Ctx<'_>, message: NamespaceServicePublisherIn) {
+    fn handle(&mut self, ctx: &Ctx<'_>, message: RegistryIn) {
         match message {
-            NamespaceServicePublisherIn::Tick => {
+            RegistryIn::NameAcknowledged { entry, peer } => {
+                let current = entry.name == DATA_DIRECTORY_SERVICE
+                    && entry.actor_addr == self.directory
+                    && !entry.tombstone
+                    && self
+                        .view
+                        .read()
+                        .expect("registry view poisoned")
+                        .entries
+                        .iter()
+                        .any(|current| {
+                            current.name == entry.name
+                                && current.actor_addr == entry.actor_addr
+                                && current.node_id == entry.node_id
+                                && current.timestamp == entry.timestamp
+                                && current.generation == entry.generation
+                                && !current.tombstone
+                        });
+                if current && let Some((peers, _)) = &mut self.rebind {
+                    peers.retain(|pending| *pending != peer);
+                    if peers.is_empty() {
+                        self.rebind = None;
+                    }
+                }
+            }
+            RegistryIn::Tick => {
                 let published = self
                     .view
                     .read()
@@ -86,13 +151,35 @@ impl ActorInterface for NamespaceServicePublisher {
                 if !published {
                     let _ = ctx.send(
                         self.registry,
-                        RegistryIn::RegisterName {
+                        RegistryIn::RegisterNameAt {
                             name: DATA_DIRECTORY_SERVICE.to_owned(),
                             actor_addr: self.directory,
+                            timestamp: self.timestamp,
                         },
                     );
                 }
+                // Recovery re-bind: keep pushing the fresh binding directly
+                // to unacknowledged workers until the bounded budget runs out.
+                // Directed gossip replaces the dead pre-crash binding on a
+                // worker within one round trip instead of one SWIM
+                // convergence period.
+                if let Some((peers, budget)) = self.rebind.take() {
+                    if !peers.is_empty() {
+                        let _ = ctx.send(
+                            self.registry,
+                            RegistryIn::DisseminateNameTo {
+                                name: DATA_DIRECTORY_SERVICE.to_owned(),
+                                peers: peers.clone(),
+                                reply_to: ctx.self_addr(),
+                            },
+                        );
+                    }
+                    if budget > 1 {
+                        self.rebind = Some((peers, budget - 1));
+                    }
+                }
             }
+            _ => {}
         }
     }
 }
@@ -106,6 +193,7 @@ impl DataNamespaceAuthority {
         stack: &DistributionRuntimeStack,
         driver: &IrohDriver,
         state_path: impl AsRef<Path>,
+        recovery_peers: Vec<swactor_transport::NodeId>,
     ) -> Result<Self, String> {
         let runtime = stack.runtime.clone();
         let source_sender: Arc<dyn BlobTransferSender> = Arc::new(IrohBlobTransferSender::new(
@@ -125,16 +213,29 @@ impl DataNamespaceAuthority {
             state_path,
             source_sender,
             source_publisher,
+            Some(Arc::new(
+                crate::contextual_process::MyelinChildRouteRegistrar::new(
+                    stack.route_view.clone(),
+                    stack.pinned_routes.clone(),
+                    stack.route_binder.clone(),
+                    runtime.clone(),
+                    stack.actors.directory,
+                    driver.connection_observer(),
+                ),
+            )),
         )
         .map_err(|error| format!("recover data namespace: {error}"))?;
         let directory = service.directory();
-        publisher.publish(directory)?;
+        let service_timestamp =
+            RECOVERED_SERVICE_TIMESTAMP_BASE.saturating_add(service.authority_epoch());
+        publisher.publish(directory, service_timestamp)?;
         runtime
             .send_to(
                 stack.actors.registry,
-                RegistryIn::RegisterName {
+                RegistryIn::RegisterNameAt {
                     name: DATA_DIRECTORY_SERVICE.to_owned(),
                     actor_addr: directory,
+                    timestamp: service_timestamp,
                 },
             )
             .map_err(|error| format!("publish data directory service: {error}"))?;
@@ -143,13 +244,19 @@ impl DataNamespaceAuthority {
                 registry: stack.actors.registry,
                 view: Arc::clone(&stack.registry_view),
                 directory,
+                timestamp: service_timestamp,
+                rebind: (!recovery_peers.is_empty())
+                    .then(|| (recovery_peers, RECOVERY_REBIND_TICKS)),
             })
             .map_err(|error| format!("spawn namespace service publisher: {error}"))?;
+        runtime
+            .send_to(service_publisher, RegistryIn::Tick)
+            .map_err(|error| format!("start namespace recovery rebind: {error}"))?;
         stack.engine.send_every(
             Duration::from_secs(1),
             runtime.create_sender(),
             service_publisher,
-            NamespaceServicePublisherIn::Tick,
+            RegistryIn::Tick,
         );
         Ok(Self { service })
     }
@@ -172,19 +279,49 @@ pub(crate) fn install_namespace_client(
     stack: &DistributionRuntimeStack,
     driver: &IrohDriver,
 ) -> Result<InstalledNamespaceClient, String> {
-    let discovery: Arc<dyn NamespaceDiscovery> = Arc::new(RegistryNamespaceDiscovery {
+    let discovery = Arc::new(RegistryNamespaceDiscovery {
         view: Arc::clone(&stack.registry_view),
+        claims: stack.directory_claims.clone(),
+        routes: Arc::clone(&stack.route_view),
+        connection_watch: OnceLock::new(),
     });
     let proxy = stack
         .runtime
         .spawn(NamespaceClientActor::new(
             stack.engine.clone(),
             stack.runtime.create_sender(),
-            discovery,
+            discovery.clone(),
             NAMESPACE_RETRY,
         ))
         .map_err(|error| format!("spawn namespace client: {error}"))?;
     stack.register_local_actor(driver.register_actor(proxy, 1));
+    let sender = stack.runtime.create_sender();
+    let wake: Arc<dyn Fn() + Send + Sync> = Arc::new(move || {
+        let _ = sender.send_to(proxy, NamespaceClientIn::Retry);
+    });
+    stack
+        .runtime
+        .send_to(
+            stack.actors.registry,
+            RegistryIn::WatchName {
+                name: DATA_DIRECTORY_SERVICE.to_owned(),
+                changed: Arc::downgrade(&wake),
+            },
+        )
+        .map_err(|error| format!("watch namespace binding: {error}"))?;
+    stack
+        .runtime
+        .send_to(
+            stack.actors.directory,
+            DirectoryIn::WatchRoutes {
+                changed: Arc::downgrade(&wake),
+            },
+        )
+        .map_err(|error| format!("watch namespace reply routes: {error}"))?;
+    discovery
+        .connection_watch
+        .set(driver.connection_observer().watch_connections(wake))
+        .expect("namespace connection watch is installed once");
     let source_publisher: Arc<dyn BlobSourcePublisher> = Arc::new(LocalActorPublisher {
         runtime: stack.runtime.clone(),
         distribution_directory: stack.actors.directory,
@@ -195,3 +332,373 @@ pub(crate) fn install_namespace_client(
         source_publisher,
     })
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use data_plane::namespace::{
+        DataDirectoryActor, DataDirectoryOut, NamespaceError, NamespaceRequest,
+        register_namespace_codecs,
+    };
+    use data_plane::path::DataPath;
+    use distribution::crypto::{Keypair, KeypairExt};
+    use distribution::node::DistributedNodeConfig;
+    use distribution::swim::actor::MembershipChanged;
+    use distribution::transport_bridge::OutFrame;
+    use distribution::types::{MemberState, NodeId};
+    use swactor_engine::{Engine, SteppingBackend};
+
+    fn node(id: NodeId) -> (Engine, SteppingBackend, DistributionRuntimeStack) {
+        let (parts, runtime, codec, router) =
+            DistributionRuntimeStack::build_runtime(register_namespace_codecs, None);
+        let backend = SteppingBackend::new();
+        let engine = Engine::new(parts, backend.clone()).unwrap();
+        let stack = DistributionRuntimeStack::new_from_runtime(
+            runtime,
+            codec,
+            router,
+            id,
+            DistributedNodeConfig::default(),
+            engine.handle(),
+        );
+        (engine, backend, stack)
+    }
+
+    fn step(backend: &SteppingBackend) {
+        for _ in 0..4 {
+            backend.step();
+        }
+    }
+
+    fn deliver(
+        frames: Vec<OutFrame>,
+        target: &DistributionRuntimeStack,
+        backend: &SteppingBackend,
+    ) {
+        let routes = target.actor_bridge_routes();
+        for frame in frames {
+            let message = target
+                .codec
+                .decode(&frame.type_tag, &frame.payload)
+                .unwrap();
+            let destination = routes.get(&frame.type_tag).copied().unwrap_or(frame.dest);
+            target.runtime.deliver_raw(destination, message).unwrap();
+        }
+        step(backend);
+    }
+
+    fn transfer(
+        source: &DistributionRuntimeStack,
+        target: &DistributionRuntimeStack,
+        backend: &SteppingBackend,
+    ) {
+        let frames = source.outbox.lock().unwrap().drain(..).collect();
+        deliver(frames, target, backend);
+    }
+
+    fn publish(
+        stack: &DistributionRuntimeStack,
+        backend: &SteppingBackend,
+        key: &Keypair,
+        store: &Path,
+    ) {
+        let directory = DataDirectoryActor::recover(store, None, |_, _| {
+            unreachable!("empty namespace has no sources to recover")
+        })
+        .unwrap();
+        let timestamp = RECOVERED_SERVICE_TIMESTAMP_BASE + directory.authority_epoch();
+        let directory = stack.runtime.spawn(directory).unwrap();
+        stack.register_local_actor(key.sign_directory_entry(directory, timestamp));
+        // Startup observation and recovery rebind may both publish the same
+        // live authority. Its registry Lamport clock advances; its epoch does not.
+        for _ in 0..2 {
+            stack
+                .runtime
+                .send_to(
+                    stack.actors.registry,
+                    RegistryIn::RegisterNameAt {
+                        name: DATA_DIRECTORY_SERVICE.to_owned(),
+                        actor_addr: directory,
+                        timestamp,
+                    },
+                )
+                .unwrap();
+        }
+        step(backend);
+    }
+
+    #[test]
+    fn recovery_acknowledgements_retire_only_exact_current_peer() {
+        use distribution::registry::{ClusterRegistry, RegistryConfig, RegistryEntry};
+
+        let owner = NodeId([1; 32]);
+        let first = NodeId([2; 32]);
+        let second = NodeId([3; 32]);
+        let (_engine, backend, stack) = node(owner);
+        let outbound = stack.runtime.new_inbox::<RegistryIn>().unwrap();
+        let entry = RegistryEntry {
+            name: DATA_DIRECTORY_SERVICE.to_owned(),
+            actor_addr: ActorAddress([4; 32]),
+            node_id: owner,
+            timestamp: RECOVERED_SERVICE_TIMESTAMP_BASE + 2,
+            generation: 2,
+            tombstone: false,
+        };
+        let mut registry = ClusterRegistry::new(RegistryConfig::default());
+        registry.merge(entry.clone());
+        let publisher = stack
+            .runtime
+            .spawn(NamespaceServicePublisher {
+                registry: *outbound.addr(),
+                view: Arc::new(std::sync::RwLock::new(registry.snapshot())),
+                directory: entry.actor_addr,
+                timestamp: entry.timestamp,
+                rebind: Some((vec![first, second], RECOVERY_REBIND_TICKS)),
+            })
+            .unwrap();
+        let tick = || {
+            stack.runtime.send_to(publisher, RegistryIn::Tick).unwrap();
+            step(&backend);
+            outbound.try_recv()
+        };
+        let acknowledge = |entry: RegistryEntry, peer| {
+            stack
+                .runtime
+                .send_to(publisher, RegistryIn::NameAcknowledged { entry, peer })
+                .unwrap();
+        };
+        let mut stale = entry.clone();
+        stale.generation -= 1;
+        acknowledge(stale, first);
+        let mut stale = entry.clone();
+        stale.actor_addr = ActorAddress([5; 32]);
+        acknowledge(stale, first);
+        let mut stale = entry.clone();
+        stale.timestamp -= 1;
+        acknowledge(stale, first);
+        let mut stale = entry.clone();
+        stale.node_id = second;
+        acknowledge(stale, first);
+        let mut stale = entry.clone();
+        stale.name = "other-service".to_owned();
+        acknowledge(stale, first);
+        let mut stale = entry.clone();
+        stale.tombstone = true;
+        acknowledge(stale, first);
+        assert!(matches!(tick(),
+            Some(RegistryIn::DisseminateNameTo { peers, .. }) if peers == vec![first, second]
+        ));
+
+        acknowledge(entry.clone(), first);
+        acknowledge(entry.clone(), first);
+        acknowledge(entry.clone(), NodeId([9; 32]));
+        assert!(matches!(tick(),
+            Some(RegistryIn::DisseminateNameTo { peers, .. }) if peers == vec![second]
+        ));
+        acknowledge(entry, second);
+        assert!(
+            tick().is_none(),
+            "all exact peer ACKs retire directed recovery"
+        );
+    }
+
+    #[test]
+    fn missing_recovery_peer_keeps_bounded_retry_budget() {
+        use distribution::registry::{ClusterRegistry, RegistryConfig};
+
+        let owner = NodeId([1; 32]);
+        let peer = NodeId([2; 32]);
+        let (_engine, backend, stack) = node(owner);
+        let outbound = stack.runtime.new_inbox::<RegistryIn>().unwrap();
+        let directory = ActorAddress([4; 32]);
+        let mut registry = ClusterRegistry::new(RegistryConfig::default());
+        registry.register_at(DATA_DIRECTORY_SERVICE.to_owned(), directory, owner, 100, 1);
+        let publisher = stack
+            .runtime
+            .spawn(NamespaceServicePublisher {
+                registry: *outbound.addr(),
+                view: Arc::new(std::sync::RwLock::new(registry.snapshot())),
+                directory,
+                timestamp: 100,
+                rebind: Some((vec![peer], RECOVERY_REBIND_TICKS)),
+            })
+            .unwrap();
+        for _ in 0..RECOVERY_REBIND_TICKS {
+            stack.runtime.send_to(publisher, RegistryIn::Tick).unwrap();
+            step(&backend);
+            assert!(matches!(outbound.try_recv(),
+                Some(RegistryIn::DisseminateNameTo { peers, .. }) if peers == vec![peer]
+            ));
+        }
+        stack.runtime.send_to(publisher, RegistryIn::Tick).unwrap();
+        step(&backend);
+        assert!(
+            outbound.try_recv().is_none(),
+            "missing peers cannot push forever"
+        );
+    }
+
+    #[test]
+    fn namespace_lookup_uses_signed_epoch_after_republish_and_recovery_without_ticks() {
+        let authority_key = Keypair::from_bytes(&[1; 32]);
+        let worker_key = Keypair::from_bytes(&[2; 32]);
+        let (_authority_engine, authority_backend, authority) = node(authority_key.node_id());
+        let (_worker_engine, worker_backend, worker) = node(worker_key.node_id());
+        let state = tempfile::tempdir().unwrap();
+        let store = state.path().join("namespace.json");
+        publish(&authority, &authority_backend, &authority_key, &store);
+
+        for (stack, peer) in [
+            (&authority, worker_key.node_id()),
+            (&worker, authority_key.node_id()),
+        ] {
+            stack
+                .runtime
+                .send_to(
+                    stack.actors.directory,
+                    DirectoryIn::Membership(MembershipChanged {
+                        node_id: peer,
+                        state: MemberState::Alive,
+                        incarnation: 1,
+                    }),
+                )
+                .unwrap();
+        }
+        let discovery = Arc::new(RegistryNamespaceDiscovery {
+            view: worker.registry_view.clone(),
+            claims: worker.directory_claims.clone(),
+            routes: worker.route_view.clone(),
+            connection_watch: OnceLock::new(),
+        });
+        let proxy = worker
+            .runtime
+            .spawn(NamespaceClientActor::new(
+                worker.engine.clone(),
+                worker.runtime.create_sender(),
+                discovery,
+                NAMESPACE_RETRY,
+            ))
+            .unwrap();
+        worker.register_local_actor(worker_key.sign_directory_entry(proxy, 1));
+        let sender = worker.runtime.create_sender();
+        let wake: Arc<dyn Fn() + Send + Sync> = Arc::new(move || {
+            sender.send_to(proxy, NamespaceClientIn::Retry).unwrap();
+        });
+        worker
+            .runtime
+            .send_to(
+                worker.actors.registry,
+                RegistryIn::WatchName {
+                    name: DATA_DIRECTORY_SERVICE.to_owned(),
+                    changed: Arc::downgrade(&wake),
+                },
+            )
+            .unwrap();
+        worker
+            .runtime
+            .send_to(
+                worker.actors.directory,
+                DirectoryIn::WatchRoutes {
+                    changed: Arc::downgrade(&wake),
+                },
+            )
+            .unwrap();
+        let replies = worker.runtime.new_inbox::<DataDirectoryOut>().unwrap();
+        let path = DataPath::parse("/cases/fresh-child/missing").unwrap();
+        let request = NamespaceClientIn::Request {
+            request: NamespaceRequest::Lookup { path: path.clone() },
+            reply_to: *replies.addr(),
+        };
+        worker.runtime.send_to(proxy, request.clone()).unwrap();
+        worker
+            .runtime
+            .send_to(
+                worker.actors.directory,
+                DirectoryIn::SyncTo {
+                    peer: authority_key.node_id(),
+                },
+            )
+            .unwrap();
+        step(&worker_backend);
+        transfer(&worker, &authority, &authority_backend);
+        assert!(replies.try_recv().is_none());
+
+        authority
+            .runtime
+            .send_to(
+                authority.actors.registry,
+                RegistryIn::SyncTo {
+                    peer: worker_key.node_id(),
+                },
+            )
+            .unwrap();
+        authority
+            .runtime
+            .send_to(
+                authority.actors.directory,
+                DirectoryIn::SyncTo {
+                    peer: worker_key.node_id(),
+                },
+            )
+            .unwrap();
+        step(&authority_backend);
+        transfer(&authority, &worker, &worker_backend);
+        transfer(&worker, &authority, &authority_backend);
+        transfer(&authority, &worker, &worker_backend);
+        assert!(matches!(
+            replies.try_recv(),
+            Some(DataDirectoryOut::LookedUp {
+                authority_epoch: 1,
+                result: Err(NamespaceError::PathNotFound(missing)),
+                ..
+            }) if missing == path
+        ));
+
+        // Hold a genuine old-authority reply across recovery. Publishing only
+        // the new name must not let the old reply through while its claim is
+        // still undiscovered; the signed new route then wakes and re-drives it.
+        worker.runtime.send_to(proxy, request).unwrap();
+        step(&worker_backend);
+        transfer(&worker, &authority, &authority_backend);
+        let stale = authority.outbox.lock().unwrap().drain(..).collect();
+        publish(&authority, &authority_backend, &authority_key, &store);
+        authority
+            .runtime
+            .send_to(
+                authority.actors.registry,
+                RegistryIn::SyncTo {
+                    peer: worker_key.node_id(),
+                },
+            )
+            .unwrap();
+        step(&authority_backend);
+        transfer(&authority, &worker, &worker_backend);
+        deliver(stale, &worker, &worker_backend);
+        assert!(
+            replies.try_recv().is_none(),
+            "old epoch must not complete the lookup"
+        );
+        authority
+            .runtime
+            .send_to(
+                authority.actors.directory,
+                DirectoryIn::SyncTo {
+                    peer: worker_key.node_id(),
+                },
+            )
+            .unwrap();
+        step(&authority_backend);
+        transfer(&authority, &worker, &worker_backend);
+        transfer(&worker, &authority, &authority_backend);
+        transfer(&authority, &worker, &worker_backend);
+        assert!(matches!(
+            replies.try_recv(),
+            Some(DataDirectoryOut::LookedUp {
+                authority_epoch: 2,
+                result: Err(NamespaceError::PathNotFound(missing)),
+                ..
+            }) if missing == path
+        ));
+        // No clock advancement, gossip tick, or namespace retry tick occurred.
+    }
+}

apps/myelin/src/node/actor.rs

@@ -1,13 +1,13 @@
+use distribution::directory_actor::DirectoryIn;
 use iroh::EndpointAddr;
+use iroh_driver::PeerConnector;
+use myelin_control_contract::{ContextualProcessEvent, ContextualProcessEventKind};
 use serde::{Deserialize, Serialize};
 use swactor::actor::{ActorAddress, ActorInterface};
 use swactor::runtime::Ctx;
 use swactor_transport::{CodecRegistry, NetworkMessage};
 
-use crate::contextual_process::{
-    ContextualNodeCommand, ContextualProcessControllerIn, ContextualProcessEventKindWire,
-    ContextualProcessEventWire,
-};
+use crate::contextual_process::{ContextualNodeCommand, ContextualProcessControllerIn};
 use crate::gguf_shard::StageShardPlan;
 use crate::run_plan;
 use crate::staging as stage;
@@ -146,7 +146,15 @@ pub(crate) enum NodeAgentMsg {
         endpoint: EndpointAddr,
         node_actor: ActorAddress,
         readiness_id: u64,
+        #[serde(default)]
+        artifact_digest: Option<String>,
+        #[serde(default)]
+        deployment_generation: Option<String>,
     },
+    JoinPeers {
+        endpoints: Vec<EndpointAddr>,
+    },
+    ResyncRoutes,
     RuntimeReadyAck {
         run_id: u64,
         node_id: u64,
@@ -346,6 +354,8 @@ pub(crate) struct NodeAgentActor {
     orchestrator: ActorAddress,
     report_to: Option<ActorAddress>,
     contextual_controller: Option<ActorAddress>,
+    peer_connector: Option<PeerConnector>,
+    directory: Option<ActorAddress>,
     control_generation: u64,
     inbound_edge: Option<StageInboundEdgeWire>,
     outbound_edge: Option<StageOutboundEdgeWire>,
@@ -366,6 +376,8 @@ impl NodeAgentActor {
             orchestrator,
             report_to,
             contextual_controller: None,
+            peer_connector: None,
+            directory: None,
             control_generation: 0,
             inbound_edge: None,
             outbound_edge: None,
@@ -380,6 +392,16 @@ impl NodeAgentActor {
         self
     }
 
+    pub(crate) fn with_peer_connector(mut self, connector: PeerConnector) -> Self {
+        self.peer_connector = Some(connector);
+        self
+    }
+
+    pub(crate) fn with_directory_actor(mut self, directory: ActorAddress) -> Self {
+        self.directory = Some(directory);
+        self
+    }
+
     fn forward_prompt_or_snapshot(&mut self, ctx: &Ctx, msg: NodeAgentMsg) -> Option<NodeAgentMsg> {
         match msg {
             NodeAgentMsg::InferPrompt {
@@ -461,7 +483,7 @@ impl NodeAgentActor {
                         } => (
                             request_id,
                             reply_to,
-                            ContextualProcessEventKindWire::SpawnRejected {
+                            ContextualProcessEventKind::SpawnRejected {
                                 error: CONTROL_UNAVAILABLE.to_owned(),
                             },
                         ),
@@ -472,7 +494,7 @@ impl NodeAgentActor {
                         } => (
                             request_id,
                             reply_to,
-                            ContextualProcessEventKindWire::StopRejected {
+                            ContextualProcessEventKind::StopRejected {
                                 error: CONTROL_UNAVAILABLE.to_owned(),
                             },
                         ),
@@ -482,14 +504,14 @@ impl NodeAgentActor {
                         } => (
                             request_id,
                             reply_to,
-                            ContextualProcessEventKindWire::ControlUnavailable {
+                            ContextualProcessEventKind::ControlUnavailable {
                                 error: CONTROL_UNAVAILABLE.to_owned(),
                             },
                         ),
                     };
                     let _ = ctx.send(
                         reply_to,
-                        OrchestratorMsg::ContextualEvent(ContextualProcessEventWire {
+                        OrchestratorMsg::ContextualEvent(ContextualProcessEvent {
                             request_id,
                             logical_node_id: self.logical_node_id,
                             event,
@@ -518,6 +540,18 @@ impl NodeAgentActor {
             return;
         };
         match msg {
+            NodeAgentMsg::JoinPeers { endpoints } => {
+                if let Some(connector) = &self.peer_connector {
+                    for endpoint in endpoints {
+                        connector.connect(endpoint);
+                    }
+                }
+            }
+            NodeAgentMsg::ResyncRoutes => {
+                if let Some(directory) = self.directory {
+                    let _ = ctx.send(directory, DirectoryIn::Resync);
+                }
+            }
             NodeAgentMsg::ProvisionStage(provision) => {
                 self.core.observe(stage::StageEvent::ProvisionStage {
                     from: stage::NodeId(provision.authorized_orchestrator),
@@ -534,6 +568,8 @@ impl NodeAgentActor {
                 endpoint,
                 node_actor,
                 readiness_id,
+                artifact_digest,
+                deployment_generation,
             } => {
                 self.core.observe(stage::StageEvent::WorkerReady);
                 let _ = ctx.send(
@@ -545,6 +581,8 @@ impl NodeAgentActor {
                         endpoint,
                         node_actor,
                         readiness_id,
+                        artifact_digest,
+                        deployment_generation,
                     },
                 );
             }

apps/myelin/src/node/worker_node_runtime.rs

@@ -6,7 +6,7 @@
 
 use std::collections::BTreeMap;
 use std::fs::{self, File, OpenOptions};
-use std::io::{BufRead, BufReader, Read, Write};
+use std::io::{Read, Write};
 use std::path::{Path, PathBuf};
 use std::process::{Child, ChildStdin, Command, ExitCode, Stdio};
 use std::sync::{
@@ -18,7 +18,8 @@ use std::time::{Duration, Instant};
 use telemetry::frame::TelemetryEvent;
 use telemetry::{
     ChannelContent, ChannelId, Lifetime, NodeId, Record, StreamDescriptor, StreamId, StreamOrigin,
-    TelemetryEndpoint, TelemetryProducer, TelemetrySubscription,
+    TelemetryEndpoint, TelemetryProducer, TelemetrySubscription, decode_record_value,
+    encode_record,
 };
 
 use crate::codecs::register_myelin_actor_codecs;
@@ -57,6 +58,7 @@ use iroh_driver::{
     TELEMETRY_ALPN, spawn_pull_server,
 };
 use iroh_driver::{EndpointAddrMask, MVP_IROH_ENDPOINT_ADDR_MASK_ENV, advertised_endpoint};
+use myelin_control_contract::DeploymentIdentity;
 use parking_lot::Mutex;
 use serde_json::{Value, json};
 use swactor::actor::{ActorAddress, ActorInterface};
@@ -166,10 +168,8 @@ fn emit_node_event(
     detail: Value,
 ) {
     let channel = telemetry.channel_by_name(channel);
-    telemetry.submit_text(
-        channel,
-        node_event_payload(config, phase, status, detail).to_string(),
-    );
+    let payload = node_event_payload(config, phase, status, detail);
+    telemetry.submit_value(channel, &payload);
     telemetry.tick();
 }
 
@@ -272,6 +272,8 @@ enum DebugJoinClientError {
     Runtime(String),
 }
 
+pub(crate) const DEBUG_JOIN_TIMEOUT: Duration = Duration::from_secs(30);
+
 fn debug_join_client_main(args: Vec<String>) -> ExitCode {
     match run_debug_join_client(args) {
         Ok(response) => {
@@ -303,6 +305,7 @@ fn debug_join_client_main(args: Vec<String>) -> ExitCode {
 fn run_debug_join_client(args: Vec<String>) -> Result<DebugJoinResponseWire, DebugJoinClientError> {
     let mut socket = None;
     let mut endpoint_json = None;
+    let mut orchestrator_actor_json = None;
     let mut read_endpoint_stdin = false;
     let mut iter = args.into_iter();
     while let Some(arg) = iter.next() {
@@ -318,16 +321,21 @@ fn run_debug_join_client(args: Vec<String>) -> Result<DebugJoinResponseWire, Deb
                 })?);
             }
             "--endpoint-json-stdin" => read_endpoint_stdin = true,
+            "--orchestrator-actor-json" => {
+                orchestrator_actor_json = Some(iter.next().ok_or_else(|| {
+                    DebugJoinClientError::Cli("--orchestrator-actor-json requires JSON".to_owned())
+                })?);
+            }
             other => {
                 return Err(DebugJoinClientError::Cli(format!(
-                    "unknown argument {other:?}; usage: debug-join --socket <path> (--endpoint-json <json> | --endpoint-json-stdin)"
+                    "unknown argument {other:?}; usage: debug-join --socket <path> (--endpoint-json <json> | --endpoint-json-stdin) [--orchestrator-actor-json <json>]"
                 )));
             }
         }
     }
     let socket = socket.ok_or_else(|| {
         DebugJoinClientError::Cli(
-            "missing --socket <path>; usage: debug-join --socket <path> (--endpoint-json <json> | --endpoint-json-stdin)".to_owned(),
+            "missing --socket <path>; usage: debug-join --socket <path> (--endpoint-json <json> | --endpoint-json-stdin) [--orchestrator-actor-json <json>]".to_owned(),
         )
     })?;
     let endpoint_json = match (endpoint_json, read_endpoint_stdin) {
@@ -353,15 +361,33 @@ fn run_debug_join_client(args: Vec<String>) -> Result<DebugJoinResponseWire, Deb
     };
     let endpoint = serde_json::from_str::<EndpointAddr>(&endpoint_json)
         .map_err(|e| DebugJoinClientError::Cli(format!("parse endpoint JSON: {e}")))?;
-    send_debug_join_request(&socket, endpoint, None).map_err(DebugJoinClientError::Runtime)
+    let orchestrator_actor = orchestrator_actor_json
+        .as_deref()
+        .map(serde_json::from_str::<ActorAddress>)
+        .transpose()
+        .map_err(|error| {
+            DebugJoinClientError::Cli(format!("parse orchestrator actor JSON: {error}"))
+        })?;
+    let deadline = Instant::now() + DEBUG_JOIN_TIMEOUT;
+    let deadline = crate::provisioning::execution_owner_deadline()
+        .map_err(DebugJoinClientError::Runtime)?
+        .map_or(deadline, |owner| owner.min(deadline));
+    send_debug_join_request(
+        &socket,
+        endpoint,
+        orchestrator_actor,
+        deadline.saturating_duration_since(Instant::now()),
+    )
+    .map_err(DebugJoinClientError::Runtime)
 }
 
 pub(crate) fn request_debug_join(
     socket: &Path,
     endpoint: EndpointAddr,
     orchestrator_actor: ActorAddress,
+    timeout: Duration,
 ) -> Result<(), String> {
-    match send_debug_join_request(socket, endpoint, Some(orchestrator_actor))? {
+    match send_debug_join_request(socket, endpoint, Some(orchestrator_actor), timeout)? {
         DebugJoinResponseWire::JoinQueued { .. } => Ok(()),
         DebugJoinResponseWire::JoinRejected { error, detail } => {
             Err(format!("worker join rejected: {error}: {detail}"))
@@ -376,18 +402,55 @@ fn send_debug_join_request(
     socket: &Path,
     endpoint: EndpointAddr,
     orchestrator_actor: Option<ActorAddress>,
+    timeout: Duration,
 ) -> Result<DebugJoinResponseWire, String> {
+    let deadline = Instant::now()
+        .checked_add(timeout)
+        .ok_or_else(|| "debug join deadline overflow".to_owned())?;
+    let remaining = || {
+        deadline
+            .checked_duration_since(Instant::now())
+            .filter(|duration| !duration.is_zero())
+            .ok_or_else(|| "debug join request deadline expired".to_owned())
+    };
     let request = debug_join_request_line(endpoint, orchestrator_actor)?;
-    let mut stream = std::os::unix::net::UnixStream::connect(socket)
-        .map_err(|e| format!("connect {}: {e}", socket.display()))?;
-    stream
-        .write_all(request.as_bytes())
-        .map_err(|e| format!("write request: {e}"))?;
-    stream.flush().map_err(|e| format!("flush request: {e}"))?;
-    let mut response_line = String::new();
-    BufReader::new(stream)
-        .read_line(&mut response_line)
-        .map_err(|e| format!("read response: {e}"))?;
+    let mut stream = connect_debug_socket(socket, remaining()?)?;
+    let mut request = request.as_bytes();
+    while !request.is_empty() {
+        stream
+            .set_write_timeout(Some(remaining()?))
+            .map_err(|error| error.to_string())?;
+        match stream.write(request) {
+            Ok(0) => return Err("debug join socket closed while writing".to_owned()),
+            Ok(written) => request = &request[written..],
+            Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue,
+            Err(error) => return Err(format!("write debug join request: {error}")),
+        }
+    }
+    let mut response = Vec::new();
+    let mut buffer = [0_u8; 4096];
+    loop {
+        stream
+            .set_read_timeout(Some(remaining()?))
+            .map_err(|error| error.to_string())?;
+        let read = match stream.read(&mut buffer) {
+            Ok(read) => read,
+            Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue,
+            Err(error) => return Err(format!("read debug join response: {error}")),
+        };
+        if read == 0 {
+            break;
+        }
+        let end = buffer[..read].iter().position(|byte| *byte == b'\n');
+        response.extend_from_slice(&buffer[..end.unwrap_or(read)]);
+        if response.len() > 64 * 1024 {
+            return Err("debug join response exceeds 64 KiB".to_owned());
+        }
+        if end.is_some() {
+            break;
+        }
+    }
+    let response_line = String::from_utf8(response).map_err(|error| error.to_string())?;
     if response_line.trim().is_empty() {
         return Err("debug join socket closed without response".to_owned());
     }
@@ -395,6 +458,56 @@ fn send_debug_join_request(
         .map_err(|e| format!("parse response JSON: {e}"))
 }
 
+fn connect_debug_socket(
+    path: &Path,
+    timeout: Duration,
+) -> Result<std::os::unix::net::UnixStream, String> {
+    #[cfg(target_os = "linux")]
+    {
+        use std::os::fd::{AsRawFd, FromRawFd};
+        use std::os::unix::ffi::OsStrExt;
+        let bytes = path.as_os_str().as_bytes();
+        let mut address: libc::sockaddr_un = unsafe { std::mem::zeroed() };
+        if bytes.len() >= address.sun_path.len() || bytes.contains(&0) {
+            return Err("debug join socket path is invalid or too long".to_owned());
+        }
+        address.sun_family = libc::AF_UNIX as _;
+        for (slot, byte) in address.sun_path.iter_mut().zip(bytes) {
+            *slot = *byte as _;
+        }
+        let fd = unsafe { libc::socket(libc::AF_UNIX, libc::SOCK_STREAM | libc::SOCK_CLOEXEC, 0) };
+        if fd < 0 {
+            return Err(std::io::Error::last_os_error().to_string());
+        }
+        let stream = unsafe { std::os::unix::net::UnixStream::from_raw_fd(fd) };
+        // Linux AF_UNIX connect observes SO_SNDTIMEO, including a full
+        // listener backlog. Configure it before the potentially blocking call.
+        stream
+            .set_write_timeout(Some(timeout))
+            .map_err(|error| error.to_string())?;
+        let result = unsafe {
+            libc::connect(
+                stream.as_raw_fd(),
+                (&address as *const libc::sockaddr_un).cast(),
+                std::mem::size_of_val(&address) as _,
+            )
+        };
+        if result != 0 {
+            return Err(format!(
+                "connect {}: {}",
+                path.display(),
+                std::io::Error::last_os_error()
+            ));
+        }
+        Ok(stream)
+    }
+    #[cfg(not(target_os = "linux"))]
+    {
+        let _ = (path, timeout);
+        Err("bounded worker debug join requires Linux AF_UNIX sockets".to_owned())
+    }
+}
+
 fn debug_join_request_line(
     endpoint: EndpointAddr,
     orchestrator_actor: Option<ActorAddress>,
@@ -652,9 +765,10 @@ fn submit_sampler_health(
     status: &str,
     detail: Value,
 ) {
-    producer.submit_text(
+    let payload = sampler_health_payload(context, sampler, sample_channel, status, detail);
+    producer.submit_bytes(
         channel,
-        sampler_health_payload(context, sampler, sample_channel, status, detail).to_string(),
+        encode_record(&payload).expect("serialize sampler health telemetry"),
     );
 }
 
@@ -1678,7 +1792,7 @@ fn run() -> Result<(), String> {
 
     // Telemetry must exist before the runtime: its stats hook is wired in
     // during runtime construction.
-    let mut telemetry = NodeTelemetry::new(&config);
+    let mut telemetry = NodeTelemetry::new(&config)?;
 
     // Build the core swactor runtime parts, clone the routing handle needed by
     // integrations, then hand the workers to the engine. The engine owns both
@@ -1722,6 +1836,7 @@ fn run() -> Result<(), String> {
         IrohDriverConfig {
             secret_key: None,
             relay_mode: config.relay_mode.clone(),
+            bind_port: None,
             node: DistributedNodeConfig::default(),
             peer_auth: None,
             additional_alpns: vec![EDGE_ALPN.to_vec(), TELEMETRY_ALPN.to_vec()],
@@ -1958,6 +2073,33 @@ fn run() -> Result<(), String> {
             return Err(format!("node report inbox: {error}"));
         }
     };
+    let report_wake_target = Arc::new(Mutex::new(None));
+    let report_forwarder = stack
+        .runtime
+        .spawn(NodeReportWakeForwarder {
+            destination: *reports.addr(),
+            target: Arc::clone(&report_wake_target),
+        })
+        .map_err(|error| format!("spawn node report wake forwarder: {error}"))?;
+    let route_target = Arc::clone(&report_wake_target);
+    let route_sender = stack.runtime.create_sender();
+    let route_wake: Arc<dyn Fn() + Send + Sync> = Arc::new(move || {
+        if let Some(target) = *route_target.lock() {
+            let _ = route_sender.send_to(target, NodeRuntimeMsg::RoutingChanged);
+        }
+    });
+    stack
+        .runtime
+        .send_to(
+            stack.actors.directory,
+            distribution::directory_actor::DirectoryIn::WatchRoutes {
+                changed: Arc::downgrade(&route_wake),
+            },
+        )
+        .map_err(|error| format!("watch runtime ready routes: {error}"))?;
+    // The synchronous entrypoint retains the transport watch until its runtime
+    // actor completes; teardown cancels the watch even on startup failure.
+    let _route_watch = driver.connection_observer().watch_connections(route_wake);
     let rejoin_replies = stack
         .runtime
         .new_inbox::<ManualControlReply>()
@@ -1977,13 +2119,26 @@ fn run() -> Result<(), String> {
             driver.endpoint_addr(),
             driver.edge_events_handle(),
         ));
-        blob_receiver.install_pump(&engine.handle(), stack.runtime.clone(), PUMP_INTERVAL);
+        blob_receiver.install_pump(
+            &engine.handle(),
+            stack.runtime.clone(),
+            PUMP_INTERVAL,
+            driver.edge_events_changed(),
+        );
         let transfer_receiver: Arc<dyn BlobTransferReceiver> = blob_receiver;
         let source_sender: Arc<dyn BlobTransferSender> = Arc::new(IrohBlobTransferSender::new(
             driver.edge_connector(),
             &engine.handle(),
             stack.runtime.clone(),
         ));
+        let routes: Arc<dyn HostRouteRegistrar> = Arc::new(MyelinChildRouteRegistrar::new(
+            stack.route_view.clone(),
+            stack.pinned_routes.clone(),
+            stack.route_binder.clone(),
+            stack.runtime.clone(),
+            stack.actors.directory,
+            driver.connection_observer(),
+        ));
         let spawner = Arc::new(build_contextual_process_spawner(
             MyelinContextualProcessConfig {
                 runtime: stack.runtime.clone(),
@@ -1994,18 +2149,11 @@ fn run() -> Result<(), String> {
                 transfer_receiver: Some(Arc::clone(&transfer_receiver)),
                 source_sender: Some(source_sender),
                 source_publisher: Some(namespace.source_publisher),
-                route_view: stack.route_view.clone(),
-                pinned_routes: stack.pinned_routes.clone(),
-                route_binder: stack.route_binder.clone(),
+                route_registrar: Arc::clone(&routes),
                 stream_transport: Some(driver.stream_transport()),
                 host_endpoint: driver.endpoint_addr(),
             },
         )?);
-        let routes: Arc<dyn HostRouteRegistrar> = Arc::new(MyelinChildRouteRegistrar::new(
-            stack.route_view.clone(),
-            stack.pinned_routes.clone(),
-            stack.route_binder.clone(),
-        ));
         let materializer = ContextualProgramMaterializer {
             namespace_proxy: namespace.client.proxy(),
             receiver: transfer_receiver,
@@ -2027,7 +2175,18 @@ fn run() -> Result<(), String> {
                     Arc::clone(spawner),
                     stack.runtime.create_sender(),
                 )
-                .with_program_materializer(materializer.clone()),
+                .with_program_materializer(materializer.clone())
+                .with_resource_probe(
+                    crate::contextual_process::ContextualResourceProbe {
+                        runtime: stack.runtime.clone(),
+                        engine: engine.handle(),
+                        arena: Arc::clone(&arena_manager),
+                        stream_transport: driver.stream_transport(),
+                        generation: telemetry.producer.stream_id().life.0,
+                        iroh_node_id: driver.node_id().to_string(),
+                        deployment: config.deployment.clone(),
+                    },
+                ),
             )
         })
         .transpose()
@@ -2035,8 +2194,10 @@ fn run() -> Result<(), String> {
     let mut node_agent = NodeAgentActor::new(
         stage::NodeId(config.logical_node_id),
         orchestrator,
-        Some(*reports.addr()),
-    );
+        Some(report_forwarder),
+    )
+    .with_directory_actor(stack.actors.directory)
+    .with_peer_connector(driver.peer_connector());
     if let Some(controller) = contextual_controller {
         node_agent = node_agent.with_contextual_controller(controller);
     }
@@ -2147,6 +2308,8 @@ fn run() -> Result<(), String> {
                 completion: completion.clone(),
             })
             .map_err(|error| format!("spawn agent node runtime actor: {error}"))?;
+        *report_wake_target.lock() = Some(runtime_actor);
+        let _ = sender.send_to(runtime_actor, NodeRuntimeMsg::RoutingChanged);
         let stop_actor = actor_runtime
             .spawn(StdinStopForwarder {
                 sender: sender.clone(),
@@ -2279,6 +2442,8 @@ fn run() -> Result<(), String> {
             completion: completion.clone(),
         })
         .map_err(|error| format!("spawn worker node runtime actor: {error}"))?;
+    *report_wake_target.lock() = Some(runtime_actor);
+    let _ = sender.send_to(runtime_actor, NodeRuntimeMsg::RoutingChanged);
     let stop_actor = actor_runtime
         .spawn(StdinStopForwarder {
             sender: sender.clone(),
@@ -2291,9 +2456,28 @@ fn run() -> Result<(), String> {
 #[derive(Clone, Copy)]
 enum NodeRuntimeMsg {
     Tick,
+    Progress,
+    RoutingChanged,
     Shutdown,
 }
 
+struct NodeReportWakeForwarder {
+    destination: ActorAddress,
+    target: Arc<Mutex<Option<ActorAddress>>>,
+}
+
+impl ActorInterface for NodeReportWakeForwarder {
+    type Incoming = NodeAgentReport;
+    type Response = ();
+
+    fn handle(&mut self, ctx: &Ctx, report: NodeAgentReport) {
+        let _ = ctx.send(self.destination, report);
+        if let Some(target) = *self.target.lock() {
+            let _ = ctx.send(target, NodeRuntimeMsg::Progress);
+        }
+    }
+}
+
 struct StdinStopForwarder {
     sender: ExternalSender,
     target: ActorAddress,
@@ -2369,7 +2553,7 @@ impl AgentNodeRuntimeEffects for AgentNodeRuntimeLive {
             }),
         );
         self.telemetry
-            .submit_text(self.telemetry.channels.node_ready, ready.to_string());
+            .submit_value(self.telemetry.channels.node_ready, ready);
     }
 
     fn tick_after_reports(
@@ -2505,6 +2689,14 @@ impl<E: AgentNodeRuntimeEffects> ActorInterface for AgentNodeRuntimeActor<E> {
                 Ok(()) => self.schedule_tick(ctx),
                 Err(error) => self.finish(ctx, Err(error)),
             },
+            NodeRuntimeMsg::Progress | NodeRuntimeMsg::RoutingChanged => {
+                if matches!(message, NodeRuntimeMsg::RoutingChanged) {
+                    self.pending_runtime_ready.next_attempt_at = Instant::now();
+                }
+                if let Err(error) = self.tick() {
+                    self.finish(ctx, Err(error));
+                }
+            }
             NodeRuntimeMsg::Shutdown => {
                 self.effects.shutdown();
                 self.finish(ctx, Ok(()));
@@ -2787,6 +2979,14 @@ impl<E: WorkerNodeRuntimeEffects> ActorInterface for WorkerNodeRuntimeActor<E> {
                 Ok(()) => self.schedule_tick(ctx),
                 Err(error) => self.finish(ctx, Err(error)),
             },
+            NodeRuntimeMsg::Progress | NodeRuntimeMsg::RoutingChanged => {
+                if matches!(message, NodeRuntimeMsg::RoutingChanged) {
+                    self.pending_runtime_ready.next_attempt_at = Instant::now();
+                }
+                if let Err(error) = self.tick() {
+                    self.finish(ctx, Err(error));
+                }
+            }
             NodeRuntimeMsg::Shutdown => {
                 self.effects.shutdown();
                 self.finish(ctx, Ok(()));
@@ -2845,21 +3045,88 @@ struct NodeTelemetry {
     archive: Option<TelemetryArchive>,
 }
 
+fn allocate_telemetry_lifetime(
+    state_root: &Path,
+    node_id: u64,
+    run_id: u64,
+) -> Result<u64, String> {
+    let directory = state_root.join(format!("node-{node_id}"));
+    fs::create_dir_all(&directory)
+        .map_err(|error| format!("create telemetry state {}: {error}", directory.display()))?;
+    // Lock a stable inode: the counter itself is atomically replaced below.
+    let lock = OpenOptions::new()
+        .create(true)
+        .truncate(false)
+        .read(true)
+        .write(true)
+        .open(directory.join("generation.lock"))
+        .map_err(|error| format!("open telemetry generation lock: {error}"))?;
+    lock.lock()
+        .map_err(|error| format!("lock telemetry generation: {error}"))?;
+    let path = directory.join("generation");
+    let previous = match fs::read_to_string(&path) {
+        Ok(value) => value
+            .trim()
+            .parse::<u64>()
+            .map_err(|error| format!("read telemetry generation {}: {error}", path.display()))?,
+        Err(error) if error.kind() == std::io::ErrorKind::NotFound => 0,
+        Err(error) => {
+            return Err(format!(
+                "read telemetry generation {}: {error}",
+                path.display()
+            ));
+        }
+    };
+    let next = previous
+        .max(run_id)
+        .checked_add(1)
+        .ok_or_else(|| format!("telemetry generation exhausted for node {node_id}"))?;
+    let pending = directory.join("generation.next");
+    let mut file = File::create(&pending)
+        .map_err(|error| format!("create telemetry generation {}: {error}", pending.display()))?;
+    file.write_all(next.to_string().as_bytes())
+        .and_then(|()| file.sync_all())
+        .map_err(|error| {
+            format!(
+                "persist telemetry generation {}: {error}",
+                pending.display()
+            )
+        })?;
+    fs::rename(&pending, &path)
+        .map_err(|error| format!("commit telemetry generation {}: {error}", path.display()))?;
+    File::open(&directory)
+        .and_then(|directory| directory.sync_all())
+        .map_err(|error| format!("sync telemetry state {}: {error}", directory.display()))?;
+    Ok(next)
+}
+
 impl NodeTelemetry {
-    fn new(config: &DeploymentConfig) -> Self {
+    fn new(config: &DeploymentConfig) -> Result<Self, String> {
+        // Lifetimes are ordered generations, not random readiness identities.
+        // Persist the next one before any producer can emit startup records.
+        let state_root = env_optional("XDG_STATE_HOME")
+            .map(PathBuf::from)
+            .or_else(|| env_optional("HOME").map(|home| PathBuf::from(home).join(".local/state")))
+            .ok_or_else(|| "telemetry requires XDG_STATE_HOME or HOME for node state".to_owned())?
+            .join("myelin/telemetry");
+        let lifetime =
+            allocate_telemetry_lifetime(&state_root, config.logical_node_id, config.run_id)?;
         let stream = StreamId::new(
             NodeId::new(config.logical_node_id.to_string()),
-            Lifetime(config.run_id),
+            Lifetime(lifetime),
+        );
+        let endpoint = Arc::new(
+            TelemetryEndpoint::with_descriptor(
+                StreamDescriptor {
+                    stream: stream.clone(),
+                    label: Some("myelin worker node".to_owned()),
+                    origin: StreamOrigin::RemoteNode,
+                },
+                16_384,
+                16_384,
+            )
+            .with_retention(16_384, 16 * 1024 * 1024),
         );
-        let endpoint = Arc::new(TelemetryEndpoint::with_descriptor(
-            StreamDescriptor {
-                stream: stream.clone(),
-                label: Some("myelin worker node".to_owned()),
-                origin: StreamOrigin::RemoteNode,
-            },
-            256,
-            1024,
-        ));
         let producer = endpoint.producer();
         let mut by_name = BTreeMap::new();
         let mut by_id = BTreeMap::new();
@@ -2883,29 +3150,29 @@ impl NodeTelemetry {
             "myelin.worker.device_object",
             "myelin.worker.shutdown",
         ] {
-            register_json_channel(&producer, &mut by_name, &mut by_id, name);
+            register_messagepack_channel(&producer, &mut by_name, &mut by_id, name);
         }
 
         let channels = TelemetryChannelSet {
-            node_ready: register_json_channel(
+            node_ready: register_messagepack_channel(
                 &producer,
                 &mut by_name,
                 &mut by_id,
                 "myelin.node.ready",
             ),
-            node_lifecycle: register_json_channel(
+            node_lifecycle: register_messagepack_channel(
                 &producer,
                 &mut by_name,
                 &mut by_id,
                 "myelin.node.lifecycle",
             ),
-            node_self_test: register_json_channel(
+            node_self_test: register_messagepack_channel(
                 &producer,
                 &mut by_name,
                 &mut by_id,
                 "myelin.node.self_test",
             ),
-            worker_stderr: register_json_channel(
+            worker_stderr: register_messagepack_channel(
                 &producer,
                 &mut by_name,
                 &mut by_id,
@@ -2956,25 +3223,26 @@ impl NodeTelemetry {
             TelemetryArchive::open(path, endpoint.subscribe_all("frame-log")).ok()
         });
 
-        Self {
+        Ok(Self {
             endpoint,
             producer,
             channels,
             by_name,
             by_id,
             archive,
-        }
+        })
     }
 
     fn channel_by_name(&mut self, name: &str) -> ChannelId {
         if let Some(id) = self.by_name.get(name).copied() {
             return id;
         }
-        register_json_channel(&self.producer, &mut self.by_name, &mut self.by_id, name)
+        register_messagepack_channel(&self.producer, &mut self.by_name, &mut self.by_id, name)
     }
 
-    fn submit_text(&self, channel: ChannelId, text: impl AsRef<[u8]>) {
-        self.producer.submit_text(channel, text);
+    fn submit_value(&self, channel: ChannelId, value: &Value) {
+        let payload = encode_record(value).expect("serialize node telemetry record");
+        self.producer.submit_bytes(channel, payload);
     }
 
     fn tick(&mut self) {
@@ -3000,7 +3268,7 @@ fn serve_telemetry_pulls(
     }
 }
 
-fn register_json_channel(
+fn register_messagepack_channel(
     producer: &TelemetryProducer,
     by_name: &mut BTreeMap<String, ChannelId>,
     by_id: &mut BTreeMap<ChannelId, String>,
@@ -3008,7 +3276,7 @@ fn register_json_channel(
 ) -> ChannelId {
     let id = producer.register_channel(
         name,
-        ChannelContent::JsonRecord {
+        ChannelContent::MessagePackRecord {
             schema: Some(name.to_owned()),
         },
     );
@@ -3048,12 +3316,15 @@ impl TelemetryArchive {
                     .get(&frame.channel.channel)
                     .cloned()
                     .unwrap_or_else(|| format!("channel#{}", frame.channel.channel.0));
+                let payload = decode_record_value(&frame.payload)
+                    .map(|value| json!({"encoding":"messagepack","value":value}))
+                    .unwrap_or_else(|_| json!({"encoding":"bytes","value":frame.payload}));
                 let record = json!({
                     "stream":frame.channel.stream.to_string(),
                     "channel":channel,
                     "channel_id":frame.channel.channel.0,
                     "position":frame.position.0,
-                    "payload":String::from_utf8_lossy(&frame.payload),
+                    "payload":payload,
                 });
                 let _ = serde_json::to_writer(&mut self.file, &record);
                 let _ = writeln!(self.file);
@@ -3095,12 +3366,21 @@ impl PendingControlRejoin {
                 run_id: config.run_id,
                 logical_node_id: config.logical_node_id,
                 attempt_id: config.attempt_id,
+                readiness_id: config.readiness_id,
                 selected_offer_id: config.selected_offer_id,
                 endpoint: serde_json::to_string(endpoint)
                     .map_err(|error| format!("serialize rejoin endpoint: {error}"))?,
                 swim_node_id,
                 stage_index: config.stage_index,
                 node_actor,
+                artifact_digest: config
+                    .deployment
+                    .as_ref()
+                    .map(|identity| identity.artifact_digest.clone()),
+                deployment_generation: config
+                    .deployment
+                    .as_ref()
+                    .map(|identity| identity.deployment_generation.clone()),
             },
             last_bound_actor: orchestrator_actor,
             pending_actor: None,
@@ -3211,6 +3491,7 @@ struct PendingRuntimeReady {
     backoff: Duration,
     acked: bool,
     swim_logged: bool,
+    deployment: Option<DeploymentIdentity>,
 }
 
 impl PendingRuntimeReady {
@@ -3225,12 +3506,13 @@ impl PendingRuntimeReady {
                 .coordinator_endpoint
                 .as_ref()
                 .map(|endpoint| DistNodeId(*endpoint.id.as_bytes())),
-            readiness_id: config.attempt_id,
+            readiness_id: config.readiness_id,
             attempts: 0,
             next_attempt_at: Instant::now(),
             backoff: RUNTIME_READY_RETRY_INITIAL,
             acked: false,
             swim_logged: false,
+            deployment: config.deployment.clone(),
         }
     }
 
@@ -3283,6 +3565,14 @@ impl PendingRuntimeReady {
                     endpoint: self.endpoint.clone(),
                     node_actor: self.node_actor,
                     readiness_id: self.readiness_id,
+                    artifact_digest: self
+                        .deployment
+                        .as_ref()
+                        .map(|identity| identity.artifact_digest.clone()),
+                    deployment_generation: self
+                        .deployment
+                        .as_ref()
+                        .map(|identity| identity.deployment_generation.clone()),
                 },
             )
             .map_err(|error| format!("signal runtime loaded: {error}"))?;
@@ -3350,10 +3640,8 @@ fn handle_node_report(
         }
         NodeAgentReport::Lifecycle(event) => {
             let event = format!("{event:?}");
-            telemetry.submit_text(
-                telemetry.channels.node_lifecycle,
-                json!({"type":"node_lifecycle","event":event}).to_string(),
-            );
+            let payload = json!({"type":"node_lifecycle","event":event});
+            telemetry.submit_value(telemetry.channels.node_lifecycle, &payload);
             node_stage(telemetry, "lifecycle", "observed", json!({"event":event}));
             Ok(NodeReportOutcome::None)
         }
@@ -4014,7 +4302,7 @@ fn publish_stage_shard_fetch_event(
 ) -> Result<(), String> {
     let channel = telemetry.channel_by_name("myelin.worker.weights");
     let payload = node_event_payload(config, "stage_shard_fetch", "event", event.clone());
-    telemetry.submit_text(channel, payload.to_string());
+    telemetry.submit_value(channel, &payload);
     emit_stdio_telemetry_frame("myelin.worker.weights", &payload)
         .map_err(|e| format!("emit stage shard fetch telemetry frame: {e}"))?;
     telemetry.tick();
@@ -4421,7 +4709,7 @@ fn run_self_test(
     )?;
     let result = worker.infer_prompt(0, prompt, config.self_test_max_tokens, config, telemetry)?;
     let record = json!({"type":"self_test_completed","prompt_bytes":prompt.len(),"result":result});
-    telemetry.submit_text(telemetry.channels.node_self_test, record.to_string());
+    telemetry.submit_value(telemetry.channels.node_self_test, &record);
     emit_node_event(
         telemetry,
         config,
@@ -4445,11 +4733,13 @@ struct DeploymentConfig {
     run_id: u64,
     logical_node_id: u64,
     attempt_id: u64,
+    readiness_id: u64,
     selected_offer_id: Option<u64>,
     stage_index: u32,
     coordinator_endpoint: Option<EndpointAddr>,
     orchestrator_actor: Option<ActorAddress>,
     telemetry_frame_log: Option<String>,
+    deployment: Option<DeploymentIdentity>,
     debug_join_socket: Option<String>,
     relay_mode: iroh::RelayMode,
     endpoint_addr_mask: EndpointAddrMask,
@@ -4482,6 +4772,41 @@ impl DeploymentConfig {
         let run_id = env_parse!("MYELIN_RUN_ID", 1)?;
         let logical_node_id = env_parse!("MYELIN_LOGICAL_NODE_ID", 1)?;
         let attempt_id = env_parse!("MYELIN_NODE_ATTEMPT_ID", 1)?;
+        let deployment = match (
+            env_optional("MYELIN_ARTIFACT_DIGEST"),
+            env_optional("MYELIN_DEPLOYMENT_GENERATION"),
+        ) {
+            (None, None) => None,
+            (Some(artifact_digest), Some(deployment_generation)) => Some(DeploymentIdentity {
+                artifact_digest,
+                deployment_generation,
+            }),
+            (Some(_), None) => {
+                return Err(
+                    "MYELIN_ARTIFACT_DIGEST requires MYELIN_DEPLOYMENT_GENERATION".to_owned(),
+                );
+            }
+            (None, Some(_)) => {
+                return Err(
+                    "MYELIN_DEPLOYMENT_GENERATION requires MYELIN_ARTIFACT_DIGEST".to_owned(),
+                );
+            }
+        };
+        let readiness_id = if let Some(identity) = &deployment {
+            let stat = std::fs::read_to_string("/proc/self/stat")
+                .map_err(|error| format!("read deployment process identity: {error}"))?;
+            let start_ticks = stat
+                .rsplit_once(')')
+                .and_then(|(_, fields)| fields.split_whitespace().nth(19))
+                .ok_or_else(|| "process stat omitted start ticks".to_owned())?;
+            crate::orchestration::daemon::deployment_readiness_id(&format!(
+                "{}:{start_ticks}:{}",
+                std::process::id(),
+                identity.deployment_generation,
+            ))
+        } else {
+            attempt_id
+        };
         let relay = relay_runtime_config_from_env(run_id)?;
         let debug_join_socket = match env_optional("MYELIN_DEBUG_JOIN_SOCKET").as_deref() {
             Some("disabled") => None,
@@ -4505,6 +4830,7 @@ impl DeploymentConfig {
             run_id,
             logical_node_id,
             attempt_id,
+            readiness_id,
             selected_offer_id: env_optional(SELECTED_OFFER_ID_ENV)
                 .map(|value| {
                     value.parse().map_err(|error| {
@@ -4526,6 +4852,7 @@ impl DeploymentConfig {
                 })
                 .transpose()?,
             telemetry_frame_log: env_optional("MYELIN_TELEMETRY_FRAME_LOG"),
+            deployment,
             debug_join_socket,
             relay_mode: relay.mode,
             endpoint_addr_mask: env_optional(MVP_IROH_ENDPOINT_ADDR_MASK_ENV)
@@ -4609,7 +4936,7 @@ fn drain_worker_stderr(
     let mut emitted = false;
     while let Ok(line) = stderr_rx.lock().try_recv() {
         let payload = node_event_payload(config, "worker_stderr", "observed", json!({"line":line}));
-        telemetry.submit_text(telemetry.channels.worker_stderr, payload.to_string());
+        telemetry.submit_value(telemetry.channels.worker_stderr, &payload);
         emitted = true;
     }
     if emitted {
@@ -4816,7 +5143,7 @@ fn wait_for_helper_event(
     let outcome = completion.wait();
     for line in outcome.stderr_lines {
         let payload = node_event_payload(config, "worker_stderr", "observed", json!({"line":line}));
-        telemetry.submit_text(telemetry.channels.worker_stderr, payload.to_string());
+        telemetry.submit_value(telemetry.channels.worker_stderr, &payload);
     }
     for (elapsed_ms, wait_cycles) in outcome.wait_samples {
         emit_node_event(
@@ -4872,7 +5199,7 @@ fn wait_for_helper_event(
             "ready",
             json!({"command_type":command_type,"expected_event_type":expected,"channel":channel_name,"line_bytes":line_bytes,"worker_event_type":worker_event_type}),
         );
-        telemetry.submit_text(channel, value.to_string());
+        telemetry.submit_value(channel, &value);
         emit_stdio_telemetry_frame(channel_name, &value)
             .map_err(|error| format!("emit worker stdio telemetry frame: {error}"))?;
         if worker_event_type != expected {
@@ -5417,6 +5744,44 @@ mod control_flow_properties {
 
     const DRIVE_PER_ACTION: usize = 16;
     const FINAL_DRIVE_BUDGET: usize = 256;
+
+    #[test]
+    fn telemetry_lifetimes_survive_restarts_and_serialize_concurrent_boots() {
+        let state = tempfile::tempdir().expect("telemetry state");
+        let run_id = 1 << 62;
+        assert_eq!(
+            allocate_telemetry_lifetime(state.path(), 7, run_id).unwrap(),
+            run_id + 1
+        );
+        let mut generations = std::thread::scope(|scope| {
+            let handles = (0..4)
+                .map(|_| {
+                    scope.spawn(|| allocate_telemetry_lifetime(state.path(), 7, run_id).unwrap())
+                })
+                .collect::<Vec<_>>();
+            handles
+                .into_iter()
+                .map(|handle| handle.join().unwrap())
+                .collect::<Vec<_>>()
+        });
+        generations.sort_unstable();
+        assert_eq!(generations, (run_id + 2..=run_id + 5).collect::<Vec<_>>());
+        // A new process opening the same state cannot regress with a lower run id.
+        assert_eq!(
+            allocate_telemetry_lifetime(state.path(), 7, 1).unwrap(),
+            run_id + 6
+        );
+        assert_eq!(
+            allocate_telemetry_lifetime(state.path(), 7, run_id + 100).unwrap(),
+            run_id + 101
+        );
+        assert!(allocate_telemetry_lifetime(state.path(), 7, u64::MAX).is_err());
+        assert_eq!(
+            allocate_telemetry_lifetime(state.path(), 7, 1).unwrap(),
+            run_id + 102
+        );
+    }
+
     #[test]
     fn sampler_health_emits_only_state_transitions() {
         let endpoint = TelemetryEndpoint::with_descriptor(
@@ -5431,7 +5796,7 @@ mod control_flow_properties {
         let producer = endpoint.producer();
         let health_channel = producer.register_channel(
             NODE_SAMPLER_CHANNEL,
-            ChannelContent::JsonRecord {
+            ChannelContent::MessagePackRecord {
                 schema: Some(NODE_SAMPLER_CHANNEL.to_owned()),
             },
         );
@@ -5503,7 +5868,7 @@ mod control_flow_properties {
             .into_iter()
             .filter_map(|event| match event {
                 TelemetryEvent::Frame(delivery) if delivery.channel.channel == health_channel => {
-                    serde_json::from_slice::<Value>(&delivery.payload)
+                    decode_record_value(&delivery.payload)
                         .ok()
                         .and_then(|value| value["status"].as_str().map(str::to_owned))
                 }
@@ -5538,7 +5903,7 @@ mod control_flow_properties {
         let sample_channel = producer.register_record::<telemetry::hardware::net::HostNetSample>();
         let health_channel = producer.register_channel(
             NODE_SAMPLER_CHANNEL,
-            ChannelContent::JsonRecord {
+            ChannelContent::MessagePackRecord {
                 schema: Some(NODE_SAMPLER_CHANNEL.to_owned()),
             },
         );
@@ -5595,7 +5960,7 @@ mod control_flow_properties {
         let sample_channel = producer.register_record::<telemetry::hardware::cpu::HostCpuSample>();
         let health_channel = producer.register_channel(
             NODE_SAMPLER_CHANNEL,
-            ChannelContent::JsonRecord {
+            ChannelContent::MessagePackRecord {
                 schema: Some(NODE_SAMPLER_CHANNEL.to_owned()),
             },
         );
@@ -5876,6 +6241,7 @@ mod control_flow_properties {
             backoff: RUNTIME_READY_RETRY_INITIAL,
             acked: false,
             swim_logged: false,
+            deployment: None,
         }
     }
 

apps/myelin/src/observability/frame_archive.rs

@@ -3,7 +3,8 @@ use std::io::{BufRead, BufReader, Write};
 use std::path::{Path, PathBuf};
 
 use serde_json::json;
-use telemetry::frame::{Frame, StreamId};
+use telemetry::decode_record_value;
+use telemetry::frame::{ChannelContent, Frame, StreamId};
 
 use crate::observability::benchmark;
 
@@ -49,11 +50,21 @@ impl FrameArchive {
         source: &str,
         stream: &StreamId,
         channel: &str,
+        content: &ChannelContent,
         frame: &Frame,
     ) -> Result<(), String> {
-        let payload = match std::str::from_utf8(&frame.payload) {
-            Ok(text) => json!({"encoding": "utf8", "value": text}),
-            Err(_) => json!({"encoding": "bytes", "value": frame.payload}),
+        let payload = match content {
+            ChannelContent::MessagePackRecord { .. } => decode_record_value(&frame.payload)
+                .map(|value| json!({"encoding": "messagepack", "value": value}))
+                .unwrap_or_else(|_| json!({"encoding": "bytes", "value": frame.payload})),
+            ChannelContent::JsonRecord { .. } => serde_json::from_slice(&frame.payload)
+                .map(|value: serde_json::Value| json!({"encoding": "json", "value": value}))
+                .unwrap_or_else(|_| json!({"encoding": "bytes", "value": frame.payload})),
+            ChannelContent::TextStream => match std::str::from_utf8(&frame.payload) {
+                Ok(text) => json!({"encoding": "utf8", "value": text}),
+                Err(_) => json!({"encoding": "bytes", "value": frame.payload}),
+            },
+            ChannelContent::Bytes => json!({"encoding": "bytes", "value": frame.payload}),
         };
         let record = json!({
             "arrival_seq": self.next_seq,

apps/myelin/src/observability/frame_collector.rs

@@ -6,16 +6,20 @@
 //! never names [`Frame`] or [`TelemetryEvent`] directly — frames reach sinks
 //! only through these closures.
 
-use iroh::EndpointAddr;
+use iroh::{EndpointAddr, PublicKey};
 use iroh_driver::telemetry_transport::PullCollectorConfig;
 use iroh_driver::{IrohDriver, PullCollectorHandle, TelemetryQuicHeader, spawn_pull_collector};
 use parking_lot::Mutex;
 use std::collections::{BTreeMap, BTreeSet};
+use std::fs::File;
+use std::io::{BufRead, BufReader};
+use std::path::Path;
 use std::sync::{Arc, mpsc};
 use swactor_engine::EngineHandle;
-use telemetry::frame::{ChannelRef, Frame, StreamId, TelemetryEvent};
+use telemetry::frame::{ChannelRef, Frame, Lifetime, NodeId, Position, StreamId, TelemetryEvent};
 use telemetry::{
-    DeliveryFanout, StreamDescriptor, SubscriptionRequest, TelemetrySnapshot, TelemetrySubscription,
+    ChannelContent, ChannelDescriptor, DeliveryFanout, StreamDescriptor, SubscriptionRequest,
+    TelemetrySnapshot, TelemetrySubscription,
 };
 
 use crate::observability::orch_telemetry::DashboardSupport;
@@ -26,6 +30,7 @@ struct CollectedTelemetryFrame {
     stream: StreamId,
     descriptor: Option<StreamDescriptor>,
     channel_name: String,
+    channel_content: ChannelContent,
     frame: Frame,
 }
 
@@ -37,10 +42,11 @@ pub(crate) struct FrameCollector {
     pull_subscription: TelemetrySubscription,
     pull_header_tx: mpsc::Sender<TelemetryQuicHeader>,
     pull_header_rx: mpsc::Receiver<TelemetryQuicHeader>,
-    pull_channels: Mutex<BTreeMap<ChannelRef, String>>,
+    pull_channels: Mutex<BTreeMap<ChannelRef, ChannelDescriptor>>,
     pull_streams: Mutex<BTreeMap<StreamId, StreamDescriptor>>,
     pull_stream_owners: Mutex<BTreeMap<StreamId, (u64, u64)>>,
-    pull_collectors: Mutex<BTreeMap<(u64, u64), PullCollectorHandle>>,
+    pull_positions: Mutex<BTreeMap<StreamId, Position>>,
+    pull_collectors: Mutex<BTreeMap<(u64, u64), (PublicKey, PullCollectorHandle)>>,
 }
 
 impl FrameCollector {
@@ -65,10 +71,65 @@ impl FrameCollector {
             pull_channels: Mutex::new(BTreeMap::new()),
             pull_streams: Mutex::new(BTreeMap::new()),
             pull_stream_owners: Mutex::new(BTreeMap::new()),
+            pull_positions: Mutex::new(BTreeMap::new()),
             pull_collectors: Mutex::new(BTreeMap::new()),
         }
     }
 
+    /// Recover exact stream cursors before reconnecting retained worker nodes.
+    /// The archive, not the prior process's receive queue, is the durable boundary.
+    pub(crate) fn restore_archive(&self, path: Option<&Path>) -> Result<(), String> {
+        let Some(path) = path else {
+            return Ok(());
+        };
+        let file = match File::open(path) {
+            Ok(file) => file,
+            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
+            Err(error) => {
+                return Err(format!(
+                    "read telemetry cursors {}: {error}",
+                    path.display()
+                ));
+            }
+        };
+        #[derive(serde::Deserialize)]
+        struct ArchivedPosition {
+            source: String,
+            stream: String,
+            position: u64,
+        }
+        let mut positions = BTreeMap::new();
+        for (index, line) in BufReader::new(file).lines().enumerate() {
+            let line = line
+                .map_err(|error| format!("read telemetry cursors {}: {error}", path.display()))?;
+            let record: ArchivedPosition = serde_json::from_str(&line).map_err(|error| {
+                format!(
+                    "read telemetry cursor {}:{}: {error}",
+                    path.display(),
+                    index + 1
+                )
+            })?;
+            if record.source != "node" {
+                continue;
+            }
+            let (node, lifetime) = record
+                .stream
+                .rsplit_once('#')
+                .ok_or_else(|| format!("invalid archived telemetry stream {}", record.stream))?;
+            let lifetime = lifetime.parse::<u64>().map_err(|error| {
+                format!(
+                    "invalid archived telemetry stream {}: {error}",
+                    record.stream
+                )
+            })?;
+            let stream = StreamId::new(NodeId::new(node), Lifetime(lifetime));
+            let position = positions.entry(stream).or_insert(Position(record.position));
+            *position = (*position).max(Position(record.position));
+        }
+        self.pull_positions.lock().extend(positions);
+        Ok(())
+    }
+
     /// Dial a bootstrapped node and retain its live telemetry subscription.
     pub(crate) fn subscribe_node(
         &self,
@@ -78,6 +139,19 @@ impl FrameCollector {
         run_id: u64,
         node_id: u64,
     ) {
+        let mut collectors = self.pull_collectors.lock();
+        if collectors
+            .get(&(run_id, node_id))
+            .is_some_and(|(peer_id, handle)| {
+                *peer_id == peer.id && !handle.is_cancelled() && !handle.is_finished()
+            })
+        {
+            return;
+        }
+        if let Some((_, previous)) = collectors.remove(&(run_id, node_id)) {
+            previous.cancel();
+        }
+        let peer_id = peer.id;
         let mut flow_id = [0_u8; 16];
         flow_id[..8].copy_from_slice(&run_id.to_le_bytes());
         flow_id[8..].copy_from_slice(&node_id.to_le_bytes());
@@ -93,17 +167,11 @@ impl FrameCollector {
             },
             self.pull_header_tx.clone(),
         );
-        if let Some(previous) = self
-            .pull_collectors
-            .lock()
-            .insert((run_id, node_id), collector)
-        {
-            previous.cancel();
-        }
+        collectors.insert((run_id, node_id), (peer_id, collector));
     }
     /// Stop retaining and reconnecting a telemetry subscription for a terminal node.
     pub(crate) fn unsubscribe_node(&self, run_id: u64, node_id: u64) {
-        if let Some(collector) = self.pull_collectors.lock().remove(&(run_id, node_id)) {
+        if let Some((_, collector)) = self.pull_collectors.lock().remove(&(run_id, node_id)) {
             collector.cancel();
         }
         let mut ended = BTreeSet::new();
@@ -140,13 +208,11 @@ impl FrameCollector {
             );
             let mut channels = self.pull_channels.lock();
             for descriptor in header.channels {
-                channels.insert(
-                    ChannelRef {
-                        stream: descriptor.stream,
-                        channel: descriptor.id,
-                    },
-                    descriptor.name,
-                );
+                let channel = ChannelRef {
+                    stream: descriptor.stream.clone(),
+                    channel: descriptor.id,
+                };
+                channels.insert(channel, descriptor);
             }
         }
         for event in self.pull_subscription.drain_available() {
@@ -157,30 +223,46 @@ impl FrameCollector {
                         .insert(descriptor.stream.clone(), descriptor);
                 }
                 TelemetryEvent::ChannelDeclared(descriptor) => {
-                    self.pull_channels.lock().insert(
-                        ChannelRef {
-                            stream: descriptor.stream,
-                            channel: descriptor.id,
-                        },
-                        descriptor.name,
-                    );
+                    let channel = ChannelRef {
+                        stream: descriptor.stream.clone(),
+                        channel: descriptor.id,
+                    };
+                    self.pull_channels.lock().insert(channel, descriptor);
                 }
                 TelemetryEvent::Frame(delivery) => {
                     let stream = delivery.channel.stream;
-                    let channel_name = self
+                    {
+                        let mut positions = self.pull_positions.lock();
+                        if positions
+                            .get(&stream)
+                            .is_some_and(|position| delivery.position <= *position)
+                        {
+                            continue;
+                        }
+                        positions.insert(stream.clone(), delivery.position);
+                    }
+                    let channel = self
                         .pull_channels
                         .lock()
                         .get(&ChannelRef {
                             stream: stream.clone(),
                             channel: delivery.channel.channel,
                         })
-                        .cloned()
-                        .unwrap_or_else(|| format!("channel#{}", delivery.channel.channel.0));
+                        .cloned();
+                    let (channel_name, channel_content) = channel
+                        .map(|descriptor| (descriptor.name, descriptor.content))
+                        .unwrap_or_else(|| {
+                            (
+                                format!("channel#{}", delivery.channel.channel.0),
+                                ChannelContent::Bytes,
+                            )
+                        });
                     let descriptor = self.pull_streams.lock().get(&stream).cloned();
                     let _ = self.tx.send(CollectedTelemetryFrame {
                         stream,
                         descriptor,
                         channel_name,
+                        channel_content,
                         frame: Frame::new(
                             delivery.channel.channel,
                             delivery.position,
@@ -208,7 +290,7 @@ impl FrameCollector {
     /// Drain queued frames, forwarding each via the closure. No progress extraction.
     pub(crate) fn drain<F>(&self, forward: F)
     where
-        F: FnMut(&StreamId, Option<&StreamDescriptor>, &str, &Frame),
+        F: FnMut(&StreamId, Option<&StreamDescriptor>, &str, &ChannelContent, &Frame),
     {
         let mut forward = forward;
         while let Ok(collected) = self.rx.try_recv() {
@@ -216,6 +298,7 @@ impl FrameCollector {
                 &collected.stream,
                 collected.descriptor.as_ref(),
                 &collected.channel_name,
+                &collected.channel_content,
                 &collected.frame,
             );
         }
@@ -241,7 +324,7 @@ fn drain_telemetry_connections(
                         stream: descriptor.stream.clone(),
                         channel: descriptor.id,
                     },
-                    descriptor.name.clone(),
+                    descriptor.clone(),
                 )
             })
             .collect::<BTreeMap<_, _>>();
@@ -251,23 +334,28 @@ fn drain_telemetry_connections(
                     streams.insert(descriptor.stream.clone(), descriptor);
                 }
                 TelemetryEvent::ChannelDeclared(descriptor) => {
-                    channels.insert(
-                        ChannelRef {
-                            stream: descriptor.stream.clone(),
-                            channel: descriptor.id,
-                        },
-                        descriptor.name,
-                    );
+                    let channel = ChannelRef {
+                        stream: descriptor.stream.clone(),
+                        channel: descriptor.id,
+                    };
+                    channels.insert(channel, descriptor);
                 }
                 TelemetryEvent::Frame(delivery) => {
                     let stream = delivery.channel.stream;
-                    let channel_name = channels
+                    let channel = channels
                         .get(&ChannelRef {
                             stream: stream.clone(),
                             channel: delivery.channel.channel,
                         })
-                        .cloned()
-                        .unwrap_or_else(|| format!("channel#{}", delivery.channel.channel.0));
+                        .cloned();
+                    let (channel_name, channel_content) = channel
+                        .map(|descriptor| (descriptor.name, descriptor.content))
+                        .unwrap_or_else(|| {
+                            (
+                                format!("channel#{}", delivery.channel.channel.0),
+                                ChannelContent::Bytes,
+                            )
+                        });
                     let frame = Frame::new(
                         delivery.channel.channel,
                         delivery.position,
@@ -278,6 +366,7 @@ fn drain_telemetry_connections(
                             descriptor: streams.get(&stream).cloned(),
                             stream,
                             channel_name,
+                            channel_content,
                             frame,
                         })
                         .is_err()
@@ -299,11 +388,12 @@ pub(crate) fn ingest_dashboard_frame(
     dashboard: Option<&DashboardSupport>,
     stream: &StreamId,
     channel: &str,
+    content: &ChannelContent,
     frame: &Frame,
     descriptor: Option<&StreamDescriptor>,
 ) {
     if let Some(dashboard) = dashboard {
-        dashboard.publish_frame(stream, descriptor, channel, frame);
+        dashboard.publish_frame(stream, descriptor, channel, content, frame);
     }
 }
 
@@ -313,6 +403,63 @@ mod tests {
     use telemetry::frame::{FrameDelivery, Lifetime, NodeId, Position, StreamOrigin};
     use telemetry::{ChannelContent, ChannelDescriptor, ChannelId};
 
+    #[test]
+    fn restored_archive_skips_replay_but_accepts_fresh_producer_positions() {
+        use crate::observability::frame_archive::FrameArchive;
+
+        let log = tempfile::NamedTempFile::new().expect("archive");
+        let retained = StreamId::new(NodeId::new("7"), Lifetime(11));
+        let restarted = StreamId::new(NodeId::new("7"), Lifetime(12));
+        let channel = ChannelId(1);
+        {
+            let mut archive = FrameArchive::open_with_label(log.path(), "test").unwrap();
+            for position in 0..2 {
+                archive
+                    .record(
+                        "node",
+                        &retained,
+                        "runtime.log",
+                        &ChannelContent::TextStream,
+                        &Frame::new(channel, Position(position), vec![position as u8]),
+                    )
+                    .unwrap();
+            }
+        }
+        let collector = FrameCollector::new();
+        collector
+            .restore_archive(Some(log.path()))
+            .expect("restore durable cursor");
+        for (stream, position) in [
+            (retained.clone(), 0),
+            (retained.clone(), 1),
+            (retained.clone(), 2),
+            (retained.clone(), 2),
+            (retained.clone(), 3),
+            (restarted.clone(), 0),
+        ] {
+            collector
+                .pull_fanout
+                .publish(TelemetryEvent::Frame(FrameDelivery {
+                    channel: ChannelRef { stream, channel },
+                    position: Position(position),
+                    payload: vec![position as u8],
+                }));
+        }
+        collector.pump_pulls();
+        let mut observed = Vec::new();
+        collector.drain(|stream, _, _, _, frame| {
+            observed.push((stream.clone(), frame.position, frame.payload.clone()));
+        });
+        assert_eq!(
+            observed,
+            vec![
+                (retained.clone(), Position(2), vec![2]),
+                (retained, Position(3), vec![3]),
+                (restarted, Position(0), vec![0]),
+            ]
+        );
+    }
+
     #[test]
     fn pulled_frame_preserves_remote_stream_metadata() {
         let collector = FrameCollector::new();
@@ -327,8 +474,10 @@ mod tests {
             id: ChannelId(9),
             name: "host.net".to_owned(),
             label: None,
-            content: ChannelContent::JsonRecord { schema: None },
+            content: ChannelContent::MessagePackRecord { schema: None },
         };
+        let payload =
+            telemetry::encode_record(&serde_json::json!({"rx": 1})).expect("encode test payload");
         collector
             .pull_header_tx
             .send(TelemetryQuicHeader::new(
@@ -346,27 +495,34 @@ mod tests {
                     channel: channel.id,
                 },
                 position: Position(11),
-                payload: br#"{"rx":1}"#.to_vec(),
+                payload: payload.clone(),
             }));
 
         collector.pump_pulls();
         let mut observed = None;
-        collector.drain(|stream, descriptor, channel, frame| {
+        collector.drain(|stream, descriptor, channel, content, frame| {
             observed = Some((
                 stream.clone(),
                 descriptor.cloned(),
                 channel.to_owned(),
+                content.clone(),
                 frame.clone(),
             ));
         });
 
-        let (observed_stream, observed_descriptor, observed_channel, observed_frame) =
-            observed.expect("pulled frame");
+        let (
+            observed_stream,
+            observed_descriptor,
+            observed_channel,
+            observed_content,
+            observed_frame,
+        ) = observed.expect("pulled frame");
         assert_eq!(observed_stream, stream);
         assert_eq!(observed_descriptor, Some(descriptor));
         assert_eq!(observed_channel, "host.net");
+        assert_eq!(observed_content, channel.content);
         assert_eq!(observed_frame.position, Position(11));
-        assert_eq!(observed_frame.payload, br#"{"rx":1}"#);
+        assert_eq!(observed_frame.payload, payload);
         collector
             .pull_fanout
             .publish(TelemetryEvent::StreamEnded(stream.clone()));
@@ -396,7 +552,7 @@ mod tests {
             id: ChannelId(3),
             name: "runtime.actors".to_owned(),
             label: None,
-            content: ChannelContent::JsonRecord { schema: None },
+            content: ChannelContent::MessagePackRecord { schema: None },
         };
         let mut flow_id = [0_u8; 16];
         flow_id[..8].copy_from_slice(&5_u64.to_le_bytes());

apps/myelin/src/observability/orch_telemetry.rs

@@ -12,7 +12,7 @@ use std::sync::Arc;
 use telemetry::frame::{Frame, StreamId};
 use telemetry::{
     ChannelContent, ChannelId, Lifetime, NodeId, Record, StreamDescriptor, StreamOrigin,
-    TelemetryEndpoint, TelemetryProducer,
+    TelemetryEndpoint, TelemetryProducer, encode_record,
 };
 
 use crate::observability::benchmark;
@@ -101,7 +101,7 @@ impl OrchTelemetry {
         }
         let id = self.producer.register_channel(
             name,
-            ChannelContent::JsonRecord {
+            ChannelContent::MessagePackRecord {
                 schema: Some(name.to_owned()),
             },
         );
@@ -129,8 +129,7 @@ impl OrchTelemetry {
         dashboard: Option<&DashboardSupport>,
         event: ProvisionEvent,
     ) {
-        let payload = serde_json::to_vec(&MyelinProvisionEventRecord::new(event))
-            .expect("serialize provisioning event");
+        let payload = MyelinProvisionEventRecord::new(event).encode();
         self.emit_bytes(dashboard, MYELIN_PROVISIONING_EVENTS, payload);
     }
 
@@ -140,8 +139,7 @@ impl OrchTelemetry {
         line: ProvisionLogLine,
     ) {
         let channel = myelin_provision_log_channel(line.node_id, line.stream);
-        let payload = serde_json::to_vec(&MyelinProvisionLogRecord::new(line))
-            .expect("serialize provision log");
+        let payload = MyelinProvisionLogRecord::new(line).encode();
         self.emit_bytes(dashboard, &channel, payload);
     }
     pub(crate) fn emit_orchestrator_log(
@@ -150,7 +148,7 @@ impl OrchTelemetry {
         stream: crate::provisioning::ProvisionLogStream,
         line: String,
     ) {
-        let payload = serde_json::to_vec(&serde_json::json!({
+        let payload = encode_record(&serde_json::json!({
             "source": format!("{stream:?}").to_ascii_lowercase(),
             "line": line,
         }))
@@ -189,7 +187,7 @@ impl OrchTelemetry {
             detail,
         } = emission;
         let benchmark = benchmark::stamp("myelin-orchestrator");
-        let payload = serde_json::to_vec(&json!({
+        let payload = encode_record(&json!({
             "schema_version": benchmark["schema_version"].clone(),
             "type":"OrchBootstrap",
             "event_type":"OrchBootstrap",
@@ -253,8 +251,18 @@ impl OrchTelemetry {
                 .get(&frame.channel)
                 .cloned()
                 .unwrap_or_else(|| format!("channel#{}", frame.channel.0));
-            ingest_dashboard_frame(dashboard, &stream, &channel, &frame, Some(&self.descriptor));
-            self.archive_frame(source, &stream, &channel, &frame);
+            let content = ChannelContent::MessagePackRecord {
+                schema: Some(channel.clone()),
+            };
+            ingest_dashboard_frame(
+                dashboard,
+                &stream,
+                &channel,
+                &content,
+                &frame,
+                Some(&self.descriptor),
+            );
+            self.archive_frame(source, &stream, &channel, &content, &frame);
         }
     }
 
@@ -263,10 +271,11 @@ impl OrchTelemetry {
         source: &str,
         stream: &StreamId,
         channel: &str,
+        content: &ChannelContent,
         frame: &Frame,
     ) {
         if let Some(archive) = &mut self.archive {
-            let _ = archive.record(source, stream, channel, frame);
+            let _ = archive.record(source, stream, channel, content, frame);
         }
     }
 
@@ -323,6 +332,7 @@ impl DashboardSupport {
         stream: &StreamId,
         descriptor: Option<&StreamDescriptor>,
         channel: &str,
+        content: &ChannelContent,
         frame: &Frame,
     ) {
         let origin = descriptor.map(|descriptor| {
@@ -343,6 +353,7 @@ impl DashboardSupport {
             channel: channel.to_owned(),
             position: frame.position.0,
             payload: frame.payload.clone(),
+            content: content.clone(),
         });
     }
 }
@@ -381,6 +392,7 @@ impl DashboardSupport {
         _stream: &StreamId,
         _descriptor: Option<&StreamDescriptor>,
         _channel: &str,
+        _content: &ChannelContent,
         _frame: &Frame,
     ) {
     }

apps/myelin/src/observability/provisioning_logs.rs

@@ -1,6 +1,6 @@
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
-use telemetry::{ChannelContent, TelemetryProducer};
+use telemetry::{ChannelContent, Record, TelemetryProducer};
 
 use crate::observability::telemetry::{
     MYELIN_PROVISIONING_LOGS, MyelinProvisionLogRecord, myelin_provision_log_channel,
@@ -67,6 +67,14 @@ impl BootstrapTelemetryBridge {
         });
     }
 
+    pub(crate) fn observe_failed(&self, reason: impl Into<String>) {
+        self.sink.observe(PluginObservation::Failed {
+            run_id: self.spec.run_id,
+            node_id: self.spec.node_id,
+            reason: reason.into(),
+        });
+    }
+
     fn submit_log(&self, stream: ProvisionLogStream, line: &str) {
         let Some(producer) = &self.producer else {
             return;
@@ -79,12 +87,11 @@ impl BootstrapTelemetryBridge {
         });
         let channel = producer.register_channel(
             myelin_provision_log_channel(self.spec.node_id, stream),
-            ChannelContent::JsonRecord {
+            ChannelContent::MessagePackRecord {
                 schema: Some(MYELIN_PROVISIONING_LOGS.to_owned()),
             },
         );
-        let payload = serde_json::to_vec(&record).expect("serialize bootstrap log record");
-        producer.submit_bytes(channel, payload);
+        producer.submit_bytes(channel, record.encode());
     }
 }
 

apps/myelin/src/orchestration/actor.rs

@@ -2,15 +2,17 @@ use std::collections::{HashMap, VecDeque};
 use std::path::PathBuf;
 
 use iroh::EndpointAddr;
+use myelin_control_contract::{
+    ContextualControlReply, ContextualEventCursor, ContextualEventRecord,
+    ContextualEventsAcknowledgement, ContextualEventsBatch, ContextualExecutionView,
+    ContextualProcessEvent, ContextualProcessEventKind, ContextualProcessSpec, ControlRevision,
+};
 use serde::{Deserialize, Serialize};
 use swactor::actor::{ActorAddress, ActorInterface};
 use swactor::runtime::Ctx;
 use swactor_transport::{CodecRegistry, NetworkMessage};
 
-use crate::contextual_process::{
-    ContextualNodeCommand, ContextualProcessEventKindWire, ContextualProcessEventWire,
-    ContextualProcessSpecWire,
-};
+use crate::contextual_process::ContextualNodeCommand;
 use crate::node_actor::NodeAgentMsg;
 use crate::orchestration::manual_control::{ManualActorControl, ManualControlMsg};
 use crate::run_fsm as core;
@@ -115,6 +117,10 @@ pub(crate) enum OrchestratorMsg {
         endpoint: EndpointAddr,
         node_actor: ActorAddress,
         readiness_id: u64,
+        #[serde(default)]
+        artifact_digest: Option<String>,
+        #[serde(default)]
+        deployment_generation: Option<String>,
     },
     ObserveNodeRuntimeReadyAck {
         run_id: u64,
@@ -163,7 +169,7 @@ pub(crate) enum OrchestratorMsg {
     ContextualSpawn {
         logical_node_id: u64,
         request_id: String,
-        spec: ContextualProcessSpecWire,
+        spec: ContextualProcessSpec,
         reply_to: ActorAddress,
     },
     ContextualStop {
@@ -182,7 +188,24 @@ pub(crate) enum OrchestratorMsg {
         after_sequence: u64,
         reply_to: ActorAddress,
     },
-    ContextualEvent(ContextualProcessEventWire),
+    ContextualEventsBatch {
+        cursors: Vec<ContextualEventCursor>,
+        wait_key: Option<String>,
+        reply_to: ActorAddress,
+    },
+    ContextualEventsAck {
+        request_id: String,
+        execution_incarnation: String,
+        through_sequence: u64,
+        reply_to: ActorAddress,
+    },
+    ControlChanges {
+        generation: Option<String>,
+        after_revision: Option<u64>,
+        wait_key: Option<String>,
+        reply_to: ActorAddress,
+    },
+    ContextualEvent(ContextualProcessEvent),
     /// An HTTP control observer timed out waiting for its reply; drop the
     /// pending entry so a hung node cannot accumulate parked control requests.
     ContextualControlCancel {
@@ -197,37 +220,16 @@ impl NetworkMessage for OrchestratorMsg {
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
-pub(crate) struct ContextualEventRecord {
-    pub sequence: u64,
-    pub observation: ContextualProcessEventWire,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
-pub(crate) struct ContextualExecutionView {
-    pub request_id: String,
-    pub logical_node_id: u64,
-    pub process: Option<ActorAddress>,
-    pub terminal: bool,
-    pub truncated_before: u64,
-    pub next_sequence: u64,
-    pub events: Vec<ContextualEventRecord>,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(tag = "type", rename_all = "snake_case")]
-pub(crate) enum ContextualControlReply {
-    Event {
-        observation: ContextualProcessEventWire,
-    },
-    Events {
-        execution: ContextualExecutionView,
-    },
-    Rejected {
-        error: String,
-    },
-    /// Locally injected by the HTTP reply observer when its deadline passes;
-    /// never produced by the orchestrator.
+pub(crate) enum ContextualReplyObserverMsg {
+    Reply(ContextualControlReply),
+    /// Locally injected by the HTTP reply observer when its deadline passes.
     TimedOut,
+    /// Local HTTP observer message, enqueued only after request registration.
+    ArmTimeout,
+}
+
+fn send_contextual_reply(ctx: &Ctx<'_>, reply_to: ActorAddress, reply: ContextualControlReply) {
+    let _ = ctx.send(reply_to, ContextualReplyObserverMsg::Reply(reply));
 }
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -298,6 +300,10 @@ pub(crate) enum OrchestratorReport {
         endpoint: EndpointAddr,
         node_actor: ActorAddress,
         readiness_id: u64,
+        #[serde(default)]
+        artifact_digest: Option<String>,
+        #[serde(default)]
+        deployment_generation: Option<String>,
     },
     NodeRuntimeReadyAck {
         run_id: u64,
@@ -336,7 +342,8 @@ const MAX_CONTEXTUAL_EVENTS_PER_EXECUTION: usize = 4096;
 
 struct ContextualExecutionState {
     logical_node_id: u64,
-    process: Option<ActorAddress>,
+    execution_incarnation: String,
+    process: Option<String>,
     terminal: bool,
     next_sequence: u64,
     events: VecDeque<ContextualEventRecord>,
@@ -345,9 +352,9 @@ struct ContextualExecutionState {
 }
 
 impl ContextualExecutionState {
-    fn record(&mut self, observation: ContextualProcessEventWire) {
-        if let ContextualProcessEventKindWire::Spawned { process, .. } = &observation.event {
-            self.process = Some(*process);
+    fn record(&mut self, observation: ContextualProcessEvent) {
+        if let ContextualProcessEventKind::Spawned { process, .. } = &observation.event {
+            self.process = Some(process.clone());
         }
         self.terminal |= observation.event.is_terminal();
         self.events.push_back(ContextualEventRecord {
@@ -363,8 +370,9 @@ impl ContextualExecutionState {
     fn view(&self, request_id: String, after_sequence: u64) -> ContextualExecutionView {
         ContextualExecutionView {
             request_id,
+            execution_incarnation: self.execution_incarnation.clone(),
             logical_node_id: self.logical_node_id,
-            process: self.process,
+            process: self.process.clone(),
             terminal: self.terminal,
             truncated_before: self
                 .events
@@ -382,7 +390,7 @@ impl ContextualExecutionState {
 }
 
 struct PendingContextualReply {
-    reply_to: ActorAddress,
+    reply_to: Option<ActorAddress>,
     target_request_id: Option<String>,
     logical_node_id: u64,
 }
@@ -395,6 +403,9 @@ pub(crate) struct OrchestratorActor {
     manual: Option<ManualActorControl>,
     contextual_executions: HashMap<String, ContextualExecutionState>,
     pending_contextual_replies: HashMap<String, PendingContextualReply>,
+    contextual_event_waiters: HashMap<String, (ActorAddress, Vec<ContextualEventCursor>)>,
+    control_revision: u64,
+    control_change_waiters: HashMap<String, ActorAddress>,
     contextual_artifact_cleanup: Option<ContextualArtifactCleanup>,
 }
 
@@ -408,6 +419,9 @@ impl OrchestratorActor {
             manual: None,
             contextual_executions: HashMap::new(),
             pending_contextual_replies: HashMap::new(),
+            contextual_event_waiters: HashMap::new(),
+            control_revision: 0,
+            control_change_waiters: HashMap::new(),
             contextual_artifact_cleanup: None,
         }
     }
@@ -436,7 +450,8 @@ impl OrchestratorActor {
         reply_to: ActorAddress,
         error: impl Into<String>,
     ) {
-        let _ = ctx.send(
+        send_contextual_reply(
+            ctx,
             reply_to,
             ContextualControlReply::Rejected {
                 error: error.into(),
@@ -464,6 +479,34 @@ impl OrchestratorActor {
 
     fn handle_contextual(&mut self, ctx: &Ctx<'_>, msg: &OrchestratorMsg) -> bool {
         match msg {
+            OrchestratorMsg::ControlChanges {
+                generation,
+                after_revision,
+                wait_key,
+                reply_to,
+            } => {
+                let current_generation = ctx.self_addr().to_string();
+                if generation.as_deref() == Some(current_generation.as_str())
+                    && *after_revision == Some(self.control_revision)
+                    && let Some(key) = wait_key
+                {
+                    if self.control_change_waiters.len() >= MAX_CONTEXTUAL_EXECUTIONS {
+                        self.contextual_rejection(ctx, *reply_to, "too many control observers");
+                    } else {
+                        self.control_change_waiters.insert(key.clone(), *reply_to);
+                    }
+                } else {
+                    send_contextual_reply(
+                        ctx,
+                        *reply_to,
+                        ContextualControlReply::ControlRevision(ControlRevision {
+                            generation: current_generation,
+                            revision: self.control_revision,
+                        }),
+                    );
+                }
+                true
+            }
             OrchestratorMsg::ContextualSpawn {
                 logical_node_id,
                 request_id,
@@ -482,10 +525,6 @@ impl OrchestratorActor {
                     );
                     return true;
                 }
-                if self.contextual_executions.len() >= MAX_CONTEXTUAL_EXECUTIONS {
-                    self.contextual_executions
-                        .retain(|_, execution| !execution.terminal);
-                }
                 if self.contextual_executions.len() >= MAX_CONTEXTUAL_EXECUTIONS {
                     self.cleanup_contextual_artifact(
                         ctx,
@@ -497,7 +536,7 @@ impl OrchestratorActor {
                     self.contextual_rejection(
                         ctx,
                         *reply_to,
-                        "too many live contextual executions",
+                        "too many unacknowledged contextual executions",
                     );
                     return true;
                 }
@@ -505,6 +544,7 @@ impl OrchestratorActor {
                     request_id.clone(),
                     ContextualExecutionState {
                         logical_node_id: *logical_node_id,
+                        execution_incarnation: ActorAddress::new_random().to_string(),
                         process: None,
                         terminal: false,
                         next_sequence: 0,
@@ -559,19 +599,19 @@ impl OrchestratorActor {
                     );
                     return true;
                 };
-                let Some(process) = execution.process else {
+                if execution.process.is_none() {
                     self.contextual_rejection(
                         ctx,
                         *reply_to,
                         format!("contextual request {request_id:?} has not spawned"),
                     );
                     return true;
-                };
+                }
                 let logical_node_id = execution.logical_node_id;
                 self.pending_contextual_replies.insert(
                     control_request_id.clone(),
                     PendingContextualReply {
-                        reply_to: *reply_to,
+                        reply_to: Some(*reply_to),
                         target_request_id: Some(request_id.clone()),
                         logical_node_id,
                     },
@@ -581,7 +621,7 @@ impl OrchestratorActor {
                     logical_node_id,
                     ContextualNodeCommand::Stop {
                         request_id: control_request_id.clone(),
-                        process,
+                        target_request_id: request_id.clone(),
                         kill_after_ms: *kill_after_ms,
                         reply_to: ctx.self_addr(),
                     },
@@ -610,7 +650,7 @@ impl OrchestratorActor {
                 self.pending_contextual_replies.insert(
                     control_request_id.clone(),
                     PendingContextualReply {
-                        reply_to: *reply_to,
+                        reply_to: Some(*reply_to),
                         target_request_id: None,
                         logical_node_id: *logical_node_id,
                     },
@@ -641,14 +681,91 @@ impl OrchestratorActor {
                     );
                     return true;
                 };
-                let terminal = execution.terminal;
                 let reply = ContextualControlReply::Events {
                     execution: execution.view(request_id.clone(), *after_sequence),
                 };
-                let _ = ctx.send(*reply_to, reply);
-                if terminal {
+                send_contextual_reply(ctx, *reply_to, reply);
+                true
+            }
+            OrchestratorMsg::ContextualEventsBatch {
+                cursors,
+                wait_key,
+                reply_to,
+            } => {
+                if cursors.len() > MAX_CONTEXTUAL_EXECUTIONS {
+                    self.contextual_rejection(ctx, *reply_to, "too many execution cursors");
+                    return true;
+                }
+                if let Some(key) = wait_key
+                    && !self.contextual_cursors_ready(cursors)
+                {
+                    if self.contextual_event_waiters.len() >= MAX_CONTEXTUAL_EXECUTIONS {
+                        self.contextual_rejection(ctx, *reply_to, "too many execution observers");
+                    } else {
+                        self.contextual_event_waiters
+                            .insert(key.clone(), (*reply_to, cursors.clone()));
+                    }
+                    return true;
+                }
+                let mut executions = Vec::with_capacity(cursors.len());
+                let mut missing = Vec::new();
+                for cursor in cursors {
+                    match self.contextual_executions.get(&cursor.request_id) {
+                        Some(execution) => executions
+                            .push(execution.view(cursor.request_id.clone(), cursor.after_sequence)),
+                        None => missing.push(cursor.request_id.clone()),
+                    }
+                }
+                send_contextual_reply(
+                    ctx,
+                    *reply_to,
+                    ContextualControlReply::EventsBatch(ContextualEventsBatch {
+                        executions,
+                        missing,
+                    }),
+                );
+                true
+            }
+            OrchestratorMsg::ContextualEventsAck {
+                request_id,
+                execution_incarnation,
+                through_sequence,
+                reply_to,
+            } => {
+                if self.pending_contextual_replies.values().any(|pending| {
+                    pending.target_request_id.as_deref() == Some(request_id.as_str())
+                }) {
+                    self.contextual_rejection(
+                        ctx,
+                        *reply_to,
+                        "acknowledgement must wait for outstanding stop outcomes",
+                    );
+                    return true;
+                }
+                if let Some(execution) = self.contextual_executions.get(request_id) {
+                    if !execution.terminal
+                        || execution.next_sequence != *through_sequence
+                        || execution.execution_incarnation != *execution_incarnation
+                    {
+                        self.contextual_rejection(
+                            ctx,
+                            *reply_to,
+                            "acknowledgement must cover the complete terminal execution",
+                        );
+                        return true;
+                    }
                     self.contextual_executions.remove(request_id);
                 }
+                // Retrying after a lost acknowledgement response is harmless.
+                send_contextual_reply(
+                    ctx,
+                    *reply_to,
+                    ContextualControlReply::Acknowledged(ContextualEventsAcknowledgement {
+                        request_id: request_id.clone(),
+                        execution_incarnation: execution_incarnation.clone(),
+                        through_sequence: *through_sequence,
+                    }),
+                );
                 true
             }
             OrchestratorMsg::ContextualEvent(observation) => {
@@ -657,14 +774,16 @@ impl OrchestratorActor {
                     .remove(&observation.request_id)
                 {
                     if pending.logical_node_id != observation.logical_node_id {
-                        self.contextual_rejection(
-                            ctx,
-                            pending.reply_to,
-                            format!(
-                                "contextual reply came from node {}, expected {}",
-                                observation.logical_node_id, pending.logical_node_id
-                            ),
-                        );
+                        if let Some(reply_to) = pending.reply_to {
+                            self.contextual_rejection(
+                                ctx,
+                                reply_to,
+                                format!(
+                                    "contextual reply came from node {}, expected {}",
+                                    observation.logical_node_id, pending.logical_node_id
+                                ),
+                            );
+                        }
                         return true;
                     }
                     if let Some(target) = pending.target_request_id {
@@ -672,7 +791,11 @@ impl OrchestratorActor {
                             self.contextual_executions
                                 .get_mut(&target)
                                 .and_then(|execution| {
-                                    execution.record(observation.clone());
+                                    // The direct reply belongs to the control request;
+                                    // the retained log belongs to its target execution.
+                                    let mut target_observation = observation.clone();
+                                    target_observation.request_id = target.clone();
+                                    execution.record(target_observation);
                                     observation
                                         .event
                                         .is_terminal()
@@ -681,12 +804,15 @@ impl OrchestratorActor {
                                 });
                         self.cleanup_contextual_artifact(ctx, &target, staged_source);
                     }
-                    let _ = ctx.send(
-                        pending.reply_to,
-                        ContextualControlReply::Event {
-                            observation: observation.clone(),
-                        },
-                    );
+                    if let Some(reply_to) = pending.reply_to {
+                        send_contextual_reply(
+                            ctx,
+                            reply_to,
+                            ContextualControlReply::Event {
+                                observation: observation.clone(),
+                            },
+                        );
+                    }
                     return true;
                 }
                 let Some(execution) = self.contextual_executions.get_mut(&observation.request_id)
@@ -699,7 +825,8 @@ impl OrchestratorActor {
                     let staged_source = execution.staged_source.take();
                     execution.terminal = true;
                     if let Some(waiter) = waiter {
-                        let _ = ctx.send(
+                        send_contextual_reply(
+                            ctx,
                             waiter,
                             ContextualControlReply::Rejected {
                                 error: format!(
@@ -714,7 +841,7 @@ impl OrchestratorActor {
                 }
                 let resolves_spawn = matches!(
                     &observation.event,
-                    ContextualProcessEventKindWire::Spawned { .. }
+                    ContextualProcessEventKind::Spawned { .. }
                 ) || observation.event.is_terminal();
                 execution.record(observation.clone());
                 let staged_source = observation
@@ -723,7 +850,8 @@ impl OrchestratorActor {
                     .then(|| execution.staged_source.take())
                     .flatten();
                 if resolves_spawn && let Some(waiter) = execution.spawn_waiter.take() {
-                    let _ = ctx.send(
+                    send_contextual_reply(
+                        ctx,
                         waiter,
                         ContextualControlReply::Event {
                             observation: observation.clone(),
@@ -734,14 +862,80 @@ impl OrchestratorActor {
                 true
             }
             OrchestratorMsg::ContextualControlCancel { control_request_id } => {
-                // The HTTP caller already observed a timeout; the pending
-                // entry is stale and must not swallow a later reply.
-                self.pending_contextual_replies.remove(control_request_id);
+                // A timed-out HTTP waiter no longer needs a direct reply, but
+                // a dispatched stop still owns a future target-log observation.
+                if let Some(pending) = self.pending_contextual_replies.get_mut(control_request_id)
+                    && pending.target_request_id.is_some()
+                {
+                    pending.reply_to = None;
+                } else {
+                    self.pending_contextual_replies.remove(control_request_id);
+                }
+                self.contextual_event_waiters.remove(control_request_id);
+                self.control_change_waiters.remove(control_request_id);
                 true
             }
             _ => false,
         }
     }
+    fn contextual_cursors_ready(&self, cursors: &[ContextualEventCursor]) -> bool {
+        let minimum_records = u64::from(cursors.len() > 1) + 1;
+        cursors.is_empty()
+            || cursors.iter().any(|cursor| {
+                self.contextual_executions
+                    .get(&cursor.request_id)
+                    .is_none_or(|execution| {
+                        execution.terminal
+                            || execution
+                                .next_sequence
+                                .saturating_sub(cursor.after_sequence)
+                                >= minimum_records
+                    })
+            })
+    }
+
+    fn wake_contextual_event_waiters(&mut self, ctx: &Ctx<'_>) {
+        if self.contextual_event_waiters.is_empty() {
+            return;
+        }
+        // Check and subscribe are serialized by this actor; no event can fall
+        // between the predicate check and registration.
+        let ready = self
+            .contextual_event_waiters
+            .iter()
+            .filter(|(_, (_, cursors))| self.contextual_cursors_ready(cursors))
+            .map(|(key, _)| key.clone())
+            .collect::<Vec<_>>();
+        for key in ready {
+            if let Some((reply_to, cursors)) = self.contextual_event_waiters.remove(&key) {
+                self.handle_contextual(
+                    ctx,
+                    &OrchestratorMsg::ContextualEventsBatch {
+                        cursors,
+                        wait_key: None,
+                        reply_to,
+                    },
+                );
+            }
+        }
+    }
+
+    fn publish_control_change(&mut self, ctx: &Ctx<'_>) {
+        self.control_revision = self
+            .control_revision
+            .checked_add(1)
+            .expect("control revision exhausted");
+        for (_, reply_to) in self.control_change_waiters.drain() {
+            send_contextual_reply(
+                ctx,
+                reply_to,
+                ContextualControlReply::ControlRevision(ControlRevision {
+                    generation: ctx.self_addr().to_string(),
+                    revision: self.control_revision,
+                }),
+            );
+        }
+    }
 
     fn fail_contextual_node(&mut self, ctx: &Ctx<'_>, logical_node_id: u64, reason: &str) {
         let mut waiters = Vec::new();
@@ -750,10 +944,10 @@ impl OrchestratorActor {
             if execution.logical_node_id != logical_node_id || execution.terminal {
                 continue;
             }
-            let observation = ContextualProcessEventWire {
+            let observation = ContextualProcessEvent {
                 request_id: request_id.clone(),
                 logical_node_id,
-                event: ContextualProcessEventKindWire::ProcessError {
+                event: ContextualProcessEventKind::ProcessError {
                     error: reason.to_owned(),
                 },
             };
@@ -769,7 +963,7 @@ impl OrchestratorActor {
             self.cleanup_contextual_artifact(ctx, &request_id, Some(source));
         }
         for (waiter, observation) in waiters {
-            let _ = ctx.send(waiter, ContextualControlReply::Event { observation });
+            send_contextual_reply(ctx, waiter, ContextualControlReply::Event { observation });
         }
 
         // Stop and query controls parked against the failed node will never
@@ -778,7 +972,7 @@ impl OrchestratorActor {
         let mut timed_out_callers = Vec::new();
         self.pending_contextual_replies.retain(|_, pending| {
             if pending.logical_node_id == logical_node_id {
-                timed_out_callers.push(pending.reply_to);
+                timed_out_callers.extend(pending.reply_to);
                 false
             } else {
                 true
@@ -825,6 +1019,9 @@ impl OrchestratorActor {
             | OrchestratorMsg::ContextualStop { .. }
             | OrchestratorMsg::ContextualQuery { .. }
             | OrchestratorMsg::ContextualEvents { .. }
+            | OrchestratorMsg::ContextualEventsBatch { .. }
+            | OrchestratorMsg::ContextualEventsAck { .. }
+            | OrchestratorMsg::ControlChanges { .. }
             | OrchestratorMsg::ContextualEvent(_)
             | OrchestratorMsg::ContextualControlCancel { .. } => {}
             OrchestratorMsg::ObserveTokenInEndpointReady => {
@@ -914,27 +1111,46 @@ impl ActorInterface for OrchestratorActor {
     }
     fn handle(&mut self, ctx: &Ctx, msg: Self::Incoming) {
         if self.handle_contextual(ctx, &msg) {
+            self.wake_contextual_event_waiters(ctx);
             return;
         }
         if let OrchestratorMsg::Manual(manual_msg) = msg.clone() {
+            let changes_state = !matches!(
+                &manual_msg,
+                ManualControlMsg::Query { .. }
+                    | ManualControlMsg::QueryFleet { .. }
+                    | ManualControlMsg::SearchOffers { .. }
+                    | ManualControlMsg::Flush { .. }
+            );
             if let ManualControlMsg::Kill { request, .. } = &manual_msg {
                 self.fail_contextual_node(
                     ctx,
                     request.logical_node_id,
                     "contextual process node termination was requested",
                 );
+                self.wake_contextual_event_waiters(ctx);
             }
             if let Some(manual) = self.manual.as_mut() {
                 manual.handle(ctx, manual_msg);
             }
+            if changes_state {
+                self.publish_control_change(ctx);
+            }
             return;
         }
+        if !matches!(
+            &msg,
+            OrchestratorMsg::Snapshot { .. } | OrchestratorMsg::AdvanceTimeMs(_)
+        ) {
+            self.publish_control_change(ctx);
+        }
         if let OrchestratorMsg::ObserveMembershipLost { node_id, .. } = &msg {
             self.fail_contextual_node(
                 ctx,
                 *node_id,
                 "contextual process node was lost from membership",
             );
+            self.wake_contextual_event_waiters(ctx);
         }
         match msg.clone() {
             OrchestratorMsg::ObserveNodeRuntimeReady {
@@ -944,28 +1160,9 @@ impl ActorInterface for OrchestratorActor {
                 endpoint,
                 node_actor,
                 readiness_id,
+                artifact_digest,
+                deployment_generation,
             } => {
-                if let Some(manual) = self.manual.as_mut() {
-                    manual.observe_runtime_ready(
-                        ctx.self_addr(),
-                        node_id,
-                        crate::orchestration::daemon::RuntimeFacts {
-                            run_id,
-                            attempt_id: manual
-                                .read_model()
-                                .nodes
-                                .iter()
-                                .find(|node| node.logical_node_id == node_id)
-                                .and_then(|node| node.spec.as_ref())
-                                .map_or(0, |spec| spec.attempt_id),
-                            endpoint: serde_json::to_string(&endpoint).unwrap_or_default(),
-                            node_actor,
-                            swim_node_id: distribution::types::NodeId(*endpoint.id.as_bytes()),
-                            stage_index,
-                            readiness_id,
-                        },
-                    );
-                }
                 if let Some(report_to) = self.report_to {
                     let _ = ctx.send(
                         report_to,
@@ -976,6 +1173,8 @@ impl ActorInterface for OrchestratorActor {
                             endpoint,
                             node_actor,
                             readiness_id,
+                            artifact_digest: artifact_digest.clone(),
+                            deployment_generation: deployment_generation.clone(),
                         },
                     );
                 }
@@ -988,7 +1187,13 @@ impl ActorInterface for OrchestratorActor {
                 readiness_id,
             } => {
                 if let Some(manual) = self.manual.as_mut() {
-                    manual.observe_node_ack(ctx.self_addr(), node_id, readiness_id);
+                    manual.observe_node_ack(
+                        ctx.self_addr(),
+                        run_id,
+                        stage_index,
+                        node_id,
+                        readiness_id,
+                    );
                 }
                 if let Some(report_to) = self.report_to {
                     let _ = ctx.send(
@@ -1142,3 +1347,299 @@ impl From<&core::TokenObjectPayload> for TokenObjectPayloadWire {
         }
     }
 }
+
+#[cfg(test)]
+mod event_observation_tests {
+    use super::*;
+    use crate::tests::fuzz_support::drive_steps;
+    use parking_lot::Mutex;
+    use std::sync::Arc;
+    use swactor::config::RuntimeConfig;
+    use swactor::runtime::{Runtime, RuntimeParts};
+    use swactor_engine::{Engine, SteppingBackend};
+
+    struct Replies(Arc<Mutex<Vec<ContextualControlReply>>>);
+
+    impl ActorInterface for Replies {
+        type Incoming = ContextualReplyObserverMsg;
+        type Response = ();
+
+        fn handle(&mut self, _: &Ctx<'_>, reply: Self::Incoming) {
+            let ContextualReplyObserverMsg::Reply(reply) = reply else {
+                panic!("unexpected observer control message");
+            };
+            self.0.lock().push(reply);
+        }
+    }
+
+    fn pending_execution() -> ContextualExecutionState {
+        ContextualExecutionState {
+            logical_node_id: 1,
+            execution_incarnation: "test-incarnation".to_owned(),
+            process: None,
+            terminal: false,
+            next_sequence: 0,
+            events: VecDeque::new(),
+            spawn_waiter: None,
+            staged_source: None,
+        }
+    }
+
+    fn terminal() -> ContextualProcessEvent {
+        ContextualProcessEvent {
+            request_id: "attempt".to_owned(),
+            logical_node_id: 1,
+            event: ContextualProcessEventKind::ProcessError {
+                error: "terminated".to_owned(),
+            },
+        }
+    }
+
+    fn send(
+        runtime: &Runtime,
+        backend: &SteppingBackend,
+        actor: ActorAddress,
+        message: OrchestratorMsg,
+        replies: &Mutex<Vec<ContextualControlReply>>,
+    ) -> Vec<ContextualControlReply> {
+        runtime.send_to(actor, message).unwrap();
+        drive_steps(backend, 64);
+        std::mem::take(&mut *replies.lock())
+    }
+
+    #[test]
+    fn stop_reply_and_execution_history_keep_distinct_request_identities() {
+        let parts = RuntimeParts::new(RuntimeConfig::default());
+        let runtime = parts.runtime().clone();
+        let backend = SteppingBackend::new();
+        let _engine = Engine::new(parts, backend.clone()).unwrap();
+        let replies = Arc::new(Mutex::new(Vec::new()));
+        let reply_to = runtime.spawn(Replies(replies.clone())).unwrap();
+        let mut actor = OrchestratorActor::new(
+            core::RunConfig {
+                run_id: core::RunId(1),
+                max_tokens: 1,
+                prompt: vec![],
+            },
+            None,
+        );
+        actor
+            .contextual_executions
+            .insert("attempt".to_owned(), pending_execution());
+        actor.pending_contextual_replies.insert(
+            "stop-command".to_owned(),
+            PendingContextualReply {
+                reply_to: Some(reply_to),
+                target_request_id: Some("attempt".to_owned()),
+                logical_node_id: 1,
+            },
+        );
+        let actor = runtime.spawn(actor).unwrap();
+        let accepted = ContextualProcessEvent {
+            request_id: "stop-command".to_owned(),
+            logical_node_id: 1,
+            event: ContextualProcessEventKind::StopAccepted {
+                process: reply_to.to_full_hex(),
+            },
+        };
+        let direct = send(
+            &runtime,
+            &backend,
+            actor,
+            OrchestratorMsg::ContextualEvent(accepted.clone()),
+            &replies,
+        );
+        assert!(
+            matches!(&direct[..], [ContextualControlReply::Event { observation }]
+            if observation == &accepted)
+        );
+        send(
+            &runtime,
+            &backend,
+            actor,
+            OrchestratorMsg::ContextualEvent(terminal()),
+            &replies,
+        );
+        let history = send(
+            &runtime,
+            &backend,
+            actor,
+            OrchestratorMsg::ContextualEvents {
+                request_id: "attempt".to_owned(),
+                after_sequence: 0,
+                reply_to,
+            },
+            &replies,
+        );
+        let [ContextualControlReply::Events { execution }] = &history[..] else {
+            panic!("missing execution history: {history:?}");
+        };
+        assert!(execution.terminal);
+        let mut retained = accepted;
+        retained.request_id = "attempt".to_owned();
+        assert_eq!(
+            execution
+                .events
+                .iter()
+                .map(|event| &event.observation)
+                .collect::<Vec<_>>(),
+            vec![&retained, &terminal()],
+        );
+    }
+
+    #[test]
+    fn terminal_history_survives_lost_response_until_complete_acknowledgement() {
+        let parts = RuntimeParts::new(RuntimeConfig::default());
+        let runtime = parts.runtime().clone();
+        let backend = SteppingBackend::new();
+        let _engine = Engine::new(parts, backend.clone()).unwrap();
+        let replies = Arc::new(Mutex::new(Vec::new()));
+        let reply_to = runtime.spawn(Replies(replies.clone())).unwrap();
+        let mut actor = OrchestratorActor::new(
+            core::RunConfig {
+                run_id: core::RunId(1),
+                max_tokens: 1,
+                prompt: vec![],
+            },
+            None,
+        );
+        let mut execution = pending_execution();
+        execution.record(terminal());
+        actor
+            .contextual_executions
+            .insert("attempt".to_owned(), execution);
+        let actor = runtime.spawn(actor).unwrap();
+        let query = || OrchestratorMsg::ContextualEvents {
+            request_id: "attempt".to_owned(),
+            after_sequence: 0,
+            reply_to,
+        };
+        let first = send(&runtime, &backend, actor, query(), &replies);
+        assert!(
+            matches!(&first[..], [ContextualControlReply::Events { execution }]
+            if execution.terminal && execution.events[0].observation == terminal())
+        );
+        assert_eq!(send(&runtime, &backend, actor, query(), &replies), first);
+        let rejected = send(
+            &runtime,
+            &backend,
+            actor,
+            OrchestratorMsg::ContextualEventsAck {
+                request_id: "attempt".to_owned(),
+                through_sequence: 0,
+                reply_to,
+                execution_incarnation: "test-incarnation".to_owned(),
+            },
+            &replies,
+        );
+        assert!(matches!(
+            &rejected[..],
+            [ContextualControlReply::Rejected { .. }]
+        ));
+        assert_eq!(send(&runtime, &backend, actor, query(), &replies), first);
+        let stale = send(
+            &runtime,
+            &backend,
+            actor,
+            OrchestratorMsg::ContextualEventsAck {
+                request_id: "attempt".to_owned(),
+                through_sequence: 1,
+                reply_to,
+                execution_incarnation: "previous-incarnation".to_owned(),
+            },
+            &replies,
+        );
+        assert!(matches!(
+            &stale[..],
+            [ContextualControlReply::Rejected { .. }]
+        ));
+        assert_eq!(send(&runtime, &backend, actor, query(), &replies), first);
+        for _ in 0..2 {
+            let ack = send(
+                &runtime,
+                &backend,
+                actor,
+                OrchestratorMsg::ContextualEventsAck {
+                    request_id: "attempt".to_owned(),
+                    through_sequence: 1,
+                    reply_to,
+                    execution_incarnation: "test-incarnation".to_owned(),
+                },
+                &replies,
+            );
+            assert!(matches!(
+                &ack[..],
+                [ContextualControlReply::Acknowledged(_)]
+            ));
+        }
+        assert!(matches!(
+            &send(&runtime, &backend, actor, query(), &replies)[..],
+            [ContextualControlReply::Rejected { .. }]
+        ));
+    }
+
+    #[test]
+    fn cursor_subscription_wakes_on_event_and_cancellation_removes_waiter() {
+        let parts = RuntimeParts::new(RuntimeConfig::default());
+        let runtime = parts.runtime().clone();
+        let backend = SteppingBackend::new();
+        let _engine = Engine::new(parts, backend.clone()).unwrap();
+        let replies = Arc::new(Mutex::new(Vec::new()));
+        let reply_to = runtime.spawn(Replies(replies.clone())).unwrap();
+        let mut actor = OrchestratorActor::new(
+            core::RunConfig {
+                run_id: core::RunId(1),
+                max_tokens: 1,
+                prompt: vec![],
+            },
+            None,
+        );
+        actor
+            .contextual_executions
+            .insert("attempt".to_owned(), pending_execution());
+        let actor = runtime.spawn(actor).unwrap();
+        for key in ["cancelled", "live"] {
+            assert!(
+                send(
+                    &runtime,
+                    &backend,
+                    actor,
+                    OrchestratorMsg::ContextualEventsBatch {
+                        cursors: vec![ContextualEventCursor {
+                            request_id: "attempt".to_owned(),
+                            after_sequence: 0,
+                        }],
+                        wait_key: Some(key.to_owned()),
+                        reply_to,
+                    },
+                    &replies
+                )
+                .is_empty()
+            );
+        }
+        assert!(
+            send(
+                &runtime,
+                &backend,
+                actor,
+                OrchestratorMsg::ContextualControlCancel {
+                    control_request_id: "cancelled".to_owned(),
+                },
+                &replies
+            )
+            .is_empty()
+        );
+        let observed = send(
+            &runtime,
+            &backend,
+            actor,
+            OrchestratorMsg::ContextualEvent(terminal()),
+            &replies,
+        );
+        assert!(
+            matches!(&observed[..], [ContextualControlReply::EventsBatch(ContextualEventsBatch { executions, missing })]
+            if missing.is_empty() && executions.len() == 1
+                && executions[0].events[0].observation == terminal())
+        );
+    }
+}

apps/myelin/src/orchestration/control.rs

@@ -15,21 +15,26 @@ use swactor::runtime::{Ctx, ExternalSender, Runtime};
 use swactor_engine::EngineHandle;
 use swactor_vastai::VastClient;
 
-use crate::contextual_process::{ContextualProcessSpecWire, ContextualProgramFileWire};
-use crate::orchestration::actor::ContextualControlReply;
-use crate::orchestration::actor::OrchestratorMsg;
+use crate::orchestration::actor::{ContextualReplyObserverMsg, OrchestratorMsg};
 use crate::orchestration::manual_control::{
     KillRequest, ManualControlMsg, ManualControlReply, OfferSearchRequest,
     ProviderConfigurationRequest, ProvisionRequest,
 };
+use myelin_control_contract::{
+    ContextualControlReply, ContextualEventsAckRequest, ContextualEventsRequest,
+    ContextualProcessEventKind, ContextualProcessSpec, ContextualProgramFile,
+    ContextualSpawnRequest, ContextualStopRequest, Versioned,
+};
 
 const CONTROL_REPLY_TIMEOUT: Duration = Duration::from_secs(2);
+const RETAINED_BLOBS_REPLY_TIMEOUT: Duration = Duration::from_secs(5);
+const NAMESPACE_CLEANUP_REPLY_TIMEOUT: Duration = Duration::from_secs(5);
 const OFFER_SEARCH_REPLY_MARGIN: Duration = Duration::from_secs(5);
 const OFFER_SEARCH_REPLY_TIMEOUT: Duration =
     VastClient::REQUEST_TIMEOUT.saturating_add(OFFER_SEARCH_REPLY_MARGIN);
 const PROGRAM_UPLOAD_LIMIT: usize = 256 * 1024;
-const PROGRAM_ATTACH_TIMEOUT: Duration = Duration::from_secs(30);
-const PROGRAM_SPAWN_REPLY_TIMEOUT: Duration = Duration::from_secs(35);
+const PROGRAM_ATTACH_TIMEOUT: Duration = Duration::from_secs(80);
+const PROGRAM_SPAWN_REPLY_TIMEOUT: Duration = Duration::from_secs(85);
 struct ControlReplyObserver {
     reply: Arc<Mutex<Option<tokio::sync::oneshot::Sender<ManualControlReply>>>>,
     engine: EngineHandle,
@@ -63,8 +68,14 @@ impl ActorInterface for ControlReplyObserver {
     }
 }
 
+#[derive(Debug, PartialEq, Eq)]
+enum ContextualResponse {
+    Reply(ContextualControlReply),
+    TimedOut,
+}
+
 struct ContextualReplyObserver {
-    reply: Arc<Mutex<Option<tokio::sync::oneshot::Sender<ContextualControlReply>>>>,
+    reply: Arc<Mutex<Option<tokio::sync::oneshot::Sender<ContextualResponse>>>>,
     engine: EngineHandle,
     sender: ExternalSender,
     timeout: Duration,
@@ -73,36 +84,40 @@ struct ContextualReplyObserver {
 }
 
 impl ActorInterface for ContextualReplyObserver {
-    type Incoming = ContextualControlReply;
+    type Incoming = ContextualReplyObserverMsg;
     type Response = ();
 
-    fn on_start(&mut self, ctx: &swactor::runtime::Ctx<'_>) {
-        self.engine.send_after(
-            self.timeout,
-            self.sender.clone(),
-            ctx.self_addr(),
-            ContextualControlReply::TimedOut,
-        );
-    }
-
-    fn handle(&mut self, ctx: &swactor::runtime::Ctx<'_>, reply: Self::Incoming) {
-        if matches!(reply, ContextualControlReply::TimedOut) {
-            // The orchestrator never answers now; drop the parked request so
-            // a later reply cannot be swallowed by a stale entry.
-            if let Some(control_request_id) = self.cancel_key.take() {
-                let _ = ctx.send(
-                    self.orchestrator,
-                    OrchestratorMsg::ContextualControlCancel { control_request_id },
+    fn handle(&mut self, ctx: &Ctx<'_>, message: Self::Incoming) {
+        let response = match message {
+            ContextualReplyObserverMsg::ArmTimeout => {
+                self.engine.send_after(
+                    self.timeout,
+                    self.sender.clone(),
+                    ctx.self_addr(),
+                    ContextualReplyObserverMsg::TimedOut,
                 );
+                return;
             }
-        }
-        if let Some(response) = self
+            ContextualReplyObserverMsg::TimedOut => {
+                // The orchestrator never answers now; drop the parked request so
+                // a later reply cannot be swallowed by a stale entry.
+                if let Some(control_request_id) = self.cancel_key.take() {
+                    let _ = ctx.send(
+                        self.orchestrator,
+                        OrchestratorMsg::ContextualControlCancel { control_request_id },
+                    );
+                }
+                ContextualResponse::TimedOut
+            }
+            ContextualReplyObserverMsg::Reply(reply) => ContextualResponse::Reply(reply),
+        };
+        if let Some(reply) = self
             .reply
             .lock()
             .unwrap_or_else(std::sync::PoisonError::into_inner)
             .take()
         {
-            let _ = response.send(reply);
+            let _ = reply.send(response);
         }
         ctx.stop_self();
     }
@@ -118,6 +133,7 @@ struct ControlHttpState {
     orchestrator: ActorAddress,
     namespace: Option<data_plane::control::DataPlaneControl>,
     upload_root: Arc<PathBuf>,
+    live_env: crate::orchestration::provider_adapters::ssh_bootstrap::BootstrapEnvSource,
 }
 
 pub(crate) fn plugin(
@@ -126,6 +142,7 @@ pub(crate) fn plugin(
     orchestrator: ActorAddress,
     namespace: data_plane::control::DataPlaneControl,
     upload_root: PathBuf,
+    live_env: crate::orchestration::provider_adapters::ssh_bootstrap::BootstrapEnvSource,
 ) -> dashboard::DashboardPlugin {
     let state = ControlHttpState {
         runtime,
@@ -133,12 +150,16 @@ pub(crate) fn plugin(
         orchestrator,
         namespace: Some(namespace),
         upload_root: Arc::new(upload_root),
+        live_env,
     };
     let routes = Router::new()
         .route(FLEET_CONTROL_SCRIPT_URL, get(fleet_control_script))
         .route("/api/control/status", get(status))
+        .route("/api/control/endpoint", get(live_endpoint))
         .route("/api/control/fleet", get(fleet_status))
         .route("/api/control/actors", get(actor_stats))
+        .route("/api/control/actors/snapshot", get(actor_snapshot))
+        .route("/api/control/changes", post(control_changes))
         .route("/api/control/provision", post(provision))
         .route("/api/control/kill", post(kill))
         .route("/api/control/provider", post(configure_provider))
@@ -154,6 +175,14 @@ pub(crate) fn plugin(
             "/api/control/contextual/{request_id}/events",
             get(contextual_events),
         )
+        .route(
+            "/api/control/contextual/events",
+            post(contextual_events_batch),
+        )
+        .route(
+            "/api/control/contextual/{request_id}/ack",
+            post(contextual_events_ack),
+        )
         .route(
             "/api/control/contextual/{request_id}/stop",
             post(contextual_stop),
@@ -162,6 +191,14 @@ pub(crate) fn plugin(
             "/api/control/contextual/nodes/{logical_node_id}",
             get(contextual_query_node),
         )
+        .route(
+            "/api/control/contextual/namespace-cleanup",
+            post(namespace_cleanup),
+        )
+        .route(
+            "/api/control/contextual/retained-blobs",
+            post(retained_blobs),
+        )
         .layer(DefaultBodyLimit::max(PROGRAM_UPLOAD_LIMIT))
         .with_state(state);
     dashboard::DashboardPlugin::new(routes).with_page(dashboard::PluginPage::new(
@@ -178,6 +215,27 @@ async fn fleet_control_script() -> impl IntoResponse {
         FLEET_CONTROL_SCRIPT,
     )
 }
+/// The orchestrator's CURRENT transport endpoint and actor address.
+///
+/// A restarted orchestrator binds a fresh port; remote workers that were
+/// still booting when the previous process died never persisted an endpoint
+/// the recovery join can dial, and they keep targeting the dead address.
+/// Operators (and the e2e harness) read this and push the live endpoint to
+/// those workers through each node's debug-join socket.
+async fn live_endpoint(State(state): State<ControlHttpState>) -> Response {
+    match (state.live_env)() {
+        Ok(env) => Json(serde_json::Map::from_iter(
+            env.into_iter()
+                .map(|(key, value)| (key, serde_json::Value::String(value))),
+        ))
+        .into_response(),
+        Err(error) => (
+            StatusCode::SERVICE_UNAVAILABLE,
+            Json(ErrorResponse { error }),
+        )
+            .into_response(),
+    }
+}
 
 async fn provision(
     State(state): State<ControlHttpState>,
@@ -296,14 +354,6 @@ async fn flush(State(state): State<ControlHttpState>) -> Response {
     .await
 }
 
-#[derive(serde::Deserialize)]
-struct ContextualSpawnRequest {
-    logical_node_id: u64,
-    request_id: String,
-    #[serde(flatten)]
-    spec: ContextualProcessSpecWire,
-}
-
 async fn contextual_spawn(
     State(state): State<ControlHttpState>,
     Json(request): Json<ContextualSpawnRequest>,
@@ -426,8 +476,8 @@ fn python_contextual_spec(
     request_id: &str,
     filename: &str,
     namespace_path: String,
-) -> ContextualProcessSpecWire {
-    ContextualProcessSpecWire {
+) -> ContextualProcessSpec {
+    ContextualProcessSpec {
         command: "python3".to_owned(),
         args: Vec::new(),
         env: BTreeMap::new(),
@@ -437,7 +487,7 @@ fn python_contextual_spec(
         read_prefixes: vec!["/models".to_owned(), format!("/runs/{request_id}/results")],
         write_prefixes: vec![format!("/runs/{request_id}/results")],
         attach_timeout_ms: PROGRAM_ATTACH_TIMEOUT.as_millis() as u64,
-        staged_program: Some(ContextualProgramFileWire { namespace_path }),
+        staged_program: Some(ContextualProgramFile { namespace_path }),
     }
 }
 
@@ -503,9 +553,177 @@ async fn contextual_events(
 }
 
 #[derive(serde::Deserialize)]
-struct ContextualStopRequest {
-    control_request_id: String,
-    kill_after_ms: Option<u64>,
+struct ActorSnapshotQuery {
+    request_id: String,
+}
+
+async fn actor_snapshot(
+    State(state): State<ControlHttpState>,
+    Query(query): Query<ActorSnapshotQuery>,
+) -> Response {
+    if query.request_id.is_empty() {
+        return (
+            StatusCode::BAD_REQUEST,
+            Json(ErrorResponse {
+                error: "actor snapshot requires a nonempty request identity".to_owned(),
+            }),
+        )
+            .into_response();
+    }
+    let request = match state.runtime.admin().list_actors() {
+        Ok(request) => request,
+        Err(error) => {
+            return (
+                StatusCode::SERVICE_UNAVAILABLE,
+                Json(ErrorResponse {
+                    error: format!("request fresh actor census: {error}"),
+                }),
+            )
+                .into_response();
+        }
+    };
+    let actors = match tokio::time::timeout(CONTROL_REPLY_TIMEOUT, request.recv()).await {
+        Ok(Ok(snapshot)) => snapshot.actors,
+        result => {
+            return (
+                StatusCode::GATEWAY_TIMEOUT,
+                Json(ErrorResponse {
+                    error: format!("fresh actor census incomplete: {result:?}"),
+                }),
+            )
+                .into_response();
+        }
+    };
+    Json(serde_json::json!({
+        "schema_version": myelin_control_contract::SCHEMA_VERSION,
+        "request_id": query.request_id,
+        "actors": actors.into_iter().map(|actor| serde_json::json!({
+            "address": actor.address.to_full_hex(),
+            "actor_type": actor.actor_type,
+            "worker_id": actor.worker_id,
+            "mailbox_depth": actor.mailbox_depth,
+            "poisoned": actor.status.poisoned,
+            "stopping": actor.status.stopping,
+        })).collect::<Vec<_>>(),
+        "runtime_stats": state.runtime.stats(),
+        "namespace_commits": data_plane::namespace_store::commit_metrics(),
+    }))
+    .into_response()
+}
+
+async fn control_changes(
+    State(state): State<ControlHttpState>,
+    Json(request): Json<Versioned<myelin_control_contract::ControlChangesRequest>>,
+) -> Response {
+    let request = match request.into_payload() {
+        Ok(request) => request,
+        Err(error) => {
+            return (StatusCode::BAD_REQUEST, Json(ErrorResponse { error })).into_response();
+        }
+    };
+    let wait_ms = request.wait_ms.min(1_000);
+    let wait_key = (wait_ms != 0).then(|| format!("control-{}", ActorAddress::new_random()));
+    let timeout = if wait_ms == 0 {
+        CONTROL_REPLY_TIMEOUT
+    } else {
+        Duration::from_millis(wait_ms)
+    };
+    let receiver =
+        match begin_contextual_request_reply(&state, wait_key.clone(), timeout, |reply_to| {
+            OrchestratorMsg::ControlChanges {
+                generation: request.generation,
+                after_revision: request.after_revision,
+                wait_key,
+                reply_to,
+            }
+        }) {
+            Ok(receiver) => receiver,
+            Err(response) => return *response,
+        };
+    contextual_response(receiver).await
+}
+
+async fn contextual_events_batch(
+    State(state): State<ControlHttpState>,
+    Json(request): Json<Versioned<ContextualEventsRequest>>,
+) -> Response {
+    let request = match request.into_payload() {
+        Ok(request) => request,
+        Err(error) => {
+            return (StatusCode::BAD_REQUEST, Json(ErrorResponse { error })).into_response();
+        }
+    };
+    let wait_ms = request.wait_ms.min(1_000);
+    let wait_key = (wait_ms != 0).then(|| format!("events-{}", ActorAddress::new_random()));
+    let timeout = if wait_ms == 0 {
+        CONTROL_REPLY_TIMEOUT
+    } else {
+        Duration::from_millis(wait_ms)
+    };
+    let cursors = request.cursors;
+    let receiver =
+        match begin_contextual_request_reply(&state, wait_key.clone(), timeout, |reply_to| {
+            OrchestratorMsg::ContextualEventsBatch {
+                cursors: cursors.clone(),
+                wait_key,
+                reply_to,
+            }
+        }) {
+            Ok(receiver) => receiver,
+            Err(response) => return *response,
+        };
+    match receiver.await {
+        Ok(ContextualResponse::TimedOut) if wait_ms != 0 => {
+            let receiver = match begin_contextual_request_reply(
+                &state,
+                None,
+                CONTROL_REPLY_TIMEOUT,
+                |reply_to| OrchestratorMsg::ContextualEventsBatch {
+                    cursors,
+                    wait_key: None,
+                    reply_to,
+                },
+            ) {
+                Ok(receiver) => receiver,
+                Err(response) => return *response,
+            };
+            match receiver.await {
+                Ok(ContextualResponse::Reply(ContextualControlReply::EventsBatch(batch)))
+                    if batch.missing.is_empty()
+                        && batch
+                            .executions
+                            .iter()
+                            .all(|execution| execution.events.is_empty()) =>
+                {
+                    contextual_response_result(Ok(ContextualResponse::TimedOut))
+                }
+                response => contextual_response_result(response),
+            }
+        }
+        response => contextual_response_result(response),
+    }
+}
+
+async fn contextual_events_ack(
+    Path(request_id): Path<String>,
+    State(state): State<ControlHttpState>,
+    Json(request): Json<Versioned<ContextualEventsAckRequest>>,
+) -> Response {
+    let request = match request.into_payload() {
+        Ok(request) => request,
+        Err(error) => {
+            return (StatusCode::BAD_REQUEST, Json(ErrorResponse { error })).into_response();
+        }
+    };
+    contextual_request_reply(&state, None, |reply_to| {
+        OrchestratorMsg::ContextualEventsAck {
+            request_id,
+            through_sequence: request.through_sequence,
+            execution_incarnation: request.execution_incarnation,
+            reply_to,
+        }
+    })
+    .await
 }
 
 async fn contextual_stop(
@@ -536,14 +754,329 @@ async fn contextual_query_node(
     Query(query): Query<ContextualNodeQuery>,
     State(state): State<ControlHttpState>,
 ) -> Response {
-    contextual_request_reply(&state, Some(query.control_request_id.clone()), |reply_to| {
-        OrchestratorMsg::ContextualQuery {
+    use myelin_control_contract::{
+        ContextualHealthEvent, ContextualHealthObservation, ContextualHealthReply,
+        ContextualHealthReplyType, HealthExecution,
+    };
+    let response = match begin_contextual_request_reply(
+        &state,
+        Some(query.control_request_id.clone()),
+        CONTROL_REPLY_TIMEOUT,
+        |reply_to| OrchestratorMsg::ContextualQuery {
             logical_node_id,
-            control_request_id: query.control_request_id,
+            control_request_id: query.control_request_id.clone(),
             reply_to,
+        },
+    ) {
+        Ok(response) => response,
+        Err(response) => return *response,
+    };
+    let reply = match response.await {
+        Ok(ContextualResponse::Reply(ContextualControlReply::Event { observation })) => observation,
+        other => {
+            return (
+                StatusCode::SERVICE_UNAVAILABLE,
+                Json(ErrorResponse {
+                    error: format!(
+                        "contextual health did not return a successful observation: {other:?}"
+                    ),
+                }),
+            )
+                .into_response();
         }
+    };
+    let ContextualProcessEventKind::LiveExecutions {
+        executions,
+        resources: Some(resources),
+    } = reply.event
+    else {
+        return (
+            StatusCode::SERVICE_UNAVAILABLE,
+            Json(ErrorResponse {
+                error: "contextual health omitted live executions or fresh resources".to_owned(),
+            }),
+        )
+            .into_response();
+    };
+    let reply = ContextualHealthReply {
+        schema_version: myelin_control_contract::SCHEMA_VERSION,
+        reply_type: ContextualHealthReplyType::Event,
+        observation: ContextualHealthObservation {
+            logical_node_id: reply.logical_node_id,
+            request_id: reply.request_id,
+            event: ContextualHealthEvent::LiveExecutions {
+                executions: executions
+                    .into_iter()
+                    .map(|execution| HealthExecution {
+                        request_id: execution.request_id,
+                        process: execution.process,
+                        execution_id: execution.identity.execution_id,
+                        generation: execution.identity.generation,
+                        started_pid: execution.started_pid,
+                        context_ready: execution.context_ready,
+                    })
+                    .collect(),
+                resources,
+            },
+        },
+    };
+    if let Err(error) = reply.validate(logical_node_id, &query.control_request_id) {
+        return (
+            StatusCode::SERVICE_UNAVAILABLE,
+            Json(ErrorResponse { error }),
+        )
+            .into_response();
+    }
+    Json(reply).into_response()
+}
+
+async fn namespace_cleanup(
+    State(state): State<ControlHttpState>,
+    Json(request): Json<myelin_control_contract::NamespaceCleanupRequest>,
+) -> Response {
+    use data_plane::control::NamespaceCleanupStatus;
+    use myelin_control_contract::{
+        NamespaceCleanupPath, NamespaceCleanupQuiescence, NamespaceCleanupReply, SCHEMA_VERSION,
+    };
+    if request.schema_version != SCHEMA_VERSION
+        || request.request_id.is_empty()
+        || request.paths.is_empty()
+        || request
+            .paths
+            .iter()
+            .collect::<std::collections::BTreeSet<_>>()
+            .len()
+            != request.paths.len()
+        || request
+            .paths
+            .iter()
+            .any(|path| !path.starts_with("/cases/") && !path.starts_with("/runs/"))
+    {
+        return (
+            StatusCode::BAD_REQUEST,
+            Json(ErrorResponse {
+                error: "invalid owned namespace cleanup request".to_owned(),
+            }),
+        )
+            .into_response();
+    }
+    let Some(namespace) = &state.namespace else {
+        return (
+            StatusCode::SERVICE_UNAVAILABLE,
+            Json(ErrorResponse {
+                error: "namespace is unavailable".to_owned(),
+            }),
+        )
+            .into_response();
+    };
+    let mut parsed = Vec::with_capacity(request.paths.len());
+    for path in &request.paths {
+        match data_plane::path::DataPath::parse(path) {
+            Ok(data_path) => parsed.push((path.clone(), data_path)),
+            Err(error) => {
+                return (
+                    StatusCode::BAD_REQUEST,
+                    Json(ErrorResponse {
+                        error: format!("invalid owned cleanup path: {error}"),
+                    }),
+                )
+                    .into_response();
+            }
+        }
+    }
+
+    let deadline = tokio::time::Instant::now() + NAMESPACE_CLEANUP_REPLY_TIMEOUT;
+    let mut cleanups = tokio::task::JoinSet::new();
+    for (path, data_path) in parsed {
+        let namespace = namespace.clone();
+        cleanups.spawn(async move {
+            let mut quiescence = None;
+            let error = loop {
+                match tokio::time::timeout_at(
+                    deadline,
+                    namespace.try_unregister_quiescent(data_path.clone()),
+                )
+                .await
+                {
+                    Ok(Ok(NamespaceCleanupStatus::ActiveStream)) => {
+                        tokio::task::yield_now().await;
+                    }
+                    Ok(Ok(NamespaceCleanupStatus::Absent)) => break None,
+                    Ok(Ok(NamespaceCleanupStatus::Removed {
+                        quiescent_stream_revision,
+                    })) => {
+                        quiescence =
+                            quiescent_stream_revision.map(|revision| NamespaceCleanupQuiescence {
+                                revision,
+                                active: false,
+                            });
+                        break None;
+                    }
+                    Ok(Err(error)) => break Some(error.to_string()),
+                    Err(error) => break Some(format!("namespace cleanup deadline: {error}")),
+                }
+            };
+            NamespaceCleanupPath {
+                record_type: "path_cleanup".to_owned(),
+                path,
+                attempted: true,
+                absent: error.is_none(),
+                quiescence,
+                error,
+            }
+        });
+    }
+    let mut paths = Vec::with_capacity(request.paths.len());
+    while let Some(result) = cleanups.join_next().await {
+        match result {
+            Ok(path) => paths.push(path),
+            Err(error) => {
+                return (
+                    StatusCode::SERVICE_UNAVAILABLE,
+                    Json(ErrorResponse {
+                        error: format!("namespace cleanup task failed: {error}"),
+                    }),
+                )
+                    .into_response();
+            }
+        }
+    }
+    paths.sort_by(|left, right| left.path.cmp(&right.path));
+    Json(NamespaceCleanupReply {
+        schema_version: SCHEMA_VERSION,
+        request_id: request.request_id,
+        paths,
     })
-    .await
+    .into_response()
+}
+
+async fn retained_blobs(
+    State(state): State<ControlHttpState>,
+    Json(request): Json<myelin_control_contract::RetainedBlobsRequest>,
+) -> Response {
+    use data_plane::control::RetainedNamespaceResources;
+    use myelin_control_contract::RetainedNonBlob;
+    use myelin_control_contract::{RetainedBlob, RetainedBlobsReply, SCHEMA_VERSION};
+    if request.schema_version != SCHEMA_VERSION
+        || request.request_id.is_empty()
+        || request
+            .paths
+            .iter()
+            .collect::<std::collections::BTreeSet<_>>()
+            .len()
+            != request.paths.len()
+    {
+        return (
+            StatusCode::BAD_REQUEST,
+            Json(ErrorResponse {
+                error: "invalid retained-blob ownership query".to_owned(),
+            }),
+        )
+            .into_response();
+    }
+    let Some(namespace) = &state.namespace else {
+        return (
+            StatusCode::SERVICE_UNAVAILABLE,
+            Json(ErrorResponse {
+                error: "namespace is unavailable".to_owned(),
+            }),
+        )
+            .into_response();
+    };
+    let mut blobs = std::collections::BTreeMap::new();
+    let mut non_blobs = std::collections::BTreeMap::new();
+    let mut pending = request
+        .paths
+        .iter()
+        .cloned()
+        .collect::<std::collections::BTreeSet<_>>();
+    let deadline = tokio::time::Instant::now() + RETAINED_BLOBS_REPLY_TIMEOUT;
+    let mut queries = tokio::task::JoinSet::new();
+    for path in request.paths {
+        let parsed = match data_plane::path::DataPath::parse(&path) {
+            Ok(path) => path,
+            Err(error) => {
+                return (
+                    StatusCode::BAD_REQUEST,
+                    Json(ErrorResponse {
+                        error: format!("invalid retained path: {error}"),
+                    }),
+                )
+                    .into_response();
+            }
+        };
+        let namespace = namespace.clone();
+        queries.spawn(async move {
+            let resources = namespace.retained_blob_resources(parsed).await;
+            (path, resources)
+        });
+    }
+    while !queries.is_empty() {
+        let joined = match tokio::time::timeout_at(deadline, queries.join_next()).await {
+            Ok(Some(joined)) => joined,
+            Ok(None) => break,
+            Err(error) => {
+                return (
+                    StatusCode::GATEWAY_TIMEOUT,
+                    Json(ErrorResponse {
+                        error: format!(
+                            "retained blob ownership query deadline: {error}; pending={pending:?}"
+                        ),
+                    }),
+                )
+                    .into_response();
+            }
+        };
+        let (path, binding) = match joined {
+            Ok(result) => result,
+            Err(error) => {
+                return (
+                    StatusCode::SERVICE_UNAVAILABLE,
+                    Json(ErrorResponse {
+                        error: format!("retained blob ownership query task failed: {error}"),
+                    }),
+                )
+                    .into_response();
+            }
+        };
+        pending.remove(&path);
+        match binding {
+            Ok(RetainedNamespaceResources::Blob(binding)) => {
+                blobs.insert(
+                    path,
+                    RetainedBlob {
+                        source: binding.source.to_full_hex(),
+                        binding: binding.owner.map(|owner| owner.to_full_hex()),
+                        source_node: binding.source_node,
+                        length: binding.length,
+                        revision: binding.revision,
+                    },
+                );
+            }
+            Ok(RetainedNamespaceResources::Absent) => {
+                non_blobs.insert(path, RetainedNonBlob::Absent);
+            }
+            Ok(RetainedNamespaceResources::QuiescentStream { revision }) => {
+                non_blobs.insert(path, RetainedNonBlob::QuiescentStream { revision });
+            }
+            Err(error) => {
+                return (
+                    StatusCode::SERVICE_UNAVAILABLE,
+                    Json(ErrorResponse {
+                        error: format!("retained blob could not be resolved: {error}"),
+                    }),
+                )
+                    .into_response();
+            }
+        }
+    }
+    Json(RetainedBlobsReply {
+        schema_version: SCHEMA_VERSION,
+        request_id: request.request_id,
+        blobs,
+        non_blobs,
+    })
+    .into_response()
 }
 
 fn route_mutation(state: &ControlHttpState, msg: ManualControlMsg) -> Response {
@@ -608,20 +1141,31 @@ async fn contextual_request_reply(
 }
 
 async fn contextual_response(
-    response_rx: tokio::sync::oneshot::Receiver<ContextualControlReply>,
+    response_rx: tokio::sync::oneshot::Receiver<ContextualResponse>,
 ) -> Response {
-    match response_rx.await {
-        Ok(ContextualControlReply::TimedOut) => (
+    contextual_response_result(response_rx.await)
+}
+
+fn contextual_response_result(
+    response: Result<ContextualResponse, tokio::sync::oneshot::error::RecvError>,
+) -> Response {
+    match response {
+        Ok(ContextualResponse::TimedOut) => (
             StatusCode::GATEWAY_TIMEOUT,
             Json(ErrorResponse {
                 error: "contextual control reply timed out".to_owned(),
             }),
         )
             .into_response(),
-        Ok(ContextualControlReply::Rejected { error }) => {
+        Ok(ContextualResponse::Reply(ContextualControlReply::Rejected { error })) => {
             (StatusCode::CONFLICT, Json(ErrorResponse { error })).into_response()
         }
-        Ok(reply) => Json(reply).into_response(),
+        Ok(ContextualResponse::Reply(
+            reply @ (ContextualControlReply::EventsBatch(_)
+            | ContextualControlReply::Acknowledged(_)
+            | ContextualControlReply::ControlRevision(_)),
+        )) => Json(Versioned::new(reply)).into_response(),
+        Ok(ContextualResponse::Reply(reply)) => Json(reply).into_response(),
         Err(error) => (
             StatusCode::SERVICE_UNAVAILABLE,
             Json(ErrorResponse {
@@ -637,7 +1181,7 @@ fn begin_contextual_request_reply(
     cancel_key: Option<String>,
     timeout: Duration,
     build: impl FnOnce(ActorAddress) -> OrchestratorMsg,
-) -> Result<tokio::sync::oneshot::Receiver<ContextualControlReply>, Box<Response>> {
+) -> Result<tokio::sync::oneshot::Receiver<ContextualResponse>, Box<Response>> {
     let (response_tx, response_rx) = tokio::sync::oneshot::channel();
     let response_tx = Arc::new(Mutex::new(Some(response_tx)));
     let reply_to = state
@@ -673,6 +1217,12 @@ fn begin_contextual_request_reply(
                 .into_response(),
         ));
     }
+    // The subscription is already queued before its timeout can enqueue
+    // cancellation. Otherwise a short timeout can cancel a not-yet-registered
+    // waiter and strand it after the observer has stopped.
+    let _ = state
+        .runtime
+        .send_to(reply_to, ContextualReplyObserverMsg::ArmTimeout);
     Ok(response_rx)
 }
 
@@ -1138,6 +1688,7 @@ mod properties {
                 orchestrator,
                 namespace: None,
                 upload_root: Arc::new(PathBuf::new()),
+                live_env: Arc::new(|| Ok(Vec::new())),
             };
             let outcome = (|| {
                 let mut responses = Vec::with_capacity(actions.len());
@@ -1286,6 +1837,7 @@ mod properties {
             orchestrator: ActorAddress::default(),
             namespace: None,
             upload_root: Arc::new(PathBuf::new()),
+            live_env: Arc::new(|| Ok(Vec::new())),
         };
         let response = match begin_request_reply(&state, Duration::from_millis(1), |reply_to| {
             ManualControlMsg::Query { reply_to }
@@ -1329,6 +1881,7 @@ mod properties {
             orchestrator,
             namespace: None,
             upload_root: Arc::new(PathBuf::new()),
+            live_env: Arc::new(|| Ok(Vec::new())),
         };
         let pending = begin_http_action(&state, 0, HttpAction::Status);
         drive_steps(&backend, STEP_BUDGET);
@@ -1373,6 +1926,7 @@ mod properties {
             orchestrator: ActorAddress::default(),
             namespace: None,
             upload_root: Arc::new(PathBuf::new()),
+            live_env: Arc::new(|| Ok(Vec::new())),
         };
         let actions = vec![HttpAction::Status];
         let responses = vec![HttpObservation {
@@ -1405,4 +1959,37 @@ mod properties {
         assert_eq!(spec.command, "python3");
         assert!(spec.args.is_empty());
     }
+
+    #[test]
+    fn subscription_timeout_starts_only_after_request_is_enqueued() {
+        let parts = RuntimeParts::new(RuntimeConfig::default());
+        let runtime = parts.runtime().clone();
+        let backend = SteppingBackend::new();
+        let engine = Engine::new(parts, backend.clone()).unwrap();
+        let (sender, mut receiver) = tokio::sync::oneshot::channel();
+        let observer = runtime
+            .spawn(ContextualReplyObserver {
+                reply: Arc::new(Mutex::new(Some(sender))),
+                engine: engine.handle(),
+                sender: runtime.create_sender(),
+                timeout: Duration::from_millis(1),
+                orchestrator: ActorAddress::default(),
+                cancel_key: Some("withheld-registration".to_owned()),
+            })
+            .unwrap();
+        drive_steps(&backend, STEP_BUDGET);
+        backend.advance_time(Duration::from_secs(1));
+        drive_steps(&backend, STEP_BUDGET);
+        assert!(matches!(
+            receiver.try_recv(),
+            Err(tokio::sync::oneshot::error::TryRecvError::Empty)
+        ));
+        runtime
+            .send_to(observer, ContextualReplyObserverMsg::ArmTimeout)
+            .unwrap();
+        drive_steps(&backend, STEP_BUDGET);
+        backend.advance_time(Duration::from_millis(1));
+        drive_steps(&backend, STEP_BUDGET);
+        assert_eq!(receiver.try_recv().unwrap(), ContextualResponse::TimedOut);
+    }
 }

apps/myelin/src/orchestration/daemon.rs

@@ -13,7 +13,7 @@ use std::path::{Path, PathBuf};
 use std::time::{SystemTime, UNIX_EPOCH};
 
 use crate::orchestration::manual_control::{CommandKind, CommandRecord, CommandState, NodePhase};
-use crate::provisioning::NodeProvisionSpec;
+use crate::provisioning::{DeploymentIdentity, NodeProvisionSpec};
 use distribution::types::NodeId as DistNodeId;
 use serde::{Deserialize, Serialize};
 use swactor::actor::ActorAddress;
@@ -64,6 +64,30 @@ pub(crate) struct RuntimeFacts {
     pub swim_node_id: DistNodeId,
     pub stage_index: u32,
     pub readiness_id: u64,
+    /// Deployment identity reported by the worker. None for providers that
+    /// do not ship deployment bundles.
+    #[serde(default)]
+    pub artifact_digest: Option<String>,
+    #[serde(default)]
+    pub deployment_generation: Option<String>,
+}
+
+impl RuntimeFacts {
+    /// Deployment identity reported by the worker, when the provider ships
+    /// deployment bundles.
+    pub(crate) fn deployment_identity(&self) -> Option<DeploymentIdentity> {
+        Some(DeploymentIdentity {
+            artifact_digest: self.artifact_digest.clone()?,
+            deployment_generation: self.deployment_generation.clone()?,
+        })
+    }
+}
+
+/// Readiness is scoped to a launched process, not its reusable provision attempt.
+pub(crate) fn deployment_readiness_id(incarnation: &str) -> u64 {
+    use sha2::{Digest, Sha256};
+    let digest = Sha256::digest(incarnation.as_bytes());
+    u64::from_be_bytes(digest[..8].try_into().expect("SHA256 prefix"))
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -311,14 +335,32 @@ impl StateDir {
     }
 
     /// Removes all state files. Explicit operator action only.
+    ///
+    /// The full inventory matters: a reset that leaves the data namespace or
+    /// process registry behind recovers bindings from the discarded run, so a
+    /// "fresh" fixture serves stale paths and typed-absence contracts fail
+    /// (observed: a quarantined run's case routes reappeared after reset).
     pub(crate) fn reset(&self) -> Result<(), String> {
-        for path in [self.identity_path(), self.snapshot_path()] {
+        for path in [
+            self.identity_path(),
+            self.snapshot_path(),
+            self.root.join("data-namespace.json"),
+            self.process_registry_path(),
+        ] {
             if let Err(error) = fs::remove_file(&path)
                 && error.kind() != std::io::ErrorKind::NotFound
             {
                 return Err(format!("remove {}: {error}", path.display()));
             }
         }
+        if let Err(error) = fs::remove_dir_all(self.root.join("contextual-uploads"))
+            && error.kind() != std::io::ErrorKind::NotFound
+        {
+            return Err(format!(
+                "remove {}: {error}",
+                self.root.join("contextual-uploads").display()
+            ));
+        }
         Ok(())
     }
 }

apps/myelin/src/orchestration/distribution_stack.rs

@@ -19,7 +19,7 @@ use swactor::std::StdExtension;
 use swactor_engine::EngineHandle;
 use swactor_transport::{CodecRegistry, CodecRemoteSink, NetworkMessage, TransportRouter};
 
-use distribution::directory_actor::{DirectoryActor, DirectoryIn};
+use distribution::directory_actor::{DirectoryActor, DirectoryClaims, DirectoryIn};
 use distribution::messages::{
     DirectoryGossip, MetadataGossip, RegistryGossip, actor_codec_registry,
 };
@@ -103,6 +103,7 @@ pub(crate) struct DistributionRuntimeStack {
     pub pinned_routes: RouteView,
     pub route_binder: Arc<OutboxRouteBinder>,
     pub registry_view: RegistryView,
+    pub directory_claims: DirectoryClaims,
     pub membership_mirror: Arc<Mutex<MemberList>>,
     pub swim_telemetry: Arc<SwimTelemetry>,
     pub swim_config: SwimConfig,
@@ -156,7 +157,7 @@ impl DistributionRuntimeStack {
         config: DistributedNodeConfig,
         engine: EngineHandle,
     ) -> Self {
-        let outbox: Outbox = Arc::new(Mutex::new(Vec::new()));
+        let outbox: Outbox = Arc::new(Default::default());
         let relay_mirror: RelayMirror = Arc::new(RwLock::new(HashMap::new()));
         let route_view: RouteView = Arc::new(RwLock::new(HashMap::new()));
         let pinned_routes: RouteView = Arc::new(RwLock::new(HashMap::new()));
@@ -201,15 +202,15 @@ impl DistributionRuntimeStack {
             Arc::clone(&transport_router),
             Arc::clone(&route_view_transport),
         ));
-        let directory_addr = runtime
-            .spawn(DirectoryActor::with_pinned_routes(
-                node_id,
-                peer_directory,
-                Arc::clone(&route_view),
-                Arc::clone(&pinned_routes),
-                route_binder.clone(),
-            ))
-            .expect("spawn DirectoryActor");
+        let directory = DirectoryActor::with_pinned_routes(
+            node_id,
+            peer_directory,
+            Arc::clone(&route_view),
+            Arc::clone(&pinned_routes),
+            route_binder.clone(),
+        );
+        let directory_claims = directory.claims();
+        let directory_addr = runtime.spawn(directory).expect("spawn DirectoryActor");
 
         let membership_mirror = Arc::new(Mutex::new(MemberList::new(NodeId([0xFF; 32]))));
         let fanout_addr = runtime
@@ -240,6 +241,7 @@ impl DistributionRuntimeStack {
             pinned_routes,
             membership_mirror,
             registry_view,
+            directory_claims,
             swim_telemetry,
             swim_config,
             actors: DistributionActorAddrs {

apps/myelin/src/orchestration/fleet_control.js

@@ -257,11 +257,33 @@
       const body = await response.json().catch(() => ({}));
       if (!response.ok) throw new Error(body.error || `HTTP ${response.status}`);
       const execution = body.execution;
+      if (Number(execution?.truncated_before || 0) > afterSequence) {
+        throw new Error('Execution event history has a gap; output is incomplete');
+      }
       let terminal = Boolean(execution?.terminal);
       for (const record of execution?.events || []) {
         terminal = appendExecutionEvent(panel, record.observation) || terminal;
       }
       if (terminal) {
+        const acknowledged = await fetch(
+          `/api/control/contextual/${encodeURIComponent(requestId)}/ack`,
+          {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+              schema_version: 1, payload: {
+                execution_incarnation: execution.execution_incarnation,
+                through_sequence: execution.next_sequence,
+              },
+            }),
+          },
+        );
+        if (!acknowledged.ok) throw new Error(`Execution acknowledgement failed: HTTP ${acknowledged.status}`);
+        const acknowledgement = await acknowledged.json();
+        if (acknowledgement.schema_version !== 1
+            || acknowledgement.payload?.type !== 'acknowledged') {
+          throw new Error('Unsupported execution acknowledgement response');
+        }
         stopExecutionPoll();
         panel.querySelector('[data-run]').disabled = false;
         return;
@@ -324,11 +346,9 @@
           const observation = body.observation;
           const requestId = observation?.request_id;
           if (!requestId) throw new Error('spawn response did not include a request ID');
-          const terminal = appendExecutionEvent(panel, observation);
-          if (terminal) {
-            run.disabled = false;
-            return;
-          }
+          // Even terminal spawn responses must drain the retained cursor and
+          // acknowledge its exact incarnation; the spawn reply is not a drain.
+          appendExecutionEvent(panel, observation);
           executionPoll = { requestId, afterSequence: 0, timer: null };
           pollExecution(panel, requestId, 0);
         } catch (error) {

apps/myelin/src/orchestration/manual_control.rs

@@ -6,6 +6,12 @@
 use std::collections::{BTreeMap, VecDeque};
 use std::sync::Arc;
 
+use myelin_control_contract::MAX_PROVISION_COUNT;
+pub(crate) use myelin_control_contract::{
+    CommandKind, CommandRecord, CommandState, KillRequest, NodePhase, Offer as OfferDto,
+    OfferSearchRequest, OfferSearchResults, ProviderConfigurationRequest, ProviderReadiness,
+    ProviderReadinessKind, ProvisionRequest,
+};
 use parking_lot::Mutex;
 use serde::{Deserialize, Serialize};
 use swactor::actor::{ActorAddress, ActorInterface};
@@ -24,115 +30,11 @@ use crate::provisioning::{
     ProvisionPlugin,
 };
 
-pub(crate) const MAX_PROVISION_COUNT: u32 = 8;
 pub(crate) const CONTROL_REGISTRY_NAME: &str = "myelin.manual-control";
 pub(crate) const SELECTED_OFFER_ID_ENV: &str = "MYELIN_SELECTED_OFFER_ID";
 pub(crate) const READ_MODEL_COMMAND_LIMIT: usize = 256;
 const OFFER_SEARCH_RECEIPT_TTL_MS: u64 = 5 * 60 * 1_000;
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(rename_all = "snake_case")]
-pub(crate) enum ProviderReadinessKind {
-    Unconfigured,
-    Validating,
-    Ready,
-    ConfigurationError,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
-pub(crate) struct ProviderReadiness {
-    pub name: String,
-    pub provisioning_mode: String,
-    pub runtime_image: String,
-    pub kind: ProviderReadinessKind,
-    pub error: Option<String>,
-}
-
-impl ProviderReadiness {
-    #[cfg(test)]
-    pub(crate) fn ready() -> Self {
-        Self::ready_for("test", "test-image")
-    }
-
-    pub(crate) fn ready_for(name: impl Into<String>, runtime_image: impl Into<String>) -> Self {
-        Self {
-            name: name.into(),
-            provisioning_mode: "real".to_owned(),
-            runtime_image: runtime_image.into(),
-            kind: ProviderReadinessKind::Ready,
-            error: None,
-        }
-    }
-
-    pub(crate) fn unconfigured_for(
-        name: impl Into<String>,
-        runtime_image: impl Into<String>,
-        error: impl Into<String>,
-    ) -> Self {
-        Self {
-            name: name.into(),
-            provisioning_mode: "real".to_owned(),
-            runtime_image: runtime_image.into(),
-            kind: ProviderReadinessKind::Unconfigured,
-            error: Some(error.into()),
-        }
-    }
-
-    pub(crate) fn with_provisioning_mode(mut self, mode: impl Into<String>) -> Self {
-        self.provisioning_mode = mode.into();
-        self
-    }
-}
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(rename_all = "snake_case")]
-pub(crate) enum CommandKind {
-    Provision,
-    Kill,
-    /// Schema-v1 accepted-command IDs had no durable kind or result.
-    Migrated,
-}
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(rename_all = "snake_case")]
-pub(crate) enum CommandState {
-    Persisting,
-    Running,
-    Succeeded,
-    Failed,
-}
-
-impl CommandState {
-    pub(crate) fn is_terminal(self) -> bool {
-        matches!(self, Self::Succeeded | Self::Failed)
-    }
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
-pub(crate) struct CommandRecord {
-    pub command_id: String,
-    pub kind: CommandKind,
-    pub state: CommandState,
-    pub node_ids: Vec<u64>,
-    pub error: Option<String>,
-}
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(rename_all = "snake_case")]
-pub(crate) enum NodePhase {
-    Requested,
-    Creating,
-    Bootstrapping,
-    Joining,
-    Acknowledging,
-    Running,
-    KillRequested,
-    Stopping,
-    StopFailed,
-    Stopped,
-    Orphan,
-}
-
 fn provision_event_kind_for_phase(phase: NodePhase) -> Option<ProvisionEventKind> {
     match phase {
         NodePhase::Requested => Some(ProvisionEventKind::Requested),
@@ -149,43 +51,12 @@ fn provision_event_kind_for_phase(phase: NodePhase) -> Option<ProvisionEventKind
     }
 }
 
-/// Runtime correction input. `api_key` is intentionally absent from Debug and
-/// never enters durable state or a response DTO.
-#[derive(Clone, PartialEq, Serialize, Deserialize)]
-pub(crate) struct ProviderConfigurationRequest {
-    pub api_key: Option<String>,
-    pub ssh_identity: Option<String>,
-    pub bootstrap_command: Option<String>,
+pub(crate) trait OfferSearchRequestExt {
+    fn browse_criteria(&self) -> OfferBrowseCriteria;
 }
 
-impl std::fmt::Debug for ProviderConfigurationRequest {
-    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        formatter
-            .debug_struct("ProviderConfigurationRequest")
-            .field("api_key", &self.api_key.as_ref().map(|_| "[redacted]"))
-            .field("ssh_identity", &self.ssh_identity)
-            .field("bootstrap_command", &self.bootstrap_command)
-            .finish()
-    }
-}
-
-#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
-pub(crate) struct OfferSearchRequest {
-    pub gpu_model: Option<String>,
-    pub min_gpu_ram_mb: Option<u64>,
-    pub min_compute_cap: Option<u64>,
-    pub min_reliability: Option<f64>,
-    pub require_verified: Option<bool>,
-    pub min_download_mbps: Option<f64>,
-    pub min_upload_mbps: Option<f64>,
-    pub max_hourly_price: Option<f64>,
-    #[serde(default)]
-    pub blacklist_hosts: Vec<u64>,
-    pub count: Option<u32>,
-}
-
-impl OfferSearchRequest {
-    pub(crate) fn browse_criteria(&self) -> OfferBrowseCriteria {
+impl OfferSearchRequestExt for OfferSearchRequest {
+    fn browse_criteria(&self) -> OfferBrowseCriteria {
         OfferBrowseCriteria {
             gpu_name_contains: self.gpu_model.clone(),
             min_gpu_ram_mb: self.min_gpu_ram_mb,
@@ -198,88 +69,22 @@ impl OfferSearchRequest {
             blacklist_hosts: self.blacklist_hosts.clone(),
         }
     }
-
-    pub(crate) fn validate(&self) -> Result<(), String> {
-        let finite_non_negative = [
-            ("min_reliability", self.min_reliability),
-            ("min_download_mbps", self.min_download_mbps),
-            ("min_upload_mbps", self.min_upload_mbps),
-            ("max_hourly_price", self.max_hourly_price),
-        ];
-        for (name, value) in finite_non_negative {
-            if value.is_some_and(|value| !value.is_finite() || value < 0.0) {
-                return Err(format!("{name} must be finite and non-negative"));
-            }
-        }
-        if self.min_reliability.is_some_and(|value| value > 1.0) {
-            return Err("min_reliability must not exceed 1".to_owned());
-        }
-        let count = self.count.unwrap_or(1);
-        if count == 0 || count > MAX_PROVISION_COUNT {
-            return Err(format!(
-                "offer count must be between 1 and {MAX_PROVISION_COUNT}"
-            ));
-        }
-        Ok(())
-    }
 }
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub(crate) struct OfferDto {
-    pub offer_id: u64,
-    pub host_id: Option<u64>,
-    pub gpu_model: String,
-    pub gpu_ram_mb: Option<f64>,
-    pub compute_cap: u64,
-    pub verification: Option<String>,
-    pub reliability: Option<f64>,
-    pub download_mbps: Option<f64>,
-    pub upload_mbps: Option<f64>,
-    pub location: Option<String>,
-    pub hourly_price: f64,
-    pub download_cost_per_tb: f64,
-    pub upload_cost_per_tb: f64,
-}
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub(crate) struct OfferSearchResults {
-    pub search_id: u64,
-    pub offers: Vec<OfferDto>,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
-pub(crate) struct ProvisionRequest {
-    pub command_id: String,
-    #[serde(default = "default_one")]
-    pub count: u32,
-    #[serde(default)]
-    pub selected_offer_ids: Vec<u64>,
-    #[serde(default)]
-    pub search_id: Option<u64>,
-    #[serde(default)]
-    pub image: Option<String>,
-}
-
-const fn default_one() -> u32 {
-    1
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
-pub(crate) struct KillRequest {
-    pub command_id: String,
-    pub logical_node_id: u64,
-}
-
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub(crate) struct RejoinHello {
     pub run_id: u64,
     pub logical_node_id: u64,
     pub attempt_id: u64,
+    pub readiness_id: u64,
     pub selected_offer_id: Option<u64>,
     pub endpoint: String,
     pub swim_node_id: distribution::types::NodeId,
     pub stage_index: u32,
     pub node_actor: ActorAddress,
+    #[serde(default)]
+    pub artifact_digest: Option<String>,
+    #[serde(default)]
+    pub deployment_generation: Option<String>,
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -701,6 +506,24 @@ impl ManualControl {
             .snapshot
             .node_mut(node_id)
             .ok_or_else(|| format!("runtime-ready for unknown node {node_id}"))?;
+        if node.phase == NodePhase::Acknowledging && node.runtime.as_ref() == Some(&facts) {
+            // A lost ACK (including the worker's reply) must not strand an
+            // accepted runtime. Coalesce replay while the durable grant or
+            // completion is already pending; never bypass persistence.
+            if self.in_flight.get(&node_id) == Some(&EffectKind::CompleteBootstrap)
+                || self.pending_persists.iter().any(|pending| {
+                    matches!(
+                        pending.after,
+                        AfterPersist::SendRuntimeReadyAck(id)
+                            | AfterPersist::CompleteBootstrap(id) if id == node_id
+                    )
+                })
+            {
+                return Ok(());
+            }
+            self.queue_persist(AfterPersist::SendRuntimeReadyAck(node_id));
+            return Ok(());
+        }
         if node.phase != NodePhase::Joining {
             return Err(format!(
                 "runtime-ready is invalid for node {node_id} in {:?}",
@@ -711,12 +534,26 @@ impl ManualControl {
             .spec
             .as_ref()
             .ok_or_else(|| format!("node {node_id} has no provision intent"))?;
+        if spec
+            .stage_index
+            .is_some_and(|stage| stage != facts.stage_index)
+        {
+            return Err(format!("runtime-ready stage mismatch for node {node_id}"));
+        }
         if facts.run_id != spec.run_id || facts.attempt_id != spec.attempt_id {
             return Err(format!(
                 "runtime-ready identity mismatch for node {node_id}: expected run/attempt {}/{}, got {}/{}",
                 spec.run_id, spec.attempt_id, facts.run_id, facts.attempt_id
             ));
         }
+        if let Some(expected) = &spec.deployment
+            && facts.deployment_identity().as_ref() != Some(expected)
+        {
+            return Err(format!(
+                "runtime-ready deployment identity mismatch for node {node_id}: expected {expected:?}, worker runs {:?}",
+                facts.deployment_identity(),
+            ));
+        }
         node.runtime = Some(facts);
         node.last_seen_unix_ms = unix_ms_now();
         self.queue_persist(AfterPersist::None);
@@ -752,6 +589,15 @@ impl ManualControl {
                 "node ACK does not match node {node_id} acknowledging attempt"
             ));
         }
+        // Replayed durable grants can produce multiple final ACKs before the
+        // provider completes. They represent one admission, not new effects.
+        if self.in_flight.get(&node_id) == Some(&EffectKind::CompleteBootstrap)
+            || self.pending_persists.iter().any(|pending| {
+                matches!(pending.after, AfterPersist::CompleteBootstrap(id) if id == node_id)
+            })
+        {
+            return Ok(());
+        }
         self.queue_persist(AfterPersist::CompleteBootstrap(node_id));
         Ok(())
     }
@@ -799,6 +645,40 @@ impl ManualControl {
                 hello.logical_node_id
             ));
         }
+        if let Some(expected) = &spec.deployment {
+            if node.phase != NodePhase::Running
+                || !node.runtime.as_ref().is_some_and(|runtime| {
+                    runtime.readiness_id == hello.readiness_id
+                        && runtime.node_actor == hello.node_actor
+                        && runtime.swim_node_id == hello.swim_node_id
+                        && runtime.stage_index == hello.stage_index
+                })
+            {
+                return Err(format!(
+                    "rejoin cannot replace receipt-validated readiness for node {}",
+                    hello.logical_node_id
+                ));
+            }
+            let reported = crate::orchestration::daemon::RuntimeFacts {
+                run_id: hello.run_id,
+                attempt_id: hello.attempt_id,
+                endpoint: hello.endpoint.clone(),
+                node_actor: hello.node_actor,
+                swim_node_id: hello.swim_node_id,
+                stage_index: hello.stage_index,
+                readiness_id: hello.readiness_id,
+                artifact_digest: hello.artifact_digest.clone(),
+                deployment_generation: hello.deployment_generation.clone(),
+            };
+            if reported.deployment_identity().as_ref() != Some(expected) {
+                return Err(format!(
+                    "rejoin deployment identity mismatch for node {}: expected {:?}, worker runs {:?}",
+                    hello.logical_node_id,
+                    expected,
+                    reported.deployment_identity(),
+                ));
+            }
+        }
         if matches!(
             node.phase,
             NodePhase::KillRequested
@@ -818,7 +698,9 @@ impl ManualControl {
             node_actor: hello.node_actor,
             swim_node_id: hello.swim_node_id,
             stage_index: hello.stage_index,
-            readiness_id: hello.attempt_id,
+            readiness_id: hello.readiness_id,
+            artifact_digest: hello.artifact_digest.clone(),
+            deployment_generation: hello.deployment_generation.clone(),
         });
         node.phase = NodePhase::Running;
         node.last_error = None;
@@ -1152,6 +1034,17 @@ impl ManualControl {
                 let node = self.snapshot.node_mut(node_id).expect("effect node exists");
                 node.provider_ref = Some(provider_ref);
                 node.last_seen_unix_ms = unix_ms_now();
+                let deployment_stale = node
+                    .spec
+                    .as_ref()
+                    .and_then(|spec| spec.deployment.clone())
+                    .is_some_and(|expected| {
+                        node.runtime
+                            .as_ref()
+                            .and_then(|facts| facts.deployment_identity())
+                            .as_ref()
+                            != Some(&expected)
+                    });
                 let after = match node.phase {
                     NodePhase::Requested | NodePhase::Creating | NodePhase::Bootstrapping => {
                         node.phase = NodePhase::Bootstrapping;
@@ -1161,6 +1054,16 @@ impl ManualControl {
                         node.phase = NodePhase::KillRequested;
                         AfterPersist::Stop(node_id)
                     }
+                    NodePhase::Joining | NodePhase::Acknowledging | NodePhase::Running
+                        if deployment_stale =>
+                    {
+                        // The retained resource runs a stale deployment.
+                        // Keep the resource, refresh its mutable state, and
+                        // require a fresh join before it may run again.
+                        node.phase = NodePhase::Bootstrapping;
+                        node.runtime = None;
+                        AfterPersist::StartBootstrap(node_id)
+                    }
                     NodePhase::Joining
                     | NodePhase::Acknowledging
                     | NodePhase::Running
@@ -1373,6 +1276,7 @@ pub(crate) enum ManualControlMsg {
     },
     JoinBarrierSatisfied {
         node_id: u64,
+        facts: RuntimeFacts,
     },
     Rejoin {
         hello: RejoinHello,
@@ -1427,6 +1331,7 @@ fn refresh_recovery_routing(
             .filter(|(key, _)| ROUTING_KEYS.contains(&key.as_str())),
     );
     persisted.args = current.args;
+    persisted.deployment = current.deployment;
     persisted
 }
 
@@ -1591,9 +1496,6 @@ impl ManualActorControl {
         }
     }
 
-    pub(crate) fn read_model(&self) -> ManualReadModel {
-        self.core.read_model()
-    }
     pub(crate) fn running_node_actor(&self, logical_node_id: u64) -> Result<ActorAddress, String> {
         let node = self
             .core
@@ -1963,8 +1865,18 @@ impl ManualActorControl {
                     self.lanes.remove(&node_id);
                 }
             }
-            ManualControlMsg::JoinBarrierSatisfied { node_id } => {
-                let _ = self.core.join_barrier_satisfied(node_id);
+            ManualControlMsg::JoinBarrierSatisfied { node_id, mut facts } => {
+                if let Some(spec) = self
+                    .core
+                    .snapshot()
+                    .node(node_id)
+                    .and_then(|node| node.spec.as_ref())
+                {
+                    facts.attempt_id = spec.attempt_id;
+                }
+                if self.core.runtime_ready(node_id, facts).is_ok() {
+                    let _ = self.core.join_barrier_satisfied(node_id);
+                }
             }
             ManualControlMsg::Rejoin { hello, reply_to } => {
                 if let Err(error) =
@@ -1985,22 +1897,23 @@ impl ManualActorControl {
         self.finish_flush_waiters(ctx);
     }
 
-    pub(crate) fn observe_runtime_ready(
-        &mut self,
-        actor: ActorAddress,
-        node_id: u64,
-        facts: RuntimeFacts,
-    ) {
-        let _ = self.core.runtime_ready(node_id, facts);
-        self.dispatch_actions(actor);
-    }
-
     pub(crate) fn observe_node_ack(
         &mut self,
         actor: ActorAddress,
+        run_id: u64,
+        stage_index: u32,
         node_id: u64,
         readiness_id: u64,
     ) {
+        if !self
+            .core
+            .snapshot()
+            .node(node_id)
+            .and_then(|node| node.runtime.as_ref())
+            .is_some_and(|facts| facts.run_id == run_id && facts.stage_index == stage_index)
+        {
+            return;
+        }
         let _ = self.core.node_ack(node_id, readiness_id);
         self.dispatch_actions(actor);
     }
@@ -2388,6 +2301,7 @@ mod tests {
 
     fn spec(node_id: u64) -> NodeProvisionSpec {
         NodeProvisionSpec {
+            deployment: None,
             run_id: 7,
             node_id,
             attempt_id: 0,
@@ -2463,6 +2377,8 @@ mod tests {
 
     fn facts(node_id: u64, readiness_id: u64) -> RuntimeFacts {
         RuntimeFacts {
+            artifact_digest: None,
+            deployment_generation: None,
             run_id: 7,
             attempt_id: 0,
             endpoint: format!("endpoint-{node_id}"),
@@ -2575,6 +2491,14 @@ mod tests {
             settle_persistence(&mut core).as_slice(),
             [ManualAction::SendRuntimeReadyAck { node_id: 1, .. }]
         ));
+        // Lose the first acknowledgement. Only a replay of the exact accepted
+        // runtime may recover it, and recovery still waits for persistence.
+        assert!(core.runtime_ready(1, facts(1, 54)).is_err());
+        core.runtime_ready(1, facts(1, 55)).unwrap();
+        assert!(matches!(
+            settle_persistence(&mut core).as_slice(),
+            [ManualAction::SendRuntimeReadyAck { node_id: 1, .. }]
+        ));
         core.node_ack(1, 55).unwrap();
         assert!(matches!(
             settle_persistence(&mut core).as_slice(),
@@ -2592,6 +2516,23 @@ mod tests {
             core.snapshot().commands["provision"].state,
             CommandState::Succeeded
         );
+        let wire =
+            serde_json::to_value(ManualControlReply::FleetStatus(core.fleet_read_model())).unwrap();
+        let myelin_control_contract::ControlReply::FleetStatus(fleet) =
+            serde_json::from_value(wire).expect("shared client must decode the native fleet reply")
+        else {
+            panic!("expected fleet reply");
+        };
+        let runtime = fleet.nodes[0].runtime.as_ref().unwrap();
+        assert_eq!(runtime.readiness_id, 55);
+        assert_eq!(
+            serde_json::to_value(&runtime.swim_node_id).unwrap(),
+            serde_json::to_value([1u8; 32]).unwrap()
+        );
+        assert_eq!(
+            serde_json::to_value(&runtime.node_actor).unwrap(),
+            serde_json::to_value(ActorAddress::default()).unwrap()
+        );
     }
 
     #[test]
@@ -2838,9 +2779,12 @@ mod tests {
         let binding = core
             .rejoin(
                 &RejoinHello {
+                    artifact_digest: None,
+                    deployment_generation: None,
                     run_id: 7,
                     logical_node_id: 1,
                     attempt_id: 0,
+                    readiness_id: 0,
                     selected_offer_id: None,
                     endpoint: "endpoint".to_owned(),
                     swim_node_id: DistNodeId([7; 32]),
@@ -2879,9 +2823,12 @@ mod tests {
         assert!(
             core.rejoin(
                 &RejoinHello {
+                    artifact_digest: None,
+                    deployment_generation: None,
                     run_id: 7,
                     logical_node_id: 1,
                     attempt_id: 1,
+                    readiness_id: 0,
                     selected_offer_id: None,
                     endpoint: "endpoint".to_owned(),
                     swim_node_id: DistNodeId([7; 32]),
@@ -4021,9 +3968,12 @@ mod tests {
                             actor,
                             ManualControlMsg::Rejoin {
                                 hello: RejoinHello {
+                                    artifact_digest: None,
+                                    deployment_generation: None,
                                     run_id: 7,
                                     logical_node_id: u64::from(selector) + 10_000,
                                     attempt_id: 0,
+                                    readiness_id: 0,
                                     selected_offer_id: None,
                                     endpoint: "generated-rejoin".to_owned(),
                                     swim_node_id: DistNodeId([selector; 32]),
@@ -4349,4 +4299,168 @@ mod tests {
         assert_eq!(runtime.stats().actors.len(), baseline);
         assert_mailboxes_drained(&runtime);
     }
+
+    #[test]
+    fn runtime_ready_rejects_stale_deployment_identity() {
+        let mut expected_spec = spec(1);
+        expected_spec.deployment = Some(crate::provisioning::DeploymentIdentity {
+            artifact_digest: "sha256:expected".to_owned(),
+            deployment_generation: "deploy-gen-2".to_owned(),
+        });
+        // The persisted provision intent already demands generation two.
+        let mut snapshot = ClusterSnapshot::fresh(7, "test");
+        snapshot.upsert_node(crate::orchestration::daemon::SnapshotNode {
+            logical_node_id: 1,
+            spec: Some(expected_spec),
+            selected_offer_id: None,
+            provider_ref: None,
+            phase: NodePhase::Joining,
+            runtime: None,
+            last_error: None,
+            last_seen_unix_ms: 0,
+        });
+        let mut core = ManualControl::new(snapshot, ProviderReadiness::ready());
+
+        let mut stale = facts(1, 55);
+        stale.artifact_digest = Some("sha256:stale".to_owned());
+        stale.deployment_generation = Some("deploy-gen-1".to_owned());
+        let rejected = core.runtime_ready(1, stale).unwrap_err();
+        assert!(rejected.contains("deployment identity mismatch"));
+
+        let mut fresh = facts(1, 55);
+        fresh.artifact_digest = Some("sha256:expected".to_owned());
+        fresh.deployment_generation = Some("deploy-gen-2".to_owned());
+        core.runtime_ready(1, fresh).unwrap();
+    }
+
+    #[test]
+    fn rejoin_rejects_stale_deployment_identity() {
+        let mut node_spec = spec(1);
+        node_spec.deployment = Some(crate::provisioning::DeploymentIdentity {
+            artifact_digest: "sha256:expected".to_owned(),
+            deployment_generation: "deploy-gen-2".to_owned(),
+        });
+        let mut node = crate::orchestration::daemon::SnapshotNode {
+            logical_node_id: 1,
+            spec: Some(node_spec.clone()),
+            selected_offer_id: None,
+            provider_ref: Some("static-ssh:node".to_owned()),
+            phase: NodePhase::Running,
+            runtime: None,
+            last_error: None,
+            last_seen_unix_ms: 0,
+        };
+        node.runtime = Some({
+            let mut facts = facts(1, 0);
+            facts.artifact_digest = Some("sha256:expected".to_owned());
+            facts.deployment_generation = Some("deploy-gen-2".to_owned());
+            facts
+        });
+        let mut snapshot = ClusterSnapshot::fresh(7, "test");
+        snapshot.upsert_node(node);
+        let mut core = ManualControl::new(snapshot, ProviderReadiness::ready());
+
+        let mut stale_hello = rejoin_hello(1);
+        stale_hello.artifact_digest = Some("sha256:expected".to_owned());
+        stale_hello.deployment_generation = Some("deploy-gen-1".to_owned());
+        let rejected = core
+            .rejoin(
+                &stale_hello,
+                ActorAddress::default(),
+                0,
+                ActorAddress::default(),
+            )
+            .unwrap_err();
+        assert!(rejected.contains("deployment identity mismatch"));
+
+        let mut fresh_hello = rejoin_hello(1);
+        fresh_hello.artifact_digest = Some("sha256:expected".to_owned());
+        fresh_hello.deployment_generation = Some("deploy-gen-2".to_owned());
+        fresh_hello.readiness_id = 99;
+        assert!(
+            core.rejoin(
+                &fresh_hello,
+                ActorAddress::default(),
+                0,
+                ActorAddress::default(),
+            )
+            .is_err()
+        );
+        fresh_hello.readiness_id = 0;
+        core.rejoin(
+            &fresh_hello,
+            ActorAddress::default(),
+            0,
+            ActorAddress::default(),
+        )
+        .unwrap();
+    }
+
+    #[test]
+    fn recovery_rebootstraps_nodes_running_a_stale_deployment() {
+        let mut node_spec = spec(1);
+        node_spec.deployment = Some(crate::provisioning::DeploymentIdentity {
+            artifact_digest: "sha256:expected".to_owned(),
+            deployment_generation: "deploy-gen-2".to_owned(),
+        });
+        let mut node = crate::orchestration::daemon::SnapshotNode {
+            logical_node_id: 1,
+            spec: Some(node_spec),
+            selected_offer_id: None,
+            provider_ref: Some("static-ssh:node".to_owned()),
+            phase: NodePhase::Running,
+            runtime: None,
+            last_error: None,
+            last_seen_unix_ms: 0,
+        };
+        // The persisted worker still runs generation one.
+        node.runtime = Some({
+            let mut facts = facts(1, 0);
+            facts.artifact_digest = Some("sha256:expected".to_owned());
+            facts.deployment_generation = Some("deploy-gen-1".to_owned());
+            facts
+        });
+        let mut snapshot = ClusterSnapshot::fresh(7, "test");
+        snapshot.upsert_node(node);
+        let mut core = ManualControl::new(snapshot, ProviderReadiness::ready());
+        core.begin_recovery();
+        assert!(matches!(
+            core.take_actions().collect::<Vec<_>>().as_slice(),
+            [ManualAction::Recover { node_id: 1, .. }]
+        ));
+        core.effect_finished(
+            1,
+            EffectKind::Recover,
+            Ok(EffectOutcome::Recovered {
+                provider_ref: Some("static-ssh:node".to_owned()),
+            }),
+        )
+        .unwrap();
+        let effects = settle_persistence(&mut core);
+        assert!(matches!(
+            effects.as_slice(),
+            [ManualAction::StartBootstrap { node_id: 1 }]
+        ));
+        assert_eq!(
+            core.snapshot().node(1).unwrap().phase,
+            NodePhase::Bootstrapping
+        );
+        assert!(core.snapshot().node(1).unwrap().runtime.is_none());
+    }
+
+    fn rejoin_hello(node_id: u64) -> RejoinHello {
+        RejoinHello {
+            run_id: 7,
+            logical_node_id: node_id,
+            attempt_id: 0,
+            readiness_id: 0,
+            selected_offer_id: None,
+            endpoint: "\"endpoint\"".to_owned(),
+            swim_node_id: distribution::types::NodeId([1; 32]),
+            stage_index: 0,
+            node_actor: ActorAddress::default(),
+            artifact_digest: None,
+            deployment_generation: None,
+        }
+    }
 }

apps/myelin/src/orchestration/mod.rs

@@ -14,5 +14,7 @@ pub(crate) mod distribution_stack;
 pub(crate) mod manual_control;
 pub(crate) mod provider_adapters {
     pub(crate) mod relay;
+    pub(crate) mod ssh_bootstrap;
+    pub(crate) mod static_ssh;
     pub(super) mod vastai;
 }

apps/myelin/src/orchestration/node_provisioning.rs

@@ -21,13 +21,18 @@ pub(crate) mod provider_kind {
         ProviderKind::new("vastai")
     }
 
+    pub(crate) fn static_ssh() -> ProviderKind {
+        ProviderKind::new("static-ssh")
+    }
+
     pub(crate) fn parse_deploy(value: &str) -> Result<ProviderKind, String> {
         match value.trim().to_ascii_lowercase().as_str() {
             "process" | "local_process" | "local-process" => Ok(process()),
             "docker" | "local_docker" | "local-docker" => Ok(docker()),
             "vastai" | "vast_ai" | "vast-ai" => Ok(vastai()),
+            "static-ssh" | "static_ssh" | "staticssh" => Ok(static_ssh()),
             other => Err(format!(
-                "unsupported provider {other:?}; use process, docker, or vastai"
+                "unsupported provider {other:?}; use process, docker, vastai, or static-ssh"
             )),
         }
     }

apps/myelin/src/orchestration/provider_adapters/static_ssh.rs

@@ -0,0 +1,540 @@
+//! Static raw-SSH fleet provider.
+//!
+//! A deployment E2E fixture: pre-created blank Docker containers, each on
+//! its own isolated network, reachable only through a published SSH port.
+//! The plugin maps deterministic node identities to fixture slots, ships a
+//! deployment bundle through the shared SSH artifact bootstrap, and stops
+//! containers through local Docker control — never through SSH.
+//!
+//! Slot mapping is derived from the logical node id (`node_id - 1`), not
+//! `stage_index`: fleet provisioning assigns every node `stage_index = 0`,
+//! while logical node ids are unique and persisted across recovery.
+
+use std::collections::BTreeMap;
+use std::path::PathBuf;
+
+use serde::Deserialize;
+use serde_json::json;
+
+use crate::orchestration::provider_adapters::ssh_bootstrap::{
+    SshArtifactBootstrapLauncher, SshBootstrapLauncher, SshEndpoint,
+};
+use crate::provisioning::{
+    AdoptedNode, NodeProvisionSpec, PluginNodeHandle, PluginObservation, PluginSink,
+    ProvisionPlugin,
+};
+
+/// Fixed install path of the worker inside a raw node.
+pub(crate) const WORKER_BIN_PATH: &str = "/opt/myelin/current/bin/myelin-worker";
+
+/// Fixture manifest written by the harness; describes pre-created raw nodes.
+#[derive(Clone, Debug, Deserialize)]
+pub(crate) struct StaticFleetManifest {
+    pub nodes: Vec<StaticFleetNode>,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+pub(crate) struct StaticFleetNode {
+    pub slot: u32,
+    pub container: String,
+    pub ssh_host: String,
+    pub ssh_port: u16,
+    #[serde(default)]
+    pub ssh_user: Option<String>,
+}
+
+impl StaticFleetManifest {
+    pub(crate) fn load(path: &std::path::Path) -> Result<Self, String> {
+        let text = std::fs::read_to_string(path)
+            .map_err(|error| format!("read static fleet manifest {}: {error}", path.display()))?;
+        let manifest: StaticFleetManifest = serde_json::from_str(&text)
+            .map_err(|error| format!("decode static fleet manifest {}: {error}", path.display()))?;
+        if manifest.nodes.is_empty() {
+            return Err(format!(
+                "static fleet manifest {} has no nodes",
+                path.display()
+            ));
+        }
+        let mut slots = std::collections::BTreeSet::new();
+        for node in &manifest.nodes {
+            if !slots.insert(node.slot) {
+                return Err(format!(
+                    "static fleet manifest {} has duplicate slot {}",
+                    path.display(),
+                    node.slot
+                ));
+            }
+        }
+        Ok(manifest)
+    }
+
+    fn node_for_slot(&self, slot: u32) -> Option<&StaticFleetNode> {
+        self.nodes.iter().find(|node| node.slot == slot)
+    }
+}
+
+/// Deterministic node → slot mapping. Logical node ids are assigned from 1.
+fn slot_for_spec(spec: &NodeProvisionSpec) -> Result<u32, String> {
+    let node_id = spec.node_id;
+    if node_id == 0 {
+        return Err("static-ssh provisioning requires a nonzero logical node id".to_owned());
+    }
+    u32::try_from(node_id - 1)
+        .map_err(|_| format!("logical node id {node_id} overflows the fixture slot space"))
+}
+
+pub(crate) struct StaticSshFleetPlugin<B>
+where
+    B: SshBootstrapLauncher,
+{
+    manifest: StaticFleetManifest,
+    bootstrap: B,
+    nodes: BTreeMap<u64, StaticNode<B>>,
+    next_handle_id: u64,
+}
+
+struct StaticNode<B>
+where
+    B: SshBootstrapLauncher,
+{
+    container: String,
+    endpoint: SshEndpoint,
+    spec: NodeProvisionSpec,
+    sink: PluginSink,
+    bootstrap: Option<B::Handle>,
+}
+
+impl StaticSshFleetPlugin<SshArtifactBootstrapLauncher> {
+    pub(crate) fn new(
+        manifest: StaticFleetManifest,
+        bootstrap: SshArtifactBootstrapLauncher,
+    ) -> Self {
+        Self::with_launcher(manifest, bootstrap)
+    }
+}
+
+impl<B> StaticSshFleetPlugin<B>
+where
+    B: SshBootstrapLauncher,
+{
+    pub(crate) fn with_launcher(manifest: StaticFleetManifest, bootstrap: B) -> Self {
+        Self {
+            manifest,
+            bootstrap,
+            nodes: BTreeMap::new(),
+            next_handle_id: 1,
+        }
+    }
+
+    fn node_line(&self, node: &StaticNode<B>, message: impl Into<String>) {
+        node.sink.observe(PluginObservation::ProviderLine {
+            run_id: node.spec.run_id,
+            node_id: node.spec.node_id,
+            line: message.into(),
+        });
+    }
+
+    /// Binds one provision attempt to its fixture slot. Idempotent: the same
+    /// logical node always maps to the same slot and container.
+    fn bind_slot(
+        &mut self,
+        spec: NodeProvisionSpec,
+        sink: &PluginSink,
+    ) -> Result<(u64, String), String> {
+        let slot = slot_for_spec(&spec)?;
+        let fixture = self.manifest.node_for_slot(slot).ok_or_else(|| {
+            format!(
+                "static fleet has no node for slot {slot} (logical node {})",
+                spec.node_id
+            )
+        })?;
+        if crate::provisioning::docker_container_is_absent(&fixture.container)? {
+            return Err(format!(
+                "substrate_lost: static fleet container {} for slot {slot} is absent; deployment cannot replace it",
+                fixture.container
+            ));
+        }
+        if !crate::provisioning::docker_container_is_running(&fixture.container)? {
+            return Err(format!(
+                "substrate_lost: static fleet container {} for slot {slot} is stopped; deployment cannot restart it",
+                fixture.container
+            ));
+        }
+        sink.observe(PluginObservation::ProviderLine {
+            run_id: spec.run_id,
+            node_id: spec.node_id,
+            line: json!({
+                "type": "StaticSshSlotBound",
+                "run_id": spec.run_id,
+                "node_id": spec.node_id,
+                "slot": slot,
+                "container": fixture.container,
+                "ssh_endpoint": format!("{}@{}:{}", fixture.ssh_user.as_deref().unwrap_or("root"), fixture.ssh_host, fixture.ssh_port),
+            })
+            .to_string(),
+        });
+        let handle_id = self.next_handle_id;
+        self.next_handle_id = self.next_handle_id.wrapping_add(1).max(1);
+        self.nodes.insert(
+            handle_id,
+            StaticNode {
+                container: fixture.container.clone(),
+                endpoint: SshEndpoint {
+                    host: fixture.ssh_host.clone(),
+                    port: fixture.ssh_port,
+                    user: fixture
+                        .ssh_user
+                        .clone()
+                        .unwrap_or_else(|| "root".to_owned()),
+                },
+                spec: spec.clone(),
+                sink: sink.clone(),
+                bootstrap: None,
+            },
+        );
+        Ok((handle_id, fixture.container.clone()))
+    }
+
+    fn stop_bootstrap_for(&mut self, handle: &PluginNodeHandle) {
+        if let Some(node) = self.nodes.get_mut(&handle.id)
+            && let Some(mut bootstrap) = node.bootstrap.take()
+        {
+            self.bootstrap.stop_bootstrap(&mut bootstrap);
+        }
+    }
+}
+
+fn docker(args: &[&str]) -> Result<std::process::Output, String> {
+    let mut command = std::process::Command::new("docker");
+    command.args(args);
+    swactor_process::command_output(&mut command)
+        .map_err(|error| format!("spawn docker {:?}: {error}", args))
+}
+
+fn kill_container(container: &str) -> Result<bool, String> {
+    if crate::provisioning::docker_container_is_absent(container)? {
+        return Ok(false);
+    }
+    let output = docker(&["kill", container])?;
+    if !output.status.success() {
+        return Err(format!(
+            "docker kill {container} exited {}: {}",
+            output.status,
+            String::from_utf8_lossy(&output.stderr)
+        ));
+    }
+    Ok(true)
+}
+
+impl<B> ProvisionPlugin for StaticSshFleetPlugin<B>
+where
+    B: SshBootstrapLauncher + 'static,
+{
+    fn create_node(
+        &mut self,
+        spec: NodeProvisionSpec,
+        sink: PluginSink,
+    ) -> Result<PluginNodeHandle, String> {
+        if spec.deployment.is_none() {
+            return Err(format!(
+                "static-ssh node {} requires a deployment identity in its spec",
+                spec.node_id
+            ));
+        }
+        let (handle_id, _container) = self.bind_slot(spec, &sink)?;
+        Ok(PluginNodeHandle {
+            id: handle_id,
+            provider_process_id: None,
+        })
+    }
+
+    fn start_bootstrap(&mut self, handle: &PluginNodeHandle) -> Result<(), String> {
+        let node = self
+            .nodes
+            .get_mut(&handle.id)
+            .ok_or_else(|| format!("static-ssh node handle {} is absent", handle.id))?;
+        if node.bootstrap.is_some() {
+            return Ok(());
+        }
+        let endpoint = node.endpoint.clone();
+        let bootstrap = self.bootstrap.start_bootstrap(
+            node.spec.clone(),
+            endpoint,
+            node.sink.clone(),
+            None,
+            swactor_vastai::LifecyclePolicy::default(),
+        )?;
+        let node = self.nodes.get_mut(&handle.id).expect("node re-registered");
+        node.bootstrap = Some(bootstrap);
+        Ok(())
+    }
+
+    fn cancel_bootstrap(&mut self, handle: &PluginNodeHandle) -> Result<(), String> {
+        self.stop_bootstrap_for(handle);
+        Ok(())
+    }
+
+    fn complete_bootstrap(&mut self, handle: &PluginNodeHandle) -> Result<(), String> {
+        let Some(node) = self.nodes.get(&handle.id) else {
+            return Ok(());
+        };
+        self.node_line(
+            node,
+            json!({
+                "type": "StaticSshBootstrapComplete",
+                "run_id": node.spec.run_id,
+                "node_id": node.spec.node_id,
+                "classification": "runtime_ready_over_data_plane",
+            })
+            .to_string(),
+        );
+        self.stop_bootstrap_for(handle);
+        Ok(())
+    }
+
+    fn stop_node(&mut self, handle: &PluginNodeHandle) -> Result<(), String> {
+        self.stop_bootstrap_for(handle);
+        let Some(node) = self.nodes.remove(&handle.id) else {
+            return Ok(());
+        };
+        let result = kill_container(&node.container);
+        self.node_line(
+            &node,
+            json!({
+                "type": "StaticSshNodeStopped",
+                "run_id": node.spec.run_id,
+                "node_id": node.spec.node_id,
+                "container": node.container,
+                "result": if result.is_ok() { "ok" } else { "failed" },
+                "error": result.as_ref().err(),
+            })
+            .to_string(),
+        );
+        match result {
+            Ok(_) => Ok(()),
+            Err(error) => {
+                self.nodes.insert(handle.id, node);
+                Err(error)
+            }
+        }
+    }
+
+    fn adopt_by_spec(
+        &mut self,
+        spec: &NodeProvisionSpec,
+        sink: PluginSink,
+    ) -> Result<Option<AdoptedNode>, String> {
+        let slot = slot_for_spec(spec)?;
+        let Some(fixture) = self.manifest.node_for_slot(slot) else {
+            return Ok(None);
+        };
+        if crate::provisioning::docker_container_is_absent(&fixture.container)? {
+            return Ok(None);
+        }
+        let (handle_id, container) = self.bind_slot(spec.clone(), &sink)?;
+        Ok(Some(AdoptedNode {
+            handle: PluginNodeHandle {
+                id: handle_id,
+                provider_process_id: None,
+            },
+            provider_ref: format!("static-ssh:{container}"),
+        }))
+    }
+
+    fn provider_ref_for(&self, spec: &NodeProvisionSpec) -> String {
+        let slot = slot_for_spec(spec).unwrap_or(u32::MAX);
+        self.manifest
+            .node_for_slot(slot)
+            .map(|fixture| format!("static-ssh:{}", fixture.container))
+            .unwrap_or_else(|| format!("static-ssh:slot-{slot}"))
+    }
+
+    fn list_managed_refs(&self) -> Result<Vec<String>, String> {
+        let mut refs = Vec::new();
+        for fixture in &self.manifest.nodes {
+            if !crate::provisioning::docker_container_is_absent(&fixture.container)? {
+                refs.push(format!("static-ssh:{}", fixture.container));
+            }
+        }
+        Ok(refs)
+    }
+
+    fn stop_by_spec(&mut self, spec: &NodeProvisionSpec, sink: PluginSink) -> Result<bool, String> {
+        let slot = slot_for_spec(spec)?;
+        let Some(fixture) = self.manifest.node_for_slot(slot) else {
+            return Ok(false);
+        };
+        let stopped = kill_container(&fixture.container)?;
+        sink.observe(PluginObservation::ProviderLine {
+            run_id: spec.run_id,
+            node_id: spec.node_id,
+            line: json!({
+                "type": "StaticSshNodeStoppedBySpec",
+                "run_id": spec.run_id,
+                "node_id": spec.node_id,
+                "container": fixture.container,
+                "existed": stopped,
+            })
+            .to_string(),
+        });
+        Ok(stopped)
+    }
+}
+
+/// Resolved static-ssh provider configuration held by the live orchestrator.
+/// The bundle path is live configuration only; durable state persists the
+/// deployment identity, never host-local paths.
+#[derive(Clone, Debug)]
+pub(crate) struct StaticSshRuntimeConfig {
+    pub manifest: StaticFleetManifest,
+    pub identity_path: PathBuf,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::orchestration::provider_adapters::ssh_bootstrap::{
+        SshBootstrapLauncher, SshEndpoint,
+    };
+    use crate::provisioning::DeploymentIdentity;
+    use std::sync::Mutex;
+    use telemetry::TelemetryProducer;
+
+    fn manifest_text(containers: &[&str]) -> String {
+        let nodes = containers
+            .iter()
+            .enumerate()
+            .map(|(slot, container)| {
+                format!(
+                    "{{\"slot\":{slot},\"container\":\"{container}\",\
+                     \"ssh_host\":\"127.0.0.1\",\"ssh_port\":{}}}",
+                    32_100 + slot
+                )
+            })
+            .collect::<Vec<_>>()
+            .join(",");
+        format!("{{\"nodes\":[{nodes}]}}")
+    }
+
+    fn manifest(containers: &[&str]) -> StaticFleetManifest {
+        let directory = tempfile::tempdir().expect("test directory");
+        let path = directory.path().join("manifest.json");
+        std::fs::write(&path, manifest_text(containers)).expect("write manifest");
+        StaticFleetManifest::load(&path).expect("valid manifest")
+    }
+
+    fn deployment_spec(node_id: u64) -> NodeProvisionSpec {
+        NodeProvisionSpec {
+            deployment: Some(DeploymentIdentity {
+                artifact_digest: "sha256:abcd".to_owned(),
+                deployment_generation: "gen".to_owned(),
+            }),
+            run_id: 7,
+            node_id,
+            attempt_id: 0,
+            stage_index: Some(0),
+            image: "raw".to_owned(),
+            env: Vec::new(),
+            args: vec!["exec /opt/myelin/current/bin/myelin-worker".to_owned()],
+            offer_criteria_json: None,
+            mounts: Vec::new(),
+        }
+    }
+
+    /// Records bootstrap launches; owns no processes.
+    struct RecordingLauncher {
+        started: Mutex<Vec<(u64, SshEndpoint)>>,
+    }
+
+    impl SshBootstrapLauncher for RecordingLauncher {
+        type Handle = u64;
+
+        fn start_bootstrap(
+            &mut self,
+            spec: NodeProvisionSpec,
+            endpoint: SshEndpoint,
+            _sink: PluginSink,
+            _producer: Option<TelemetryProducer>,
+            _lifecycle: swactor_vastai::LifecyclePolicy,
+        ) -> Result<Self::Handle, String> {
+            self.started
+                .lock()
+                .expect("recorder")
+                .push((spec.node_id, endpoint));
+            Ok(spec.node_id)
+        }
+
+        fn stop_bootstrap(&mut self, _handle: &mut Self::Handle) {}
+    }
+
+    #[test]
+    fn slot_mapping_is_node_id_minus_one() {
+        assert_eq!(slot_for_spec(&deployment_spec(1)).unwrap(), 0);
+        assert_eq!(slot_for_spec(&deployment_spec(5)).unwrap(), 4);
+        assert!(slot_for_spec(&deployment_spec(0)).is_err());
+        let mut overflow = deployment_spec(1);
+        overflow.node_id = u64::MAX;
+        assert!(slot_for_spec(&overflow).is_err());
+    }
+
+    #[test]
+    fn manifest_rejects_duplicates_and_empty_fleets() {
+        let directory = tempfile::tempdir().expect("test directory");
+        let duplicate = directory.path().join("duplicate.json");
+        let mut text = manifest_text(&["a"]);
+        text.truncate(text.len() - 2);
+        text.push_str(
+            ",{\"slot\":0,\"container\":\"b\",\"ssh_host\":\"127.0.0.1\",\"ssh_port\":1}]}",
+        );
+        std::fs::write(&duplicate, text).unwrap();
+        assert!(StaticFleetManifest::load(&duplicate).is_err());
+        let empty = directory.path().join("empty.json");
+        std::fs::write(&empty, "{\"nodes\":[]}").unwrap();
+        assert!(StaticFleetManifest::load(&empty).is_err());
+    }
+
+    #[test]
+    fn provider_ref_is_stable_and_slot_scoped() {
+        let fleet = manifest(&["raw-a", "raw-b"]);
+        let launcher = RecordingLauncher {
+            started: Mutex::new(Vec::new()),
+        };
+        let plugin = StaticSshFleetPlugin::with_launcher(fleet, launcher);
+        assert_eq!(
+            plugin.provider_ref_for(&deployment_spec(1)),
+            "static-ssh:raw-a"
+        );
+        assert_eq!(
+            plugin.provider_ref_for(&deployment_spec(2)),
+            "static-ssh:raw-b"
+        );
+    }
+
+    #[test]
+    fn create_node_requires_an_identity_and_a_live_slot() {
+        let fleet = manifest(&["myelin-test-absent-container"]);
+        let launcher = RecordingLauncher {
+            started: Mutex::new(Vec::new()),
+        };
+        let mut plugin = StaticSshFleetPlugin::with_launcher(fleet, launcher);
+        let sink = PluginSink::new(std::sync::Arc::new(NullSink));
+        let mut unidentified = deployment_spec(1);
+        unidentified.deployment = None;
+        assert!(plugin.create_node(unidentified, sink.clone()).is_err());
+        // Slot 2 has no fixture node at all.
+        let out_of_range = plugin
+            .create_node(deployment_spec(3), sink.clone())
+            .unwrap_err();
+        assert!(out_of_range.contains("no node for slot 2"));
+        // Slot 0 exists in the manifest but its container is absent, so the
+        // attempt fails semantically instead of retrying.
+        let absent = plugin.create_node(deployment_spec(1), sink).unwrap_err();
+        assert!(absent.contains("absent"));
+    }
+
+    struct NullSink;
+
+    impl crate::provisioning::PluginObservationSink for NullSink {
+        fn observe(&self, _observation: PluginObservation) {}
+    }
+}

apps/myelin/src/orchestration/provisioning.rs

@@ -6,11 +6,13 @@
 
 use std::collections::BTreeMap;
 use std::fs::{self, File, OpenOptions};
-use std::io::Write;
+use std::io::Read;
+use std::os::fd::{AsRawFd, FromRawFd, OwnedFd};
+use std::os::unix::ffi::OsStrExt;
 use std::path::{Path, PathBuf};
 use std::process::{Child, ChildStdin, Command, Stdio};
 use std::sync::{Arc, Mutex};
-use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
 
 #[cfg(target_os = "linux")]
 use std::os::unix::process::CommandExt;
@@ -21,10 +23,12 @@ pub use ::provisioning::plugin::{
     ProvisionLogStream, ProvisionPlugin,
 };
 use iroh::EndpointAddr;
+pub use myelin_control_contract::DeploymentIdentity;
 use swactor::actor::{ActorAddress, ActorInterface};
 use swactor::runtime::{Ctx, Runtime};
+use swactor_process::ProcessWatch;
 
-use crate::node::worker_node_runtime::request_debug_join;
+use crate::node::worker_node_runtime::{DEBUG_JOIN_TIMEOUT, request_debug_join};
 use crate::observability::provisioning_logs::BootstrapTelemetryBridge;
 use crate::orchestration::manual_control::SELECTED_OFFER_ID_ENV;
 
@@ -51,6 +55,7 @@ trait DockerLifecycleBackend: Send + Sync {
         prefix: &str,
         spec: &NodeProvisionSpec,
     ) -> Result<Option<(String, bool)>, String>;
+    fn rejoin(&self, node: &LocalDockerNode) -> Result<(), String>;
     fn observe(&self, runtime: &Runtime, node: &LocalDockerNode, tail: &str) -> Result<(), String>;
     fn list_managed(&self, prefix: &str) -> Result<Vec<String>, String>;
 }
@@ -88,6 +93,8 @@ struct LocalProcessNode {
     sink: PluginSink,
     pid: Option<u32>,
     runtime: Option<LocalProcessRuntime>,
+    observer: Option<ProcessWatch>,
+    output: Option<ProcessOutput>,
 }
 
 struct LocalProcessRuntime {
@@ -96,6 +103,157 @@ struct LocalProcessRuntime {
     exit_actor: Option<ActorAddress>,
 }
 
+fn process_exit_fd(pid: u32) -> Option<OwnedFd> {
+    let fd = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, 0) as i32 };
+    (fd >= 0).then(|| unsafe { OwnedFd::from_raw_fd(fd) })
+}
+
+fn file_change_fd(path: &Path) -> Option<OwnedFd> {
+    let path = std::ffi::CString::new(path.as_os_str().as_bytes()).ok()?;
+    let raw = unsafe { libc::inotify_init1(libc::IN_CLOEXEC | libc::IN_NONBLOCK) };
+    if raw < 0 {
+        return None;
+    }
+    let fd = unsafe { OwnedFd::from_raw_fd(raw) };
+    let mask = libc::IN_MODIFY
+        | libc::IN_CLOSE_WRITE
+        | libc::IN_CREATE
+        | libc::IN_MOVED_TO
+        | libc::IN_DELETE_SELF
+        | libc::IN_MOVE_SELF;
+    (unsafe { libc::inotify_add_watch(fd.as_raw_fd(), path.as_ptr(), mask) } >= 0).then_some(fd)
+}
+
+fn drain_change(fd: &OwnedFd) {
+    let mut bytes = [0_u8; 4096];
+    while unsafe { libc::read(fd.as_raw_fd(), bytes.as_mut_ptr().cast(), bytes.len()) } > 0 {}
+}
+
+fn wait_fds<const N: usize>(
+    fds: &[Option<&OwnedFd>; N],
+    timeout: Duration,
+) -> std::io::Result<[bool; N]> {
+    let mut descriptors: [libc::pollfd; N] = std::array::from_fn(|index| libc::pollfd {
+        fd: fds[index].map_or(-1, AsRawFd::as_raw_fd),
+        events: libc::POLLIN,
+        revents: 0,
+    });
+    let millis = timeout.as_millis().min(i32::MAX as u128) as i32;
+    let result = unsafe {
+        libc::poll(
+            descriptors.as_mut_ptr(),
+            descriptors.len() as libc::nfds_t,
+            millis,
+        )
+    };
+    if result < 0 {
+        let error = std::io::Error::last_os_error();
+        if error.kind() != std::io::ErrorKind::Interrupted {
+            return Err(error);
+        }
+    }
+    Ok(std::array::from_fn(|index| descriptors[index].revents != 0))
+}
+
+fn cancellation_fd() -> Result<Arc<OwnedFd>, String> {
+    let fd = unsafe { libc::eventfd(0, libc::EFD_CLOEXEC | libc::EFD_NONBLOCK) };
+    if fd < 0 {
+        return Err(format!(
+            "create process observer cancellation: {}",
+            std::io::Error::last_os_error()
+        ));
+    }
+    Ok(Arc::new(unsafe { OwnedFd::from_raw_fd(fd) }))
+}
+
+fn cancel_observer(fd: &OwnedFd) {
+    let value = 1_u64;
+    unsafe {
+        libc::write(
+            fd.as_raw_fd(),
+            (&value as *const u64).cast(),
+            std::mem::size_of::<u64>(),
+        );
+    }
+}
+
+struct FollowLocalFile {
+    file: File,
+    identity: swactor_process::ProcessIdentity,
+    changed: Option<OwnedFd>,
+    exited: Option<OwnedFd>,
+    cancel: Arc<OwnedFd>,
+}
+
+impl FollowLocalFile {
+    fn new(file: File, identity: swactor_process::ProcessIdentity, cancel: Arc<OwnedFd>) -> Self {
+        // Watch the actual open inode, not a path that may be replaced.
+        let changed = file_change_fd(Path::new(&format!("/proc/self/fd/{}", file.as_raw_fd())));
+        let exited = process_exit_fd(identity.pid);
+        Self {
+            file,
+            identity,
+            changed,
+            exited,
+            cancel,
+        }
+    }
+}
+
+impl Read for FollowLocalFile {
+    fn read(&mut self, buffer: &mut [u8]) -> std::io::Result<usize> {
+        loop {
+            if wait_fds(&[Some(&self.cancel)], Duration::ZERO)?[0] {
+                return Ok(0);
+            }
+            let count = self.file.read(buffer)?;
+            if count != 0 || !self.identity.matches() {
+                return Ok(count);
+            }
+            let ready = wait_fds(
+                &[
+                    self.changed.as_ref(),
+                    self.exited.as_ref(),
+                    Some(&self.cancel),
+                ],
+                if self.changed.is_some() {
+                    Duration::from_secs(30)
+                } else {
+                    Duration::from_millis(50)
+                },
+            )?;
+            if ready[2] {
+                return Ok(0);
+            }
+            if ready[1] {
+                return self.file.read(buffer);
+            }
+            if ready[0] {
+                if let Some(changed) = &self.changed {
+                    drain_change(changed);
+                }
+            }
+        }
+    }
+}
+
+struct ProcessOutput {
+    cancel: Arc<OwnedFd>,
+    readers: Vec<swactor_process::LineReaderHandle>,
+    actor: ActorAddress,
+    runtime: Runtime,
+}
+
+impl Drop for ProcessOutput {
+    fn drop(&mut self) {
+        cancel_observer(&self.cancel);
+        for reader in self.readers.drain(..) {
+            reader.join();
+        }
+        let _ = self.runtime.stop_actor(self.actor);
+    }
+}
+
 #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
 struct LocalProcessRecord {
     pid: u32,
@@ -414,6 +572,9 @@ fn discover_process_record(
             spec.node_id.to_string(),
         ),
     ];
+    // Linux /proc does not provide reliable inotify creation/exec events.
+    // Retain bounded discovery reconciliation; socket and child waits below
+    // use kernel notifications once an exact identity has been discovered.
     let mut matches = swactor_process::find_process_identities_with_retry(
         &environment,
         true,
@@ -467,32 +628,89 @@ fn remove_debug_join_socket(spec: &NodeProvisionSpec) -> Result<(), String> {
     }
 }
 
-fn request_process_rejoin(spec: &NodeProvisionSpec) -> Result<(), String> {
+pub(crate) const EXECUTION_OWNER_DEADLINE_ENV: &str = "MYELIN_E2E_EXECUTION_DEADLINE_MONOTONIC_MS";
+
+pub(crate) fn execution_owner_deadline() -> Result<Option<Instant>, String> {
+    let Some(value) = std::env::var_os(EXECUTION_OWNER_DEADLINE_ENV) else {
+        return Ok(None);
+    };
+    let deadline = value
+        .to_str()
+        .and_then(|value| value.parse::<u64>().ok())
+        .ok_or_else(|| "invalid execution owner monotonic deadline".to_owned())?;
+    let started = Instant::now();
+    let mut now: libc::timespec = unsafe { std::mem::zeroed() };
+    if unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut now) } != 0 {
+        return Err(format!(
+            "read execution owner clock: {}",
+            std::io::Error::last_os_error()
+        ));
+    }
+    let now = Duration::from_secs(now.tv_sec as u64) + Duration::from_nanos(now.tv_nsec as u64);
+    let remaining = Duration::from_millis(deadline).saturating_sub(now);
+    if remaining.is_zero() {
+        Err("execution budget expired; pending owned operation/child exit".to_owned())
+    } else {
+        Ok(Some(started + remaining))
+    }
+}
+
+fn recovery_route_env(spec: &NodeProvisionSpec) -> Result<Option<(&str, &str)>, String> {
     let Some(endpoint_json) = spec
         .env
         .iter()
         .find_map(|(key, value)| (key == "MYELIN_COORDINATOR_ENDPOINT").then_some(value.as_str()))
     else {
-        return Ok(());
+        return Ok(None);
     };
-    let endpoint = serde_json::from_str::<EndpointAddr>(endpoint_json)
-        .map_err(|error| format!("parse recovery coordinator endpoint: {error}"))?;
     let actor_json = spec
         .env
         .iter()
         .find_map(|(key, value)| (key == "MYELIN_ORCHESTRATOR_ACTOR").then_some(value.as_str()))
         .ok_or_else(|| "recovery spec has no orchestrator actor".to_owned())?;
+    Ok(Some((endpoint_json, actor_json)))
+}
+
+fn request_process_rejoin(spec: &NodeProvisionSpec) -> Result<(), String> {
+    let Some((endpoint_json, actor_json)) = recovery_route_env(spec)? else {
+        return Ok(());
+    };
+    let endpoint = serde_json::from_str::<EndpointAddr>(endpoint_json)
+        .map_err(|error| format!("parse recovery coordinator endpoint: {error}"))?;
     let orchestrator_actor = serde_json::from_str::<ActorAddress>(actor_json)
         .map_err(|error| format!("parse recovery orchestrator actor: {error}"))?;
     let socket = debug_join_socket_path(spec);
-    if !swactor_process::wait_for_path(&socket, 40, Duration::from_millis(50)) {
+    let change = socket.parent().and_then(file_change_fd);
+    let deadline = Instant::now() + Duration::from_secs(2);
+    while !socket.exists() && Instant::now() < deadline {
+        wait_fds(
+            &[change.as_ref()],
+            deadline
+                .saturating_duration_since(Instant::now())
+                .min(if change.is_some() {
+                    Duration::from_secs(2)
+                } else {
+                    Duration::from_millis(50)
+                }),
+        )
+        .map_err(|error| format!("wait rejoin socket {}: {error}", socket.display()))?;
+        if let Some(change) = &change {
+            drain_change(change);
+        }
+    }
+    if !socket.exists() {
         return Err(format!(
             "rejoin socket {} did not become ready",
             socket.display()
         ));
     }
-    request_debug_join(&socket, endpoint, orchestrator_actor)
-        .map_err(|error| format!("rejoin local worker through {}: {error}", socket.display()))
+    request_debug_join(
+        &socket,
+        endpoint,
+        orchestrator_actor,
+        deadline.saturating_duration_since(Instant::now()),
+    )
+    .map_err(|error| format!("rejoin local worker through {}: {error}", socket.display()))
 }
 
 fn observe_process_files(
@@ -500,7 +718,7 @@ fn observe_process_files(
     spec: NodeProvisionSpec,
     sink: PluginSink,
     record: LocalProcessRecord,
-) -> Result<(), String> {
+) -> Result<ProcessOutput, String> {
     let stdout = File::open(&record.stdout_path).map_err(|error| {
         format!(
             "open local process stdout {}: {error}",
@@ -513,21 +731,34 @@ fn observe_process_files(
             record.stderr_path.display()
         )
     })?;
-    observe_output_streams(
-        runtime,
-        spec,
-        sink,
-        swactor_process::FollowProcessFile::new(
-            stdout,
-            record.identity(),
-            Duration::from_millis(50),
+    let cancel = cancellation_fd()?;
+    let actor = runtime
+        .spawn(BootstrapOutputActor {
+            bridge: BootstrapTelemetryBridge::new(spec, sink, None),
+            closed: 0,
+        })
+        .map_err(|error| format!("spawn local process output actor: {error}"))?;
+    let sender = runtime.create_sender();
+    let readers = vec![
+        swactor_process::spawn_line_reader(
+            swactor_process::ProcessStream::Stdout,
+            FollowLocalFile::new(stdout, record.identity(), Arc::clone(&cancel)),
+            sender.clone(),
+            actor,
         ),
-        swactor_process::FollowProcessFile::new(
-            stderr,
-            record.identity(),
-            Duration::from_millis(50),
+        swactor_process::spawn_line_reader(
+            swactor_process::ProcessStream::Stderr,
+            FollowLocalFile::new(stderr, record.identity(), Arc::clone(&cancel)),
+            sender,
+            actor,
         ),
-    )
+    ];
+    Ok(ProcessOutput {
+        cancel,
+        readers,
+        actor,
+        runtime: runtime.clone(),
+    })
 }
 
 fn observe_output_streams(
@@ -607,7 +838,7 @@ fn docker_inspect_error_is_absent(stderr: &str) -> bool {
     stderr.contains("no such object") || stderr.contains("no such container")
 }
 
-fn docker_container_is_absent(name: &str) -> Result<bool, String> {
+pub(crate) fn docker_container_is_absent(name: &str) -> Result<bool, String> {
     let output = swactor_process::command_output(Command::new("docker").arg("inspect").arg(name))
         .map_err(|error| format!("inspect Docker container {name}: {error}"))?;
     if output.status.success() {
@@ -625,7 +856,7 @@ fn docker_container_is_absent(name: &str) -> Result<bool, String> {
     }
 }
 
-fn docker_container_is_running(name: &str) -> Result<bool, String> {
+pub(crate) fn docker_container_is_running(name: &str) -> Result<bool, String> {
     let output = swactor_process::command_output(
         Command::new("docker")
             .args(["inspect", "-f", "{{.State.Running}}"])
@@ -884,29 +1115,23 @@ fn docker_status_vec(args: Vec<String>, label: &str) -> Result<(), String> {
 }
 
 fn stop_owned_process(runtime: &mut LocalProcessRuntime) -> Result<Option<i32>, String> {
-    let _ = runtime.stdin.write_all(b"shutdown\n");
-    let _ = runtime.stdin.flush();
-    swactor_process::wait_shared_child_or_kill(
+    swactor_process::stop_shared_child_with_input(
         &runtime.child,
-        Duration::from_secs(2),
-        true,
-        Duration::from_millis(50),
+        &mut runtime.stdin,
+        b"shutdown\n",
+        Instant::now() + Duration::from_secs(2),
     )
     .map(|status| status.and_then(|status| status.code()))
-    .map_err(|error| format!("stop local process node: {error}"))
+    .map_err(|error| format!("stop local process: {error}"))
 }
 
 fn stop_adopted_process(record: &LocalProcessRecord) -> Result<(), String> {
-    #[cfg(target_os = "linux")]
-    {
-        swactor_process::terminate_process_group(
-            &record.identity(),
-            Duration::from_secs(2),
-            Duration::from_millis(50),
-        )
-    }
-    #[cfg(not(target_os = "linux"))]
-    Ok(())
+    swactor_process::terminate_process_group(
+        &record.identity(),
+        Duration::from_secs(2),
+        Duration::from_millis(50),
+    )
+    .map_err(|error| format!("stop adopted node {}: {error}", record.pid))
 }
 
 fn observe_adopted_process(
@@ -916,7 +1141,7 @@ fn observe_adopted_process(
     registry_path: PathBuf,
     provider_ref: String,
     record: LocalProcessRecord,
-) -> Result<(), String> {
+) -> Result<ProcessWatch, String> {
     let actor = runtime
         .spawn(ProcessExitActor {
             spec,
@@ -924,13 +1149,8 @@ fn observe_adopted_process(
             registry: Some((registry_path, provider_ref)),
         })
         .map_err(|error| format!("spawn adopted process observer actor: {error}"))?;
-    swactor_process::spawn_identity_exit_wait(
-        record.identity(),
-        Duration::from_millis(100),
-        runtime.create_sender(),
-        actor,
-    );
-    Ok(())
+    swactor_process::watch_process(record.identity(), None, runtime.create_sender(), actor)
+        .map_err(|error| format!("watch adopted process: {error}"))
 }
 
 impl ProvisionPlugin for LocalProcessPlugin {
@@ -951,6 +1171,8 @@ impl ProvisionPlugin for LocalProcessPlugin {
                 sink,
                 pid: None,
                 runtime: None,
+                observer: None,
+                output: None,
             },
         );
         Ok(handle)
@@ -1068,7 +1290,12 @@ impl ProvisionPlugin for LocalProcessPlugin {
             child: Arc::clone(&child),
             exit_actor: None,
         });
-        observe_process_files(&self.runtime, spec.clone(), sink.clone(), record)?;
+        node.output = Some(observe_process_files(
+            &self.runtime,
+            spec.clone(),
+            sink.clone(),
+            record.clone(),
+        )?);
         let exit_actor = self
             .runtime
             .spawn(ProcessExitActor {
@@ -1081,11 +1308,14 @@ impl ProvisionPlugin for LocalProcessPlugin {
             .as_mut()
             .expect("local process runtime was installed before its exit observer")
             .exit_actor = Some(exit_actor);
-        swactor_process::spawn_shared_child_wait(
-            child,
-            Duration::from_millis(100),
-            self.runtime.create_sender(),
-            exit_actor,
+        node.observer = Some(
+            swactor_process::watch_process(
+                record.identity(),
+                Some(child),
+                self.runtime.create_sender(),
+                exit_actor,
+            )
+            .map_err(|error| format!("watch local process: {error}"))?,
         );
         Ok(())
     }
@@ -1197,8 +1427,9 @@ impl ProvisionPlugin for LocalProcessPlugin {
             return Ok(None);
         };
         request_process_rejoin(spec)?;
-        observe_process_files(&self.runtime, spec.clone(), sink.clone(), record.clone())?;
-        observe_adopted_process(
+        let output =
+            observe_process_files(&self.runtime, spec.clone(), sink.clone(), record.clone())?;
+        let observer = observe_adopted_process(
             &self.runtime,
             spec.clone(),
             sink.clone(),
@@ -1218,6 +1449,8 @@ impl ProvisionPlugin for LocalProcessPlugin {
                 sink,
                 pid: Some(record.pid),
                 runtime: None,
+                observer: Some(observer),
+                output: Some(output),
             },
         );
         Ok(Some(AdoptedNode {
@@ -1462,6 +1695,44 @@ impl DockerLifecycleBackend for SystemDockerLifecycle {
         Ok(Some((name, running)))
     }
 
+    fn rejoin(&self, node: &LocalDockerNode) -> Result<(), String> {
+        let Some((endpoint_json, actor_json)) = recovery_route_env(&node.spec)? else {
+            return Ok(());
+        };
+        // Docker keeps the original worker environment on adoption. Send the
+        // live endpoint and control actor through its existing debug socket;
+        // replaying old readiness logs cannot reconnect either control route.
+        let deadline = Instant::now() + DEBUG_JOIN_TIMEOUT;
+        let deadline = execution_owner_deadline()?.map_or(deadline, |owner| owner.min(deadline));
+        let mut command = Command::new("docker");
+        command.arg("exec");
+        if std::env::var_os(EXECUTION_OWNER_DEADLINE_ENV).is_some() {
+            // The container shares the host monotonic clock, but docker exec
+            // otherwise inherits the old container environment, not this owner.
+            command.arg("--env").arg(EXECUTION_OWNER_DEADLINE_ENV);
+        }
+        command
+            .arg(&node.container_name)
+            .args(["/usr/local/bin/myelin-node", "debug-join", "--socket"])
+            .arg(debug_join_socket_path(&node.spec))
+            .arg("--endpoint-json")
+            .arg(endpoint_json)
+            .arg("--orchestrator-actor-json")
+            .arg(actor_json);
+        let output = swactor_process::command_output_until(&mut command, deadline)
+            .map_err(|error| format!("rejoin Docker worker {}: {error}", node.container_name))?;
+        if !output.status.success() {
+            return Err(format!(
+                "rejoin Docker worker {} exited with {}: stdout={}, stderr={}",
+                node.container_name,
+                output.status,
+                String::from_utf8_lossy(&output.stdout).trim(),
+                String::from_utf8_lossy(&output.stderr).trim()
+            ));
+        }
+        Ok(())
+    }
+
     fn observe(&self, runtime: &Runtime, node: &LocalDockerNode, tail: &str) -> Result<(), String> {
         observe_docker_container(
             runtime,
@@ -1617,6 +1888,10 @@ impl ProvisionPlugin for LocalDockerPlugin {
             node.container_name == docker_container_name(&self.container_name_prefix, spec)
         }) {
             node.sink = sink;
+            node.spec = spec.clone();
+            if self.backend.container_state(&node.container_name)? == Some(true) {
+                self.backend.rejoin(node)?;
+            }
             return Ok(Some(AdoptedNode {
                 handle: PluginNodeHandle {
                     id,
@@ -1641,6 +1916,7 @@ impl ProvisionPlugin for LocalDockerPlugin {
             container_name: container_name.clone(),
         };
         if running {
+            self.backend.rejoin(&node)?;
             // Replay this container's bootstrap log into the fresh daemon,
             // then follow new output. The replay supplies runtime facts when
             // the prior daemon died before persisting readiness.
@@ -1799,6 +2075,7 @@ mod tests {
 
     fn test_spec() -> NodeProvisionSpec {
         NodeProvisionSpec {
+            deployment: None,
             run_id: 5,
             node_id: 7,
             attempt_id: 11,
@@ -1933,6 +2210,29 @@ mod tests {
     #[cfg(target_os = "linux")]
     #[test]
     fn local_process_stop_releases_all_observer_actors() {
+        // /proc/self/fd is process-wide; sibling tests must not contribute
+        // descriptors to this resource-ownership assertion.
+        const CHILD_ENV: &str = "MYELIN_LOCAL_PROCESS_FD_TEST_CHILD";
+        if std::env::var_os(CHILD_ENV).is_none() {
+            let output = swactor_process::command_output_until(
+                std::process::Command::new(std::env::current_exe().unwrap())
+                    .args([
+                        "--exact",
+                        "provisioning::tests::local_process_stop_releases_all_observer_actors",
+                        "--nocapture",
+                    ])
+                    .env(CHILD_ENV, "1"),
+                std::time::Instant::now() + Duration::from_secs(15),
+            )
+            .expect("complete isolated local-process resource check");
+            assert!(
+                output.status.success(),
+                "isolated local-process resource check failed:\n{}\n{}",
+                String::from_utf8_lossy(&output.stdout),
+                String::from_utf8_lossy(&output.stderr),
+            );
+            return;
+        }
         let (_engine, runtime) = test_runtime();
         let baseline_actors = runtime.stats().actors.len();
         let baseline_fds = fs::read_dir("/proc/self/fd").unwrap().count();
@@ -2163,6 +2463,7 @@ mod tests {
         next_resource_id: u64,
         fail_next_start: bool,
         fail_next_remove: bool,
+        fail_next_rejoin: bool,
         started_specs: Vec<NodeProvisionSpec>,
     }
 
@@ -2389,6 +2690,13 @@ mod tests {
             Ok(self.container_state(&name)?.map(|running| (name, running)))
         }
 
+        fn rejoin(&self, _node: &LocalDockerNode) -> Result<(), String> {
+            if std::mem::take(&mut self.state.lock().fail_next_rejoin) {
+                return Err("scripted Docker worker rejected rejoin".to_owned());
+            }
+            Ok(())
+        }
+
         fn observe(
             &self,
             runtime: &Runtime,
@@ -2426,6 +2734,59 @@ mod tests {
         }
     }
 
+    #[test]
+    fn docker_adoption_requires_rejoin_without_replacing_retained_resource() {
+        let parts = RuntimeParts::new(swactor::config::RuntimeConfig::default());
+        let runtime = parts.runtime().clone();
+        let stepping = SteppingBackend::new();
+        let _engine = Engine::new(parts, stepping.clone()).expect("stepping engine");
+        let baseline_actors = runtime.stats().actors.len();
+        let sink = PluginSink::new(Arc::new(RecordingDockerSink::default()));
+        let backend = Arc::new(ScriptedDockerBackend::default());
+        let mut plugin =
+            LocalDockerPlugin::with_backend("myelin", runtime.clone(), backend.clone());
+        let spec = test_spec();
+        let name = docker_container_name("myelin", &spec);
+        backend.seed_orphan(&name, true).unwrap();
+        let retained_id = backend.state.lock().resources[0].id;
+        backend.state.lock().fail_next_rejoin = true;
+
+        // Existing bootstrap logs are not proof of a current control binding.
+        // A rejected rejoin must not report a successful adoption or dispose
+        // of the live worker, so recovery can retry the same external resource.
+        assert!(plugin.adopt_by_spec(&spec, sink.clone()).is_err());
+        assert!(plugin.nodes.is_empty());
+        assert_eq!(plugin.list_managed_refs().unwrap(), [name.clone()]);
+        assert_eq!(backend.container_state(&name).unwrap(), Some(true));
+        assert_eq!(backend.state.lock().resources[0].id, retained_id);
+        assert_eq!(runtime.stats().actors.len(), baseline_actors);
+
+        let adopted = plugin
+            .adopt_by_spec(&spec, sink.clone())
+            .unwrap()
+            .expect("retained container adopted after rejoin");
+        assert_eq!(adopted.provider_ref, name);
+        assert_eq!(backend.state.lock().resources[0].id, retained_id);
+
+        // An already registered handle also has to reconnect on recovery;
+        // failure retains its ownership rather than silently accepting it.
+        backend.state.lock().fail_next_rejoin = true;
+        assert!(plugin.adopt_by_spec(&spec, sink.clone()).is_err());
+        let retried = plugin
+            .adopt_by_spec(&spec, sink)
+            .unwrap()
+            .expect("registered container rejoined");
+        assert_eq!(retried.handle.id, adopted.handle.id);
+        assert_eq!(plugin.list_managed_refs().unwrap(), [name.clone()]);
+        assert_eq!(backend.state.lock().resources[0].id, retained_id);
+
+        plugin.stop_node(&retried.handle).unwrap();
+        backend.finish_all(&runtime).unwrap();
+        settle_docker_actors(&stepping);
+        assert!(plugin.list_managed_refs().unwrap().is_empty());
+        assert_eq!(runtime.stats().actors.len(), baseline_actors);
+    }
+
     #[test]
     fn mock_vastai_realizes_selected_image_in_one_docker_resource() {
         let (_engine, runtime) = test_runtime();

apps/myelin/src/tests/contextual_process_guarantees.rs

@@ -2,19 +2,22 @@ use std::collections::HashMap;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
+use data_plane::host::HostRouteRegistrar;
 use data_plane::path::SessionAccess;
-use swactor::actor::{ActorInterface, Ctx};
+use distribution::directory_actor::{DirectoryIn, Located};
+use distribution::types::NodeId;
+use myelin_control_contract::{ContextualProcessEventKind, ContextualProcessSpec};
+use swactor::actor::{ActorAddress, ActorInterface, Ctx};
 use swactor::runtime::ExternalSender;
 use swactor_process::{ProcessOutput, ProcessSpec};
 use swactor_process_context::{
     ContextualProcessOutput, ContextualProcessOutputConfig, ContextualProcessSpawner,
-    ContextualProcessSpec,
+    ContextualProcessSpec as RuntimeContextualProcessSpec,
 };
 
 use crate::contextual_process::{
     ContextualNodeCommand, ContextualProcessController, ContextualProcessControllerIn,
-    ContextualProcessEventKindWire, ContextualProcessSpecWire, MyelinContextualProcessConfig,
-    build_contextual_process_spawner,
+    MyelinChildRouteRegistrar, MyelinContextualProcessConfig, build_contextual_process_spawner,
 };
 use crate::orchestration::actor::OrchestratorMsg;
 use crate::tests::harness::build_iroh_composition;
@@ -22,7 +25,7 @@ use crate::tests::harness::build_iroh_composition;
 struct Launch {
     spawner: Arc<ContextualProcessSpawner>,
     sender: ExternalSender,
-    spec: Option<ContextualProcessSpec>,
+    spec: Option<RuntimeContextualProcessSpec>,
     output: Option<ContextualProcessOutputConfig>,
 }
 
@@ -45,6 +48,69 @@ impl ActorInterface for Launch {
     fn handle(&mut self, _ctx: &Ctx<'_>, _message: ()) {}
 }
 
+#[test]
+fn shared_source_route_survives_one_owner_release_and_cleans_up_after_last() {
+    let (_engine, driver, stack) = build_iroh_composition(Duration::from_millis(5));
+    let host_reader: Arc<dyn HostRouteRegistrar> = Arc::new(MyelinChildRouteRegistrar::new(
+        stack.route_view.clone(),
+        stack.pinned_routes.clone(),
+        stack.route_binder.clone(),
+        stack.runtime.clone(),
+        stack.actors.directory,
+        driver.connection_observer(),
+    ));
+    let program_transfer = Arc::clone(&host_reader);
+    let source = ActorAddress([23; 32]);
+    let source_node = NodeId([17; 32]);
+    let replies = stack.runtime.new_inbox::<Located>().unwrap();
+    let resolve_after_republish = || {
+        // Rebuild from the directory's own claims and live pins. Merely reading
+        // the old route view would hide an incorrectly removed shared pin.
+        stack
+            .runtime
+            .send_to(stack.actors.directory, DirectoryIn::Resync)
+            .unwrap();
+        stack
+            .runtime
+            .send_to(
+                stack.actors.directory,
+                DirectoryIn::Resolve {
+                    actor: source,
+                    reply: *replies.addr(),
+                },
+            )
+            .unwrap();
+        let deadline = Instant::now() + Duration::from_secs(10);
+        loop {
+            if let Some(located) = replies.try_recv() {
+                assert_eq!(located.actor, source);
+                break located.host;
+            }
+            assert!(
+                Instant::now() < deadline,
+                "directory did not resolve source"
+            );
+            std::thread::yield_now();
+        }
+    };
+
+    host_reader.retain_source(source, source_node.0).unwrap();
+    program_transfer
+        .retain_source(source, source_node.0)
+        .unwrap();
+    assert_eq!(resolve_after_republish(), Some(source_node));
+
+    host_reader.release_source(source);
+    drop(host_reader);
+    assert_eq!(resolve_after_republish(), Some(source_node));
+    assert!(program_transfer.is_routable(source));
+
+    program_transfer.release_source(source);
+    assert_eq!(resolve_after_republish(), None);
+    assert!(!program_transfer.is_routable(source));
+    driver.shutdown();
+}
+
 #[test]
 fn unaware_native_child_fails_bootstrap_before_terminal_output() {
     let (engine, driver, stack) = build_iroh_composition(Duration::from_millis(5));
@@ -58,9 +124,14 @@ fn unaware_native_child_fails_bootstrap_before_terminal_output() {
             transfer_receiver: None,
             source_sender: None,
             source_publisher: None,
-            route_view: stack.route_view.clone(),
-            pinned_routes: stack.pinned_routes.clone(),
-            route_binder: stack.route_binder.clone(),
+            route_registrar: Arc::new(MyelinChildRouteRegistrar::new(
+                stack.route_view.clone(),
+                stack.pinned_routes.clone(),
+                stack.route_binder.clone(),
+                stack.runtime.clone(),
+                stack.actors.directory,
+                driver.connection_observer(),
+            )),
             stream_transport: Some(driver.stream_transport()),
             host_endpoint: driver.endpoint_addr(),
         })
@@ -75,7 +146,7 @@ fn unaware_native_child_fails_bootstrap_before_terminal_output() {
         .spawn(Launch {
             spawner,
             sender: stack.runtime.create_sender(),
-            spec: Some(ContextualProcessSpec {
+            spec: Some(RuntimeContextualProcessSpec {
                 process: ProcessSpec {
                     command: "/bin/true".to_owned(),
                     args: Vec::new(),
@@ -147,9 +218,14 @@ fn worker_contextual_controller_spawns_queries_stops_and_reclaims_processes() {
             transfer_receiver: None,
             source_sender: None,
             source_publisher: None,
-            route_view: stack.route_view.clone(),
-            pinned_routes: stack.pinned_routes.clone(),
-            route_binder: stack.route_binder.clone(),
+            route_registrar: Arc::new(MyelinChildRouteRegistrar::new(
+                stack.route_view.clone(),
+                stack.pinned_routes.clone(),
+                stack.route_binder.clone(),
+                stack.runtime.clone(),
+                stack.actors.directory,
+                driver.connection_observer(),
+            )),
             stream_transport: Some(driver.stream_transport()),
             host_endpoint: driver.endpoint_addr(),
         })
@@ -170,7 +246,7 @@ fn worker_contextual_controller_spawns_queries_stops_and_reclaims_processes() {
             controller,
             ContextualProcessControllerIn::Command(ContextualNodeCommand::Spawn {
                 request_id: "execution".to_owned(),
-                spec: ContextualProcessSpecWire {
+                spec: ContextualProcessSpec {
                     command: "/bin/sh".to_owned(),
                     args: vec!["-c".to_owned(), "sleep 30".to_owned()],
                     env: Default::default(),
@@ -198,8 +274,8 @@ fn worker_contextual_controller_spawns_queries_stops_and_reclaims_processes() {
             && event.request_id == "execution"
         {
             match event.event {
-                ContextualProcessEventKindWire::Spawned { process, .. } => break process,
-                ContextualProcessEventKindWire::SpawnRejected { error } => {
+                ContextualProcessEventKind::Spawned { process, .. } => break process,
+                ContextualProcessEventKind::SpawnRejected { error } => {
                     panic!("contextual spawn rejected: {error}")
                 }
                 _ => {}
@@ -221,7 +297,7 @@ fn worker_contextual_controller_spawns_queries_stops_and_reclaims_processes() {
         assert!(Instant::now() < spawn_deadline, "live query timed out");
         if let Some(OrchestratorMsg::ContextualEvent(event)) = events.try_recv()
             && event.request_id == "query-live"
-            && let ContextualProcessEventKindWire::LiveExecutions { executions } = event.event
+            && let ContextualProcessEventKind::LiveExecutions { executions, .. } = event.event
         {
             assert_eq!(executions.len(), 1);
             assert_eq!(executions[0].process, process);
@@ -236,26 +312,26 @@ fn worker_contextual_controller_spawns_queries_stops_and_reclaims_processes() {
             controller,
             ContextualProcessControllerIn::Command(ContextualNodeCommand::Stop {
                 request_id: "stop".to_owned(),
-                process,
-                kill_after_ms: None,
+                target_request_id: "execution".to_owned(),
+                kill_after_ms: Some(100),
                 reply_to: *events.addr(),
             }),
         )
         .unwrap();
     let mut stop_accepted = false;
+    let mut stop_events = Vec::new();
     let stop_deadline = Instant::now() + Duration::from_secs(20);
     let mut terminal = false;
     while !terminal {
         assert!(
             Instant::now() < stop_deadline,
-            "stopped process did not terminate"
+            "stopped process did not terminate; stop_accepted={stop_accepted}; events={stop_events:?}"
         );
         if let Some(OrchestratorMsg::ContextualEvent(event)) = events.try_recv() {
+            stop_events.push((event.request_id.clone(), format!("{:?}", event.event)));
             if event.request_id == "stop" {
-                stop_accepted |= matches!(
-                    event.event,
-                    ContextualProcessEventKindWire::StopAccepted { .. }
-                );
+                stop_accepted |=
+                    matches!(event.event, ContextualProcessEventKind::StopAccepted { .. });
             } else if event.request_id == "execution" {
                 terminal = event.event.is_terminal();
             }
@@ -279,7 +355,7 @@ fn worker_contextual_controller_spawns_queries_stops_and_reclaims_processes() {
         assert!(Instant::now() < query_deadline, "empty query timed out");
         if let Some(OrchestratorMsg::ContextualEvent(event)) = events.try_recv()
             && event.request_id == "query-empty"
-            && let ContextualProcessEventKindWire::LiveExecutions { executions } = event.event
+            && let ContextualProcessEventKind::LiveExecutions { executions, .. } = event.event
         {
             assert!(executions.is_empty());
             break;
@@ -290,7 +366,7 @@ fn worker_contextual_controller_spawns_queries_stops_and_reclaims_processes() {
 }
 
 #[test]
-fn bootstrap_failure_is_terminal_and_reclaims_execution() {
+fn bootstrap_failure_reports_native_exit_and_reclaims_execution() {
     let (engine, driver, stack) = build_iroh_composition(Duration::from_millis(5));
     let spawner = Arc::new(
         build_contextual_process_spawner(MyelinContextualProcessConfig {
@@ -302,9 +378,14 @@ fn bootstrap_failure_is_terminal_and_reclaims_execution() {
             transfer_receiver: None,
             source_sender: None,
             source_publisher: None,
-            route_view: stack.route_view.clone(),
-            pinned_routes: stack.pinned_routes.clone(),
-            route_binder: stack.route_binder.clone(),
+            route_registrar: Arc::new(MyelinChildRouteRegistrar::new(
+                stack.route_view.clone(),
+                stack.pinned_routes.clone(),
+                stack.route_binder.clone(),
+                stack.runtime.clone(),
+                stack.actors.directory,
+                driver.connection_observer(),
+            )),
             stream_transport: Some(driver.stream_transport()),
             host_endpoint: driver.endpoint_addr(),
         })
@@ -327,7 +408,7 @@ fn bootstrap_failure_is_terminal_and_reclaims_execution() {
             controller,
             ContextualProcessControllerIn::Command(ContextualNodeCommand::Spawn {
                 request_id: "bootstrap-failure".to_owned(),
-                spec: ContextualProcessSpecWire {
+                spec: ContextualProcessSpec {
                     command: "/bin/sh".to_owned(),
                     args: vec!["-c".to_owned(), "exit 0".to_owned()],
                     env: Default::default(),
@@ -345,6 +426,7 @@ fn bootstrap_failure_is_terminal_and_reclaims_execution() {
         .unwrap();
 
     let deadline = Instant::now() + Duration::from_secs(20);
+    let mut saw_bootstrap_failure = false;
     let mut reclaimed = false;
     while !reclaimed {
         assert!(
@@ -355,18 +437,35 @@ fn bootstrap_failure_is_terminal_and_reclaims_execution() {
         if let Some(OrchestratorMsg::ContextualEvent(event)) = events.try_recv()
             && event.request_id == "bootstrap-failure"
         {
-            if let ContextualProcessEventKindWire::BootstrapFailed { .. } = event.event {
-                assert!(
-                    event.event.is_terminal(),
-                    "BootstrapFailed must be terminal, saw {:?}",
-                    event.event
-                );
-                reclaimed = true;
-            } else if event.event.is_terminal() {
-                panic!(
-                    "unexpected terminal outcome before bootstrap failure: {:?}",
-                    event.event
-                );
+            match event.event {
+                ContextualProcessEventKind::BootstrapFailed { .. } => {
+                    assert!(
+                        !event.event.is_terminal(),
+                        "BootstrapFailed must stay non-terminal until the native process is reaped, saw {:?}",
+                        event.event
+                    );
+                    saw_bootstrap_failure = true;
+                }
+                ContextualProcessEventKind::Exited { .. } => {
+                    assert!(
+                        saw_bootstrap_failure,
+                        "bootstrap failure exited without reporting why: {:?}",
+                        event.event
+                    );
+                    assert!(
+                        event.event.is_terminal(),
+                        "Exited must be terminal, saw {:?}",
+                        event.event
+                    );
+                    reclaimed = true;
+                }
+                other if other.is_terminal() => {
+                    panic!(
+                        "unexpected terminal outcome before bootstrap failure: {:?}",
+                        other
+                    );
+                }
+                _ => {}
             }
         }
         std::thread::yield_now();
@@ -390,7 +489,7 @@ fn bootstrap_failure_is_terminal_and_reclaims_execution() {
         );
         if let Some(OrchestratorMsg::ContextualEvent(event)) = events.try_recv()
             && event.request_id == "query-after-failure"
-            && let ContextualProcessEventKindWire::LiveExecutions { executions } = event.event
+            && let ContextualProcessEventKind::LiveExecutions { executions, .. } = event.event
         {
             assert!(
                 executions.is_empty(),

apps/myelin/src/tests/data_namespace_guarantees.rs

@@ -44,7 +44,12 @@ impl DataNode {
             driver.endpoint_addr(),
             driver.edge_events_handle(),
         ));
-        receiver.install_pump(&engine.handle(), stack.runtime.clone(), POLL);
+        receiver.install_pump(
+            &engine.handle(),
+            stack.runtime.clone(),
+            POLL,
+            driver.edge_events_changed(),
+        );
         let receiver: Arc<dyn BlobTransferReceiver> = receiver;
         let sender: Arc<dyn BlobTransferSender> = Arc::new(IrohBlobTransferSender::new(
             driver.edge_connector(),
@@ -105,6 +110,9 @@ impl DataNode {
                         self.stack.route_view.clone(),
                         self.stack.pinned_routes.clone(),
                         self.stack.route_binder.clone(),
+                        self.stack.runtime.clone(),
+                        self.stack.actors.directory,
+                        self.driver.connection_observer(),
                     ))),
                     stream_transport: None,
                 })
@@ -133,6 +141,7 @@ fn control_and_session_publications_cross_real_iroh_and_outlive_the_producer() {
         &node_a.stack,
         &node_a.driver,
         state.path().join("namespace.json"),
+        Vec::new(),
     )
     .expect("namespace authority");
 

apps/myelin/src/tests/engine_composition.rs

@@ -68,6 +68,7 @@ fn build_composition() -> (Engine, IrohDriver, DistributionRuntimeStack) {
         IrohDriverConfig {
             secret_key: None,
             relay_mode: RelayMode::Disabled,
+            bind_port: None,
             node: DistributedNodeConfig::default(),
             peer_auth: None,
             additional_alpns: vec![],

apps/myelin/src/tests/harness.rs

@@ -69,6 +69,7 @@ pub(crate) fn build_iroh_composition(
         IrohDriverConfig {
             secret_key: None,
             relay_mode: RelayMode::Disabled,
+            bind_port: None,
             node: DistributedNodeConfig::default(),
             peer_auth: None,
             additional_alpns: vec![EDGE_ALPN.to_vec()],

apps/myelin/src/tests/node_guarantees.rs

@@ -32,6 +32,8 @@ fn node_agent_runtime_loaded_reports_orchestrator() {
         .send_to(
             actor,
             NodeAgentMsg::RuntimeLoaded {
+                artifact_digest: None,
+                deployment_generation: None,
                 run_id: 7,
                 node_id: 11,
                 stage_index: 3,
@@ -46,6 +48,8 @@ fn node_agent_runtime_loaded_reports_orchestrator() {
     assert_eq!(
         orchestrator_inbox.try_recv(),
         Some(OrchestratorMsg::ObserveNodeRuntimeReady {
+            artifact_digest: None,
+            deployment_generation: None,
             run_id: 7,
             node_id: 11,
             stage_index: 3,

apps/myelin/src/tests/orchestration_guarantees.rs

@@ -523,3 +523,44 @@ mod run_fsm {
         assert!(fault_pos < torn_down_pos);
     }
 }
+
+mod state_reset_guarantees {
+    //! Reset must remove the complete persisted-state inventory: a partial
+    //! reset recovers a discarded run's bindings onto a "fresh" fixture.
+
+    use crate::orchestration::daemon::StateDir;
+
+    #[test]
+    fn reset_removes_the_complete_state_inventory() {
+        let root = tempfile::tempdir().expect("state dir");
+        let state = StateDir::new(root.path());
+        std::fs::create_dir_all(root.path().join("contextual-uploads/scratch")).unwrap();
+        std::fs::write(root.path().join("identity.key"), [0u8; 32]).unwrap();
+        std::fs::write(root.path().join("data-namespace.json"), "{}").unwrap();
+        std::fs::write(root.path().join("process-nodes.json"), "{}").unwrap();
+        std::fs::write(root.path().join("cluster.json"), "{}").unwrap();
+        std::fs::write(
+            root.path().join("contextual-uploads/scratch/blob.bin"),
+            b"stale",
+        )
+        .unwrap();
+
+        state.reset().expect("reset state");
+
+        for name in [
+            "identity.key",
+            "data-namespace.json",
+            "process-nodes.json",
+            "cluster.json",
+            "contextual-uploads",
+        ] {
+            assert!(
+                !root.path().join(name).exists(),
+                "reset left {name} behind; a fresh fixture would recover stale state"
+            );
+        }
+
+        // Reset is idempotent: an already-clean state dir succeeds again.
+        state.reset().expect("reset clean state");
+    }
+}

crates/bindings/python/Cargo.toml

@@ -26,10 +26,10 @@ distribution = { path = "../../distribution" }
 iroh-driver = { path = "../../iroh-driver" }
 libc = "0.2"
 futures-lite = "2"
-iroh = "0.98"
+iroh.workspace = true
 parking_lot = "0.12"
 serde_json = "1"
-pyo3 = { version = "0.23", features = ["extension-module"] }
+pyo3 = { version = "0.23", features = ["extension-module", "abi3-py312"] }
 pyo3-async-runtimes = { version = "0.23", features = ["tokio-runtime"] }
 tokio.workspace = true
 

crates/bindings/python/src/context.rs

@@ -4,7 +4,7 @@ use std::ffi::{CString, c_int, c_void};
 use std::os::fd::{AsRawFd, IntoRawFd, OwnedFd, RawFd};
 use std::ptr;
 use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::{Arc, Mutex, RwLock};
+use std::sync::{Arc, RwLock, Weak};
 use std::time::Duration;
 #[cfg(all(feature = "test-host", target_os = "linux"))]
 use std::time::Instant;
@@ -19,7 +19,7 @@ use data_plane::data_plane::{
     BlobWriter, DataPlane, DataPlaneBootstrap, Descriptor, DescriptorMapping, MapRequest,
     MapTarget, Protection, Sharing, StreamReader, StreamWriter,
 };
-use data_plane::namespace::EntryKind;
+use data_plane::namespace::{EntryKind, StreamIncarnation};
 use data_plane::path::DataPath;
 #[cfg(all(feature = "test-host", target_os = "linux"))]
 use data_plane::protocol::SessionCapability;
@@ -40,12 +40,14 @@ use pyo3::exceptions::{PyBufferError, PyOSError, PyPermissionError, PyRuntimeErr
 use pyo3::ffi;
 use pyo3::prelude::*;
 use pyo3::types::{PyAny, PyBytes, PyModule};
-use swactor::actor::{ActorAddress, ActorInterface, Ctx};
+use swactor::actor::ActorAddress;
+#[cfg(all(feature = "test-host", target_os = "linux"))]
+use swactor::actor::{ActorInterface, Ctx};
 use swactor::config::RuntimeConfig;
 use swactor::runtime::{Runtime, RuntimeParts};
 #[cfg(all(feature = "test-host", target_os = "linux"))]
 use swactor_engine::BlockingWorkSender;
-use swactor_engine::{ActorCompletion, Engine, EngineHandle, TokioBackend, TokioConfig};
+use swactor_engine::{ActorCompletion, Engine, TokioBackend, TokioConfig};
 use swactor_transport::{CodecRegistry, CodecRemoteSink, TransportRouter};
 
 const ROUTE_POLL: Duration = Duration::from_millis(5);
@@ -111,6 +113,9 @@ fn bootstrap_error(message: impl Into<String>) -> PyErr {
 fn data_plane_error(error: DataPlaneError) -> PyErr {
     match error {
         DataPlaneError::InvalidPath(reason) => PyErr::new::<DataPathError, _>(reason),
+        DataPlaneError::PathReplaced(path) => {
+            PyOSError::new_err((libc::ESTALE, format!("stream path was replaced: {path}")))
+        }
         DataPlaneError::Unauthorized { path, access } => PyPermissionError::new_err((
             libc::EACCES,
             format!("{access:?} is not authorized for {path}"),
@@ -120,7 +125,7 @@ fn data_plane_error(error: DataPlaneError) -> PyErr {
         }
         DataPlaneError::Blob(reason) => PyErr::new::<BlobError, _>(format!("{reason:?}")),
         DataPlaneError::WrongEntryType { .. }
-        | DataPlaneError::PathReplaced(_)
+        | DataPlaneError::BrokenPipe
         | DataPlaneError::PeerLost
         | DataPlaneError::StreamFault(_)
         | DataPlaneError::StreamClosed => PyErr::new::<StreamError, _>(error.to_string()),
@@ -165,55 +170,6 @@ impl Drop for ContextRouting {
     }
 }
 
-#[derive(Clone)]
-struct RoutePoll;
-
-struct RouteReadinessActor {
-    driver: Arc<IrohDriver>,
-    host_node: swactor_transport::NodeId,
-    engine: EngineHandle,
-    sender: swactor::runtime::ExternalSender,
-    completion: ActorCompletion<Result<(), String>>,
-    deadline: std::time::Instant,
-}
-
-impl RouteReadinessActor {
-    fn schedule(&self, ctx: &Ctx<'_>) {
-        self.engine
-            .send_after(ROUTE_POLL, self.sender.clone(), ctx.self_addr(), RoutePoll);
-    }
-
-    fn fail(&self, reason: String) {
-        let _ = self.completion.complete(Err(reason));
-    }
-}
-
-impl ActorInterface for RouteReadinessActor {
-    type Incoming = RoutePoll;
-    type Response = ();
-
-    fn on_start(&mut self, ctx: &Ctx<'_>) {
-        self.schedule(ctx);
-    }
-
-    fn handle(&mut self, ctx: &Ctx<'_>, _message: RoutePoll) {
-        if self.driver.has_active_connection(&self.host_node) {
-            let _ = self.completion.complete(Ok(()));
-            ctx.stop_self();
-            return;
-        }
-        if std::time::Instant::now() >= self.deadline {
-            self.fail(format!(
-                "data-plane route to host {} did not become ready within {ROUTE_READY_TIMEOUT:?}",
-                self.host_node
-            ));
-            ctx.stop_self();
-            return;
-        }
-        self.schedule(ctx);
-    }
-}
-
 fn build_child_routing(
     host_session: ActorAddress,
     host_endpoint: EndpointAddr,
@@ -239,7 +195,14 @@ fn build_child_routing(
         engine.handle(),
         IrohDriverConfig {
             secret_key: None,
-            relay_mode: RelayMode::Default,
+            // A contextual child communicates with its advertised host, not
+            // unrelated public relays. Preserve the host's relay topology.
+            relay_mode: if host_endpoint.relay_urls().next().is_some() {
+                RelayMode::Custom(host_endpoint.relay_urls().cloned().collect())
+            } else {
+                RelayMode::Disabled
+            },
+            bind_port: None,
             node: DistributedNodeConfig::default(),
             peer_auth: None,
             additional_alpns: Vec::new(),
@@ -256,7 +219,7 @@ fn build_child_routing(
             .expect("relay mirror")
             .insert(host_node, relay.to_string());
     }
-    let outbox: Outbox = Arc::new(Mutex::new(Vec::new()));
+    let outbox: Outbox = Arc::new(Default::default());
     let route_transport = Arc::new(RouteViewTransport::new(route_view.clone(), outbox.clone()));
     let binder = OutboxRouteBinder::new(router, route_transport);
     binder.ensure_routable(host_session);
@@ -274,36 +237,32 @@ fn build_child_routing(
     driver.connect_peer(host_endpoint.clone());
 
     let driver = Arc::new(driver);
-    let completion: ActorCompletion<Result<(), String>> = ActorCompletion::new();
-    runtime
-        .spawn(RouteReadinessActor {
-            driver: driver.clone(),
-            host_node,
-            engine: engine.handle(),
-            sender: runtime.create_sender(),
-            completion: completion.clone(),
-            deadline: std::time::Instant::now() + ROUTE_READY_TIMEOUT,
-        })
-        .map_err(|error| bootstrap_error(format!("start route readiness actor: {error}")))?;
-    match completion.wait_deadline(ROUTE_READY_TIMEOUT + ROUTE_POLL) {
-        Some(Ok(())) => {}
-        Some(Err(reason)) => return Err(bootstrap_error(reason)),
-        None => {
-            return Err(bootstrap_error(format!(
-                "data-plane route to host did not become ready within {ROUTE_READY_TIMEOUT:?}"
-            )));
-        }
+    let routing = ContextRouting {
+        _driver: Arc::clone(&driver),
+        _engine: engine,
+    };
+    let completion: ActorCompletion<()> = ActorCompletion::new();
+    let readiness_driver = Arc::downgrade(&driver);
+    let readiness_completion = completion.clone();
+    let watch = driver
+        .connection_observer()
+        .watch_connections(Arc::new(move || {
+            if readiness_driver
+                .upgrade()
+                .is_some_and(|driver| driver.has_active_connection(&host_node))
+            {
+                let _ = readiness_completion.complete(());
+            }
+        }));
+    if completion.wait_deadline(ROUTE_READY_TIMEOUT).is_none() {
+        return Err(bootstrap_error(format!(
+            "data-plane route to host did not become ready within {ROUTE_READY_TIMEOUT:?}"
+        )));
     }
+    drop(watch);
 
     let child_node = driver.node_id().0;
-    Ok((
-        runtime,
-        ContextRouting {
-            _driver: driver,
-            _engine: engine,
-        },
-        child_node,
-    ))
+    Ok((runtime, routing, child_node))
 }
 
 /// Static namespace facts for one path.
@@ -336,10 +295,22 @@ impl PyNamespaceEntry {
 }
 
 /// Namespace, blob, and stream operations for the attached context.
+///
+/// Driver and engine teardown is owned solely by `swactor::run`, which drops
+/// them before returning to Python. Python-visible handles therefore retain
+/// only a weak reference: cycles or delayed interpreter garbage collection
+/// cannot move native thread teardown into interpreter finalization.
 #[pyclass(name = "DataPlane")]
 pub struct PyDataPlane {
-    inner: Arc<DataPlane>,
-    _context_routing: Arc<ContextRouting>,
+    inner: Weak<DataPlane>,
+}
+
+impl PyDataPlane {
+    fn data_plane(&self) -> PyResult<Arc<DataPlane>> {
+        self.inner
+            .upgrade()
+            .ok_or_else(|| PyRuntimeError::new_err("data-plane context is closed"))
+    }
 }
 
 #[pymethods]
@@ -356,7 +327,7 @@ impl PyDataPlane {
             raw_data_plane_error(DataPlaneError::InvalidPath(error.to_string()))
         })?;
         let options = python_open_options(flags, length)?;
-        let data_plane = self.inner.clone();
+        let data_plane = self.data_plane()?;
         pyo3_async_runtimes::tokio::future_into_py(py, async move {
             let descriptor = data_plane
                 .open(&path, options)
@@ -378,7 +349,7 @@ impl PyDataPlane {
     /// Read a whole blob into bytes. Raises FileNotFoundError for a
     /// missing path.
     fn read_blob<'py>(&self, py: Python<'py>, path: String) -> PyResult<Bound<'py, PyAny>> {
-        let data_plane = self.inner.clone();
+        let data_plane = self.data_plane()?;
         pyo3_async_runtimes::tokio::future_into_py(py, async move {
             let blob = data_plane
                 .read_blob_path(&path)
@@ -407,7 +378,7 @@ impl PyDataPlane {
         let path = DataPath::parse(path).map_err(|error| {
             raw_data_plane_error(DataPlaneError::InvalidPath(error.to_string()))
         })?;
-        let data_plane = self.inner.clone();
+        let data_plane = self.data_plane()?;
         pyo3_async_runtimes::tokio::future_into_py(py, async move {
             let node = data_plane
                 .lookup(&path)
@@ -431,7 +402,7 @@ impl PyDataPlane {
         let path = DataPath::parse(path).map_err(|error| {
             raw_data_plane_error(DataPlaneError::InvalidPath(error.to_string()))
         })?;
-        let data_plane = self.inner.clone();
+        let data_plane = self.data_plane()?;
         pyo3_async_runtimes::tokio::future_into_py(py, async move {
             data_plane.unlink(&path).await.map_err(raw_data_plane_error)
         })
@@ -453,7 +424,7 @@ impl PyDataPlane {
         let destination = DataPath::parse(destination).map_err(|error| {
             raw_data_plane_error(DataPlaneError::InvalidPath(error.to_string()))
         })?;
-        let data_plane = self.inner.clone();
+        let data_plane = self.data_plane()?;
         pyo3_async_runtimes::tokio::future_into_py(py, async move {
             data_plane
                 .rename(&source, &destination, replace)
@@ -466,7 +437,7 @@ impl PyDataPlane {
     fn read_stream<'py>(&self, py: Python<'py>, path: String) -> PyResult<Bound<'py, PyAny>> {
         let path = DataPath::parse(path)
             .map_err(|error| PyErr::new::<DataPathError, _>(error.to_string()))?;
-        let data_plane = self.inner.clone();
+        let data_plane = self.data_plane()?;
         pyo3_async_runtimes::tokio::future_into_py(py, async move {
             let reader = data_plane
                 .read_stream(&path)
@@ -476,6 +447,7 @@ impl PyDataPlane {
                 Py::new(
                     py,
                     PyStreamReader {
+                        incarnation: reader.incarnation(),
                         reader: Arc::new(tokio::sync::Mutex::new(reader)),
                     },
                 )
@@ -858,7 +830,7 @@ struct PyWriteState {
 /// Async context manager that owns one blob write.
 #[pyclass(name = "_BlobWriteContext")]
 pub struct PyWriteBlobContext {
-    data_plane: Arc<DataPlane>,
+    data_plane: Weak<DataPlane>,
     path: String,
     length: u64,
     state: Arc<ParkingMutex<PyWriteState>>,
@@ -867,6 +839,10 @@ pub struct PyWriteBlobContext {
 #[pymethods]
 impl PyWriteBlobContext {
     fn __aenter__<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
+        let data_plane = self
+            .data_plane
+            .upgrade()
+            .ok_or_else(|| PyRuntimeError::new_err("data-plane context is closed"))?;
         {
             let mut state = self.state.lock();
             if state.entered {
@@ -876,7 +852,7 @@ impl PyWriteBlobContext {
             }
             state.entered = true;
         }
-        let data_plane = self.data_plane.clone();
+        let data_plane = data_plane;
         let path = self.path.clone();
         let length = self.length;
         let state = self.state.clone();
@@ -1089,11 +1065,24 @@ unsafe fn release_buffer_format(view: *mut ffi::Py_buffer) {
 
 #[pyclass(name = "StreamReader")]
 pub struct PyStreamReader {
+    incarnation: StreamIncarnation,
     reader: Arc<tokio::sync::Mutex<StreamReader>>,
 }
 
 #[pymethods]
 impl PyStreamReader {
+    /// Namespace revision of the attached stream, unchanged after EOF or replacement.
+    #[getter]
+    fn incarnation(&self) -> u64 {
+        self.incarnation.revision
+    }
+
+    /// Namespace authority epoch of the attached stream.
+    #[getter]
+    fn authority_epoch(&self) -> u64 {
+        self.incarnation.authority_epoch
+    }
+
     fn read<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
         let reader = self.reader.clone();
         pyo3_async_runtimes::tokio::future_into_py(py, async move {
@@ -1125,11 +1114,24 @@ impl PyStreamReader {
 
 #[pyclass(name = "StreamWriter")]
 pub struct PyStreamWriter {
+    incarnation: StreamIncarnation,
     stream: Arc<tokio::sync::Mutex<Option<StreamWriter>>>,
 }
 
 #[pymethods]
 impl PyStreamWriter {
+    /// Namespace revision of the attached stream, unchanged after close or replacement.
+    #[getter]
+    fn incarnation(&self) -> u64 {
+        self.incarnation.revision
+    }
+
+    /// Namespace authority epoch of the attached stream.
+    #[getter]
+    fn authority_epoch(&self) -> u64 {
+        self.incarnation.authority_epoch
+    }
+
     fn write<'py>(&self, py: Python<'py>, bytes: Vec<u8>) -> PyResult<Bound<'py, PyAny>> {
         let stream = self.stream.clone();
         pyo3_async_runtimes::tokio::future_into_py(py, async move {
@@ -1145,7 +1147,7 @@ impl PyStreamWriter {
 /// Async context manager that owns one stream write.
 #[pyclass(name = "_StreamWriteContext")]
 pub struct PyStreamContext {
-    data_plane: Arc<DataPlane>,
+    data_plane: Weak<DataPlane>,
     path: DataPath,
     replace: bool,
     stream: Arc<tokio::sync::Mutex<Option<StreamWriter>>>,
@@ -1155,12 +1157,16 @@ pub struct PyStreamContext {
 #[pymethods]
 impl PyStreamContext {
     fn __aenter__<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
+        let data_plane = self
+            .data_plane
+            .upgrade()
+            .ok_or_else(|| PyRuntimeError::new_err("data-plane context is closed"))?;
         if self.entered.swap(true, Ordering::AcqRel) {
             return Err(PyRuntimeError::new_err(
                 "stream write context cannot be entered twice",
             ));
         }
-        let data_plane = self.data_plane.clone();
+        let data_plane = data_plane;
         let path = self.path.clone();
         let state = self.stream.clone();
         let entered = self.entered.clone();
@@ -1173,8 +1179,17 @@ impl PyStreamContext {
             };
             match opened {
                 Ok(writer) => {
+                    let incarnation = writer.incarnation();
                     *state.lock().await = Some(writer);
-                    Python::with_gil(|py| Py::new(py, PyStreamWriter { stream: state }))
+                    Python::with_gil(|py| {
+                        Py::new(
+                            py,
+                            PyStreamWriter {
+                                incarnation,
+                                stream: state,
+                            },
+                        )
+                    })
                 }
                 Err(error) => {
                     entered.store(false, Ordering::Release);
@@ -1256,6 +1271,8 @@ fn run(py: Python<'_>, main: Bound<'_, PyAny>) -> PyResult<()> {
             return Err(bootstrap_error(error.to_string()));
         }
     };
+    // `routing` owns the driver and engine; it remains alive through
+    // acknowledged session shutdown and is then torn down explicitly.
     let (runtime, routing, child_node) =
         match build_child_routing(material.host_session, host_endpoint) {
             Ok(routing) => routing,
@@ -1288,22 +1305,26 @@ fn run(py: Python<'_>, main: Bound<'_, PyAny>) -> PyResult<()> {
     let data = Py::new(
         py,
         PyDataPlane {
-            inner: Arc::clone(&data_plane),
-            _context_routing: Arc::new(routing),
+            inner: Arc::downgrade(&data_plane),
         },
     )?;
     let context = Py::new(py, PyContext { data })?;
-    // Run the guest coroutine, then close the attached session before
-    // propagating either failure: the session must not outlive a `main` that
-    // failed to start or raised.
-    let main_result = (|| -> PyResult<_> {
-        let coroutine = main.call1((context,))?;
+    // Run the guest coroutine while retaining our own context reference, then
+    // release its data-plane ownership before acknowledged session shutdown
+    // and routing teardown.
+    let main_result = (|| -> PyResult<()> {
+        let coroutine = main.call1((context.clone_ref(py),))?;
         let asyncio = PyModule::import(py, "asyncio")?;
-        Ok(asyncio.call_method1("run", (coroutine,)))
+        asyncio.call_method1("run", (coroutine,))?;
+        Ok(())
     })();
-    let close = data_plane.close().map_err(data_plane_error);
-    main_result??;
-    close
+    drop(context);
+    let close_result = future::block_on(data_plane.close_acknowledged()).map_err(data_plane_error);
+    drop(data_plane);
+    // Driver and engine destruction can block; release the GIL while keeping
+    // teardown synchronous with this call.
+    py.allow_threads(move || drop(routing));
+    main_result.and(close_result)
 }
 
 #[cfg(all(feature = "test-host", target_os = "linux"))]
@@ -1408,12 +1429,12 @@ impl data_plane::namespace::NamespaceDiscovery for DebugNamespaceDiscovery {
 }
 
 #[cfg(all(feature = "test-host", target_os = "linux"))]
-struct DebugSourceRegistrar;
+struct DebugSourceRegistrar([u8; 32]);
 
 #[cfg(all(feature = "test-host", target_os = "linux"))]
 impl data_plane::source::BlobSourcePublisher for DebugSourceRegistrar {
-    fn publish_source(&self, _source: ActorAddress) -> Result<(), String> {
-        Ok(())
+    fn publish_source(&self, _source: ActorAddress) -> Result<[u8; 32], String> {
+        Ok(self.0)
     }
 }
 
@@ -1728,6 +1749,7 @@ impl PyTestDataPlaneHost {
                 Py::new(
                     py,
                     PyStreamReader {
+                        incarnation: reader.incarnation(),
                         reader: Arc::new(tokio::sync::Mutex::new(reader)),
                     },
                 )
@@ -1791,6 +1813,7 @@ fn _test_data_plane_host() -> PyResult<PyTestDataPlaneHost> {
         IrohDriverConfig {
             secret_key: None,
             relay_mode: RelayMode::Disabled,
+            bind_port: None,
             node: DistributedNodeConfig::default(),
             peer_auth: None,
             additional_alpns: Vec::new(),
@@ -1800,7 +1823,7 @@ fn _test_data_plane_host() -> PyResult<PyTestDataPlaneHost> {
 
     let route_view: RouteView = Arc::new(RwLock::new(HashMap::new()));
     let relay_mirror: RelayMirror = Arc::new(RwLock::new(HashMap::new()));
-    let outbox: Outbox = Arc::new(Mutex::new(Vec::new()));
+    let outbox: Outbox = Arc::new(Default::default());
     let route_transport = Arc::new(RouteViewTransport::new(route_view.clone(), outbox.clone()));
     let route_binder = Arc::new(OutboxRouteBinder::new(router, route_transport));
     let registrar = Arc::new(DebugHostRouteRegistrar {
@@ -1834,13 +1857,14 @@ fn _test_data_plane_host() -> PyResult<PyTestDataPlaneHost> {
             runtime: runtime.clone(),
         });
     let source_publisher: Arc<dyn data_plane::source::BlobSourcePublisher> =
-        Arc::new(DebugSourceRegistrar);
+        Arc::new(DebugSourceRegistrar(*driver.endpoint_addr().id.as_bytes()));
     let namespace_service = data_plane::control::DataNamespaceService::recover(
         runtime.clone(),
         engine.handle(),
         namespace_root.join("namespace.json"),
         Arc::clone(&source_sender),
         Arc::clone(&source_publisher),
+        None,
     )
     .map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
     let directory = namespace_service.directory();

crates/bindings/python/tests/test_bootstrap.py

@@ -136,6 +136,18 @@ def test_run_attaches_before_main_and_maps_blob_buffer_directly(host):
         os.fstat(inherited)
 
 
+def test_python_handle_cannot_retain_native_session_teardown(host):
+    install_bootstrap(host)
+    retained = []
+
+    async def main(ctx):
+        retained.append(ctx.data)
+
+    swactor.run(main)
+    with pytest.raises(RuntimeError, match="data-plane context is closed"):
+        retained[0].lookup("/models/tiny-linear/weights")
+
+
 def test_write_blob_seals_cleanly_and_exception_aborts(host):
     install_bootstrap(host)
 

crates/dashboard/src/control_plane.rs

@@ -165,9 +165,10 @@ impl DashboardView for ControlPlaneView {
 
     fn ingest(&self, _stream: &StreamId, _frame: &Frame, event: &FrameEvent) {
         let now = Instant::now();
+        let decoded_payload = event.decoded_payload();
         let mut state = self.state.write();
 
-        if let Some(routed) = provisioning_output(event) {
+        if let Some(routed) = provisioning_output(event, decoded_payload.as_ref()) {
             let key = stream_key(&routed.stream);
             if routed.stream.origin.as_deref() == Some("bootstrap") {
                 let terminal = matches!(routed.phase.as_str(), "failed" | "stopped");
@@ -219,8 +220,9 @@ impl DashboardView for ControlPlaneView {
             if let Some(output) = joined_output {
                 node.output = output;
             }
-            node.hardware.update(&event.channel, &event.payload, now);
-            node.actors.update(&event.channel, &event.payload, now);
+            node.hardware.update(event, decoded_payload.as_ref(), now);
+            node.actors
+                .update(&event.channel, decoded_payload.as_ref(), now);
             if let Some((source, phase)) = process_output_channel(&event.channel) {
                 node.output.push_chunk(source, &phase, &event.payload, now);
             }
@@ -347,8 +349,11 @@ fn ensure_node<'a>(
         })
 }
 
-fn provisioning_output(event: &FrameEvent) -> Option<RoutedProvisionOutput> {
-    let value = serde_json::from_slice::<Value>(&event.payload).ok()?;
+fn provisioning_output(
+    event: &FrameEvent,
+    decoded_payload: Option<&Value>,
+) -> Option<RoutedProvisionOutput> {
+    let value = decoded_payload?;
     if event.channel == ORCHESTRATOR_LOGS {
         return Some(RoutedProvisionOutput {
             stream: event.stream.clone(),
@@ -824,7 +829,7 @@ fn percent_decode(value: &str) -> String {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use telemetry::frame::{ChannelId, Lifetime, NodeId, Position};
+    use telemetry::frame::{ChannelContent, ChannelId, Lifetime, NodeId, Position};
 
     fn ingest_json(
         view: &ControlPlaneView,
@@ -844,6 +849,31 @@ mod tests {
             channel: channel.to_string(),
             position,
             payload: frame.payload.clone(),
+            content: ChannelContent::JsonRecord { schema: None },
+        };
+        view.ingest(stream, &frame, &event);
+    }
+
+    fn ingest_messagepack(
+        view: &ControlPlaneView,
+        stream: &StreamId,
+        position: u64,
+        channel: &str,
+        payload: &Value,
+    ) {
+        let payload = telemetry::encode_record(payload).expect("MessagePack payload");
+        let frame = Frame::new(ChannelId(1), Position(position), payload);
+        let event = FrameEvent {
+            stream: crate::StreamEvent {
+                node: stream.node.as_str().to_string(),
+                life: stream.life.0,
+                origin: None,
+                label: None,
+            },
+            channel: channel.to_string(),
+            position,
+            payload: frame.payload.clone(),
+            content: ChannelContent::MessagePackRecord { schema: None },
         };
         view.ingest(stream, &frame, &event);
     }
@@ -863,6 +893,7 @@ mod tests {
             channel: "runtime.actors".to_owned(),
             position: 0,
             payload: frame.payload.clone(),
+            content: ChannelContent::JsonRecord { schema: None },
         };
         view.ingest(&stream, &frame, &event);
 
@@ -967,9 +998,11 @@ mod tests {
                 "line": "SSH identity registered"
             }))
             .unwrap(),
+            content: ChannelContent::JsonRecord { schema: None },
         };
 
-        let routed = provisioning_output(&event).expect("orchestrator log route");
+        let decoded = event.decoded_payload();
+        let routed = provisioning_output(&event, decoded.as_ref()).expect("orchestrator log route");
 
         assert_eq!(routed.stream.node, "myelin-orchestrator");
         assert_eq!(routed.stream.origin.as_deref(), Some("orchestrator"));
@@ -1106,6 +1139,7 @@ mod tests {
                 channel: "runtime.actors".to_owned(),
                 position: 0,
                 payload: frame.payload.clone(),
+                content: ChannelContent::JsonRecord { schema: None },
             };
             view.ingest(stream, &frame, &event);
         }
@@ -1268,13 +1302,7 @@ mod tests {
             (4, "host.net", net_sample(1, 2_000, 2_000, 3_500)),
             (5, "host.storage", storage),
         ] {
-            ingest_json(
-                &view,
-                &stream,
-                position,
-                channel,
-                serde_json::to_vec(&payload).expect("hardware payload"),
-            );
+            ingest_messagepack(&view, &stream, position, channel, &payload);
         }
 
         let snapshot = view.snapshot_json();
@@ -1289,6 +1317,12 @@ mod tests {
             node["storage"]["filesystems"][0]["used_percent"],
             json!(80.0)
         );
+        for section in ["cpu", "gpu", "memory", "net", "storage"] {
+            assert!(
+                node[section].get("error").is_none(),
+                "{section} emitted a null error field"
+            );
+        }
         assert_eq!(node["history"][0]["cpu_cores_percent"], json!([25.0, 60.0]));
         assert_eq!(
             node["history"][0]["memory_pressure_some_avg10"],

crates/dashboard/src/demo_control.rs

@@ -65,7 +65,7 @@ impl DashboardView for DemoControlView {
     }
 
     fn ingest(&self, _stream: &StreamId, _frame: &Frame, event: &FrameEvent) {
-        let Ok(payload) = serde_json::from_slice::<Value>(&event.payload) else {
+        let Some(payload) = event.decoded_payload() else {
             return;
         };
         if event.channel == "myelin.provisioning.events" {

crates/dashboard/src/hardware_view.rs

@@ -1,7 +1,9 @@
 use std::collections::{BTreeMap, VecDeque};
+use std::fmt::Display;
 use std::time::{Duration, Instant};
 
-use telemetry::Record;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
 use telemetry::hardware::cpu::{
     CpuCoreSample, CpuHostSample, CpuProcessSample, HOST_CPU_CHANNEL, HostCpuSample,
 };
@@ -12,7 +14,7 @@ use telemetry::hardware::memory::{HOST_MEMORY_CHANNEL, HostMemorySample};
 use telemetry::hardware::net::{HOST_NET_CHANNEL, HostNetSample, NetInterfaceSample};
 use telemetry::hardware::storage::{HOST_STORAGE_CHANNEL, HostStorageSample};
 
-use serde::Serialize;
+use crate::FrameEvent;
 
 const HISTORY_CAP: usize = 300;
 const HISTORY_MIN_INTERVAL: Duration = Duration::from_millis(900);
@@ -51,10 +53,16 @@ impl NodeHardwareState {
         }
     }
 
-    pub(crate) fn update(&mut self, channel: &str, payload: &[u8], now: Instant) {
+    pub(crate) fn update(
+        &mut self,
+        event: &FrameEvent,
+        decoded_payload: Option<&Value>,
+        now: Instant,
+    ) {
+        let channel = event.channel.as_str();
         self.last_seen = now;
         match channel {
-            HOST_CPU_CHANNEL => match HostCpuSample::decode(payload) {
+            HOST_CPU_CHANNEL => match Self::decode_sample(decoded_payload) {
                 Ok(sample) => {
                     self.cpu = Some(sample);
                     self.decode_errors.remove(HOST_CPU_CHANNEL);
@@ -62,7 +70,7 @@ impl NodeHardwareState {
                 }
                 Err(error) => self.store_decode_error(HOST_CPU_CHANNEL, error),
             },
-            HOST_GPU_CHANNEL => match HostGpuSample::decode(payload) {
+            HOST_GPU_CHANNEL => match Self::decode_sample(decoded_payload) {
                 Ok(sample) => {
                     self.gpu = Some(sample);
                     self.decode_errors.remove(HOST_GPU_CHANNEL);
@@ -70,7 +78,7 @@ impl NodeHardwareState {
                 }
                 Err(error) => self.store_decode_error(HOST_GPU_CHANNEL, error),
             },
-            HOST_MEMORY_CHANNEL => match HostMemorySample::decode(payload) {
+            HOST_MEMORY_CHANNEL => match Self::decode_sample(decoded_payload) {
                 Ok(sample) => {
                     self.memory = Some(sample);
                     self.decode_errors.remove(HOST_MEMORY_CHANNEL);
@@ -78,7 +86,7 @@ impl NodeHardwareState {
                 }
                 Err(error) => self.store_decode_error(HOST_MEMORY_CHANNEL, error),
             },
-            HOST_NET_CHANNEL => match HostNetSample::decode(payload) {
+            HOST_NET_CHANNEL => match Self::decode_sample(decoded_payload) {
                 Ok(sample) => {
                     self.net = Some(NetSnapshot::from_sample(sample, self.net.as_ref()));
                     self.decode_errors.remove(HOST_NET_CHANNEL);
@@ -86,7 +94,7 @@ impl NodeHardwareState {
                 }
                 Err(error) => self.store_decode_error(HOST_NET_CHANNEL, error),
             },
-            HOST_STORAGE_CHANNEL => match HostStorageSample::decode(payload) {
+            HOST_STORAGE_CHANNEL => match Self::decode_sample(decoded_payload) {
                 Ok(sample) => {
                     self.storage = Some(sample);
                     self.decode_errors.remove(HOST_STORAGE_CHANNEL);
@@ -96,13 +104,13 @@ impl NodeHardwareState {
             },
             _ => {
                 if channel.starts_with("proc.") && channel.ends_with(".lifecycle") {
-                    self.process = decode_process_snapshot(payload);
+                    self.process = decoded_payload.and_then(decode_process_snapshot);
                     // Lifecycle frames also prove liveness.
                     self.last_seen = now;
                 } else if channel == "node.status" {
                     // Liveness heartbeat from the supervisor.
                     self.last_seen = now;
-                    if let Some(status) = decode_process_snapshot(payload) {
+                    if let Some(status) = decoded_payload.and_then(decode_process_snapshot) {
                         self.process.get_or_insert(status);
                     }
                 }
@@ -110,7 +118,15 @@ impl NodeHardwareState {
         }
     }
 
-    fn store_decode_error(&mut self, channel: &'static str, error: serde_json::Error) {
+    fn decode_sample<'a, T>(payload: Option<&'a Value>) -> Result<T, String>
+    where
+        T: Deserialize<'a>,
+    {
+        let payload = payload.ok_or_else(|| "payload codec decode failed".to_owned())?;
+        T::deserialize(payload).map_err(|error| error.to_string())
+    }
+
+    fn store_decode_error(&mut self, channel: &'static str, error: impl Display) {
         self.decode_errors
             .insert(channel, format!("{channel} decode error: {error}"));
     }
@@ -270,8 +286,7 @@ impl NodeHardwareState {
 }
 
 /// Decode a `swactor_process.lifecycle.v1` payload into fleet-card state.
-pub(crate) fn decode_process_snapshot(payload: &[u8]) -> Option<ProcessSnapshot> {
-    let value: serde_json::Value = serde_json::from_slice(payload).ok()?;
+pub(crate) fn decode_process_snapshot(value: &Value) -> Option<ProcessSnapshot> {
     let pid = value
         .get("pid")
         .and_then(serde_json::Value::as_u64)
@@ -296,6 +311,7 @@ pub(crate) struct NetSnapshot {
     seq: u64,
     sample_unix_ms: u64,
     pub(crate) interfaces: Vec<NetInterfaceSnapshot>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     error: Option<String>,
 }
 
@@ -430,6 +446,7 @@ pub(crate) struct CpuSnapshot {
     pub(crate) host: Option<CpuHostSample>,
     pub(crate) cores: Vec<CpuCoreSample>,
     pub(crate) processes: Vec<CpuProcessSample>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub(crate) error: Option<String>,
 }
 
@@ -454,6 +471,7 @@ pub(crate) struct GpuSnapshot {
     pub(crate) query_elapsed_ms: Option<u64>,
     pub(crate) gpus: Vec<GpuDeviceSample>,
     pub(crate) processes: Vec<GpuProcessSample>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub(crate) error: Option<String>,
 }
 

crates/dashboard/src/lib.rs

@@ -14,8 +14,9 @@ pub use control_plane::ControlPlaneView;
 
 use std::sync::Arc;
 
-use serde::Serialize;
-use telemetry::frame::{ChannelId, Frame, Lifetime, NodeId, Position, StreamId};
+use serde::ser::SerializeStruct;
+use serde::{Serialize, Serializer};
+use telemetry::frame::{ChannelContent, ChannelId, Frame, Lifetime, NodeId, Position, StreamId};
 use tokio::sync::broadcast;
 
 use crate::store::DashboardStore;
@@ -90,13 +91,14 @@ impl Default for DashboardConfig {
     }
 }
 
-/// JSON shape emitted for each incoming telemetry frame.
-#[derive(Debug, Clone, Serialize)]
+/// Shape emitted for each incoming telemetry frame.
+#[derive(Debug, Clone)]
 pub struct FrameEvent {
     pub stream: StreamEvent,
     pub channel: String,
     pub position: u64,
     pub payload: Vec<u8>,
+    pub content: ChannelContent,
 }
 
 #[derive(Debug, Clone, Serialize)]
@@ -124,11 +126,26 @@ impl StreamEvent {
 
 impl FrameEvent {
     pub fn new(stream: &StreamId, frame: &Frame) -> Self {
+        Self::new_with_content(stream, frame, ChannelContent::Bytes)
+    }
+
+    pub fn new_with_content(stream: &StreamId, frame: &Frame, content: ChannelContent) -> Self {
         Self {
             stream: StreamEvent::new(stream),
             channel: frame.channel.to_string(),
             position: frame.position.0,
             payload: frame.payload.clone(),
+            content,
+        }
+    }
+
+    pub fn decoded_payload(&self) -> Option<serde_json::Value> {
+        match &self.content {
+            ChannelContent::JsonRecord { .. } => serde_json::from_slice(&self.payload).ok(),
+            ChannelContent::MessagePackRecord { .. } => {
+                telemetry::decode_record_value(&self.payload).ok()
+            }
+            ChannelContent::Bytes | ChannelContent::TextStream => None,
         }
     }
 
@@ -144,6 +161,22 @@ impl FrameEvent {
     }
 }
 
+impl Serialize for FrameEvent {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut frame = serializer.serialize_struct("FrameEvent", 6)?;
+        frame.serialize_field("stream", &self.stream)?;
+        frame.serialize_field("channel", &self.channel)?;
+        frame.serialize_field("position", &self.position)?;
+        frame.serialize_field("payload", &self.payload)?;
+        frame.serialize_field("content", &self.content)?;
+        frame.serialize_field("decoded_payload", &self.decoded_payload())?;
+        frame.end()
+    }
+}
+
 /// Handle to the read-only dashboard server.
 ///
 /// The handle's data path publishes observed telemetry frames to HTTP clients

crates/dashboard/src/live_explorer.rs

@@ -156,7 +156,7 @@ impl DashboardView for LiveTelemetryExplorer {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use telemetry::frame::{ChannelId, Lifetime, NodeId, Position};
+    use telemetry::frame::{ChannelContent, ChannelId, Lifetime, NodeId, Position};
 
     #[test]
     fn retains_quiet_channels_and_bounds_each_channel_independently() {
@@ -252,6 +252,7 @@ mod tests {
                 channel: channel.to_owned(),
                 position: position as u64,
                 payload: payload.clone(),
+                content: ChannelContent::Bytes,
             };
             view.ingest_at(&event, Instant::now());
         }
@@ -286,6 +287,7 @@ mod tests {
             channel: channel.to_owned(),
             position,
             payload: frame.payload.clone(),
+            content: ChannelContent::Bytes,
         }
     }
 

crates/dashboard/src/live_explorer_page.html

@@ -179,8 +179,12 @@ RISK: terminal cosplay if decoration creeps past data; held by the palette law.
     return String(value).replace(/[&<>'"]/g, ch => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', "'": '&#39;', '"': '&quot;' }[ch]));
   }
 
-  function decodePayload(payload) {
+  function decodePayload(payload, decodedPayload) {
     const bytes = Array.isArray(payload) ? Uint8Array.from(payload) : new Uint8Array();
+    if (decodedPayload !== null && decodedPayload !== undefined) {
+      const compact = JSON.stringify(decodedPayload);
+      return { bytes: bytes.length, kind: 'json', text: compact, value: decodedPayload, preview: compact.length > 120 ? compact.slice(0, 120) + '…' : compact };
+    }
     const text = decoder.decode(bytes);
     if (!text) return { bytes: bytes.length, kind: 'binary', text: '', value: null, preview: `${bytes.length} bytes` };
     try {
@@ -200,7 +204,7 @@ RISK: terminal cosplay if decoration creeps past data; held by the palette law.
       channel: String(event.channel || 'unknown'),
       position: Number(event.position || 0),
       seq: ++state.seq,
-      ...decodePayload(event.payload),
+      ...decodePayload(event.payload, event.decoded_payload),
     };
   }
 

crates/dashboard/src/swactor/actor_view.rs

@@ -75,9 +75,9 @@ impl RuntimeState {
         }
     }
 
-    pub(crate) fn update(&mut self, channel: &str, payload: &[u8], now: Instant) {
+    pub(crate) fn update(&mut self, channel: &str, value: Option<&Value>, now: Instant) {
         self.last_seen = now;
-        let Ok(value) = serde_json::from_slice::<Value>(payload) else {
+        let Some(value) = value else {
             return;
         };
         match channel {
@@ -633,7 +633,7 @@ mod tests {
     use super::*;
 
     fn apply_protocol(runtime: &mut RuntimeState, value: Value, at: Instant) {
-        runtime.update(RUNTIME_ACTORS, value.to_string().as_bytes(), at);
+        runtime.update(RUNTIME_ACTORS, Some(&value), at);
     }
 
     #[test]

crates/data-plane/Cargo.toml

@@ -5,6 +5,10 @@ edition = "2024"
 license = "AGPL-3.0-only"
 publish = false
 
+[features]
+# Diagnostic tracing of the namespace directory actor's request lifecycle.
+directory-trace = []
+
 [dependencies]
 telemetry = { path = "../telemetry" }
 serde = { version = "1", features = ["derive"] }

crates/data-plane/src/blob_transfer.rs

@@ -39,6 +39,7 @@ pub enum BlobTransferEvent {
     Released(Result<(), DataPlaneError>),
     Cancel,
     RouteRetry,
+    RouteChanged,
 }
 
 impl NetworkMessage for BlobTransferEvent {

crates/data-plane/src/byte_ring.rs

@@ -16,7 +16,7 @@
 //! | 16     | generation u64                 | fixed at install       |
 //! | 24     | commit u64 (atomic)            | producer writes only   |
 //! | 32     | consume u64 (atomic)           | consumer writes only   |
-//! | 40..   | reserved (zero)                | fixed at install       |
+//! | 40     | peer terminal reason u64 (atomic) | host writes only       |
 //! | 128    | data[capacity]                 | protocol               |
 //!
 //! Memory model (property P4): the producer copies bytes and then
@@ -105,6 +105,12 @@ pub enum Role {
     Consumer,
 }
 
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum PeerTermination {
+    Closed,
+    PathReplaced,
+}
+
 /// A reserved, not-yet-committed span of the data region.
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub struct Reservation {
@@ -376,6 +382,9 @@ pub enum FlowError {
         found: u64,
     },
     BadRecord(RecordError),
+    InvalidPeerTermination {
+        found: u64,
+    },
     Io,
 }
 
@@ -585,9 +594,40 @@ pub fn attach_mapped(
 pub fn mark_peer_terminated_mapped(
     arena: &crate::mapped_arena::MappedArena,
     handle: RingHandle,
+    termination: PeerTermination,
 ) -> Result<(), AttachError> {
-    let endpoint = attach_mapped(arena, handle, Role::Producer)?;
-    endpoint.atomic(OFF_TERMINAL).store(1, Ordering::Release);
+    let total = DATA_OFFSET
+        .checked_add(handle.capacity)
+        .ok_or(AttachError::OutOfBounds {
+            end: u64::MAX,
+            arena_len: arena.len() as u64,
+        })?;
+    let range =
+        arena
+            .checked_range(handle.offset, total)
+            .map_err(|_| AttachError::OutOfBounds {
+                end: handle.offset.saturating_add(total),
+                arena_len: arena.len() as u64,
+            })?;
+    let endpoint = Endpoint {
+        header: arena.ptr_at(range.start),
+        info: handle,
+        role: Role::Producer,
+    };
+    endpoint.validate_fixed().map_err(AttachError::Header)?;
+    endpoint
+        .validate_generation(handle.generation)
+        .map_err(AttachError::Header)?;
+    let marker = match termination {
+        PeerTermination::Closed => 1,
+        PeerTermination::PathReplaced => 2,
+    };
+    // Cursor fields change independently while a stream is active. Validating
+    // a multi-atomic cursor snapshot here can spuriously reject the terminal
+    // marker; only fixed identity is required for this host-owned field.
+    endpoint
+        .atomic(OFF_TERMINAL)
+        .fetch_max(marker, Ordering::Release);
     Ok(())
 }
 
@@ -621,11 +661,16 @@ impl Endpoint {
         self.role
     }
 
-    pub fn peer_terminated(&self) -> Result<bool, FlowError> {
+    pub fn peer_termination(&self) -> Result<Option<PeerTermination>, FlowError> {
         self.validate_fixed().map_err(FlowError::Corrupt)?;
         self.validate_generation(self.info.generation)
             .map_err(FlowError::Corrupt)?;
-        Ok(self.atomic(OFF_TERMINAL).load(Ordering::Acquire) != 0)
+        match self.atomic(OFF_TERMINAL).load(Ordering::Acquire) {
+            0 => Ok(None),
+            1 => Ok(Some(PeerTermination::Closed)),
+            2 => Ok(Some(PeerTermination::PathReplaced)),
+            found => Err(FlowError::InvalidPeerTermination { found }),
+        }
     }
 
     /// Current published producer and consumer positions. This role-neutral

crates/data-plane/src/control.rs

@@ -10,6 +10,7 @@ use swactor_engine::EngineHandle;
 
 use crate::blob::FileRegistration;
 use crate::blob_transfer::BlobTransferSender;
+use crate::host::HostRouteRegistrar;
 use crate::namespace::{
     DataDirectoryActor, DirectoryClient, NamespaceError, OperationId, RetirementRetry,
 };
@@ -25,6 +26,7 @@ const RETIREMENT_RETRY_PERIOD: Duration = Duration::from_millis(250);
 pub struct DataNamespaceService {
     directory: ActorAddress,
     control: DataPlaneControl,
+    authority_epoch: u64,
 }
 
 impl DataNamespaceService {
@@ -34,12 +36,17 @@ impl DataNamespaceService {
         store_path: impl AsRef<Path>,
         source_sender: Arc<dyn BlobTransferSender>,
         source_publisher: Arc<dyn BlobSourcePublisher>,
+        routes: Option<Arc<dyn HostRouteRegistrar>>,
     ) -> Result<Self, NamespaceError> {
         let recovery_runtime = runtime.clone();
         let recovery_sender = Arc::clone(&source_sender);
         let recovery_publisher = Arc::clone(&source_publisher);
-        let retire_retry =
-            RetirementRetry::new(engine, runtime.create_sender(), RETIREMENT_RETRY_PERIOD);
+        let retire_retry = RetirementRetry::new(
+            engine,
+            runtime.create_sender(),
+            RETIREMENT_RETRY_PERIOD,
+            routes,
+        );
         let directory = DataDirectoryActor::recover(
             store_path,
             Some(retire_retry),
@@ -57,16 +64,20 @@ impl DataNamespaceService {
                             path.display()
                         ))
                     })?;
-                    if let Err(error) = recovery_publisher.publish_source(source) {
-                        let _ = recovery_runtime
-                            .send_to(source, BlobSourceIn::Retire { reply_to: None });
-                        return Err(NamespaceError::SourceRecovery(error));
-                    }
-                    Ok(source)
+                    let node = match recovery_publisher.publish_source(source) {
+                        Ok(node) => node,
+                        Err(error) => {
+                            let _ = recovery_runtime
+                                .send_to(source, BlobSourceIn::Retire { reply_to: None });
+                            return Err(NamespaceError::SourceRecovery(error));
+                        }
+                    };
+                    Ok((source, node))
                 }
-                SourceRecovery::Actor { actor } => Ok(*actor),
+                SourceRecovery::Actor { actor, node, .. } => Ok((*actor, *node)),
             },
         )?;
+        let authority_epoch = directory.authority_epoch();
         let directory = runtime.spawn(directory).map_err(|error| {
             NamespaceError::SourceRecovery(format!("spawn data directory: {error}"))
         })?;
@@ -76,7 +87,11 @@ impl DataNamespaceService {
             source_sender,
             source_publisher,
         };
-        Ok(Self { directory, control })
+        Ok(Self {
+            directory,
+            authority_epoch,
+            control,
+        })
     }
 
     pub fn directory(&self) -> ActorAddress {
@@ -86,6 +101,10 @@ impl DataNamespaceService {
     pub fn control(&self) -> DataPlaneControl {
         self.control.clone()
     }
+
+    pub fn authority_epoch(&self) -> u64 {
+        self.authority_epoch
+    }
 }
 
 #[derive(Clone)]
@@ -96,7 +115,50 @@ pub struct DataPlaneControl {
     source_publisher: Arc<dyn BlobSourcePublisher>,
 }
 
+pub enum RetainedNamespaceResources {
+    Absent,
+    QuiescentStream { revision: u64 },
+    Blob(crate::namespace::BlobBinding),
+}
+
+pub enum NamespaceCleanupStatus {
+    Absent,
+    ActiveStream,
+    Removed {
+        quiescent_stream_revision: Option<u64>,
+    },
+}
+
 impl DataPlaneControl {
+    pub async fn retained_blob_resources(
+        &self,
+        path: DataPath,
+    ) -> Result<RetainedNamespaceResources, NamespaceError> {
+        let node = match self.directory.lookup(path.clone()).await {
+            Ok(node) => node,
+            Err(NamespaceError::PathNotFound(_)) => return Ok(RetainedNamespaceResources::Absent),
+            Err(error) => return Err(error),
+        };
+        if node.kind == crate::namespace::EntryKind::Stream {
+            return if node.active {
+                Err(NamespaceError::Protocol(
+                    "retained stream is still active".to_owned(),
+                ))
+            } else {
+                Ok(RetainedNamespaceResources::QuiescentStream {
+                    revision: node.revision,
+                })
+            };
+        }
+        let binding = self.directory.resolve(path).await?;
+        if binding.revision != node.revision {
+            return Err(NamespaceError::Protocol(
+                "retained blob changed during ownership lookup".to_owned(),
+            ));
+        }
+        Ok(RetainedNamespaceResources::Blob(binding))
+    }
+
     pub async fn ensure(
         &self,
         path: DataPath,
@@ -132,11 +194,19 @@ impl DataPlaneControl {
             source,
             armed: true,
         };
-        self.source_publisher
+        let source_node = self
+            .source_publisher
             .publish_source(source)
             .map_err(NamespaceError::SourceRecovery)?;
         self.directory
-            .register(path, source, length, recovery, random_operation_id())
+            .register(
+                path,
+                source,
+                source_node,
+                length,
+                recovery,
+                random_operation_id(),
+            )
             .await?;
         cleanup.armed = false;
         Ok(())
@@ -149,6 +219,36 @@ impl DataPlaneControl {
         Ok(())
     }
 
+    /// Remove one namespace entry only after any stream writer has quiesced,
+    /// then prove the committed directory state no longer contains the path.
+    pub async fn try_unregister_quiescent(
+        &self,
+        path: DataPath,
+    ) -> Result<NamespaceCleanupStatus, NamespaceError> {
+        let node = match self.directory.lookup(path.clone()).await {
+            Ok(node) => node,
+            Err(NamespaceError::PathNotFound(_)) => return Ok(NamespaceCleanupStatus::Absent),
+            Err(error) => return Err(error),
+        };
+        if node.kind == crate::namespace::EntryKind::Stream && node.active {
+            return Ok(NamespaceCleanupStatus::ActiveStream);
+        }
+        let quiescent_stream_revision =
+            (node.kind == crate::namespace::EntryKind::Stream).then_some(node.revision);
+        self.directory
+            .unregister(path.clone(), random_operation_id())
+            .await?;
+        match self.directory.lookup(path).await {
+            Err(NamespaceError::PathNotFound(_)) => Ok(NamespaceCleanupStatus::Removed {
+                quiescent_stream_revision,
+            }),
+            Ok(_) => Err(NamespaceError::Protocol(
+                "namespace entry remained after unregister acknowledgement".to_owned(),
+            )),
+            Err(error) => Err(error),
+        }
+    }
+
     pub async fn rename(
         &self,
         source: DataPath,

crates/data-plane/src/data_plane.rs

@@ -15,9 +15,11 @@ use crate::blob::{
     WritableViewObserver,
 };
 use crate::byte_ring::{
-    Endpoint, FlowError, RecordCursor, RecordKind, RingHandle, Role, attach_mapped,
+    Endpoint, FlowError, HeaderError, PeerTermination, RecordCursor, RecordKind, RingHandle, Role,
+    attach_mapped,
 };
 use crate::mapped_arena::MappedArena;
+use crate::namespace::StreamIncarnation;
 use crate::path::DataPath;
 pub use crate::protocol::{
     AccessMode, BlobAllocation, DataPlaneError, DescriptorCapabilities, DescriptorKind, Errno,
@@ -105,6 +107,8 @@ impl DataPlaneBootstrap {
                 stream_operations: HashMap::new(),
                 pending_blob_releases: 0,
                 deferred_blob_opens: VecDeque::new(),
+                close_replies: Vec::new(),
+                close_result: None,
             })
             .map_err(|error| DataPlaneError::SessionFailed(error.to_string()))?;
 
@@ -176,8 +180,10 @@ enum DescriptorOpenGrant {
         cancellation: Arc<DescriptorGrantCancellation>,
     },
     Stream {
+        path: DataPath,
         operation: ActorAddress,
         host_binding: ActorAddress,
+        incarnation: StreamIncarnation,
         ring: RingHandle,
         role: Role,
         cancellation: Arc<DescriptorGrantCancellation>,
@@ -882,8 +888,10 @@ impl DataPlane {
                 Ok(Descriptor::write_blob(writer, access))
             }
             DescriptorOpenGrant::Stream {
+                path,
                 operation,
                 host_binding,
+                incarnation,
                 ring,
                 role,
                 cancellation,
@@ -897,15 +905,19 @@ impl DataPlane {
                         child_session: self.child_session,
                         operation,
                         host_binding,
+                        incarnation,
                         endpoint,
                         terminal: None,
                         pending_record: None,
+                        path,
                     }),
                     Role::Producer => Descriptor::write_stream(StreamWriter {
                         runtime: self.runtime.clone(),
                         child_session: self.child_session,
+                        path,
                         operation,
                         host_binding,
+                        incarnation,
                         endpoint,
                         closed: false,
                     }),
@@ -1042,9 +1054,28 @@ impl DataPlane {
 
     pub fn close(&self) -> Result<(), DataPlaneError> {
         self.runtime
-            .send_to(self.child_session, ChildSessionIn::Close)
+            .send_to(self.child_session, ChildSessionIn::Close { reply_to: None })
             .map_err(|error| DataPlaneError::SessionFailed(error.to_string()))
     }
+
+    /// Closes the child and host session and waits for the host to finish
+    /// revoking its resources. This future has no internal timeout; callers
+    /// that require a bound must apply one.
+    pub async fn close_acknowledged(&self) -> Result<(), DataPlaneError> {
+        let inbox = self
+            .runtime
+            .new_inbox::<Result<(), DataPlaneError>>()
+            .map_err(|error| DataPlaneError::SessionFailed(error.to_string()))?;
+        self.runtime
+            .send_to(
+                self.child_session,
+                ChildSessionIn::Close {
+                    reply_to: Some(*inbox.addr()),
+                },
+            )
+            .map_err(|error| DataPlaneError::SessionFailed(error.to_string()))?;
+        inbox.recv().await
+    }
 }
 
 pub trait StreamConsumer: Send + Sync + 'static {
@@ -1280,16 +1311,34 @@ pub(crate) enum ChildStreamIn {
     Wake(Result<(), DataPlaneError>),
 }
 
+fn stream_flow_error(path: &DataPath, action: &str, error: FlowError) -> DataPlaneError {
+    match error {
+        FlowError::Corrupt(HeaderError::GenerationMismatch { .. }) => {
+            DataPlaneError::PathReplaced(path.clone())
+        }
+        error => DataPlaneError::StreamFault(format!("{action}: {error:?}")),
+    }
+}
+
 pub struct StreamWriter {
     runtime: Runtime,
     child_session: ActorAddress,
     operation: ActorAddress,
+    path: DataPath,
     host_binding: ActorAddress,
+    incarnation: StreamIncarnation,
     endpoint: Endpoint,
     closed: bool,
 }
 
 impl StreamWriter {
+    /// The authoritative namespace identity of this attached stream.
+    ///
+    /// This identity remains unchanged after close or namespace replacement.
+    pub fn incarnation(&self) -> StreamIncarnation {
+        self.incarnation
+    }
+
     pub fn capacity(&self) -> u64 {
         self.endpoint.capacity()
     }
@@ -1439,15 +1488,17 @@ impl StreamWriter {
             return Ok(0);
         }
         loop {
-            if self.endpoint.peer_terminated().map_err(|error| {
-                DataPlaneError::StreamFault(format!(
-                    "observe stream peer terminal state: {error:?}"
-                ))
+            match self.endpoint.peer_termination().map_err(|error| {
+                stream_flow_error(&self.path, "observe stream peer terminal state", error)
             })? {
-                return Err(DataPlaneError::BrokenPipe);
+                None => {}
+                Some(PeerTermination::Closed) => return Err(DataPlaneError::BrokenPipe),
+                Some(PeerTermination::PathReplaced) => {
+                    return Err(DataPlaneError::PathReplaced(self.path.clone()));
+                }
             }
             let available = self.endpoint.writable_payload_capacity().map_err(|error| {
-                DataPlaneError::StreamFault(format!("observe writable stream capacity: {error:?}"))
+                stream_flow_error(&self.path, "observe writable stream capacity", error)
             })?;
             if available != 0 {
                 let count = bytes
@@ -1457,9 +1508,7 @@ impl StreamWriter {
                     .endpoint
                     .reserve_record(RecordKind::Data, count as u64)
                     .map_err(|error| {
-                        DataPlaneError::StreamFault(format!(
-                            "reserve partial stream record: {error:?}"
-                        ))
+                        stream_flow_error(&self.path, "reserve partial stream record", error)
                     })?;
                 let (first, second) = record.spans_mut();
                 first.copy_from_slice(&bytes[..first.len()]);
@@ -1479,7 +1528,7 @@ impl StreamWriter {
                 reply_to: *inbox.addr(),
             })?;
             if self.endpoint.writable_payload_capacity().map_err(|error| {
-                DataPlaneError::StreamFault(format!("recheck writable stream capacity: {error:?}"))
+                stream_flow_error(&self.path, "recheck writable stream capacity", error)
             })? != 0
             {
                 continue;
@@ -1592,12 +1641,21 @@ pub struct StreamReader {
     child_session: ActorAddress,
     operation: ActorAddress,
     host_binding: ActorAddress,
+    incarnation: StreamIncarnation,
+    path: DataPath,
     endpoint: Endpoint,
     terminal: Option<StreamReadTerminal>,
     pending_record: Option<(RecordCursor, u64)>,
 }
 
 impl StreamReader {
+    /// The authoritative namespace identity of this attached stream.
+    ///
+    /// This identity remains unchanged after EOF or namespace replacement.
+    pub fn incarnation(&self) -> StreamIncarnation {
+        self.incarnation
+    }
+
     pub fn capacity(&self) -> u64 {
         self.endpoint.capacity()
     }
@@ -1640,9 +1698,7 @@ impl StreamReader {
     fn release_record(&mut self, cursor: RecordCursor) -> Result<(), DataPlaneError> {
         self.endpoint
             .release_record_cursor(cursor)
-            .map_err(|error| {
-                DataPlaneError::StreamFault(format!("consume stream ring: {error:?}"))
-            })?;
+            .map_err(|error| stream_flow_error(&self.path, "consume stream ring", error))?;
         self.send_control(HostStreamIn::CapacityAvailable)
     }
 
@@ -1655,9 +1711,10 @@ impl StreamReader {
         }
         loop {
             if self.pending_record.is_none()
-                && let Some(cursor) = self.endpoint.record_cursor().map_err(|error| {
-                    DataPlaneError::StreamFault(format!("read stream ring: {error:?}"))
-                })?
+                && let Some(cursor) = self
+                    .endpoint
+                    .record_cursor()
+                    .map_err(|error| stream_flow_error(&self.path, "read stream ring", error))?
             {
                 match cursor.kind() {
                     RecordKind::Data if cursor.is_empty() => {
@@ -1724,6 +1781,17 @@ impl StreamReader {
                 return Ok(count);
             }
 
+            let peer_termination = self.endpoint.peer_termination().map_err(|error| {
+                stream_flow_error(&self.path, "observe stream peer terminal state", error)
+            })?;
+            match peer_termination {
+                None => {}
+                Some(PeerTermination::Closed) => return Err(DataPlaneError::PeerLost),
+                Some(PeerTermination::PathReplaced) => {
+                    return Err(DataPlaneError::PathReplaced(self.path.clone()));
+                }
+            }
+
             let inbox = self
                 .runtime
                 .new_inbox::<ChildStreamIn>()
@@ -1734,9 +1802,7 @@ impl StreamReader {
             if self
                 .endpoint
                 .record_cursor()
-                .map_err(|error| {
-                    DataPlaneError::StreamFault(format!("recheck stream ring: {error:?}"))
-                })?
+                .map_err(|error| stream_flow_error(&self.path, "recheck stream ring", error))?
                 .is_some()
             {
                 continue;
@@ -2083,6 +2149,8 @@ pub struct ChildDataPlaneSessionActor {
     stream_operations: HashMap<ActorAddress, ActorAddress>,
     pending_blob_releases: usize,
     deferred_blob_opens: VecDeque<ChildSessionIn>,
+    close_replies: Vec<ActorAddress>,
+    close_result: Option<Result<(), DataPlaneError>>,
 }
 
 impl ChildDataPlaneSessionActor {
@@ -2090,6 +2158,74 @@ impl ChildDataPlaneSessionActor {
         self.state
     }
 
+    fn complete_close(&mut self, ctx: &Ctx<'_>, result: Result<(), DataPlaneError>) {
+        self.state = ChildSessionState::Closed;
+        self.close_result = Some(result.clone());
+        for reply_to in self.close_replies.drain(..) {
+            let _ = ctx.send(reply_to, result.clone());
+        }
+    }
+
+    fn begin_close(&mut self, ctx: &Ctx<'_>, reply_to: Option<ActorAddress>) {
+        if self.state == ChildSessionState::Closed {
+            if let Some(reply_to) = reply_to {
+                let result = self.close_result.clone().unwrap_or(Ok(()));
+                let _ = ctx.send(reply_to, result);
+            }
+            return;
+        }
+        if let Some(reply_to) = reply_to {
+            self.close_replies.push(reply_to);
+        }
+        if self.state == ChildSessionState::Closing {
+            return;
+        }
+
+        self.state = ChildSessionState::Closing;
+        if let Some(reply_to) = self.attach_reply.take() {
+            let _ = ctx.send(reply_to, Err::<u64, _>(DataPlaneError::SessionNotRunning));
+        }
+        for operation in self.operations.drain() {
+            let _ = ctx.stop_actor(operation);
+        }
+        self.open_operations.clear();
+        self.stream_operations.clear();
+        for reply_to in self.namespace_operations.drain() {
+            let _ = ctx.send(
+                self.host_session,
+                HostSessionIn::CancelNamespace {
+                    operation: reply_to,
+                },
+            );
+            let _ = ctx.send(
+                reply_to,
+                Err::<NamespaceOperationResult, _>(DataPlaneError::SessionNotRunning),
+            );
+        }
+        for deferred in self.deferred_blob_opens.drain(..) {
+            if let ChildSessionIn::Open { reply_to, .. } = deferred {
+                let _ = ctx.send(
+                    reply_to,
+                    Err::<DescriptorOpenGrant, _>(DataPlaneError::OperationCancelled),
+                );
+            }
+        }
+
+        if let Err(error) = ctx.send(
+            self.host_session,
+            HostSessionIn::CloseChild {
+                child_session: ctx.self_addr(),
+            },
+        ) {
+            self.complete_close(
+                ctx,
+                Err(DataPlaneError::SessionFailed(format!(
+                    "close host data-plane session: {error}"
+                ))),
+            );
+        }
+    }
+
     fn start_stream_open(
         &mut self,
         ctx: &Ctx<'_>,
@@ -2359,6 +2495,7 @@ impl ActorInterface for ChildDataPlaneSessionActor {
             ChildSessionIn::StreamOpened {
                 operation,
                 host_binding,
+                incarnation,
                 ring,
                 role,
             } => {
@@ -2367,6 +2504,7 @@ impl ActorInterface for ChildDataPlaneSessionActor {
                         operation,
                         ChildOperationIn::StreamOpened {
                             host_binding,
+                            incarnation,
                             ring,
                             role,
                         },
@@ -2395,33 +2533,11 @@ impl ActorInterface for ChildDataPlaneSessionActor {
                 self.stream_operations
                     .retain(|_, stream_operation| *stream_operation != operation);
             }
-            ChildSessionIn::Close => {
-                if matches!(
-                    self.state,
-                    ChildSessionState::Closing | ChildSessionState::Closed
-                ) {
-                    return;
+            ChildSessionIn::Close { reply_to } => self.begin_close(ctx, reply_to),
+            ChildSessionIn::CloseCompleted { result } => {
+                if self.state == ChildSessionState::Closing {
+                    self.complete_close(ctx, result);
                 }
-                self.state = ChildSessionState::Closing;
-                for operation in self.operations.iter().copied() {
-                    let _ = ctx.stop_actor(operation);
-                }
-                self.open_operations.clear();
-                self.stream_operations.clear();
-                for reply_to in self.namespace_operations.drain() {
-                    let _ = ctx.send(
-                        self.host_session,
-                        HostSessionIn::CancelNamespace {
-                            operation: reply_to,
-                        },
-                    );
-                    let _ = ctx.send(
-                        reply_to,
-                        Err::<NamespaceOperationResult, _>(DataPlaneError::SessionNotRunning),
-                    );
-                }
-                let _ = ctx.send(self.host_session, HostSessionIn::Revoke);
-                self.state = ChildSessionState::Closed;
             }
             _ => {}
         }
@@ -2442,6 +2558,7 @@ enum ChildOperationIn {
     },
     StreamOpened {
         host_binding: ActorAddress,
+        incarnation: StreamIncarnation,
         ring: RingHandle,
         role: Role,
     },
@@ -2513,6 +2630,7 @@ impl ActorInterface for StreamOpenOperationActor {
         match message {
             ChildOperationIn::StreamOpened {
                 host_binding,
+                incarnation: _,
                 ring,
                 role,
             } if role == self.role => self.finish(
@@ -2717,6 +2835,7 @@ impl ActorInterface for DescriptorOpenOperationActor {
             }
             ChildOperationIn::StreamOpened {
                 host_binding,
+                incarnation,
                 ring,
                 role,
             } if self.state == WriteOperationState::Opening => {
@@ -2753,8 +2872,10 @@ impl ActorInterface for DescriptorOpenOperationActor {
                 self.finish_open(
                     ctx,
                     Ok(DescriptorOpenGrant::Stream {
+                        path: self.path.clone(),
                         operation: ctx.self_addr(),
                         host_binding,
+                        incarnation,
                         ring,
                         role,
                         cancellation,

crates/data-plane/src/namespace_store.rs

@@ -5,14 +5,74 @@ use std::fmt;
 use std::fs::{self, File, OpenOptions};
 use std::io::{Read, Write};
 use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicU64, Ordering};
 
 use serde::{Deserialize, Serialize};
 use swactor::actor::ActorAddress;
 
+use crate::namespace::StreamIncarnation;
 use crate::path::DataPath;
 
 const SCHEMA_VERSION: u32 = 1;
 
+static TRACE_COMMITS: AtomicU64 = AtomicU64::new(0);
+static TRACE_COMMIT_MICROS: AtomicU64 = AtomicU64::new(0);
+static TRACE_WORST_COMMIT_MICROS: AtomicU64 = AtomicU64::new(0);
+static COMMIT_BYTES: AtomicU64 = AtomicU64::new(0);
+static COMMIT_FAILURES: AtomicU64 = AtomicU64::new(0);
+static SNAPSHOT_BINDINGS: AtomicU64 = AtomicU64::new(0);
+static SNAPSHOT_OPERATIONS: AtomicU64 = AtomicU64::new(0);
+
+#[derive(Clone, Debug, Serialize)]
+pub struct NamespaceCommitMetrics {
+    pub schema_version: u32,
+    pub attempts: u64,
+    pub failures: u64,
+    pub total_micros: u64,
+    pub worst_micros: u64,
+    pub serialized_bytes: u64,
+    pub bindings: u64,
+    pub operations: u64,
+}
+
+/// Process-local attribution only; these counters never decide durability or
+/// namespace correctness. Readers need no namespace actor scheduling round.
+pub fn commit_metrics() -> NamespaceCommitMetrics {
+    NamespaceCommitMetrics {
+        schema_version: 1,
+        attempts: TRACE_COMMITS.load(Ordering::Relaxed),
+        failures: COMMIT_FAILURES.load(Ordering::Relaxed),
+        total_micros: TRACE_COMMIT_MICROS.load(Ordering::Relaxed),
+        worst_micros: TRACE_WORST_COMMIT_MICROS.load(Ordering::Relaxed),
+        serialized_bytes: COMMIT_BYTES.load(Ordering::Relaxed),
+        bindings: SNAPSHOT_BINDINGS.load(Ordering::Relaxed),
+        operations: SNAPSHOT_OPERATIONS.load(Ordering::Relaxed),
+    }
+}
+
+fn trace_commit_observed(elapsed: std::time::Duration) {
+    let micros = elapsed.as_micros().min(u128::from(u64::MAX)) as u64;
+    TRACE_COMMITS.fetch_add(1, Ordering::Relaxed);
+    TRACE_COMMIT_MICROS.fetch_add(micros, Ordering::Relaxed);
+    TRACE_WORST_COMMIT_MICROS.fetch_max(micros, Ordering::Relaxed);
+}
+
+#[cfg(feature = "directory-trace")]
+pub fn trace_commit_count() -> u64 {
+    TRACE_COMMITS.load(Ordering::Relaxed)
+}
+
+#[cfg(feature = "directory-trace")]
+pub fn trace_commit_avg_ms() -> f64 {
+    let commits = TRACE_COMMITS.load(Ordering::Relaxed).max(1);
+    TRACE_COMMIT_MICROS.load(Ordering::Relaxed) as f64 / commits as f64 / 1000.0
+}
+
+#[cfg(feature = "directory-trace")]
+pub fn trace_commit_worst_ms() -> f64 {
+    TRACE_WORST_COMMIT_MICROS.load(Ordering::Relaxed) as f64 / 1000.0
+}
+
 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub struct OperationId([u8; 16]);
 
@@ -54,8 +114,16 @@ impl<'de> Deserialize<'de> for OperationId {
 
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub enum SourceRecovery {
-    File { path: PathBuf },
-    Actor { actor: ActorAddress },
+    File {
+        path: PathBuf,
+    },
+    Actor {
+        actor: ActorAddress,
+        #[serde(default)]
+        node: [u8; 32],
+        #[serde(default)]
+        owner: Option<ActorAddress>,
+    },
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -121,6 +189,11 @@ pub struct NamespaceSnapshot {
     pub operations: BTreeMap<OperationId, PersistedOperation>,
     #[serde(default)]
     pub retirements: Vec<ActorAddress>,
+    /// Endpoints of stream generations displaced by a replacement whose
+    /// fence has not been acknowledged yet; the directory re-fans the
+    /// displacement until every endpoint acknowledges.
+    #[serde(default)]
+    pub stream_retirements: Vec<(ActorAddress, StreamIncarnation)>,
 }
 
 impl Default for NamespaceSnapshot {
@@ -133,6 +206,7 @@ impl Default for NamespaceSnapshot {
             operations: BTreeMap::new(),
             stream_nodes: BTreeMap::new(),
             retirements: Vec::new(),
+            stream_retirements: Vec::new(),
         }
     }
 }
@@ -258,8 +332,15 @@ impl NamespaceStore {
     }
 
     pub fn commit(&mut self, next: NamespaceSnapshot) -> Result<(), NamespaceStoreError> {
-        next.validate()?;
-        self.persist(&next)?;
+        let started = std::time::Instant::now();
+        let result = next.validate().and_then(|()| self.persist(&next));
+        trace_commit_observed(started.elapsed());
+        if result.is_err() {
+            COMMIT_FAILURES.fetch_add(1, Ordering::Relaxed);
+        }
+        result?;
+        SNAPSHOT_BINDINGS.store(next.bindings.len() as u64, Ordering::Relaxed);
+        SNAPSHOT_OPERATIONS.store(next.operations.len() as u64, Ordering::Relaxed);
         self.snapshot = next;
         Ok(())
     }
@@ -283,6 +364,7 @@ impl NamespaceStore {
         let temporary = parent.join(format!(".{}.tmp", file_name.to_string_lossy()));
         let bytes = serde_json::to_vec(snapshot)
             .map_err(|error| NamespaceStoreError::Corrupt(error.to_string()))?;
+        COMMIT_BYTES.fetch_add(bytes.len() as u64, Ordering::Relaxed);
 
         let write_result = (|| -> Result<(), NamespaceStoreError> {
             let mut file = OpenOptions::new()
@@ -366,6 +448,8 @@ mod tests {
                 revision: 1,
                 recovery: SourceRecovery::Actor {
                     actor: ActorAddress([7; 32]),
+                    node: [8; 32],
+                    owner: None,
                 },
             },
         );

crates/data-plane/src/protocol.rs

@@ -464,6 +464,9 @@ pub enum HostSessionIn {
     Close {
         reply_to: Option<ActorAddress>,
     },
+    CloseChild {
+        child_session: ActorAddress,
+    },
 }
 
 impl NetworkMessage for HostSessionIn {
@@ -537,6 +540,7 @@ pub enum ChildSessionIn {
     StreamOpened {
         operation: ActorAddress,
         host_binding: ActorAddress,
+        incarnation: StreamIncarnation,
         ring: RingHandle,
         role: Role,
     },
@@ -553,7 +557,12 @@ pub enum ChildSessionIn {
     OperationDone {
         operation: ActorAddress,
     },
-    Close,
+    Close {
+        reply_to: Option<ActorAddress>,
+    },
+    CloseCompleted {
+        result: Result<(), DataPlaneError>,
+    },
 }
 
 impl NetworkMessage for ChildSessionIn {
@@ -576,6 +585,9 @@ pub enum HostStreamIn {
     PeerOfferRetry {
         incarnation: StreamIncarnation,
     },
+    RouteChanged {
+        incarnation: StreamIncarnation,
+    },
     Transport(StreamTransportEvent),
     DataAvailable,
     CapacityAvailable,
@@ -600,6 +612,10 @@ pub enum HostStreamIn {
     PeerTerminationRetry {
         incarnation: StreamIncarnation,
     },
+    Displaced {
+        incarnation: StreamIncarnation,
+        reply_to: Option<ActorAddress>,
+    },
     ReleaseComplete(Result<(), DataPlaneError>),
 }
 

crates/data-plane/src/source.rs

@@ -32,6 +32,10 @@ pub enum BlobSourceIn {
     Retire {
         reply_to: Option<ActorAddress>,
     },
+    InspectResources {
+        request_id: String,
+        reply_to: ActorAddress,
+    },
 }
 
 impl NetworkMessage for BlobSourceIn {
@@ -40,6 +44,21 @@ impl NetworkMessage for BlobSourceIn {
     }
 }
 
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct BlobSourceResources {
+    pub request_id: String,
+    pub source: ActorAddress,
+    pub binding: Option<ActorAddress>,
+    pub active_transfers: usize,
+    pub retiring: bool,
+}
+
+impl NetworkMessage for BlobSourceResources {
+    fn type_tag() -> &'static str {
+        "data-plane.blob-source.resources.v1"
+    }
+}
+
 struct ActorTransferCompletion {
     runtime: Runtime,
     source: ActorAddress,
@@ -63,11 +82,12 @@ impl BlobTransferCompletion for ActorTransferCompletion {
 }
 
 pub trait BlobSourcePublisher: Send + Sync + 'static {
-    fn publish_source(&self, source: ActorAddress) -> Result<(), String>;
+    fn publish_source(&self, source: ActorAddress) -> Result<[u8; 32], String>;
 }
 
 pub trait BlobSourceRetirement: Send + Sync + 'static {
     fn retired(&self);
+    fn binding(&self) -> ActorAddress;
 }
 
 pub struct FileBlobSourceActor {
@@ -248,6 +268,24 @@ impl ActorInterface for FileBlobSourceActor {
 
     fn handle(&mut self, ctx: &Ctx<'_>, message: BlobSourceIn) {
         match message {
+            BlobSourceIn::InspectResources {
+                request_id,
+                reply_to,
+            } => {
+                let _ = ctx.send(
+                    reply_to,
+                    BlobSourceResources {
+                        request_id,
+                        source: ctx.self_addr(),
+                        binding: self
+                            .retirement
+                            .as_ref()
+                            .map(|retirement| retirement.binding()),
+                        active_transfers: self.active.len(),
+                        retiring: self.retiring,
+                    },
+                );
+            }
             BlobSourceIn::BeginTransfer { offer } => {
                 if self.retiring {
                     self.fail_destination(ctx, &offer, "blob source is retired".to_owned());
@@ -330,4 +368,7 @@ pub fn register_blob_source_codecs(registry: &mut CodecRegistry) {
     registry
         .register::<BlobSourceIn, _>(JsonCodec::default())
         .expect("unique codec registration");
+    registry
+        .register::<BlobSourceResources, _>(JsonCodec::default())
+        .expect("unique codec registration");
 }

crates/data-plane/tests/actor_blob_guarantees.rs

@@ -564,6 +564,67 @@ fn peer_replacement_requires_and_supports_a_fresh_incarnation() {
     });
 }
 
+#[test]
+fn active_stream_replacement_fences_both_old_endpoints() {
+    let _stream_test = STREAM_TEST_LOCK.lock();
+    let harness = harness(2 << 20);
+    let data_plane = harness.bootstrap.data_plane.clone();
+    let logical = path("/runs/self/results/active-replacement");
+
+    future::block_on(async {
+        let mut first_reader_open = Box::pin(data_plane.read_stream(&logical));
+        assert!(
+            future::poll_once(first_reader_open.as_mut())
+                .await
+                .is_none()
+        );
+        let mut first_writer = data_plane
+            .write_stream(&logical)
+            .await
+            .expect("first writer");
+        let mut first_reader = first_reader_open.await.expect("first reader");
+        first_writer.write(b"old").await.expect("old write");
+        assert_eq!(
+            first_reader.read().await.expect("old read"),
+            Some(b"old".to_vec())
+        );
+
+        let mut replacement_writer_open = Box::pin(data_plane.write_stream_replacing(&logical));
+        assert!(
+            future::poll_once(replacement_writer_open.as_mut())
+                .await
+                .is_none()
+        );
+        let mut replacement_reader = loop {
+            match data_plane.read_stream(&logical).await {
+                Ok(reader) => break reader,
+                Err(DataPlaneError::SessionFailed(reason))
+                    if reason.contains("already has a Sink") => {}
+                Err(error) => panic!("replacement reader: {error}"),
+            }
+        };
+        let mut replacement_writer = replacement_writer_open.await.expect("replacement writer");
+
+        assert!(matches!(
+            first_writer.write(b"stale").await,
+            Err(DataPlaneError::PathReplaced(path)) if path == logical
+        ));
+        let stale_read = first_reader.read().await;
+        assert!(
+            matches!(
+                &stale_read,
+                Err(DataPlaneError::PathReplaced(path)) if path == &logical
+            ),
+            "old reader remained usable after replacement: {stale_read:?}"
+        );
+        replacement_writer.write(b"new").await.expect("new write");
+        assert_eq!(
+            replacement_reader.read().await.expect("new read"),
+            Some(b"new".to_vec())
+        );
+    });
+}
+
 #[test]
 fn actor_stream_consumer_registers_before_writer_and_collects_to_eof() {
     let _stream_test = STREAM_TEST_LOCK.lock();

crates/data-plane/tests/data_plane_test_support.inc

@@ -150,8 +150,8 @@ impl data_plane::namespace::NamespaceDiscovery for StaticDiscovery {
 struct NoopSourceRegistrar;
 
 impl data_plane::source::BlobSourcePublisher for NoopSourceRegistrar {
-    fn publish_source(&self, _source: ActorAddress) -> Result<(), String> {
-        Ok(())
+    fn publish_source(&self, _source: ActorAddress) -> Result<[u8; 32], String> {
+        Ok([9; 32])
     }
 }
 
@@ -272,6 +272,7 @@ fn harness_with_transport(
         future::block_on(directory_client.register(
             path(logical),
             source,
+            [9; 32],
             length,
             recovery,
             data_plane::namespace::OperationId::from_u128(u128::from(length) + 1),

crates/data-plane/tests/namespace_host_read_guarantees.rs

@@ -120,8 +120,8 @@ impl BlobTransferReceiver for DirectReceiver {
 struct NoopSourceRegistrar;
 
 impl BlobSourcePublisher for NoopSourceRegistrar {
-    fn publish_source(&self, _source: ActorAddress) -> Result<(), String> {
-        Ok(())
+    fn publish_source(&self, _source: ActorAddress) -> Result<[u8; 32], String> {
+        Ok([9; 32])
     }
 }
 
@@ -231,6 +231,7 @@ fn host_read_resolves_file_source_and_seals_final_arena_lease() {
         &store_path,
         Arc::clone(&sender),
         Arc::clone(&source_publisher),
+        None,
     )
     .unwrap();
     let directory = service.directory();
@@ -333,6 +334,7 @@ fn host_read_resolves_file_source_and_seals_final_arena_lease() {
         future::block_on(direct.register(
             fault_path.clone(),
             source,
+            [9; 32],
             length,
             recovery,
             OperationId::from_u128(10 + u128::from(mode)),

crates/distribution/src/directory_actor.rs

@@ -24,7 +24,9 @@
 //! [`MetadataActor`]: crate::node_metadata_actor::MetadataActor
 
 use std::collections::{BTreeSet, HashMap, HashSet};
-use std::sync::Arc;
+use std::sync::{Arc, Weak};
+
+use parking_lot::RwLock;
 
 use swactor::actor::{ActorAddress, ActorInterface};
 use swactor::runtime::Ctx;
@@ -65,6 +67,20 @@ pub enum DirectoryIn {
         actor: ActorAddress,
         reply: ActorAddress,
     },
+    /// Local: re-arm every cached claim for dissemination. A restarted peer
+    /// keeps the same node id, so membership never transitions and ordinary
+    /// gossip has no rejoin edge to tell it that the peer lost its map.
+    Resync,
+    /// Local: send the cached signed claims directly over a newly established
+    /// peer connection. A stable-id restart need not produce a membership delta,
+    /// and the peer may have lost every application reply route.
+    SyncTo { peer: NodeId },
+    /// Local, lifetime-bound wakeup after routes or verified claims change.
+    /// The subscriber retains the callback; dropping it ends observations.
+    /// A wakeup is not a location claim: callbacks must re-read the views.
+    WatchRoutes {
+        changed: Weak<dyn Fn() + Send + Sync>,
+    },
     /// Clock: disseminate one batch to one peer.
     Tick,
 }
@@ -76,10 +92,26 @@ pub struct Located {
     pub host: Option<NodeId>,
 }
 
+/// Read-only access to the directory's verified winning claims. A cached claim
+/// authenticates its host and generation, not the host's current reachability.
+#[derive(Clone, Default)]
+pub struct DirectoryClaims {
+    entries: Arc<RwLock<HashMap<ActorAddress, DirectoryEntry>>>,
+}
+
+impl DirectoryClaims {
+    pub fn location(&self, actor: &ActorAddress) -> Option<(NodeId, u64)> {
+        self.entries
+            .read()
+            .get(actor)
+            .map(|claim| (claim.node_id, claim.generation))
+    }
+}
+
 pub struct DirectoryActor {
     self_id: NodeId,
     /// The location map: one signed claim per actor. The only writer is [`Self::merge_one`].
-    map: HashMap<ActorAddress, DirectoryEntry>,
+    map: DirectoryClaims,
     /// Alive peers (excludes self), folded from the membership stream — the
     /// dissemination fan-out set and the `cluster_size` budget basis.
     alive: BTreeSet<NodeId>,
@@ -103,6 +135,7 @@ pub struct DirectoryActor {
     /// runtime attached to this host). Directory republishing always preserves
     /// these entries.
     pinned_routes: RouteView,
+    watchers: Vec<Weak<dyn Fn() + Send + Sync>>,
 }
 
 impl DirectoryActor {
@@ -130,7 +163,7 @@ impl DirectoryActor {
     ) -> Self {
         Self {
             self_id,
-            map: HashMap::new(),
+            map: DirectoryClaims::default(),
             alive: BTreeSet::new(),
             hot: HashMap::new(),
             cursor: 0,
@@ -139,9 +172,14 @@ impl DirectoryActor {
             route_binder,
             bound_remote: HashSet::new(),
             pinned_routes,
+            watchers: Vec::new(),
         }
     }
 
+    pub fn claims(&self) -> DirectoryClaims {
+        self.map.clone()
+    }
+
     /// Merge one claim under the supersession rule — the only writer of `map`.
     ///
     /// A claim wins iff it is strictly newer by `(generation, node_id)`: a higher
@@ -150,11 +188,12 @@ impl DirectoryActor {
     /// stale, equal, or forged claim is ignored. Merging the same claim twice is a
     /// no-op — it does not re-arm dissemination, which is what lets the cluster go
     /// quiet. Idempotent and commutative.
-    fn merge_one(&mut self, claim: DirectoryEntry) {
+    fn merge_one(&mut self, claim: DirectoryEntry) -> bool {
         if !verify_directory_entry(&claim) {
-            return; // not signed by the host it names — drop it
+            return false; // not signed by the host it names — drop it
         }
-        let supersedes = match self.map.get(&claim.actor_addr) {
+        let mut map = self.map.entries.write();
+        let supersedes = match map.get(&claim.actor_addr) {
             None => true,
             Some(cur) => {
                 claim.generation > cur.generation
@@ -164,20 +203,22 @@ impl DirectoryActor {
         if supersedes {
             let budget = self.budget();
             self.hot.insert(claim.actor_addr, budget); // arm for dissemination
-            self.map.insert(claim.actor_addr, claim);
+            map.insert(claim.actor_addr, claim);
         }
+        supersedes
     }
 
     fn register(&mut self, claim: DirectoryEntry) {
-        self.merge_one(claim);
-        self.republish();
+        let changed = self.merge_one(claim);
+        self.republish(changed);
     }
 
     fn merge_batch(&mut self, claims: Vec<DirectoryEntry>) {
+        let mut changed = false;
         for c in claims {
-            self.merge_one(c);
+            changed |= self.merge_one(c);
         }
-        self.republish();
+        self.republish(changed);
     }
 
     fn on_membership(&mut self, change: MembershipChanged) {
@@ -188,7 +229,8 @@ impl DirectoryActor {
                     // peer is caught up — without a full-cluster reflood (only the
                     // actors we hold, and only via the lazy push).
                     let budget = self.budget();
-                    let actors: Vec<ActorAddress> = self.map.keys().copied().collect();
+                    let actors: Vec<ActorAddress> =
+                        self.map.entries.read().keys().copied().collect();
                     for actor in actors {
                         self.hot.insert(actor, budget);
                     }
@@ -202,7 +244,7 @@ impl DirectoryActor {
             }
             _ => {} // Suspect, or self: ignore (suspicion is SWIM's transient state)
         }
-        self.republish();
+        self.republish(false);
     }
 
     fn tick(&mut self, ctx: &Ctx) {
@@ -228,12 +270,27 @@ impl DirectoryActor {
         }
     }
 
+    fn sync_to(&self, ctx: &Ctx, peer: NodeId) {
+        let Some(addr) = self.peer_directory.resolve(&peer) else {
+            return;
+        };
+        let map = self.map.entries.read();
+        let mut claims = map.values();
+        loop {
+            let batch: Vec<_> = claims.by_ref().take(BATCH).cloned().collect();
+            if batch.is_empty() {
+                break;
+            }
+            let _ = ctx.send(addr, DirectoryIn::Gossip(DirectoryGossip { claims: batch }));
+        }
+    }
+
     /// Republish the §5 route view: every actor whose host is reachable right now
     /// (self, or an alive peer). A dead host's actors are omitted, so the egress
     /// never routes to a host SWIM has buried; the claim remains cached for
     /// recovery. Actors that left the remotely-routed set are unbound so the
     /// binder and transport router do not grow without bound.
-    fn republish(&mut self) {
+    fn republish(&mut self, claims_changed: bool) {
         let pinned = self
             .pinned_routes
             .read()
@@ -244,7 +301,8 @@ impl DirectoryActor {
             .filter_map(|(actor, node)| (*node != self.self_id).then_some(*actor))
             .collect::<HashSet<_>>();
         drop(pinned);
-        for (actor, claim) in &self.map {
+        let map = self.map.entries.read();
+        for (actor, claim) in map.iter() {
             if view.contains_key(actor) {
                 continue;
             }
@@ -255,7 +313,13 @@ impl DirectoryActor {
                 remote.insert(*actor);
             }
         }
-        *self.route_view.write().expect("route view poisoned") = view;
+        drop(map);
+        let changed = {
+            let mut published = self.route_view.write().expect("route view poisoned");
+            let changed = *published != view;
+            *published = view;
+            changed
+        };
         // Unbind actors that fell out of the remotely-routed set (host left
         // the cluster or claim superseded): without this diff the binder and
         // router retain every ever-seen actor forever.
@@ -268,6 +332,16 @@ impl DirectoryActor {
             self.route_binder.ensure_routable(actor);
         }
         self.bound_remote = remote;
+        if changed || claims_changed {
+            self.watchers.retain(|watcher| {
+                if let Some(watcher) = watcher.upgrade() {
+                    watcher();
+                    true
+                } else {
+                    false
+                }
+            });
+        }
     }
 
     /// Take up to `limit` armed claims for this tick's batch, spending one unit of
@@ -275,8 +349,9 @@ impl DirectoryActor {
     fn take_hot(&mut self, limit: usize) -> Vec<DirectoryEntry> {
         let actors: Vec<ActorAddress> = self.hot.keys().copied().take(limit).collect();
         let mut claims = Vec::with_capacity(actors.len());
+        let map = self.map.entries.read();
         for actor in actors {
-            if let Some(c) = self.map.get(&actor) {
+            if let Some(c) = map.get(&actor) {
                 claims.push(c.clone());
             }
             if let Some(left) = self.hot.get_mut(&actor) {
@@ -308,6 +383,22 @@ impl ActorInterface for DirectoryActor {
             DirectoryIn::Register(claim) => self.register(claim),
             DirectoryIn::Gossip(batch) => self.merge_batch(batch.claims),
             DirectoryIn::Membership(change) => self.on_membership(change),
+            DirectoryIn::SyncTo { peer } => self.sync_to(ctx, peer),
+            DirectoryIn::WatchRoutes { changed } => {
+                self.watchers.retain(|watcher| watcher.strong_count() != 0);
+                if let Some(watcher) = changed.upgrade() {
+                    watcher();
+                    self.watchers.push(changed);
+                }
+            }
+            DirectoryIn::Resync => {
+                let budget = self.budget();
+                let actors: Vec<ActorAddress> = self.map.entries.read().keys().copied().collect();
+                for actor in actors {
+                    self.hot.insert(actor, budget);
+                }
+                self.republish(false);
+            }
             DirectoryIn::Tick => self.tick(ctx),
             DirectoryIn::Resolve { actor, reply } => {
                 let host = self

crates/distribution/src/messages.rs

@@ -3,6 +3,7 @@
 
 use serde::{Deserialize, Serialize};
 use swactor::Error;
+use swactor::actor::ActorAddress;
 use swactor_transport::{CodecRegistry, JsonCodec, NetworkMessage};
 
 use crate::node_metadata::NodeMetadataEntry;
@@ -127,6 +128,10 @@ pub struct MembershipUpdate {
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct RegistryGossip {
     pub entries: Vec<RegistryEntry>,
+    /// Directed recovery delivery uses the same gossip route as ordinary
+    /// replication. Acknowledgements never merge entries back into the CRDT.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub delivery: Option<RegistryDelivery>,
 }
 
 impl NetworkMessage for RegistryGossip {
@@ -135,6 +140,17 @@ impl NetworkMessage for RegistryGossip {
     }
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum RegistryDelivery {
+    /// Acknowledge only entries that are exact live winners after merging.
+    Request { reply_to: ActorAddress },
+    /// `entries` contains the exact winners installed by this peer.
+    Acknowledged {
+        reply_to: ActorAddress,
+        peer: NodeId,
+    },
+}
+
 /// Node-metadata gossip — a batch of per-node metadata entries (relay URL,
 /// node name) disseminated independently of SWIM membership.
 #[derive(Debug, Clone, Serialize, Deserialize)]

crates/distribution/src/registry.rs

@@ -74,6 +74,8 @@ pub struct RegistryEntrySnapshot {
     pub name: String,
     pub actor_addr: ActorAddress,
     pub node_id: NodeId,
+    pub timestamp: u64,
+    pub generation: u64,
     pub tombstone: bool,
 }
 
@@ -150,6 +152,31 @@ impl ClusterRegistry {
         self.merge_and_enqueue(entry, cluster_size);
     }
 
+    /// Register with a caller-supplied logical timestamp. A restarted node's
+    /// fresh CRDT clock starts below entries it previously disseminated; an
+    /// epoch-derived high timestamp makes the recovered binding win without
+    /// reading ambient time inside the actor.
+    pub fn register_at(
+        &mut self,
+        name: String,
+        actor_addr: ActorAddress,
+        node_id: NodeId,
+        timestamp: u64,
+        cluster_size: usize,
+    ) {
+        self.clock = self.clock.saturating_add(1).max(timestamp);
+        let generation = self.next_generation(&name);
+        let entry = RegistryEntry {
+            name,
+            actor_addr,
+            node_id,
+            timestamp: self.clock,
+            generation,
+            tombstone: false,
+        };
+        self.merge_and_enqueue(entry, cluster_size);
+    }
+
     /// Unregister a name (create a tombstone).
     pub fn unregister(&mut self, name: &str, node_id: NodeId, cluster_size: usize) {
         self.clock += 1;
@@ -210,7 +237,11 @@ impl ClusterRegistry {
 
     /// Merge a batch of entries received from gossip.
     /// Changed entries are re-enqueued for further dissemination.
-    pub fn merge_batch(&mut self, entries: Vec<RegistryEntry>, cluster_size: usize) {
+    pub fn merge_batch(
+        &mut self,
+        entries: impl IntoIterator<Item = RegistryEntry>,
+        cluster_size: usize,
+    ) {
         for entry in entries {
             if self.merge(entry.clone()) {
                 self.enqueue(entry, cluster_size);
@@ -334,6 +365,8 @@ impl ClusterRegistry {
                 name: e.name.clone(),
                 actor_addr: e.actor_addr,
                 node_id: e.node_id,
+                timestamp: e.timestamp,
+                generation: e.generation,
                 tombstone: e.tombstone,
             })
             .collect();

crates/distribution/src/registry_actor.rs

@@ -10,13 +10,13 @@
 //! node returns.
 
 use std::collections::BTreeSet;
-use std::sync::{Arc, RwLock};
+use std::sync::{Arc, RwLock, Weak};
 
 use swactor::actor::{ActorAddress, ActorInterface};
 use swactor::runtime::Ctx;
 
-use crate::messages::RegistryGossip;
-use crate::registry::{ClusterRegistry, RegistryConfig, RegistrySnapshot};
+use crate::messages::{RegistryDelivery, RegistryGossip};
+use crate::registry::{ClusterRegistry, RegistryConfig, RegistryEntry, RegistrySnapshot};
 use crate::swim::actor::{MembershipChanged, PeerDirectory};
 use crate::types::{MemberState, NodeId};
 
@@ -38,12 +38,43 @@ pub enum RegistryIn {
         name: String,
         actor_addr: ActorAddress,
     },
+    /// Local: register with a caller-supplied logical timestamp. Used by a
+    /// recovered service whose fresh logical clock would otherwise lose to the
+    /// binding it disseminated before restarting.
+    RegisterNameAt {
+        name: String,
+        actor_addr: ActorAddress,
+        timestamp: u64,
+    },
+    /// Local: push this registry's current live entry for `name` directly to
+    /// `peers` now, bypassing SWIM-gated dissemination. Recovery re-bind: a
+    /// restarted authority must replace the dead binding workers still serve
+    /// without waiting for membership rounds to re-form (observed stall: a
+    /// dead directory binding served for the full namespace request deadline
+    /// while membership re-converged).
+    DisseminateNameTo {
+        name: String,
+        peers: Vec<NodeId>,
+        reply_to: ActorAddress,
+    },
+    /// Local: a peer installed this exact live name/binding/generation.
+    /// Delivered only while it remains the origin registry's current winner.
+    NameAcknowledged { entry: RegistryEntry, peer: NodeId },
     /// Local: unregister a name (writes a tombstone).
     UnregisterName { name: String },
     /// Local request: resolve `name`; the result is sent to `reply`.
     ResolveName { name: String, reply: ActorAddress },
     /// Gossip from a peer: a batch of CRDT entries to merge.
     Gossip(RegistryGossip),
+    /// Local: send all current winners, including tombstones, directly over a
+    /// newly established peer connection, even if membership never changed.
+    SyncTo { peer: NodeId },
+    /// Local observer, retained only while its owner keeps the callback alive.
+    /// Runs after publication and once on registration to close the check/subscribe race.
+    WatchName {
+        name: String,
+        changed: Weak<dyn Fn() + Send + Sync>,
+    },
     /// Clock: run GC and disseminate a pending batch to one peer.
     Tick,
 }
@@ -68,6 +99,7 @@ pub struct RegistryActor {
     /// Optional read-mirror the node's telemetry tick observes. Republished
     /// after each registry change. `None` when no one is observing.
     view: Option<RegistryView>,
+    watchers: Vec<(String, Option<RegistryEntry>, Weak<dyn Fn() + Send + Sync>)>,
 }
 
 impl RegistryActor {
@@ -83,6 +115,7 @@ impl RegistryActor {
             alive: BTreeSet::new(),
             fanout_cursor: 0,
             view: None,
+            watchers: Vec::new(),
         }
     }
 
@@ -96,10 +129,21 @@ impl RegistryActor {
     }
 
     /// Republish the registry snapshot to the read-mirror, if one is installed.
-    fn publish(&self) {
+    fn publish(&mut self) {
         if let Some(view) = &self.view {
             *view.write().expect("registry view poisoned") = self.registry.snapshot();
         }
+        self.watchers.retain_mut(|(name, previous, changed)| {
+            let Some(changed) = changed.upgrade() else {
+                return false;
+            };
+            let current = self.registry.entries().find(|entry| entry.name == *name);
+            if current != previous.as_ref() {
+                *previous = current.cloned();
+                changed();
+            }
+            true
+        });
     }
 
     fn cluster_size(&self) -> usize {
@@ -122,7 +166,33 @@ impl RegistryActor {
         let peer = peers[self.fanout_cursor % peers.len()];
         self.fanout_cursor = self.fanout_cursor.wrapping_add(1);
         if let Some(addr) = self.peer_directory.resolve(&peer) {
-            let _ = ctx.send(addr, RegistryIn::Gossip(RegistryGossip { entries }));
+            let _ = ctx.send(
+                addr,
+                RegistryIn::Gossip(RegistryGossip {
+                    entries,
+                    delivery: None,
+                }),
+            );
+        }
+    }
+
+    fn sync_to(&self, ctx: &Ctx, peer: NodeId) {
+        let Some(addr) = self.peer_directory.resolve(&peer) else {
+            return;
+        };
+        let mut entries = self.registry.entries();
+        loop {
+            let batch: Vec<_> = entries.by_ref().take(8).cloned().collect();
+            if batch.is_empty() {
+                break;
+            }
+            let _ = ctx.send(
+                addr,
+                RegistryIn::Gossip(RegistryGossip {
+                    entries: batch,
+                    delivery: None,
+                }),
+            );
         }
     }
 }
@@ -155,20 +225,106 @@ impl ActorInterface for RegistryActor {
                 self.registry.register(name, actor_addr, self.self_id, size);
                 self.publish();
             }
+            RegistryIn::RegisterNameAt {
+                name,
+                actor_addr,
+                timestamp,
+            } => {
+                let size = self.cluster_size();
+                self.registry
+                    .register_at(name, actor_addr, self.self_id, timestamp, size);
+                self.publish();
+            }
+            RegistryIn::DisseminateNameTo {
+                name,
+                peers,
+                reply_to,
+            } => {
+                let Some(entry) = self
+                    .registry
+                    .entries()
+                    .find(|entry| {
+                        entry.name == name && !entry.tombstone && entry.node_id == self.self_id
+                    })
+                    .cloned()
+                else {
+                    return;
+                };
+                for peer in peers {
+                    if let Some(addr) = self.peer_directory.resolve(&peer) {
+                        let _ = ctx.send(
+                            addr,
+                            RegistryIn::Gossip(RegistryGossip {
+                                entries: vec![entry.clone()],
+                                delivery: Some(RegistryDelivery::Request { reply_to }),
+                            }),
+                        );
+                    }
+                }
+            }
             RegistryIn::UnregisterName { name } => {
                 let size = self.cluster_size();
                 self.registry.unregister(&name, self.self_id, size);
                 self.publish();
             }
+            RegistryIn::SyncTo { peer } => self.sync_to(ctx, peer),
+            RegistryIn::WatchName { name, changed } => {
+                let current = self
+                    .registry
+                    .entries()
+                    .find(|entry| entry.name == name)
+                    .cloned();
+                if let Some(notify) = changed.upgrade() {
+                    notify();
+                    self.watchers.push((name, current, changed));
+                }
+            }
             RegistryIn::ResolveName { name, reply } => {
                 let binding = self.registry.resolve(&name);
                 let _ = ctx.send(reply, NameResolved { name, binding });
             }
             RegistryIn::Gossip(g) => {
                 let size = self.cluster_size();
-                self.registry.merge_batch(g.entries, size);
-                self.publish();
+                match g.delivery {
+                    Some(RegistryDelivery::Acknowledged { reply_to, peer }) => {
+                        for entry in g.entries {
+                            if !entry.tombstone
+                                && entry.node_id == self.self_id
+                                && self.registry.entries().any(|current| current == &entry)
+                            {
+                                let _ = ctx
+                                    .send(reply_to, RegistryIn::NameAcknowledged { entry, peer });
+                            }
+                        }
+                    }
+                    Some(RegistryDelivery::Request { reply_to }) => {
+                        self.registry.merge_batch(g.entries.iter().cloned(), size);
+                        self.publish();
+                        for entry in g.entries {
+                            if !entry.tombstone
+                                && self.registry.entries().any(|current| current == &entry)
+                                && let Some(addr) = self.peer_directory.resolve(&entry.node_id)
+                            {
+                                let _ = ctx.send(
+                                    addr,
+                                    RegistryIn::Gossip(RegistryGossip {
+                                        entries: vec![entry],
+                                        delivery: Some(RegistryDelivery::Acknowledged {
+                                            reply_to,
+                                            peer: self.self_id,
+                                        }),
+                                    }),
+                                );
+                            }
+                        }
+                    }
+                    None => {
+                        self.registry.merge_batch(g.entries, size);
+                        self.publish();
+                    }
+                }
             }
+            RegistryIn::NameAcknowledged { .. } => {}
             RegistryIn::Tick => {
                 self.registry.gc_tick();
                 self.disseminate(ctx);

crates/distribution/src/transport_bridge.rs

@@ -17,7 +17,7 @@
 //! only the distribution-side routing and outbound queue.
 
 use std::collections::{HashMap, HashSet};
-use std::sync::{Arc, Mutex, RwLock};
+use std::sync::{Arc, LockResult, Mutex, MutexGuard, RwLock};
 
 use parking_lot::Mutex as ParkingMutex;
 
@@ -70,7 +70,33 @@ pub struct OutFrame {
 
 /// Shared queue of outbound frames, written by worker threads (via
 /// [`OutboxPeerTransport`]) and drained by the concrete network driver.
-pub type Outbox = Arc<Mutex<Vec<OutFrame>>>;
+pub type Outbox = Arc<OutboxQueue>;
+
+#[derive(Default)]
+pub struct OutboxQueue {
+    frames: Mutex<Vec<OutFrame>>,
+    wake: RwLock<Option<Arc<dyn Fn() + Send + Sync>>>,
+}
+
+impl OutboxQueue {
+    pub fn lock(&self) -> LockResult<MutexGuard<'_, Vec<OutFrame>>> {
+        self.frames.lock()
+    }
+
+    /// Install the owning driver's wakeup and reconcile anything queued
+    /// before installation. Notifications never substitute for draining.
+    pub fn set_wake(&self, wake: Arc<dyn Fn() + Send + Sync>) {
+        *self.wake.write().expect("outbox wake poisoned") = Some(wake.clone());
+        wake();
+    }
+
+    fn push(&self, frame: OutFrame) {
+        self.frames.lock().expect("outbox poisoned").push(frame);
+        if let Some(wake) = self.wake.read().expect("outbox wake poisoned").as_ref() {
+            wake();
+        }
+    }
+}
 
 /// Per-peer egress transport. The runtime hands it an already-encoded
 /// [`WireEnvelope`] bound for this peer; it just records the frame on the shared
@@ -83,7 +109,7 @@ struct OutboxPeerTransport {
 
 impl Transport for OutboxPeerTransport {
     fn send(&self, envelope: WireEnvelope) -> Result<(), Error> {
-        self.outbox.lock().expect("outbox poisoned").push(OutFrame {
+        self.outbox.push(OutFrame {
             to: self.node_id,
             dest: envelope.dest,
             type_tag: envelope.type_tag,
@@ -161,7 +187,7 @@ impl Transport for RouteViewTransport {
             .get(&envelope.dest)
             .copied();
         if let Some(node) = host {
-            self.outbox.lock().expect("outbox poisoned").push(OutFrame {
+            self.outbox.push(OutFrame {
                 to: node,
                 dest: envelope.dest,
                 type_tag: envelope.type_tag,
@@ -254,7 +280,7 @@ mod tests {
     fn an_encoded_frame_is_enqueued_for_the_peer_it_targets() {
         // The egress contract: a routed WireEnvelope becomes an OutFrame the
         // driver can write, preserving the target peer, wire tag, and bytes.
-        let outbox: Outbox = Arc::new(Mutex::new(Vec::new()));
+        let outbox: Outbox = Arc::new(Default::default());
         let peer = id(7);
         let transport = OutboxPeerTransport {
             node_id: peer,
@@ -278,7 +304,7 @@ mod tests {
     #[test]
     fn route_view_transport_drops_missing_route_without_error() {
         let route_view: RouteView = Arc::new(RwLock::new(HashMap::new()));
-        let outbox: Outbox = Arc::new(Mutex::new(Vec::new()));
+        let outbox: Outbox = Arc::new(Default::default());
         let transport = RouteViewTransport::new(route_view, outbox.clone());
 
         let result = transport.send(WireEnvelope {
@@ -300,7 +326,7 @@ mod tests {
             .write()
             .expect("route view poisoned")
             .insert(actor, node);
-        let outbox: Outbox = Arc::new(Mutex::new(Vec::new()));
+        let outbox: Outbox = Arc::new(Default::default());
         let transport = RouteViewTransport::new(route_view, outbox.clone());
 
         let result = transport.send(WireEnvelope {

crates/distribution/tests/directory_unbind.rs

@@ -108,3 +108,75 @@ fn republish_unbinds_routes_of_departed_hosts() {
         "returning host's actor must be re-bound: {events:?}"
     );
 }
+
+#[test]
+fn verified_generation_changes_wake_observers_without_route_or_clock_changes() {
+    let parts =
+        RuntimeParts::new(RuntimeConfig::default()).with_extension(Arc::new(StdExtension::new()));
+    let rt = parts.runtime().clone();
+    let backend = SteppingBackend::new();
+    let _engine = Engine::new(parts, backend.clone()).unwrap();
+    let key = Keypair::from_bytes(&[3; 32]);
+    let actor = ActorAddress([9; 32]);
+    let directory = DirectoryActor::new(
+        key.node_id(),
+        Arc::new(SharedPeerDirectory::new()),
+        Arc::new(std::sync::RwLock::new(HashMap::new())),
+        Arc::new(RecordingBinder {
+            events: Mutex::new(Vec::new()),
+        }),
+    );
+    let claims = directory.claims();
+    let directory = rt.spawn(directory).unwrap();
+    let observations = Arc::new(Mutex::new(Vec::new()));
+    let observed = observations.clone();
+    let changed: Arc<dyn Fn() + Send + Sync> =
+        Arc::new(move || observed.lock().push(claims.location(&actor)));
+    rt.send_to(
+        directory,
+        DirectoryIn::WatchRoutes {
+            changed: Arc::downgrade(&changed),
+        },
+    )
+    .unwrap();
+    backend.step();
+    rt.send_to(
+        directory,
+        DirectoryIn::Register(key.sign_directory_entry(actor, 1)),
+    )
+    .unwrap();
+    backend.step();
+    let mut forged = key.sign_directory_entry(actor, 1);
+    forged.generation = 99;
+    rt.send_to(directory, DirectoryIn::Register(forged))
+        .unwrap();
+    rt.send_to(
+        directory,
+        DirectoryIn::Register(key.sign_directory_entry(actor, 2)),
+    )
+    .unwrap();
+    backend.step();
+    rt.send_to(
+        directory,
+        DirectoryIn::Register(key.sign_directory_entry(actor, 1)),
+    )
+    .unwrap();
+    backend.step();
+    assert_eq!(
+        *observations.lock(),
+        vec![None, Some((key.node_id(), 1)), Some((key.node_id(), 2))],
+        "only verified winning generations may wake the authority observer"
+    );
+    drop(changed);
+    rt.send_to(
+        directory,
+        DirectoryIn::Register(key.sign_directory_entry(actor, 3)),
+    )
+    .unwrap();
+    backend.step();
+    assert_eq!(
+        *observations.lock(),
+        vec![None, Some((key.node_id(), 1)), Some((key.node_id(), 2))],
+        "dropping the observation must end callbacks"
+    );
+}

crates/distribution/tests/gossip_data.rs

@@ -50,6 +50,35 @@ mod registry_crdt {
         assert_eq!(reg.resolve("svc"), Some((addr_new, node_id)));
     }
 
+    #[test]
+    fn recovered_authoritative_registration_wins_over_older_disseminated_clock() {
+        let mut restarted = ClusterRegistry::new(RegistryConfig::default());
+        let old_address = ActorAddress::new_random();
+        let recovered_address = ActorAddress::new_random();
+        let node_id = NodeId([7; 32]);
+
+        restarted.merge(RegistryEntry {
+            name: "data-directory".into(),
+            actor_addr: old_address,
+            node_id,
+            timestamp: 10_000,
+            generation: 7,
+            tombstone: false,
+        });
+        restarted.register_at(
+            "data-directory".into(),
+            recovered_address,
+            node_id,
+            1_u64 << 63,
+            2,
+        );
+
+        assert_eq!(
+            restarted.resolve("data-directory"),
+            Some((recovered_address, node_id))
+        );
+    }
+
     // ─── LWW tiebreak — generation then node_id ────────────────────────────────
 
     #[test]
@@ -518,6 +547,212 @@ mod standalone_gossip_transport {
             "registered name did not propagate over the transport"
         );
     }
+    #[test]
+    fn directed_rebind_replaces_a_stale_binding_without_gossip_rounds() {
+        use distribution::messages::RegistryGossip;
+        use distribution::registry::RegistryEntry;
+
+        let c = GossipCluster::new(2);
+        let acknowledgements = c.nodes[0].rt.new_inbox::<RegistryIn>().unwrap();
+        let name = "swactor.data-directory".to_owned();
+        let dead = ActorAddress([0xDD; 32]);
+        let fresh = ActorAddress([0xF3; 32]);
+
+        // The peer still serves the authority's pre-crash binding.
+        c.nodes[1]
+            .rt
+            .send_to(
+                c.nodes[1].registry,
+                RegistryIn::Gossip(RegistryGossip {
+                    entries: vec![RegistryEntry {
+                        name: name.clone(),
+                        actor_addr: dead,
+                        node_id: c.ids[0],
+                        timestamp: (1 << 63) + 4,
+                        generation: 1,
+                        tombstone: false,
+                    }],
+                    delivery: None,
+                }),
+            )
+            .unwrap();
+        c.pump(2);
+        assert_eq!(c.resolve_name(1, &name), Some((dead, c.ids[0])));
+
+        // The restarted authority registers its recovered binding and pushes
+        // it directly to the persisted peer — no Tick, no gossip round, no
+        // membership convergence in between.
+        c.nodes[0]
+            .rt
+            .send_to(
+                c.nodes[0].registry,
+                RegistryIn::RegisterNameAt {
+                    name: name.clone(),
+                    actor_addr: fresh,
+                    timestamp: (1 << 63) + 5,
+                },
+            )
+            .unwrap();
+        c.nodes[0].backend.step();
+        c.nodes[0]
+            .rt
+            .send_to(
+                c.nodes[0].registry,
+                RegistryIn::DisseminateNameTo {
+                    name,
+                    peers: vec![c.ids[1]],
+                    reply_to: *acknowledgements.addr(),
+                },
+            )
+            .unwrap();
+        c.pump(4);
+
+        assert_eq!(
+            c.resolve_name(1, "swactor.data-directory"),
+            Some((fresh, c.ids[0])),
+            "directed re-bind must replace the dead binding without gossip rounds"
+        );
+        assert!(matches!(
+            acknowledgements.try_recv(),
+            Some(RegistryIn::NameAcknowledged { entry, peer })
+                if entry.name == "swactor.data-directory"
+                    && entry.actor_addr == fresh
+                    && entry.node_id == c.ids[0]
+                    && entry.timestamp == (1 << 63) + 5
+                    && entry.generation == 1
+                    && !entry.tombstone
+                    && peer == c.ids[1]
+        ));
+
+        // Losing an ACK must not strand recovery: an unchanged duplicate
+        // publication still acknowledges the installed winner.
+        c.nodes[0]
+            .rt
+            .send_to(
+                c.nodes[0].registry,
+                RegistryIn::DisseminateNameTo {
+                    name: "swactor.data-directory".to_owned(),
+                    peers: vec![c.ids[1]],
+                    reply_to: *acknowledgements.addr(),
+                },
+            )
+            .unwrap();
+        c.pump(4);
+        assert!(matches!(acknowledgements.try_recv(),
+            Some(RegistryIn::NameAcknowledged { entry, peer })
+                if entry.actor_addr == fresh && peer == c.ids[1]
+        ));
+
+        // A peer that has a newer winning binding must not ACK mere receipt
+        // of the stale directed request.
+        c.nodes[1]
+            .rt
+            .send_to(
+                c.nodes[1].registry,
+                RegistryIn::RegisterNameAt {
+                    name: "swactor.data-directory".to_owned(),
+                    actor_addr: ActorAddress([0xF4; 32]),
+                    timestamp: (1 << 63) + 20,
+                },
+            )
+            .unwrap();
+        c.nodes[1].backend.step();
+        c.nodes[0]
+            .rt
+            .send_to(
+                c.nodes[0].registry,
+                RegistryIn::DisseminateNameTo {
+                    name: "swactor.data-directory".to_owned(),
+                    peers: vec![c.ids[1]],
+                    reply_to: *acknowledgements.addr(),
+                },
+            )
+            .unwrap();
+        c.pump(4);
+        assert!(acknowledgements.try_recv().is_none());
+    }
+
+    #[test]
+    fn directed_acknowledgement_rejects_superseded_generation() {
+        use distribution::messages::{RegistryDelivery, RegistryGossip};
+
+        let c = GossipCluster::new(2);
+        let replies = c.nodes[0].rt.new_inbox::<RegistryIn>().unwrap();
+        let name = "swactor.data-directory".to_owned();
+        let actor_addr = ActorAddress([0xF3; 32]);
+        for timestamp in [100, 101] {
+            c.nodes[0]
+                .rt
+                .send_to(
+                    c.nodes[0].registry,
+                    RegistryIn::RegisterNameAt {
+                        name: name.clone(),
+                        actor_addr,
+                        timestamp,
+                    },
+                )
+                .unwrap();
+        }
+        c.nodes[0].backend.step();
+        // Observe the actual publication; the registry advances its logical
+        // clock while merging, so requested timestamps are not clock snapshots.
+        c.nodes[0].dir.bind(c.ids[1], *replies.addr(), 1);
+        c.nodes[0]
+            .rt
+            .send_to(c.nodes[0].registry, RegistryIn::SyncTo { peer: c.ids[1] })
+            .unwrap();
+        c.nodes[0].backend.step();
+        let Some(RegistryIn::Gossip(publication)) = replies.try_recv() else {
+            panic!("registry did not publish its current binding");
+        };
+        let current = publication
+            .entries
+            .into_iter()
+            .find(|entry| entry.name == name)
+            .unwrap();
+        let acknowledge = |entry| {
+            c.nodes[0]
+                .rt
+                .send_to(
+                    c.nodes[0].registry,
+                    RegistryIn::Gossip(RegistryGossip {
+                        entries: vec![entry],
+                        delivery: Some(RegistryDelivery::Acknowledged {
+                            reply_to: *replies.addr(),
+                            peer: c.ids[1],
+                        }),
+                    }),
+                )
+                .unwrap();
+            c.nodes[0].backend.step();
+        };
+        let mut stale = current.clone();
+        stale.generation -= 1;
+        acknowledge(stale);
+        assert!(
+            replies.try_recv().is_none(),
+            "same binding is not the same generation"
+        );
+        let mut stale = current.clone();
+        stale.timestamp -= 1;
+        acknowledge(stale);
+        assert!(
+            replies.try_recv().is_none(),
+            "stale publication cannot finish recovery"
+        );
+        let mut future = current.clone();
+        future.timestamp += 1;
+        acknowledge(future);
+        assert!(
+            replies.try_recv().is_none(),
+            "ACKs cannot publish unseen future entries"
+        );
+        acknowledge(current.clone());
+        assert!(matches!(replies.try_recv(),
+            Some(RegistryIn::NameAcknowledged { entry, peer })
+                if entry == current && peer == c.ids[1]
+        ));
+    }
 
     #[test]
     fn a_relay_url_propagates_to_a_peer_over_the_transport() {

crates/distribution/tests/routing.rs

@@ -772,7 +772,7 @@ mod directory_route_path {
                 let engine =
                     Engine::new(parts, backend.clone()).expect("create stepping actor engine");
 
-                let outbox: Outbox = Arc::new(Mutex::new(Vec::new()));
+                let outbox: Outbox = Arc::new(Default::default());
                 let route_view: RouteView = Arc::new(RwLock::new(HashMap::new()));
                 // Gossip egress (directory → peer) and app egress (RouteView → host)
                 // both feed the one outbox, just like the live driver.

crates/distribution/tests/wire.rs

@@ -125,7 +125,10 @@ mod codec_contract {
     fn registry_and_metadata_gossip_round_trip() {
         let codecs = distribution_codec_registry();
 
-        let rg = RegistryGossip { entries: vec![] };
+        let rg = RegistryGossip {
+            entries: vec![],
+            delivery: None,
+        };
         let (tag, bytes) = codecs
             .encode(TypeId::of::<RegistryGossip>(), Box::new(rg))
             .unwrap();
@@ -148,7 +151,7 @@ mod route_view_egress {
     use distribution::transport_bridge::{Outbox, RouteView, RouteViewTransport, peer_addr};
     use distribution::types::NodeId;
     use std::collections::HashMap;
-    use std::sync::{Arc, Mutex, RwLock};
+    use std::sync::{Arc, RwLock};
     use swactor::actor::ActorAddress;
     use swactor_transport::{Transport, WireEnvelope};
 
@@ -173,7 +176,7 @@ mod route_view_egress {
         let first_host = id(1);
         let second_host = id(2);
         let route_view: RouteView = Arc::new(RwLock::new(HashMap::new()));
-        let outbox: Outbox = Arc::new(Mutex::new(Vec::new()));
+        let outbox: Outbox = Arc::new(Default::default());
         let transport = RouteViewTransport::new(route_view.clone(), outbox.clone());
 
         transport

crates/engine/src/engine.rs

@@ -16,10 +16,10 @@ use swactor::runtime::{ExternalSender, Runtime, RuntimeParts};
 /// Construct with [`Engine::new`]; obtain a scheduler handle with
 /// [`Engine::handle`].
 pub struct Engine {
+    backend: Arc<dyn ExecutionBackend>,
     /// Retained so the engine owns the runtime handle it drives for its full
     /// lifetime. Core workers are moved into substrate tasks at construction.
     _runtime: Runtime,
-    backend: Arc<dyn ExecutionBackend>,
 }
 
 impl Engine {
@@ -41,8 +41,8 @@ impl Engine {
         // disappear when an alternate backend is used (ENGINE_SPEC.md).
         crate::core_driver::install(workers, &backend);
         Ok(Engine {
-            _runtime: runtime,
             backend,
+            _runtime: runtime,
         })
     }
 

crates/iroh-driver/Cargo.toml

@@ -18,15 +18,18 @@ telemetry = { path = "../telemetry" }
 crossbeam-channel = "0.5"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
-iroh = "0.98"
+lz4_flex = { version = "0.11", default-features = false, features = ["std"] }
+postcard = { version = "1", features = ["alloc"] }
+iroh.workspace = true
+zstd = { version = "0.13", default-features = false }
 # `test-utils` exposes `CaRootsConfig::insecure_skip_verify()` so clients can
 # trust operator-controlled custom relays with self-signed QAD certs.
-iroh-relay = { version = "0.98", features = ["test-utils"] }
+iroh-relay.workspace = true
 tokio.workspace = true
 parking_lot = "0.12"
 
 [dev-dependencies]
-iroh-relay = { version = "0.98", features = ["server", "test-utils"] }
+iroh-relay = { workspace = true, features = ["server"] }
 
 [lints]
 workspace = true

crates/iroh-driver/IROH_DRIVER_SPEC.md

@@ -1,7 +1,7 @@
 # Iroh Driver Fixed Specification
 
 Id: 5
-Last modified: b887e941cbe6f1e209339abd0375507aca9bfe52
+Last modified: 74ba89c0cedae04b05a4b8b4af9a3856adfd5875
 Last reviewed:
 > Any edit to this spec must update `Last modified` above to the current `git HEAD` commit.
 
@@ -451,7 +451,7 @@ Target wire record:
 Telemetry records use:
 
 ```text
-swactor/telemetry/0
+swactor/telemetry/1
 ```
 
 The telemetry transport opens unidirectional streams over a connection negotiated with this ALPN.
@@ -461,42 +461,51 @@ The telemetry transport opens unidirectional streams over a connection negotiate
 A telemetry unidirectional stream begins with a header:
 
 ```text
-magic = "DSQ1"
+magic = "DSQ2"
 flow_id: [u8; 16]
 token_len: u16 LE
 token bytes
-stream descriptor JSON
-channel descriptors JSON
+stream descriptor: unsigned-varint length + Postcard bytes
+channel descriptors: unsigned-varint length + Postcard bytes
 ```
 
-Then zero or more length-prefixed records follow:
-
-```text
-record_len: u32 LE
-record bytes
-```
-
-Record tags:
+Compact event records follow directly, without an outer fixed-width length:
 
 ```text
 0x01 = channel declared
-0x02 = frame
+       unsigned-varint descriptor length + Postcard descriptor bytes
+0x02 = raw frame batch
+       unsigned-varint batch length + frame entries
 0x03 = stream ended
+0x04 = LZ4 frame batch (accepted from older senders)
+       unsigned-varint raw length
+       unsigned-varint compressed length
+       compressed frame entries
+0x05 = Zstandard frame batch
+       unsigned-varint raw length
+       unsigned-varint compressed length
+       compressed frame entries
 ```
 
-A frame record contains:
+Each frame entry contains:
 
 ```text
-tag
-channel_id: u32 LE
-position: u64 LE
-payload_len: u32 LE
+channel_id: unsigned varint
+position: unsigned varint (absolute for the first entry, then delta within the batch)
+payload_len: unsigned varint
 payload bytes
 ```
 
-Each telemetry record is bounded by `16 * 1024 * 1024` bytes.
+The writer drains available events without waiting, limits one batch to 256 KiB
+or 1024 events, and uses Zstandard level 1 only when the complete compressed
+record is smaller than the raw record. Readers continue to accept LZ4 batches
+from older senders. Compression is byte-preserving: the reader reconstructs the
+same channel id, absolute position, and opaque payload bytes.
 
-`StreamDeclared` events are represented in the stream header and are not emitted as individual records by the current writer.
+Each decoded record or batch is bounded by `16 * 1024 * 1024` bytes.
+
+`StreamDeclared` events are represented in the stream header and are not
+emitted as individual records by the current writer.
 
 ### 4.6 Node Identity and Endpoint Addressing
 

crates/iroh-driver/src/blob_transfer.rs

@@ -150,13 +150,19 @@ impl IrohBlobTransferReceiver {
         engine: &EngineHandle,
         runtime: Runtime,
         period: Duration,
+        mut activity: tokio::sync::watch::Receiver<()>,
     ) {
         let receiver = Arc::clone(self);
         let engine = engine.clone();
         engine.clone().spawn(async move {
             let mut interval = engine.interval(period);
             loop {
-                (&mut interval).await;
+                tokio::select! {
+                    _ = &mut interval => {},
+                    changed = activity.changed() => {
+                        if changed.is_err() { break; }
+                    },
+                }
                 receiver.drain(&runtime);
             }
         });

crates/iroh-driver/src/edge_transport.rs

@@ -28,6 +28,38 @@ type CompletionReceiver = std::sync::mpsc::Receiver<Result<(), String>>;
 pub struct EdgeSendHandle {
     tx: tokio_mpsc::UnboundedSender<Vec<u8>>,
     completion: Arc<Mutex<Option<CompletionReceiver>>>,
+    owner: Arc<EdgeSendOwner>,
+}
+
+struct EdgeSendOwner(tokio::sync::watch::Sender<bool>);
+
+impl Drop for EdgeSendOwner {
+    fn drop(&mut self) {
+        self.0.send_replace(true);
+    }
+}
+
+/// A cancelled write must reset, not implicitly finish buffered partial
+/// records when Quinn's send stream drops.
+pub(crate) struct ResetOnDrop(pub(crate) iroh::endpoint::SendStream);
+
+impl std::ops::Deref for ResetOnDrop {
+    type Target = iroh::endpoint::SendStream;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl std::ops::DerefMut for ResetOnDrop {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+
+impl Drop for ResetOnDrop {
+    fn drop(&mut self) {
+        let _ = self.0.reset(iroh::endpoint::VarInt::from_u32(0));
+    }
 }
 
 impl EdgeSendHandle {
@@ -37,14 +69,22 @@ impl EdgeSendHandle {
             .map_err(|_| "edge sender task stopped".to_owned())
     }
     pub fn finish(self, timeout: std::time::Duration) -> Result<(), String> {
-        let Self { tx, completion } = self;
+        let Self {
+            tx,
+            completion,
+            owner,
+        } = self;
         drop(tx);
-        completion
+        let result = completion
             .lock()
             .take()
             .ok_or_else(|| "edge sender completion was already observed".to_owned())?
             .recv_timeout(timeout)
-            .map_err(|error| format!("edge sender completion: {error}"))?
+            .map_err(|error| format!("edge sender completion: {error}"));
+        if result.is_err() {
+            owner.0.send_replace(true);
+        }
+        result?
     }
 }
 
@@ -65,18 +105,21 @@ pub(crate) fn spawn_edge_send_pump(
     let (ready_tx, ready_rx) = std::sync::mpsc::channel::<Result<(), String>>();
     let (completion_tx, completion_rx) = std::sync::mpsc::channel::<Result<(), String>>();
     let engine_handle = engine.clone();
+    let (cancel, mut cancellation) = tokio::sync::watch::channel(false);
+    let owner = Arc::new(EdgeSendOwner(cancel));
     engine.spawn(async move {
-        let result: Result<(), String> = async {
+        let operation = async {
             macro_rules! open_edge_stream {
                 () => {{
                     let conn = endpoint
                         .connect(peer.clone(), EDGE_ALPN)
                         .await
                         .map_err(|e| format!("connect edge {edge_id}: {e}"))?;
-                    let mut send = conn
-                        .open_uni()
-                        .await
-                        .map_err(|e| format!("open edge stream {edge_id}: {e}"))?;
+                    let mut send = ResetOnDrop(
+                        conn.open_uni()
+                            .await
+                            .map_err(|e| format!("open edge stream {edge_id}: {e}"))?,
+                    );
                     send.write_all(&encode_edge_preamble(edge_id))
                         .await
                         .map_err(|e| format!("write edge preamble {edge_id}: {e}"))?;
@@ -137,8 +180,12 @@ pub(crate) fn spawn_edge_send_pump(
                 Some(code) => Err(format!("peer stopped edge stream {edge_id}: {code}")),
                 None => Ok(()),
             }
-        }
-        .await;
+        };
+        let result: Result<(), String> = tokio::select! {
+            biased;
+            _ = cancellation.changed() => Err(format!("edge {edge_id} owner cancelled")),
+            result = operation => result,
+        };
         if let Err(error) = &result {
             let _ = ready_tx.send(Err(error.clone()));
         }
@@ -155,6 +202,7 @@ pub(crate) fn spawn_edge_send_pump(
     Ok(EdgeSendHandle {
         tx,
         completion: Arc::new(Mutex::new(Some(completion_rx))),
+        owner,
     })
 }
 
@@ -163,6 +211,7 @@ pub(crate) fn spawn_edge_recv_pump(
     conn: Connection,
     events: Arc<Mutex<Vec<WireEvent>>>,
     stream_group: u64,
+    activity: tokio::sync::watch::Sender<()>,
 ) {
     engine.spawn(async move {
         let mut next_uni_stream_id = stream_group << 32;
@@ -176,6 +225,7 @@ pub(crate) fn spawn_edge_recv_pump(
                     stream_id: Some(current_stream_id),
                     reason: WireFault::ProtocolError,
                 });
+                activity.send_replace(());
                 continue;
             }
             let edge_id = EdgeId(u64::from_le_bytes(preamble));
@@ -183,6 +233,7 @@ pub(crate) fn spawn_edge_recv_pump(
                 edge_id,
                 stream_id: current_stream_id,
             });
+            activity.send_replace(());
             let mut chunk = vec![0u8; 4096];
             loop {
                 match recv.read(&mut chunk).await {
@@ -191,6 +242,7 @@ pub(crate) fn spawn_edge_recv_pump(
                             edge_id,
                             stream_id: current_stream_id,
                         });
+                        activity.send_replace(());
                         break;
                     }
                     Ok(Some(n)) => {
@@ -199,6 +251,7 @@ pub(crate) fn spawn_edge_recv_pump(
                             stream_id: current_stream_id,
                             bytes: chunk[..n].to_vec(),
                         });
+                        activity.send_replace(());
                     }
                     Err(_) => {
                         events.lock().push(WireEvent::StreamFault {
@@ -206,6 +259,7 @@ pub(crate) fn spawn_edge_recv_pump(
                             stream_id: Some(current_stream_id),
                             reason: WireFault::ReadError,
                         });
+                        activity.send_replace(());
                         break;
                     }
                 }

crates/iroh-driver/src/lib.rs

@@ -21,8 +21,9 @@ pub use endpoint_advertisement::{
     EndpointAddrMask, MVP_IROH_ENDPOINT_ADDR_MASK_ENV, advertised_endpoint,
 };
 pub use iroh_driver::{
-    ActorBridgeConfig, ActorRegistrar, ConnType, EdgeConnector, IrohDriver, IrohDriverConfig,
-    JoinPhase, JoinStatus, TelemetryPublishHandle, conn_type_of, discover_lan_ips,
+    ActorBridgeConfig, ActorRegistrar, ConnType, ConnectionObserver, ConnectionWatch,
+    EdgeConnector, IrohDriver, IrohDriverConfig, JoinPhase, JoinStatus, PeerConnector,
+    TelemetryPublishHandle, conn_type_of, discover_lan_ips,
 };
 
 pub use edge_transport::{EDGE_ALPN, EdgeSendHandle};
@@ -30,9 +31,10 @@ pub use stream_transport::{IrohStreamTransport, STREAM_ALPN};
 
 pub use telemetry_transport::{
     PullCollectorConfig, PullCollectorHandle, TELEMETRY_ALPN, TelemetryQuicHeader,
-    TelemetryQuicRead, TelemetryQuicWriteStats, read_events_from_stream, read_next_event,
-    read_next_uni_from_connection, read_pull_request, read_stream_header, read_stream_into_fanout,
-    spawn_connection_reader, spawn_pull_collector, spawn_pull_collector_to_actor,
-    spawn_pull_server, spawn_subscription_writer, write_available_subscription, write_event,
-    write_pull_request, write_subscription_until_closed,
+    TelemetryQuicRead, TelemetryQuicWriteStats, decode_event_records, encode_event_batch,
+    encode_event_record, read_events_from_stream, read_next_events, read_next_uni_from_connection,
+    read_pull_request, read_stream_header, read_stream_into_fanout, spawn_connection_reader,
+    spawn_pull_collector, spawn_pull_collector_to_actor, spawn_pull_server,
+    spawn_subscription_writer, write_available_subscription, write_event, write_pull_request,
+    write_subscription_until_closed,
 };

crates/iroh-driver/src/stream_transport.rs

@@ -28,6 +28,7 @@ struct TaskControl {
     wake: mpsc::Sender<()>,
     progress_pending: Arc<AtomicBool>,
     cancelled: Arc<AtomicBool>,
+    cancellation: Arc<tokio::sync::Notify>,
 }
 
 impl TaskControl {
@@ -38,6 +39,7 @@ impl TaskControl {
                 wake,
                 progress_pending: Arc::new(AtomicBool::new(false)),
                 cancelled: Arc::new(AtomicBool::new(false)),
+                cancellation: Arc::new(tokio::sync::Notify::new()),
             },
             receiver,
         )
@@ -51,9 +53,16 @@ impl TaskControl {
 
     fn cancel(&self) {
         self.cancelled.store(true, Ordering::Release);
+        self.cancellation.notify_one();
         self.progress();
     }
 
+    async fn cancelled(&self) {
+        if !self.cancelled.load(Ordering::Acquire) {
+            self.cancellation.notified().await;
+        }
+    }
+
     async fn wait(&self, receiver: &mut mpsc::Receiver<()>) -> bool {
         if self.cancelled.load(Ordering::Acquire) {
             return false;
@@ -71,14 +80,25 @@ struct PendingSink {
     notifier: Arc<dyn StreamTransportNotifier>,
 }
 
+struct PendingInbound {
+    stream: RecvStream,
+    connection: Connection,
+    token: u64,
+}
+
 #[derive(Default)]
 struct TransportState {
     pending_sinks: BTreeMap<StreamIncarnation, PendingSink>,
-    pending_inbound: BTreeMap<StreamIncarnation, RecvStream>,
+    pending_inbound: BTreeMap<StreamIncarnation, PendingInbound>,
     controls: BTreeMap<StreamIncarnation, Vec<TaskControl>>,
     source_probes: BTreeMap<StreamIncarnation, RingProbe>,
     sink_probes: BTreeMap<StreamIncarnation, RingProbe>,
     local_incarnations: BTreeSet<StreamIncarnation>,
+    next_pending_token: u64,
+    // Incarnation identities are never reused by an authority. Retain exact
+    // fences for this endpoint lifetime: elapsed time cannot prove a delayed
+    // stream belongs to a live owner.
+    retired: BTreeSet<StreamIncarnation>,
 }
 
 struct Inner {
@@ -106,33 +126,64 @@ impl IrohStreamTransport {
         }
     }
 
+    /// Current ownership, read under the same lock as install/terminate.
+    /// Retired identity fences are history, not live transport resources.
+    pub fn resource_snapshot(&self) -> serde_json::Value {
+        let state = self.inner.state.lock();
+        serde_json::json!({
+            "pending_sinks": state.pending_sinks.len(),
+            "pending_inbound": state.pending_inbound.len(),
+            "active_controls": state.controls.values().map(Vec::len).sum::<usize>(),
+            "source_probes": state.source_probes.len(),
+            "sink_probes": state.sink_probes.len(),
+            "local_incarnations": state.local_incarnations.len(),
+            "retired_incarnations": state.retired.len(),
+        })
+    }
+
     pub(crate) fn accept_connection(&self, connection: Connection) {
         let transport = self.clone();
         self.inner.engine.spawn(async move {
             while let Ok(mut recv) = connection.accept_uni().await {
                 let mut preamble = [0_u8; PREAMBLE_LEN];
-                if recv.read_exact(&mut preamble).await.is_err() {
+                if !transport
+                    .inner
+                    .engine
+                    .timeout(PENDING_INBOUND_TIMEOUT, recv.read_exact(&mut preamble))
+                    .await
+                    .is_ok_and(|result| result.is_ok())
+                {
                     continue;
                 }
                 let incarnation = decode_incarnation(preamble);
                 let mut recv = Some(recv);
                 let (pending, retained) = {
                     let mut state = transport.inner.state.lock();
+                    if state.retired.contains(&incarnation) {
+                        continue;
+                    }
                     if let Some(pending) = state.pending_sinks.remove(&incarnation) {
-                        (Some(pending), false)
+                        (Some(pending), None)
                     } else if !state.pending_inbound.contains_key(&incarnation)
                         && state.pending_inbound.len() < MAX_PENDING_INBOUND
                     {
-                        state
-                            .pending_inbound
-                            .insert(incarnation, recv.take().expect("unclaimed inbound stream"));
-                        (None, true)
+                        state.next_pending_token = state.next_pending_token.wrapping_add(1);
+                        let token = state.next_pending_token;
+                        state.pending_inbound.insert(
+                            incarnation,
+                            PendingInbound {
+                                stream: recv.take().expect("unclaimed inbound stream"),
+                                connection: connection.clone(),
+                                token,
+                            },
+                        );
+                        (None, Some(token))
                     } else {
-                        (None, false)
+                        (None, None)
                     }
                 };
-                if retained {
-                    transport.schedule_pending_timeout(incarnation);
+                if let Some(token) = retained {
+                    transport.schedule_pending_timeout(incarnation, token);
                 }
                 if let Some(pending) = pending {
                     transport.start_sink(
@@ -142,22 +193,36 @@ impl IrohStreamTransport {
                     );
                 }
             }
+            // A closed owner cannot complete unmatched incarnations. Remove
+            // only its streams; another connection may already have rejoined.
+            transport
+                .inner
+                .state
+                .lock()
+                .pending_inbound
+                .retain(|_, pending| pending.connection.stable_id() != connection.stable_id());
         });
     }
 
-    fn register_control(&self, incarnation: StreamIncarnation, control: TaskControl) {
-        self.inner
-            .state
-            .lock()
-            .controls
-            .entry(incarnation)
-            .or_default()
-            .push(control);
-    }
-
     fn start_sink(&self, incarnation: StreamIncarnation, recv: RecvStream, pending: PendingSink) {
         let (control, receiver) = TaskControl::pair();
-        self.register_control(incarnation, control.clone());
+        {
+            let mut state = self.inner.state.lock();
+            if !state.sink_probes.contains_key(&incarnation) {
+                // Termination raced the accepted stream's transfer out of
+                // pending_sinks. Hand back quiescence, never revive its task.
+                drop(state);
+                drop(recv);
+                drop(pending.endpoint);
+                pending.notifier.notify(StreamTransportEvent::Quiesced);
+                return;
+            }
+            state
+                .controls
+                .entry(incarnation)
+                .or_default()
+                .push(control.clone());
+        }
         self.inner.engine.spawn(run_sink(
             incarnation,
             recv,
@@ -168,18 +233,19 @@ impl IrohStreamTransport {
         ));
     }
 
-    fn schedule_pending_timeout(&self, incarnation: StreamIncarnation) {
+    fn schedule_pending_timeout(&self, incarnation: StreamIncarnation, token: u64) {
         let engine = self.inner.engine.clone();
         let transport = self.clone();
         engine.clone().spawn(async move {
-            let mut deadline = engine.interval(PENDING_INBOUND_TIMEOUT);
-            (&mut deadline).await;
-            transport
-                .inner
-                .state
-                .lock()
+            engine.timer(PENDING_INBOUND_TIMEOUT).await;
+            let mut state = transport.inner.state.lock();
+            if state
                 .pending_inbound
-                .remove(&incarnation);
+                .get(&incarnation)
+                .is_some_and(|pending| pending.token == token)
+            {
+                state.pending_inbound.remove(&incarnation);
+            }
         });
     }
 
@@ -194,6 +260,9 @@ impl IrohStreamTransport {
         let incarnation = request.incarnation;
         let pending = {
             let mut state = self.inner.state.lock();
+            if state.retired.contains(&incarnation) {
+                return Err("iroh stream incarnation is retired".to_owned());
+            }
             state.local_incarnations.insert(incarnation);
             state.pending_inbound.remove(&incarnation);
             state.sink_probes.remove(&incarnation);
@@ -241,14 +310,23 @@ impl StreamTransport for IrohStreamTransport {
             return self.install_local_source(request);
         }
         let (control, receiver) = TaskControl::pair();
-        self.register_control(request.incarnation, control.clone());
         let endpoint = self.inner.endpoint.clone();
         let probe = request.endpoint.probe();
-        self.inner
-            .state
-            .lock()
-            .source_probes
-            .insert(request.incarnation, probe);
+        {
+            let mut state = self.inner.state.lock();
+            if state.retired.contains(&request.incarnation) {
+                return Err("iroh stream incarnation is retired".to_owned());
+            }
+            if state.source_probes.contains_key(&request.incarnation) {
+                return Err("iroh stream source is already installed".to_owned());
+            }
+            state
+                .controls
+                .entry(request.incarnation)
+                .or_default()
+                .push(control.clone());
+            state.source_probes.insert(request.incarnation, probe);
+        }
         self.inner.engine.spawn(run_source(
             request.incarnation,
             endpoint,
@@ -279,12 +357,15 @@ impl StreamTransport for IrohStreamTransport {
         });
         let inbound = {
             let mut state = self.inner.state.lock();
+            if state.retired.contains(&incarnation) {
+                return Err("iroh stream incarnation is retired".to_owned());
+            }
             if state.sink_probes.contains_key(&incarnation) {
                 return Err("iroh stream sink is already installed".to_owned());
             }
             state.sink_probes.insert(incarnation, probe);
             if let Some(inbound) = state.pending_inbound.remove(&incarnation) {
-                Some(inbound)
+                Some(inbound.stream)
             } else {
                 state
                     .pending_sinks
@@ -369,6 +450,7 @@ impl StreamTransport for IrohStreamTransport {
     fn terminate(&self, incarnation: StreamIncarnation) {
         let (local, pending, _inbound, controls) = {
             let mut state = self.inner.state.lock();
+            state.retired.insert(incarnation);
             let local = state.local_incarnations.remove(&incarnation);
             let pending = state.pending_sinks.remove(&incarnation);
             let inbound = state.pending_inbound.remove(&incarnation);
@@ -382,6 +464,7 @@ impl StreamTransport for IrohStreamTransport {
             return;
         }
         if let Some(pending) = pending {
+            drop(pending.endpoint);
             pending.notifier.notify(StreamTransportEvent::Quiesced);
         }
         for control in controls {
@@ -399,15 +482,17 @@ async fn run_source(
     control: TaskControl,
     mut receiver: mpsc::Receiver<()>,
 ) {
-    let result = async {
+    let operation = async {
         let connection = endpoint
             .connect(peer, STREAM_ALPN)
             .await
             .map_err(|error| format!("connect stream incarnation: {error}"))?;
-        let mut send = connection
-            .open_uni()
-            .await
-            .map_err(|error| format!("open stream incarnation: {error}"))?;
+        let mut send = crate::edge_transport::ResetOnDrop(
+            connection
+                .open_uni()
+                .await
+                .map_err(|error| format!("open stream incarnation: {error}"))?,
+        );
         send.write_all(&encode_incarnation(incarnation))
             .await
             .map_err(|error| format!("write stream preamble: {error}"))?;
@@ -455,8 +540,14 @@ async fn run_source(
                 return Ok(());
             }
         }
-    }
-    .await;
+    };
+    let result = tokio::select! {
+        biased;
+        _ = control.cancelled() => Ok(()),
+        result = operation => result,
+    };
+    // Quiesced transfers ownership of the ring back to the arena allocator.
+    drop(source);
     if let Err(reason) = result {
         notifier.notify(StreamTransportEvent::Fault(reason));
     }
@@ -498,7 +589,7 @@ async fn run_sink(
     mut receiver: mpsc::Receiver<()>,
 ) {
     notifier.notify(StreamTransportEvent::Ready);
-    let result = async {
+    let operation = async {
         loop {
             if control.cancelled.load(Ordering::Acquire) {
                 return Ok(());
@@ -548,8 +639,14 @@ async fn run_sink(
                 }
             }
         }
-    }
-    .await;
+    };
+    let result = tokio::select! {
+        biased;
+        _ = control.cancelled() => Ok(()),
+        result = operation => result,
+    };
+    drop(recv);
+    drop(sink);
     if let Err(reason) = result {
         notifier.notify(StreamTransportEvent::Fault(reason));
     }

crates/iroh-driver/src/telemetry_transport.rs

@@ -11,17 +11,21 @@ use swactor::actor::ActorAddress;
 use swactor::runtime::ExternalSender;
 use swactor_engine::EngineHandle;
 use telemetry::frame::{
-    ChannelDescriptor, ChannelId, ChannelRef, FrameDelivery, Position, StreamDescriptor,
+    ChannelDescriptor, ChannelId, ChannelRef, FrameDelivery, Position, StreamDescriptor, StreamId,
     TelemetryEvent,
 };
 use telemetry::{TelemetrySnapshot, TelemetrySubscription};
 
-pub const TELEMETRY_ALPN: &[u8] = b"swactor/telemetry/0";
+pub const TELEMETRY_ALPN: &[u8] = b"swactor/telemetry/1";
 
-const MAGIC: &[u8; 4] = b"DSQ1";
+const MAGIC: &[u8; 4] = b"DSQ2";
 const TAG_CHANNEL_DECLARED: u8 = 0x01;
-const TAG_FRAME: u8 = 0x02;
+const TAG_FRAME_BATCH: u8 = 0x02;
 const TAG_STREAM_ENDED: u8 = 0x03;
+const TAG_LZ4_FRAME_BATCH: u8 = 0x04;
+const TAG_ZSTD_FRAME_BATCH: u8 = 0x05;
+const WRITE_BATCH_TARGET_BYTES: usize = 256 * 1024;
+const WRITE_BATCH_MAX_EVENTS: usize = 1024;
 
 // ─── Pull model: collector-initiated subscriptions ───────────────────────────
 //
@@ -91,7 +95,7 @@ pub fn spawn_pull_server(
         let Ok((_flow_id, token, request)) = read_pull_request(&mut recv).await else {
             return;
         };
-        let subscription = endpoint.subscribe("supervisor-pull", request);
+        let subscription = endpoint.subscribe_retained("supervisor-pull", request);
         let header =
             match TelemetryQuicHeader::from_snapshot(_flow_id, token, subscription.snapshot()) {
                 Ok(header) => header,
@@ -100,9 +104,12 @@ pub fn spawn_pull_server(
         let Ok(send) = conn.open_uni().await else {
             return;
         };
-        let _ =
-            write_subscription_until_closed(&engine_handle, send, header, subscription, idle_sleep)
-                .await;
+        tokio::select! {
+            _ = conn.closed() => {}
+            _ = write_subscription_until_closed(
+                &engine_handle, send, header, subscription, idle_sleep,
+            ) => {}
+        }
     });
 }
 
@@ -215,6 +222,7 @@ fn spawn_pull_collector_with_sink(
         let _completion = PullCollectorCompletion(completion);
         let peer_id = peer.id.to_string();
         let mut retry_delay = Duration::from_millis(250);
+        let mut cursor = None;
         loop {
             if *cancellation_rx.borrow() {
                 return;
@@ -223,6 +231,7 @@ fn spawn_pull_collector_with_sink(
                 _ = cancellation_rx.changed() => return,
                 result = collect_pull_once(
                     &endpoint, &peer, flow_id, &token, &request, &fanout, &on_header,
+                    &mut cursor,
                 ) => result,
             };
             match result {
@@ -258,6 +267,7 @@ async fn collect_pull_once(
     request: &telemetry::SubscriptionRequest,
     fanout: &telemetry::DeliveryFanout,
     on_header: &PullHeaderSink,
+    cursor: &mut Option<(StreamId, Position)>,
 ) -> Result<(), String> {
     let conn = endpoint
         .connect(peer.clone(), TELEMETRY_ALPN)
@@ -281,11 +291,45 @@ async fn collect_pull_once(
         return Ok(());
     }
     let stream = header.stream;
+    if cursor
+        .as_ref()
+        .is_some_and(|(previous, _)| previous != &stream.stream)
+    {
+        *cursor = None;
+    }
     loop {
-        match read_next_event(&mut recv, &stream).await {
-            Ok(Some(event)) => {
-                let ended = matches!(event, TelemetryEvent::StreamEnded(_));
-                fanout.publish(event);
+        match read_next_events(&mut recv, &stream).await {
+            Ok(Some(mut events)) => {
+                // The endpoint replays its retained suffix on every connection.
+                // A cursor belongs to this producer stream, not a channel, and
+                // survives reconnects so a real frame is delivered only once.
+                // Do not commit it past a locally dropped delivery batch: the
+                // reconnect must replay that batch from its previous boundary.
+                let cursor_before_batch = cursor.clone();
+                events.retain(|event| {
+                    let TelemetryEvent::Frame(frame) = event else {
+                        return true;
+                    };
+                    if cursor
+                        .as_ref()
+                        .is_some_and(|(_, position)| frame.position <= *position)
+                    {
+                        return false;
+                    }
+                    *cursor = Some((frame.channel.stream.clone(), frame.position));
+                    true
+                });
+                let ended = events
+                    .iter()
+                    .any(|event| matches!(event, TelemetryEvent::StreamEnded(_)));
+                let delivery = fanout.publish_batch(events);
+                if delivery.dropped_for_subscribers != 0 {
+                    *cursor = cursor_before_batch;
+                    return Err(format!(
+                        "collector fanout dropped {} telemetry events",
+                        delivery.dropped_for_subscribers
+                    ));
+                }
                 if ended {
                     return Ok(());
                 }
@@ -388,18 +432,25 @@ pub async fn write_subscription_until_closed(
 ) -> Result<TelemetryQuicWriteStats, BoxError> {
     write_header(&mut send, &header).await?;
     let mut stats = TelemetryQuicWriteStats::default();
+    let mut compressor = zstd::bulk::Compressor::new(1)?;
+    let mut raw = Vec::with_capacity(WRITE_BATCH_TARGET_BYTES);
+    let mut bytes = Vec::with_capacity(WRITE_BATCH_TARGET_BYTES);
+    let mut events = Vec::with_capacity(WRITE_BATCH_MAX_EVENTS);
     loop {
+        let dropped = subscription.dropped();
+        if dropped != 0 {
+            return Err(format!("telemetry subscription dropped {dropped} events").into());
+        }
         match subscription.try_recv() {
-            Ok(event) => {
-                let bytes = write_event(&mut send, &event).await?;
-                if bytes > 0 {
-                    stats.bytes += bytes;
-                    stats.events += 1;
-                }
-            }
-            Err(TryRecvError::Empty) => {
-                engine.timer(idle_sleep).await;
+            Ok(first) => {
+                take_event_batch(first, &subscription, &mut events);
+                let written =
+                    write_event_batch(&mut send, &events, &mut compressor, &mut raw, &mut bytes)
+                        .await?;
+                stats.events += written.events;
+                stats.bytes += written.bytes;
             }
+            Err(TryRecvError::Empty) => engine.timer(idle_sleep).await,
             Err(TryRecvError::Disconnected) => break,
         }
     }
@@ -416,14 +467,23 @@ async fn write_subscription_inner(
 ) -> Result<TelemetryQuicWriteStats, BoxError> {
     write_header(&mut send, header).await?;
     let mut stats = TelemetryQuicWriteStats::default();
+    let mut compressor = zstd::bulk::Compressor::new(1)?;
+    let mut raw = Vec::with_capacity(WRITE_BATCH_TARGET_BYTES);
+    let mut bytes = Vec::with_capacity(WRITE_BATCH_TARGET_BYTES);
+    let mut events = Vec::with_capacity(WRITE_BATCH_MAX_EVENTS);
     loop {
+        let dropped = subscription.dropped();
+        if dropped != 0 {
+            return Err(format!("telemetry subscription dropped {dropped} events").into());
+        }
         match subscription.try_recv() {
-            Ok(event) => {
-                let bytes = write_event(&mut send, &event).await?;
-                if bytes > 0 {
-                    stats.bytes += bytes;
-                    stats.events += 1;
-                }
+            Ok(first) => {
+                take_event_batch(first, subscription, &mut events);
+                let written =
+                    write_event_batch(&mut send, &events, &mut compressor, &mut raw, &mut bytes)
+                        .await?;
+                stats.events += written.events;
+                stats.bytes += written.bytes;
             }
             Err(TryRecvError::Empty) => match idle_sleep {
                 Some(delay) => engine.timer(delay).await,
@@ -436,30 +496,192 @@ async fn write_subscription_inner(
     Ok(stats)
 }
 
-pub async fn write_event(send: &mut SendStream, event: &TelemetryEvent) -> Result<usize, BoxError> {
-    let mut bytes = Vec::new();
+fn take_event_batch(
+    first: TelemetryEvent,
+    subscription: &TelemetrySubscription,
+    events: &mut Vec<TelemetryEvent>,
+) {
+    let mut estimated_bytes = event_size_hint(&first);
+    events.clear();
+    events.push(first);
+    while estimated_bytes < WRITE_BATCH_TARGET_BYTES && events.len() < WRITE_BATCH_MAX_EVENTS {
+        let Ok(event) = subscription.try_recv() else {
+            break;
+        };
+        estimated_bytes = estimated_bytes.saturating_add(event_size_hint(&event));
+        events.push(event);
+    }
+}
+
+fn event_size_hint(event: &TelemetryEvent) -> usize {
     match event {
-        TelemetryEvent::StreamDeclared(_) => return Ok(0),
-        TelemetryEvent::ChannelDeclared(descriptor) => {
-            bytes.push(TAG_CHANNEL_DECLARED);
-            put_json(&mut bytes, descriptor)?;
-        }
-        TelemetryEvent::Frame(delivery) => {
-            bytes.push(TAG_FRAME);
-            bytes.extend_from_slice(&delivery.channel.channel.0.to_le_bytes());
-            bytes.extend_from_slice(&delivery.position.0.to_le_bytes());
-            put_bytes(&mut bytes, &delivery.payload)?;
-        }
-        TelemetryEvent::StreamEnded(_) => {
-            bytes.push(TAG_STREAM_ENDED);
+        TelemetryEvent::StreamDeclared(_) => 0,
+        TelemetryEvent::ChannelDeclared(descriptor) => descriptor.name.len() + 64,
+        TelemetryEvent::Frame(delivery) => delivery.payload.len() + 16,
+        TelemetryEvent::StreamEnded(_) => 1,
+    }
+}
+
+async fn write_event_batch(
+    send: &mut SendStream,
+    events: &[TelemetryEvent],
+    compressor: &mut zstd::bulk::Compressor<'_>,
+    raw: &mut Vec<u8>,
+    bytes: &mut Vec<u8>,
+) -> Result<TelemetryQuicWriteStats, BoxError> {
+    bytes.clear();
+    bytes.reserve(
+        events
+            .iter()
+            .map(event_size_hint)
+            .sum::<usize>()
+            .min(WRITE_BATCH_TARGET_BYTES * 2),
+    );
+    let stats = encode_event_batch_with(events, bytes, compressor, raw)?;
+    if !bytes.is_empty() {
+        send.write_all(bytes).await?;
+    }
+    Ok(stats)
+}
+
+/// Append telemetry events in the compact batched subscription wire format.
+pub fn encode_event_batch(
+    events: &[TelemetryEvent],
+    out: &mut Vec<u8>,
+) -> Result<TelemetryQuicWriteStats, BoxError> {
+    let mut compressor = zstd::bulk::Compressor::new(1)?;
+    let mut raw = Vec::new();
+    encode_event_batch_with(events, out, &mut compressor, &mut raw)
+}
+
+fn encode_event_batch_with(
+    events: &[TelemetryEvent],
+    out: &mut Vec<u8>,
+    compressor: &mut zstd::bulk::Compressor<'_>,
+    raw: &mut Vec<u8>,
+) -> Result<TelemetryQuicWriteStats, BoxError> {
+    let start = out.len();
+    let mut encoded_events = 0;
+    let mut index = 0;
+    while index < events.len() {
+        match &events[index] {
+            TelemetryEvent::StreamDeclared(_) => {
+                index += 1;
+            }
+            TelemetryEvent::ChannelDeclared(descriptor) => {
+                let bytes = postcard::to_allocvec(descriptor)?;
+                if bytes.len() > MAX_RECORD_BYTES {
+                    return Err("telemetry channel declaration exceeds max size".into());
+                }
+                out.push(TAG_CHANNEL_DECLARED);
+                put_varint(out, bytes.len() as u64);
+                out.extend_from_slice(&bytes);
+                encoded_events += 1;
+                index += 1;
+            }
+            TelemetryEvent::Frame(_) => {
+                let start_index = index;
+                let mut estimated_bytes = 0_usize;
+                while index < events.len()
+                    && matches!(events[index], TelemetryEvent::Frame(_))
+                    && index - start_index < WRITE_BATCH_MAX_EVENTS
+                {
+                    let next_bytes = event_size_hint(&events[index]);
+                    if index > start_index
+                        && estimated_bytes.saturating_add(next_bytes) > WRITE_BATCH_TARGET_BYTES
+                    {
+                        break;
+                    }
+                    estimated_bytes = estimated_bytes.saturating_add(next_bytes);
+                    index += 1;
+                }
+                encode_frame_batch(&events[start_index..index], out, raw, compressor)?;
+                encoded_events += index - start_index;
+            }
+            TelemetryEvent::StreamEnded(_) => {
+                out.push(TAG_STREAM_ENDED);
+                encoded_events += 1;
+                index += 1;
+            }
         }
     }
-    if bytes.len() > MAX_RECORD_BYTES {
-        return Err("telemetry QUIC record exceeds max size".into());
+    Ok(TelemetryQuicWriteStats {
+        events: encoded_events,
+        bytes: out.len() - start,
+    })
+}
+
+fn encode_frame_batch(
+    events: &[TelemetryEvent],
+    out: &mut Vec<u8>,
+    raw: &mut Vec<u8>,
+    compressor: &mut zstd::bulk::Compressor<'_>,
+) -> Result<(), BoxError> {
+    let raw_capacity = events.iter().fold(0_usize, |total, event| {
+        let TelemetryEvent::Frame(frame) = event else {
+            unreachable!("frame batch contains only frames");
+        };
+        total.saturating_add(frame.payload.len() + 16)
+    });
+    if raw_capacity > MAX_RECORD_BYTES {
+        return Err("telemetry frame batch exceeds max size".into());
     }
-    send.write_all(&(bytes.len() as u32).to_le_bytes()).await?;
-    send.write_all(&bytes).await?;
-    Ok(4 + bytes.len())
+    raw.clear();
+    raw.reserve(raw_capacity);
+    let mut previous_position = None;
+    for event in events {
+        let TelemetryEvent::Frame(frame) = event else {
+            unreachable!("frame batch contains only frames");
+        };
+        put_varint(raw, u64::from(frame.channel.channel.0));
+        let position = frame.position.0;
+        let encoded_position = match previous_position {
+            Some(previous) => position
+                .checked_sub(previous)
+                .ok_or("telemetry frame positions are not monotonic")?,
+            None => position,
+        };
+        put_varint(raw, encoded_position);
+        put_varint(raw, frame.payload.len() as u64);
+        raw.extend_from_slice(&frame.payload);
+        previous_position = Some(position);
+    }
+
+    let compressed = compressor.compress(raw)?;
+    let compressed_size =
+        1 + varint_size(raw.len() as u64) + varint_size(compressed.len() as u64) + compressed.len();
+    let raw_size = 1 + varint_size(raw.len() as u64) + raw.len();
+    if compressed_size < raw_size {
+        out.push(TAG_ZSTD_FRAME_BATCH);
+        put_varint(out, raw.len() as u64);
+        put_varint(out, compressed.len() as u64);
+        out.extend_from_slice(&compressed);
+    } else {
+        out.push(TAG_FRAME_BATCH);
+        put_varint(out, raw.len() as u64);
+        out.extend_from_slice(&raw);
+    }
+    Ok(())
+}
+
+/// Append one telemetry event in the compact subscription wire format.
+pub fn encode_event_record(event: &TelemetryEvent, out: &mut Vec<u8>) -> Result<usize, BoxError> {
+    Ok(encode_event_batch(std::slice::from_ref(event), out)?.bytes)
+}
+
+pub async fn write_event(send: &mut SendStream, event: &TelemetryEvent) -> Result<usize, BoxError> {
+    let mut compressor = zstd::bulk::Compressor::new(1)?;
+    let mut raw = Vec::new();
+    let mut bytes = Vec::new();
+    Ok(write_event_batch(
+        send,
+        std::slice::from_ref(event),
+        &mut compressor,
+        &mut raw,
+        &mut bytes,
+    )
+    .await?
+    .bytes)
 }
 
 pub async fn read_stream_header(recv: &mut RecvStream) -> Result<TelemetryQuicHeader, BoxError> {
@@ -469,8 +691,8 @@ pub async fn read_stream_header(recv: &mut RecvStream) -> Result<TelemetryQuicHe
 pub async fn read_events_from_stream(mut recv: RecvStream) -> Result<TelemetryQuicRead, BoxError> {
     let header = read_header(&mut recv).await?;
     let mut events = Vec::new();
-    while let Some(event) = read_next_event(&mut recv, &header.stream).await? {
-        events.push(event);
+    while let Some(batch) = read_next_events(&mut recv, &header.stream).await? {
+        events.extend(batch);
     }
     Ok(TelemetryQuicRead { header, events })
 }
@@ -514,8 +736,8 @@ async fn write_header(send: &mut SendStream, header: &TelemetryQuicHeader) -> Re
     send.write_all(&(header.token.len() as u16).to_le_bytes())
         .await?;
     send.write_all(&header.token).await?;
-    write_json(send, &header.stream).await?;
-    write_json(send, &header.channels).await?;
+    write_postcard(send, &header.stream).await?;
+    write_postcard(send, &header.channels).await?;
     Ok(())
 }
 
@@ -532,8 +754,8 @@ async fn read_header(recv: &mut RecvStream) -> Result<TelemetryQuicHeader, BoxEr
     let token_len = u16::from_le_bytes(token_len) as usize;
     let mut token = vec![0u8; token_len];
     recv.read_exact(&mut token).await?;
-    let stream = read_json(recv).await?;
-    let channels = read_json(recv).await?;
+    let stream = read_postcard(recv).await?;
+    let channels = read_postcard(recv).await?;
     Ok(TelemetryQuicHeader {
         flow_id,
         token,
@@ -542,63 +764,139 @@ async fn read_header(recv: &mut RecvStream) -> Result<TelemetryQuicHeader, BoxEr
     })
 }
 
-pub async fn read_next_event(
+/// Read the next compact event batch from a telemetry subscription stream.
+pub async fn read_next_events(
     recv: &mut RecvStream,
     stream: &StreamDescriptor,
-) -> Result<Option<TelemetryEvent>, BoxError> {
-    let mut len = [0u8; 4];
-    if recv.read_exact(&mut len).await.is_err() {
+) -> Result<Option<Vec<TelemetryEvent>>, BoxError> {
+    let mut tag = [0_u8; 1];
+    if recv.read_exact(&mut tag).await.is_err() {
         return Ok(None);
     }
-    let len = u32::from_le_bytes(len) as usize;
-    if len > MAX_RECORD_BYTES {
-        return Err("telemetry QUIC record exceeds max size".into());
-    }
-    let mut buf = vec![0u8; len];
-    recv.read_exact(&mut buf).await?;
-    decode_record(&buf, stream).map(Some)
-}
-
-fn decode_record(buf: &[u8], stream: &StreamDescriptor) -> Result<TelemetryEvent, BoxError> {
-    if buf.is_empty() {
-        return Err("empty telemetry QUIC record".into());
-    }
-    match buf[0] {
+    match tag[0] {
         TAG_CHANNEL_DECLARED => {
-            let descriptor: ChannelDescriptor = serde_json::from_slice(&buf[1..])?;
-            Ok(TelemetryEvent::ChannelDeclared(descriptor))
+            let len = read_varint(recv).await?;
+            let bytes = read_sized(recv, len, "telemetry channel declaration").await?;
+            let descriptor = postcard::from_bytes(&bytes)?;
+            Ok(Some(vec![TelemetryEvent::ChannelDeclared(descriptor)]))
         }
-        TAG_FRAME => {
-            if buf.len() < 1 + 4 + 8 + 4 {
-                return Err("telemetry QUIC frame record truncated".into());
-            }
-            let channel = ChannelId(u32::from_le_bytes([buf[1], buf[2], buf[3], buf[4]]));
-            let position = Position(u64::from_le_bytes([
-                buf[5], buf[6], buf[7], buf[8], buf[9], buf[10], buf[11], buf[12],
-            ]));
-            let mut len = [0u8; 4];
-            len.copy_from_slice(&buf[13..17]);
-            let payload_len = u32::from_le_bytes(len) as usize;
-            let payload = buf
-                .get(17..17 + payload_len)
-                .ok_or("telemetry QUIC frame payload truncated")?;
-            if 17 + payload_len != buf.len() {
-                return Err("bytes remain after telemetry QUIC frame record".into());
-            }
-            Ok(TelemetryEvent::Frame(FrameDelivery {
-                channel: ChannelRef {
-                    stream: stream.stream.clone(),
-                    channel,
-                },
-                position,
-                payload: payload.to_vec(),
-            }))
+        TAG_FRAME_BATCH => {
+            let len = read_varint(recv).await?;
+            let raw = read_sized(recv, len, "telemetry frame batch").await?;
+            decode_frame_batch(&raw, stream).map(Some)
         }
-        TAG_STREAM_ENDED => Ok(TelemetryEvent::StreamEnded(stream.stream.clone())),
+        TAG_LZ4_FRAME_BATCH | TAG_ZSTD_FRAME_BATCH => {
+            let raw_len = checked_record_len(read_varint(recv).await?, "telemetry frame batch")?;
+            let compressed_len = read_varint(recv).await?;
+            let compressed =
+                read_sized(recv, compressed_len, "compressed telemetry frame batch").await?;
+            let raw = if tag[0] == TAG_LZ4_FRAME_BATCH {
+                lz4_flex::block::decompress(&compressed, raw_len).map_err(|error| {
+                    std::io::Error::new(
+                        std::io::ErrorKind::InvalidData,
+                        format!("invalid LZ4 telemetry frame batch: {error}"),
+                    )
+                })?
+            } else {
+                decompress_zstd_batch(&compressed, raw_len)?
+            };
+            decode_frame_batch(&raw, stream).map(Some)
+        }
+        TAG_STREAM_ENDED => Ok(Some(vec![TelemetryEvent::StreamEnded(
+            stream.stream.clone(),
+        )])),
         _ => Err("unknown telemetry QUIC record tag".into()),
     }
 }
 
+/// Decode complete compact event batches from one subscription stream.
+pub fn decode_event_records(
+    bytes: &[u8],
+    stream: &StreamDescriptor,
+) -> Result<Vec<TelemetryEvent>, BoxError> {
+    let mut cursor = RecordCursor::new(bytes);
+    let mut events = Vec::new();
+    while !cursor.is_empty() {
+        match cursor.take_u8()? {
+            TAG_CHANNEL_DECLARED => {
+                let len = cursor.take_record_len("telemetry channel declaration")?;
+                let descriptor: ChannelDescriptor = postcard::from_bytes(cursor.take(len)?)?;
+                events.push(TelemetryEvent::ChannelDeclared(descriptor));
+            }
+            TAG_FRAME_BATCH => {
+                let len = cursor.take_record_len("telemetry frame batch")?;
+                events.extend(decode_frame_batch(cursor.take(len)?, stream)?);
+            }
+            tag @ (TAG_LZ4_FRAME_BATCH | TAG_ZSTD_FRAME_BATCH) => {
+                let raw_len = cursor.take_record_len("telemetry frame batch")?;
+                let compressed_len = cursor.take_record_len("compressed telemetry frame batch")?;
+                let compressed = cursor.take(compressed_len)?;
+                let raw = if tag == TAG_LZ4_FRAME_BATCH {
+                    lz4_flex::block::decompress(compressed, raw_len).map_err(|error| {
+                        std::io::Error::new(
+                            std::io::ErrorKind::InvalidData,
+                            format!("invalid LZ4 telemetry frame batch: {error}"),
+                        )
+                    })?
+                } else {
+                    decompress_zstd_batch(compressed, raw_len)?
+                };
+                events.extend(decode_frame_batch(&raw, stream)?);
+            }
+            TAG_STREAM_ENDED => {
+                events.push(TelemetryEvent::StreamEnded(stream.stream.clone()));
+            }
+            _ => return Err("unknown telemetry QUIC record tag".into()),
+        }
+    }
+    Ok(events)
+}
+
+fn decompress_zstd_batch(compressed: &[u8], raw_len: usize) -> Result<Vec<u8>, BoxError> {
+    let raw = zstd::bulk::decompress(compressed, raw_len).map_err(|error| {
+        std::io::Error::new(
+            std::io::ErrorKind::InvalidData,
+            format!("invalid zstd telemetry frame batch: {error}"),
+        )
+    })?;
+    if raw.len() != raw_len {
+        return Err("zstd telemetry frame batch length mismatch".into());
+    }
+    Ok(raw)
+}
+
+fn decode_frame_batch(
+    bytes: &[u8],
+    stream: &StreamDescriptor,
+) -> Result<Vec<TelemetryEvent>, BoxError> {
+    let mut cursor = RecordCursor::new(bytes);
+    let mut events = Vec::new();
+    let mut previous_position: Option<u64> = None;
+    while !cursor.is_empty() {
+        let channel =
+            u32::try_from(cursor.take_varint()?).map_err(|_| "telemetry channel id exceeds u32")?;
+        let encoded_position = cursor.take_varint()?;
+        let position = match previous_position {
+            Some(previous) => previous
+                .checked_add(encoded_position)
+                .ok_or("telemetry frame position overflow")?,
+            None => encoded_position,
+        };
+        let payload_len = cursor.take_record_len("telemetry frame payload")?;
+        let payload = cursor.take(payload_len)?.to_vec();
+        events.push(TelemetryEvent::Frame(FrameDelivery {
+            channel: ChannelRef {
+                stream: stream.stream.clone(),
+                channel: ChannelId(channel),
+            },
+            position: Position(position),
+            payload,
+        }));
+        previous_position = Some(position);
+    }
+    Ok(events)
+}
+
 pub async fn read_stream_into_fanout(
     recv: RecvStream,
     fanout: Arc<telemetry::DeliveryFanout>,
@@ -608,18 +906,125 @@ pub async fn read_stream_into_fanout(
     Ok(read.header)
 }
 
-fn put_json<T: serde::Serialize>(out: &mut Vec<u8>, value: &T) -> Result<(), BoxError> {
-    out.extend_from_slice(&serde_json::to_vec(value)?);
+fn put_varint(out: &mut Vec<u8>, mut value: u64) {
+    while value >= 0x80 {
+        out.push((value as u8) | 0x80);
+        value >>= 7;
+    }
+    out.push(value as u8);
+}
+
+fn varint_size(value: u64) -> usize {
+    ((u64::BITS - value.leading_zeros()).max(1) as usize).div_ceil(7)
+}
+
+fn checked_record_len(value: u64, label: &'static str) -> Result<usize, BoxError> {
+    let len = usize::try_from(value).map_err(|_| "telemetry record length exceeds usize")?;
+    if len > MAX_RECORD_BYTES {
+        return Err(format!("{label} exceeds max size").into());
+    }
+    Ok(len)
+}
+
+async fn read_varint(recv: &mut RecvStream) -> Result<u64, BoxError> {
+    let mut value = 0_u64;
+    for shift in (0..u64::BITS).step_by(7) {
+        let mut byte = [0_u8; 1];
+        recv.read_exact(&mut byte).await?;
+        if shift == 63 && byte[0] & 0x7e != 0 {
+            return Err("telemetry varint exceeds u64".into());
+        }
+        value |= u64::from(byte[0] & 0x7f) << shift;
+        if byte[0] & 0x80 == 0 {
+            return Ok(value);
+        }
+    }
+    Err("telemetry varint exceeds u64".into())
+}
+
+async fn read_sized(
+    recv: &mut RecvStream,
+    len: u64,
+    label: &'static str,
+) -> Result<Vec<u8>, BoxError> {
+    let len = checked_record_len(len, label)?;
+    let mut bytes = vec![0_u8; len];
+    recv.read_exact(&mut bytes).await?;
+    Ok(bytes)
+}
+
+struct RecordCursor<'a> {
+    bytes: &'a [u8],
+    position: usize,
+}
+
+impl<'a> RecordCursor<'a> {
+    fn new(bytes: &'a [u8]) -> Self {
+        Self { bytes, position: 0 }
+    }
+
+    fn is_empty(&self) -> bool {
+        self.position == self.bytes.len()
+    }
+
+    fn take_u8(&mut self) -> Result<u8, BoxError> {
+        Ok(self.take(1)?[0])
+    }
+
+    fn take(&mut self, len: usize) -> Result<&'a [u8], BoxError> {
+        let end = self
+            .position
+            .checked_add(len)
+            .ok_or("telemetry record length overflow")?;
+        let bytes = self
+            .bytes
+            .get(self.position..end)
+            .ok_or("telemetry record truncated")?;
+        self.position = end;
+        Ok(bytes)
+    }
+
+    fn take_varint(&mut self) -> Result<u64, BoxError> {
+        let mut value = 0_u64;
+        for shift in (0..u64::BITS).step_by(7) {
+            let byte = self.take_u8()?;
+            if shift == 63 && byte & 0x7e != 0 {
+                return Err("telemetry varint exceeds u64".into());
+            }
+            value |= u64::from(byte & 0x7f) << shift;
+            if byte & 0x80 == 0 {
+                return Ok(value);
+            }
+        }
+        Err("telemetry varint exceeds u64".into())
+    }
+
+    fn take_record_len(&mut self, label: &'static str) -> Result<usize, BoxError> {
+        checked_record_len(self.take_varint()?, label)
+    }
+}
+
+async fn write_postcard<T: serde::Serialize>(
+    send: &mut SendStream,
+    value: &T,
+) -> Result<(), BoxError> {
+    let bytes = postcard::to_allocvec(value)?;
+    if bytes.len() > MAX_RECORD_BYTES {
+        return Err("telemetry header exceeds max size".into());
+    }
+    let mut len = Vec::with_capacity(10);
+    put_varint(&mut len, bytes.len() as u64);
+    send.write_all(&len).await?;
+    send.write_all(&bytes).await?;
     Ok(())
 }
 
-fn put_bytes(out: &mut Vec<u8>, bytes: &[u8]) -> Result<(), BoxError> {
-    if bytes.len() > u32::MAX as usize {
-        return Err("telemetry delivery exceeds u32 length prefix".into());
-    }
-    out.extend_from_slice(&(bytes.len() as u32).to_le_bytes());
-    out.extend_from_slice(bytes);
-    Ok(())
+async fn read_postcard<T: serde::de::DeserializeOwned>(
+    recv: &mut RecvStream,
+) -> Result<T, BoxError> {
+    let len = read_varint(recv).await?;
+    let bytes = read_sized(recv, len, "telemetry header").await?;
+    Ok(postcard::from_bytes(&bytes)?)
 }
 
 async fn write_json<T: serde::Serialize>(send: &mut SendStream, value: &T) -> Result<(), BoxError> {

crates/iroh-driver/tests/blob_transfer.rs

@@ -1,7 +1,7 @@
 use std::collections::HashMap;
 use std::fs::File;
 use std::sync::atomic::{AtomicU64, Ordering};
-use std::sync::{Arc, Mutex, RwLock};
+use std::sync::{Arc, RwLock};
 use std::time::{Duration, Instant};
 
 use data_plane::blob_transfer::{
@@ -57,6 +57,7 @@ fn node() -> Node {
         IrohDriverConfig {
             secret_key: None,
             relay_mode: RelayMode::Disabled,
+            bind_port: None,
             node: distribution::node::DistributedNodeConfig::default(),
             peer_auth: None,
             additional_alpns: vec![EDGE_ALPN.to_vec()],
@@ -73,7 +74,7 @@ fn node() -> Node {
         swim: ActorAddress::default(),
         relay_mirror: Arc::new(RwLock::new(HashMap::new())),
         route_view: Arc::new(RwLock::new(HashMap::new())),
-        outbox: Arc::new(Mutex::new(Vec::new())),
+        outbox: Arc::new(Default::default()),
     });
     driver.install_actor_bridge_pump(Duration::from_millis(5));
     Node {
@@ -95,6 +96,7 @@ fn real_iroh_transfer_delivers_exact_file_bytes() {
         &destination.engine.handle(),
         destination.runtime.clone(),
         Duration::from_millis(5),
+        destination.driver.edge_events_changed(),
     );
     let sender = IrohBlobTransferSender::new(
         source.driver.edge_connector(),

crates/iroh-driver/tests/common/iroh.rs

@@ -19,7 +19,7 @@
 
 use std::collections::HashMap;
 use std::ops::{Index, IndexMut};
-use std::sync::{Arc, Mutex as StdMutex, RwLock};
+use std::sync::{Arc, RwLock};
 use std::time::{Duration, Instant};
 
 use iroh::{EndpointAddr, PublicKey, RelayMode};
@@ -132,7 +132,7 @@ impl IrohNode {
         let metadata_lambda = node_config.metadata_lambda;
 
         // Shared egress state.
-        let outbox: Outbox = Arc::new(StdMutex::new(Vec::new()));
+        let outbox: Outbox = Arc::new(Default::default());
         let relay_mirror: RelayMirror = Arc::new(RwLock::new(HashMap::new()));
         let route_view: RouteView = Arc::new(RwLock::new(HashMap::new()));
         let peer_directory = Arc::new(OutboxPeerDirectory::new(
@@ -309,6 +309,7 @@ pub fn make_driver() -> IrohNode {
     IrohNode::from_config(IrohDriverConfig {
         secret_key: None,
         relay_mode: RelayMode::Disabled,
+        bind_port: None,
         node: test_config(),
         peer_auth: None,
         additional_alpns: vec![],
@@ -319,6 +320,7 @@ pub fn make_driver_with_auth(auth: Arc<Mutex<PeerAllowList>>) -> IrohNode {
     IrohNode::from_config(IrohDriverConfig {
         secret_key: None,
         relay_mode: RelayMode::Disabled,
+        bind_port: None,
         node: test_config(),
         peer_auth: Some(auth),
         additional_alpns: vec![],
@@ -329,6 +331,7 @@ pub fn make_driver_with_relay(relay_url: iroh::RelayUrl) -> IrohNode {
     IrohNode::from_config(IrohDriverConfig {
         secret_key: None,
         relay_mode: RelayMode::Custom(relay_url.into()),
+        bind_port: None,
         node: test_config(),
         peer_auth: None,
         additional_alpns: vec![],
@@ -419,18 +422,12 @@ pub fn spawn_test_relay() -> (iroh::RelayUrl, RelayGuard) {
         .unwrap();
     let server = rt
         .block_on(async {
-            iroh_relay::server::Server::spawn(iroh_relay::server::ServerConfig::<(), ()> {
-                relay: Some(iroh_relay::server::RelayConfig {
-                    http_bind_addr: (std::net::Ipv4Addr::LOCALHOST, 0).into(),
-                    tls: None,
-                    limits: Default::default(),
-                    key_cache_capacity: Some(256),
-                    access: iroh_relay::server::AccessConfig::Everyone,
-                }),
-                quic: None,
-                metrics_addr: None,
-            })
-            .await
+            let mut relay =
+                iroh_relay::server::RelayConfig::new((std::net::Ipv4Addr::LOCALHOST, 0));
+            relay.key_cache_capacity = Some(256);
+            let mut config = iroh_relay::server::ServerConfig::default();
+            config.relay = Some(relay);
+            iroh_relay::server::Server::spawn(config).await
         })
         .unwrap();
     let url = server.http_url().expect("relay has no HTTP URL");

crates/iroh-driver/tests/iroh_driver.rs

@@ -284,6 +284,7 @@ fn driver_rejects_engine_without_io() {
         IrohDriverConfig {
             secret_key: None,
             relay_mode: RelayMode::Disabled,
+            bind_port: None,
             node: DistributedNodeConfig::default(),
             peer_auth: None,
             additional_alpns: vec![],

crates/iroh-driver/tests/telemetry_transport.rs

@@ -4,16 +4,17 @@ use std::time::{Duration, Instant};
 
 use iroh::{Endpoint, EndpointAddr, RelayMode};
 use iroh_driver::{
-    PullCollectorConfig, TELEMETRY_ALPN, TelemetryQuicHeader, read_next_uni_from_connection,
-    spawn_pull_collector, write_available_subscription,
+    PullCollectorConfig, TELEMETRY_ALPN, TelemetryQuicHeader, decode_event_records,
+    encode_event_batch, read_next_uni_from_connection, spawn_pull_collector, spawn_pull_server,
+    write_available_subscription,
 };
 use swactor::config::RuntimeConfig;
 use swactor::runtime::RuntimeParts;
 use swactor_engine::{Engine, TokioBackend, TokioConfig};
-use telemetry::frame::TelemetryEvent;
+use telemetry::frame::{ChannelRef, FrameDelivery, StreamOrigin, TelemetryEvent};
 use telemetry::{
-    ChannelContent, DeliveryFanout, Lifetime, NodeId, Position, StreamId, SubscriptionRequest,
-    TelemetryEndpoint,
+    ChannelContent, ChannelId, DeliveryFanout, Lifetime, NodeId, Position, StreamDescriptor,
+    StreamId, SubscriptionRequest, TelemetryEndpoint, TelemetrySnapshot, TelemetrySubscription,
 };
 
 /// Telemetry transport test scheduled through `EngineHandle`, not an ambient
@@ -117,6 +118,41 @@ fn iroh_telemetry_alpn_carries_catalog_and_numeric_frames() {
     }
 }
 
+#[test]
+fn zstd_batch_preserves_exact_frames_and_rejects_truncation() {
+    let stream = StreamId::new(NodeId::new("zstd-source"), Lifetime(4));
+    let descriptor = StreamDescriptor {
+        stream: stream.clone(),
+        label: Some("compression test".to_owned()),
+        origin: StreamOrigin::RemoteNode,
+    };
+    let events = (0..128)
+        .map(|position| {
+            TelemetryEvent::Frame(FrameDelivery {
+                channel: ChannelRef {
+                    stream: stream.clone(),
+                    channel: ChannelId(7),
+                },
+                position: Position(position),
+                payload: vec![position as u8; 2048],
+            })
+        })
+        .collect::<Vec<_>>();
+
+    let mut encoded = Vec::new();
+    let stats = encode_event_batch(&events, &mut encoded).expect("encode zstd telemetry batch");
+    assert_eq!(stats.events, events.len());
+    assert_eq!(encoded.first().copied(), Some(0x05));
+    assert!(encoded.len() < events.len() * 2048 / 16);
+    assert_eq!(
+        decode_event_records(&encoded, &descriptor).expect("decode zstd telemetry batch"),
+        events
+    );
+
+    encoded.pop();
+    assert!(decode_event_records(&encoded, &descriptor).is_err());
+}
+
 #[test]
 fn pull_collector_cancellation_interrupts_inflight_io() {
     let parts = RuntimeParts::new(RuntimeConfig::default());
@@ -170,6 +206,136 @@ fn pull_collector_cancellation_interrupts_inflight_io() {
     drop((collector_endpoint, silent_peer));
 }
 
+#[test]
+fn pull_replays_startup_and_disconnect_frames_without_duplicates() {
+    let engine = Engine::new(
+        RuntimeParts::new(RuntimeConfig::default()),
+        TokioBackend::new(TokioConfig::default()).expect("test backend"),
+    )
+    .expect("test engine");
+    let handle = engine.handle();
+    let task_handle = handle.clone();
+    let (done_tx, done_rx) = std::sync::mpsc::channel();
+    handle.spawn(async move {
+        let timeout_handle = task_handle.clone();
+        let result = timeout_handle
+            .timeout(Duration::from_secs(15), async move {
+                let source = test_endpoint().await;
+                let sink = test_endpoint().await;
+                let endpoint = Arc::new(
+                    TelemetryEndpoint::with_capacity(
+                        StreamId::new(NodeId::new("retained-node"), Lifetime(9)),
+                        8,
+                        8,
+                    )
+                    .with_retention(32, 4096),
+                );
+                let producer = endpoint.producer();
+                let channel = producer.register_channel("runtime.log", ChannelContent::TextStream);
+                producer.submit_text(channel, "before-ready");
+                endpoint.tick();
+
+                let fanout = Arc::new(DeliveryFanout::new(8));
+                let subscription = fanout.subscribe_all(
+                    "archive",
+                    TelemetrySnapshot {
+                        streams: Vec::new(),
+                        channels: Vec::new(),
+                    },
+                );
+                let (first_tx, first_rx) = tokio::sync::oneshot::channel();
+                {
+                    let source = source.clone();
+                    let endpoint = Arc::clone(&endpoint);
+                    let server_handle = task_handle.clone();
+                    task_handle.spawn(async move {
+                        let mut first_tx = Some(first_tx);
+                        for _ in 0..2 {
+                            let conn = source
+                                .accept()
+                                .await
+                                .expect("incoming pull")
+                                .await
+                                .expect("accepted pull");
+                            if let Some(first_tx) = first_tx.take() {
+                                let _ = first_tx.send(conn.clone());
+                            }
+                            spawn_pull_server(
+                                &server_handle,
+                                conn,
+                                Arc::clone(&endpoint),
+                                Duration::from_millis(1),
+                            );
+                        }
+                    });
+                }
+                let (header_tx, _header_rx) = std::sync::mpsc::channel();
+                let collector = spawn_pull_collector(
+                    &task_handle,
+                    PullCollectorConfig {
+                        endpoint: sink.clone(),
+                        peer: endpoint_addr(&source),
+                        flow_id: [9; 16],
+                        token: Vec::new(),
+                        request: SubscriptionRequest::all(),
+                        fanout,
+                    },
+                    header_tx,
+                );
+                let first = next_pulled_frame(&task_handle, &subscription).await;
+                assert_eq!(
+                    (first.position, first.payload),
+                    (Position(0), b"before-ready".to_vec())
+                );
+                producer.submit_text(channel, "before-disconnect");
+                endpoint.tick();
+                let second = next_pulled_frame(&task_handle, &subscription).await;
+                assert_eq!(
+                    (second.position, second.payload),
+                    (Position(1), b"before-disconnect".to_vec())
+                );
+
+                first_rx
+                    .await
+                    .expect("first server connection")
+                    .close(0u32.into(), b"test disconnect");
+                producer.submit_text(channel, "while-disconnected");
+                endpoint.tick();
+                let third = next_pulled_frame(&task_handle, &subscription).await;
+                assert_eq!(
+                    (third.position, third.payload),
+                    (Position(2), b"while-disconnected".to_vec())
+                );
+                assert_eq!(
+                    endpoint.subscriber_count(),
+                    1,
+                    "closed pull kept a source subscription"
+                );
+                collector.cancel();
+                source.close().await;
+                sink.close().await;
+            })
+            .await;
+        done_tx.send(result).expect("test completion");
+    });
+    done_rx
+        .recv()
+        .expect("pull replay task completed")
+        .expect("pull replay deadline");
+}
+
+async fn next_pulled_frame(
+    engine: &swactor_engine::EngineHandle,
+    subscription: &TelemetrySubscription,
+) -> FrameDelivery {
+    loop {
+        if let Ok(TelemetryEvent::Frame(frame)) = subscription.try_recv() {
+            return frame;
+        }
+        engine.timer(Duration::from_millis(1)).await;
+    }
+}
+
 async fn test_endpoint() -> Endpoint {
     Endpoint::builder(iroh::endpoint::presets::Minimal)
         .relay_mode(RelayMode::Disabled)

crates/myelin-control-contract/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "myelin-control-contract"
+version = "0.1.0"
+edition = "2024"
+license = "AGPL-3.0-only"
+publish = false
+
+[dependencies]
+serde = { version = "1", features = ["derive"] }
+
+[dev-dependencies]
+serde_json = "1"
+
+[lints]
+workspace = true

crates/process-context/src/actor.rs

@@ -352,14 +352,19 @@ impl ContextualProcessActor {
                 owner: ctx.self_addr(),
             })
             .map_err(|error| format!("spawn session close relay: {error}"))?;
-        self.sender
-            .send_to(
-                active.host_session,
-                HostSessionIn::Close {
-                    reply_to: Some(relay),
-                },
-            )
-            .map_err(|error| format!("close contextual host session: {error}"))
+        match self.sender.send_to(
+            active.host_session,
+            HostSessionIn::Close {
+                reply_to: Some(relay),
+            },
+        ) {
+            Ok(()) => Ok(()),
+            Err(error) => {
+                let _ = ctx.stop_actor(relay);
+                self.session_closed(ctx, Err(format!("close contextual host session: {error}")));
+                Ok(())
+            }
+        }
     }
 
     fn session_closed(&mut self, ctx: &Ctx<'_>, result: Result<(), String>) {

crates/process/src/lib.rs

@@ -10,6 +10,8 @@ mod types;
 
 pub use lifecycle::{ProcessLifecycleObservability, ProcessOutputConfig};
 pub use message::{ProcessCommand, ProcessOutput};
+#[cfg(target_os = "linux")]
+pub use operations::command_output_until;
 #[cfg(unix)]
 pub use operations::request_child_termination;
 #[cfg(target_os = "linux")]
@@ -20,14 +22,16 @@ pub use operations::spawn_unix_stream_listener;
 pub use operations::terminate_process_group;
 pub use operations::{
     CommandOutputObservation, FollowProcessFile, LineReaderHandle, ProcessExitObservation,
-    ProcessIdentity, ProcessStopSignal, ProcessStream, ProcessStreamObservation, child_kill,
-    child_try_wait, child_wait, child_wait_with_output, command_output, command_spawn,
-    command_status, find_process_identities_by_environment, find_process_identities_with_retry,
-    spawn_child_wait, spawn_command_output, spawn_detached_command_status,
-    spawn_identity_exit_wait, spawn_line_channel, spawn_line_reader, spawn_mapped_line_channel,
-    spawn_mapped_line_reader, spawn_shared_child_wait, spawn_stdin_command_wait,
-    spawn_stop_channel_wait, wait_for_path, wait_shared_child_or_kill,
+    ProcessIdentity, ProcessStopSignal, ProcessStream, ProcessStreamObservation,
+    SupervisedProcessObservation, child_kill, child_try_wait, child_wait, child_wait_with_output,
+    command_output, command_spawn, command_status, find_process_identities_by_environment,
+    find_process_identities_with_retry, spawn_child_wait, spawn_command_output,
+    spawn_detached_command_status, spawn_identity_exit_wait, spawn_line_channel, spawn_line_reader,
+    spawn_mapped_line_channel, spawn_mapped_line_reader, spawn_shared_child_wait,
+    spawn_stdin_command_wait, spawn_stop_channel_wait, wait_for_path,
 };
+#[cfg(target_os = "linux")]
+pub use operations::{ProcessWatch, SupervisedChild, stop_shared_child_with_input, watch_process};
 #[cfg(unix)]
 pub use resources::{ProcessResourceError, ProcessSpawnResources};
 #[cfg(unix)]

crates/process/src/operations.rs

@@ -5,7 +5,11 @@
 //! boundary.
 
 use std::fs::File;
-use std::io::{self, BufRead, BufReader, Read};
+use std::io::{self, BufRead, BufReader, Read, Write};
+#[cfg(target_os = "linux")]
+use std::os::fd::{AsRawFd, FromRawFd, OwnedFd};
+#[cfg(target_os = "linux")]
+use std::os::unix::process::CommandExt;
 use std::path::Path;
 use std::process::{Child, Command, ExitStatus, Output};
 use std::sync::mpsc::Receiver;
@@ -20,6 +24,64 @@ pub fn command_output(command: &mut Command) -> io::Result<Output> {
     command.output()
 }
 
+/// Capture a command's exact output within an absolute owner deadline.
+///
+/// Uses the same process-group, parent-death and joined-I/O ownership as
+/// [`SupervisedChild`], without requiring an actor mailbox to make progress.
+/// Timeout kills and reaps the owned child before returning.
+#[cfg(target_os = "linux")]
+pub fn command_output_until(command: &mut Command, deadline: Instant) -> io::Result<Output> {
+    let output = Arc::new(Mutex::new(CapturedCommandOutput::default()));
+    let (finished, completion) = std::sync::mpsc::channel();
+    let owner = SupervisedChild::spawn_observed(
+        command.stdin(std::process::Stdio::null()),
+        None,
+        Some(deadline),
+        SupervisedOutputObserver::Capture {
+            output: Arc::clone(&output),
+            finished,
+        },
+    )?;
+    let completed = completion.recv_timeout(deadline.saturating_duration_since(Instant::now()));
+    // On timeout this cancels the same owner; on completion it joins the
+    // waiter that has already reaped the child and joined both output readers.
+    drop(owner);
+    let status = match completed {
+        Ok(result) => result.map_err(|error| {
+            io::Error::new(
+                if Instant::now() >= deadline {
+                    io::ErrorKind::TimedOut
+                } else {
+                    io::ErrorKind::Other
+                },
+                error,
+            )
+        })?,
+        Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {
+            return Err(io::Error::new(
+                io::ErrorKind::TimedOut,
+                "process owner deadline expired; pending command output/exit",
+            ));
+        }
+        Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => {
+            return Err(io::Error::other(
+                "process owner finished without an exit result",
+            ));
+        }
+    };
+    let mut output = output
+        .lock()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    if let Some(error) = output.error.take() {
+        return Err(error);
+    }
+    Ok(Output {
+        status,
+        stdout: std::mem::take(&mut output.stdout),
+        stderr: std::mem::take(&mut output.stderr),
+    })
+}
+
 pub fn command_status(command: &mut Command) -> io::Result<ExitStatus> {
     command.status()
 }
@@ -88,6 +150,29 @@ impl LineReaderHandle {
     }
 }
 
+fn read_process_lines<R: Read>(
+    stream: ProcessStream,
+    reader: R,
+    mut observe: impl FnMut(ProcessStreamObservation) -> bool,
+) {
+    for next in BufReader::new(reader).lines() {
+        let observation = match next {
+            Ok(line) => ProcessStreamObservation::Line { stream, line },
+            Err(error) => {
+                observe(ProcessStreamObservation::Error {
+                    stream,
+                    error: error.to_string(),
+                });
+                break;
+            }
+        };
+        if !observe(observation) {
+            return;
+        }
+    }
+    observe(ProcessStreamObservation::Closed { stream });
+}
+
 /// Read one child stream and deliver typed observations to an actor relay.
 pub fn spawn_line_reader<R>(
     stream: ProcessStream,
@@ -99,25 +184,9 @@ where
     R: Read + Send + 'static,
 {
     let join = thread::spawn(move || {
-        for next in BufReader::new(reader).lines() {
-            let observation = match next {
-                Ok(line) => ProcessStreamObservation::Line { stream, line },
-                Err(error) => {
-                    let _ = sender.send_to(
-                        actor,
-                        ProcessStreamObservation::Error {
-                            stream,
-                            error: error.to_string(),
-                        },
-                    );
-                    break;
-                }
-            };
-            if sender.send_to(actor, observation).is_err() {
-                return;
-            }
-        }
-        let _ = sender.send_to(actor, ProcessStreamObservation::Closed { stream });
+        read_process_lines(stream, reader, |observation| {
+            sender.send_to(actor, observation).is_ok()
+        });
     });
     LineReaderHandle { join }
 }
@@ -239,6 +308,502 @@ pub struct ProcessExitObservation {
     pub error: Option<String>,
 }
 
+#[cfg(target_os = "linux")]
+fn process_exit_fd(pid: u32) -> io::Result<OwnedFd> {
+    let fd = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, 0) as i32 };
+    if fd < 0 {
+        Err(io::Error::last_os_error())
+    } else {
+        Ok(unsafe { OwnedFd::from_raw_fd(fd) })
+    }
+}
+
+#[cfg(target_os = "linux")]
+fn wait_process_fds<const N: usize>(
+    fds: &[Option<&OwnedFd>; N],
+    timeout: Duration,
+) -> io::Result<[bool; N]> {
+    let mut descriptors: [libc::pollfd; N] = std::array::from_fn(|index| libc::pollfd {
+        fd: fds[index].map_or(-1, AsRawFd::as_raw_fd),
+        events: libc::POLLIN,
+        revents: 0,
+    });
+    let millis = timeout.as_millis().min(i32::MAX as u128) as i32;
+    let result = unsafe {
+        libc::poll(
+            descriptors.as_mut_ptr(),
+            descriptors.len() as libc::nfds_t,
+            millis,
+        )
+    };
+    if result < 0 {
+        let error = io::Error::last_os_error();
+        if error.kind() != io::ErrorKind::Interrupted {
+            return Err(error);
+        }
+    }
+    Ok(std::array::from_fn(|index| descriptors[index].revents != 0))
+}
+
+/// A cancellable exit subscription. Dropping it joins its observer without
+/// stopping the process, so adopted workers may outlive their observation.
+#[cfg(target_os = "linux")]
+pub struct ProcessWatch {
+    cancel: Arc<OwnedFd>,
+    thread: Option<JoinHandle<()>>,
+}
+
+#[cfg(target_os = "linux")]
+impl Drop for ProcessWatch {
+    fn drop(&mut self) {
+        let value = 1_u64;
+        unsafe {
+            libc::write(
+                self.cancel.as_raw_fd(),
+                (&value as *const u64).cast(),
+                std::mem::size_of::<u64>(),
+            );
+        }
+        if let Some(thread) = self.thread.take() {
+            let _ = thread.join();
+        }
+    }
+}
+
+#[cfg(target_os = "linux")]
+pub fn watch_process(
+    identity: ProcessIdentity,
+    child: Option<Arc<Mutex<Option<Child>>>>,
+    sender: ExternalSender,
+    actor: ActorAddress,
+) -> io::Result<ProcessWatch> {
+    let exit = process_exit_fd(identity.pid).ok();
+    let fd = unsafe { libc::eventfd(0, libc::EFD_CLOEXEC | libc::EFD_NONBLOCK) };
+    if fd < 0 {
+        return Err(io::Error::last_os_error());
+    }
+    let cancel = Arc::new(unsafe { OwnedFd::from_raw_fd(fd) });
+    let cancelled = Arc::clone(&cancel);
+    let thread = thread::Builder::new().spawn(move || {
+        let mut error = None;
+        let status = loop {
+            if let Some(child) = &child {
+                let mut slot = child
+                    .lock()
+                    .unwrap_or_else(std::sync::PoisonError::into_inner);
+                let Some(child) = slot.as_mut() else {
+                    return;
+                };
+                match child.try_wait() {
+                    Ok(Some(status)) => {
+                        *slot = None;
+                        break status.code();
+                    }
+                    Ok(None) => {}
+                    Err(reason) => {
+                        error = Some(reason.to_string());
+                        break None;
+                    }
+                }
+            } else if !identity.matches() {
+                break None;
+            }
+            let delay = if exit.is_some() {
+                Duration::from_secs(30)
+            } else {
+                Duration::from_millis(100)
+            };
+            match wait_process_fds(&[exit.as_ref(), Some(&cancelled)], delay) {
+                Ok(ready) if ready[1] => return,
+                Ok(ready) if ready[0] && child.is_none() => break None,
+                Ok(_) => {}
+                Err(reason) => {
+                    error = Some(reason.to_string());
+                    break None;
+                }
+            }
+        };
+        let _ = sender.send_to(actor, ProcessExitObservation { status, error });
+    })?;
+    Ok(ProcessWatch {
+        cancel,
+        thread: Some(thread),
+    })
+}
+
+/// Send a best-effort shutdown command without blocking on stdin, then observe
+/// exit until the absolute deadline. Kill and reap the owned group on expiry.
+#[cfg(target_os = "linux")]
+pub fn stop_shared_child_with_input(
+    child: &Arc<Mutex<Option<Child>>>,
+    stdin: &mut std::process::ChildStdin,
+    input: &[u8],
+    deadline: Instant,
+) -> io::Result<Option<ExitStatus>> {
+    let fd = stdin.as_raw_fd();
+    let flags = unsafe { libc::fcntl(fd, libc::F_GETFL) };
+    if flags >= 0 && unsafe { libc::fcntl(fd, libc::F_SETFL, flags | libc::O_NONBLOCK) } >= 0 {
+        let _ = stdin.write_all(input);
+        let _ = stdin.flush();
+    }
+    let exit = child
+        .lock()
+        .unwrap_or_else(std::sync::PoisonError::into_inner)
+        .as_ref()
+        .and_then(|child| process_exit_fd(child.id()).ok());
+    loop {
+        let mut slot = child
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        let Some(child) = slot.as_mut() else {
+            return Ok(None);
+        };
+        if let Some(status) = child.try_wait()? {
+            *slot = None;
+            return Ok(Some(status));
+        }
+        if Instant::now() >= deadline {
+            terminate_owned_child(child);
+            let status = child.wait()?;
+            *slot = None;
+            return Ok(Some(status));
+        }
+        drop(slot);
+        let left = deadline.saturating_duration_since(Instant::now());
+        wait_process_fds(
+            &[exit.as_ref()],
+            if exit.is_some() {
+                left
+            } else {
+                left.min(Duration::from_millis(50))
+            },
+        )?;
+    }
+}
+
+#[cfg(target_os = "linux")]
+fn terminate_owned_child(child: &mut Child) {
+    // The unreaped leader pins the process-group number against PID reuse.
+    unsafe {
+        libc::kill(-(child.id() as i32), libc::SIGKILL);
+    }
+    let _ = child.kill();
+}
+
+#[derive(Clone, Debug)]
+pub enum SupervisedProcessObservation {
+    Stream(ProcessStreamObservation),
+    Exited {
+        operation: u64,
+        result: Result<ExitStatus, String>,
+    },
+}
+
+/// Owns an entire command attempt: child, input, incremental output and waiter.
+/// Completion and cancellation both reap the group and join every I/O thread.
+#[cfg(target_os = "linux")]
+pub struct SupervisedChild {
+    child: Arc<Mutex<Option<Child>>>,
+    waiter: Option<JoinHandle<()>>,
+}
+
+#[cfg(target_os = "linux")]
+#[derive(Default)]
+struct CapturedCommandOutput {
+    stdout: Vec<u8>,
+    stderr: Vec<u8>,
+    error: Option<io::Error>,
+}
+
+#[cfg(target_os = "linux")]
+#[derive(Clone)]
+enum SupervisedOutputObserver {
+    Actor {
+        sender: ExternalSender,
+        actor: ActorAddress,
+        operation: u64,
+    },
+    Capture {
+        output: Arc<Mutex<CapturedCommandOutput>>,
+        finished: std::sync::mpsc::Sender<Result<ExitStatus, String>>,
+    },
+}
+
+#[cfg(target_os = "linux")]
+impl SupervisedOutputObserver {
+    fn close_stream(&self, stream: ProcessStream) {
+        if let Self::Actor { sender, actor, .. } = self {
+            let _ = sender.send_to(
+                *actor,
+                SupervisedProcessObservation::Stream(ProcessStreamObservation::Closed { stream }),
+            );
+        }
+    }
+
+    fn exited(&self, result: Result<ExitStatus, String>) {
+        match self {
+            Self::Actor {
+                sender,
+                actor,
+                operation,
+            } => {
+                let _ = sender.send_to(
+                    *actor,
+                    SupervisedProcessObservation::Exited {
+                        operation: *operation,
+                        result,
+                    },
+                );
+            }
+            Self::Capture { finished, .. } => {
+                let _ = finished.send(result);
+            }
+        }
+    }
+}
+
+#[cfg(target_os = "linux")]
+fn spawn_supervised_reader<R: Read + Send + 'static>(
+    stream: ProcessStream,
+    mut reader: R,
+    observer: SupervisedOutputObserver,
+) -> io::Result<JoinHandle<()>> {
+    thread::Builder::new().spawn(move || match observer {
+        SupervisedOutputObserver::Actor { sender, actor, .. } => {
+            read_process_lines(stream, reader, |observation| {
+                sender
+                    .send_to(actor, SupervisedProcessObservation::Stream(observation))
+                    .is_ok()
+            });
+        }
+        SupervisedOutputObserver::Capture { output, .. } => {
+            let mut bytes = Vec::new();
+            let result = reader.read_to_end(&mut bytes);
+            let mut output = output
+                .lock()
+                .unwrap_or_else(std::sync::PoisonError::into_inner);
+            match stream {
+                ProcessStream::Stdout => output.stdout = bytes,
+                ProcessStream::Stderr => output.stderr = bytes,
+            }
+            if let Err(error) = result {
+                output.error = Some(error);
+            }
+        }
+    })
+}
+
+#[cfg(target_os = "linux")]
+impl SupervisedChild {
+    pub fn spawn(
+        command: &mut Command,
+        input: Option<Arc<[u8]>>,
+        deadline: Option<Instant>,
+        sender: ExternalSender,
+        actor: ActorAddress,
+        operation: u64,
+    ) -> io::Result<Self> {
+        Self::spawn_observed(
+            command,
+            input,
+            deadline,
+            SupervisedOutputObserver::Actor {
+                sender,
+                actor,
+                operation,
+            },
+        )
+    }
+
+    fn spawn_observed(
+        command: &mut Command,
+        input: Option<Arc<[u8]>>,
+        deadline: Option<Instant>,
+        observer: SupervisedOutputObserver,
+    ) -> io::Result<Self> {
+        if deadline.is_some_and(|deadline| deadline <= Instant::now()) {
+            return Err(io::Error::new(
+                io::ErrorKind::TimedOut,
+                "process owner deadline expired",
+            ));
+        }
+        command.process_group(0);
+        let parent = unsafe { libc::getpid() };
+        unsafe {
+            command.pre_exec(move || {
+                if libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL) != 0 {
+                    return Err(io::Error::last_os_error());
+                }
+                if libc::getppid() != parent {
+                    return Err(io::Error::other("process owner exited during spawn"));
+                }
+                Ok(())
+            });
+        }
+        command
+            .stdout(std::process::Stdio::piped())
+            .stderr(std::process::Stdio::piped());
+        if input.is_some() {
+            command.stdin(std::process::Stdio::piped());
+        }
+        let mut child = command.spawn()?;
+        let exit = match process_exit_fd(child.id()) {
+            Ok(exit) => exit,
+            Err(error) => {
+                terminate_owned_child(&mut child);
+                let _ = child.wait();
+                return Err(error);
+            }
+        };
+        let stdout = child
+            .stdout
+            .take()
+            .expect("supervisor configures piped stdout");
+        let stderr = child
+            .stderr
+            .take()
+            .expect("supervisor configures piped stderr");
+        let stdin = input.map(|bytes| {
+            (
+                child
+                    .stdin
+                    .take()
+                    .expect("supervisor configures piped stdin"),
+                bytes,
+            )
+        });
+        let child = Arc::new(Mutex::new(Some(child)));
+        let owned = Arc::clone(&child);
+        let waiter = match thread::Builder::new().spawn(move || {
+            let mut setup_error = None;
+            let readers = [
+                (
+                    ProcessStream::Stdout,
+                    spawn_supervised_reader(ProcessStream::Stdout, stdout, observer.clone()),
+                ),
+                (
+                    ProcessStream::Stderr,
+                    spawn_supervised_reader(ProcessStream::Stderr, stderr, observer.clone()),
+                ),
+            ]
+            .map(|(stream, reader)| {
+                (
+                    stream,
+                    match reader {
+                        Ok(reader) => Some(reader),
+                        Err(error) => {
+                            setup_error = Some(error.to_string());
+                            observer.close_stream(stream);
+                            None
+                        }
+                    },
+                )
+            });
+            let writer = stdin.and_then(|(mut stdin, bytes)| {
+                match thread::Builder::new()
+                    .spawn(move || stdin.write_all(&bytes).and_then(|()| stdin.flush()))
+                {
+                    Ok(writer) => Some(writer),
+                    Err(error) => {
+                        setup_error = Some(error.to_string());
+                        None
+                    }
+                }
+            });
+            let waited = if let Some(error) = setup_error {
+                Err(error)
+            } else {
+                loop {
+                    let left = deadline
+                        .map_or(Duration::from_millis(i32::MAX as u64), |deadline| {
+                            deadline.saturating_duration_since(Instant::now())
+                        });
+                    if left.is_zero() {
+                        break Err("process owner deadline expired; pending child exit".to_owned());
+                    }
+                    match wait_process_fds(&[Some(&exit)], left) {
+                        Ok([true]) => break Ok(()),
+                        Ok([false]) => {}
+                        Err(error) => break Err(error.to_string()),
+                    }
+                }
+            };
+            let status = {
+                let mut slot = owned
+                    .lock()
+                    .unwrap_or_else(std::sync::PoisonError::into_inner);
+                slot.take().map(|mut child| {
+                    terminate_owned_child(&mut child);
+                    child.wait().map_err(|error| error.to_string())
+                })
+            };
+            let input_result = writer
+                .map(|writer| {
+                    writer
+                        .join()
+                        .map_err(|_| "process stdin writer panicked".to_owned())?
+                        .map_err(|error| format!("write process stdin: {error}"))
+                })
+                .unwrap_or(Ok(()));
+            let mut reader_error = None;
+            for (stream, reader) in readers {
+                if reader.is_some_and(|reader| reader.join().is_err()) {
+                    reader_error = Some("process output reader panicked".to_owned());
+                    observer.close_stream(stream);
+                }
+            }
+            if let Some(status) = status {
+                let result = waited.and(status).and_then(|status| {
+                    if let Some(error) = reader_error {
+                        return Err(error);
+                    }
+                    if status.success() {
+                        input_result.map(|()| status)
+                    } else {
+                        Ok(status)
+                    }
+                });
+                observer.exited(result);
+            }
+        }) {
+            Ok(waiter) => waiter,
+            Err(error) => {
+                if let Some(mut child) = child
+                    .lock()
+                    .unwrap_or_else(std::sync::PoisonError::into_inner)
+                    .take()
+                {
+                    terminate_owned_child(&mut child);
+                    let _ = child.wait();
+                }
+                return Err(error);
+            }
+        };
+        Ok(Self {
+            child,
+            waiter: Some(waiter),
+        })
+    }
+}
+
+#[cfg(target_os = "linux")]
+impl Drop for SupervisedChild {
+    fn drop(&mut self) {
+        let mut slot = self
+            .child
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        if let Some(mut child) = slot.take() {
+            terminate_owned_child(&mut child);
+            let _ = child.wait();
+        }
+        drop(slot);
+        if let Some(waiter) = self.waiter.take() {
+            let _ = waiter.join();
+        }
+    }
+}
+
 pub fn spawn_shared_child_wait(
     child: Arc<Mutex<Option<Child>>>,
     poll_interval: Duration,
@@ -278,46 +843,6 @@ pub fn spawn_shared_child_wait(
     });
 }
 
-pub fn wait_shared_child_or_kill(
-    child: &Arc<Mutex<Option<Child>>>,
-    timeout: Duration,
-    process_group: bool,
-    poll_interval: Duration,
-) -> io::Result<Option<ExitStatus>> {
-    let deadline = Instant::now() + timeout;
-    while Instant::now() < deadline {
-        let mut slot = child
-            .lock()
-            .unwrap_or_else(std::sync::PoisonError::into_inner);
-        let Some(child) = slot.as_mut() else {
-            return Ok(None);
-        };
-        match child.try_wait()? {
-            Some(status) => {
-                *slot = None;
-                return Ok(Some(status));
-            }
-            None => drop(slot),
-        }
-        thread::sleep(poll_interval);
-    }
-
-    let mut slot = child
-        .lock()
-        .unwrap_or_else(std::sync::PoisonError::into_inner);
-    let Some(child) = slot.as_mut() else {
-        return Ok(None);
-    };
-    #[cfg(target_os = "linux")]
-    if process_group {
-        let _ = unsafe { libc::kill(-(child.id() as i32), libc::SIGKILL) };
-    }
-    child.kill()?;
-    let status = child.wait()?;
-    *slot = None;
-    Ok(Some(status))
-}
-
 #[cfg(target_os = "linux")]
 pub fn terminate_process_group(
     identity: &ProcessIdentity,
@@ -327,7 +852,11 @@ pub fn terminate_process_group(
     if !identity.matches() {
         return Ok(());
     }
-    if unsafe { libc::kill(-(identity.pid as i32), libc::SIGTERM) } != 0 {
+    let exit = process_exit_fd(identity.pid).ok();
+    if !identity.matches() {
+        return Ok(());
+    }
+    if unsafe { libc::kill(-(identity.pid as i32), libc::SIGTERM) } != 0 && identity.matches() {
         return Err(format!(
             "terminate process group {}: {}",
             identity.pid,
@@ -335,18 +864,52 @@ pub fn terminate_process_group(
         ));
     }
     let deadline = Instant::now() + timeout;
-    while Instant::now() < deadline {
-        if !identity.matches() {
+    while identity.matches() && Instant::now() < deadline {
+        let left = deadline.saturating_duration_since(Instant::now());
+        if wait_process_fds(
+            &[exit.as_ref()],
+            if exit.is_some() {
+                left
+            } else {
+                left.min(poll_interval)
+            },
+        )
+        .map_err(|error| format!("wait process group {}: {error}", identity.pid))?[0]
+        {
             return Ok(());
         }
-        thread::sleep(poll_interval);
     }
-    if unsafe { libc::kill(-(identity.pid as i32), libc::SIGKILL) } != 0 && identity.matches() {
-        return Err(format!(
-            "kill process group {}: {}",
-            identity.pid,
-            io::Error::last_os_error()
-        ));
+    if identity.matches() {
+        if unsafe { libc::kill(-(identity.pid as i32), libc::SIGKILL) } != 0 && identity.matches() {
+            return Err(format!(
+                "kill process group {}: {}",
+                identity.pid,
+                io::Error::last_os_error()
+            ));
+        }
+        if let Some(exit) = &exit {
+            let deadline = Instant::now() + timeout;
+            loop {
+                let left = deadline.saturating_duration_since(Instant::now());
+                if left.is_zero() {
+                    return Err(format!(
+                        "process group {} did not exit after KILL",
+                        identity.pid
+                    ));
+                }
+                if wait_process_fds(&[Some(exit)], left).map_err(|error| {
+                    format!("observe killed process group {}: {error}", identity.pid)
+                })?[0]
+                {
+                    break;
+                }
+            }
+        } else if identity.matches() {
+            return Err(format!(
+                "process group {} exit cannot be confirmed",
+                identity.pid
+            ));
+        }
     }
     Ok(())
 }
@@ -688,6 +1251,82 @@ mod properties {
     use super::*;
 
     const DRIVER_BUDGET: usize = 64;
+
+    #[cfg(target_os = "linux")]
+    #[test]
+    fn bounded_command_output_kills_and_reaps_a_withheld_response() {
+        let pid_path = std::env::temp_dir().join(format!(
+            "swactor-command-deadline-{}-{}.pid",
+            std::process::id(),
+            std::time::SystemTime::now()
+                .duration_since(std::time::UNIX_EPOCH)
+                .unwrap()
+                .as_nanos()
+        ));
+        let mut command = Command::new("/bin/sh");
+        command
+            .args([
+                "-c",
+                "trap '' TERM; printf '%s' \"$$\" > \"$1\"; printf partial; printf diagnostic >&2; exec sleep 30",
+                "withheld-response",
+            ])
+            .arg(&pid_path);
+        let started = Instant::now();
+        let result = command_output_until(&mut command, started + Duration::from_secs(1));
+        let elapsed = started.elapsed();
+        let pid = std::fs::read_to_string(&pid_path);
+        let _ = std::fs::remove_file(&pid_path);
+
+        assert_eq!(result.unwrap_err().kind(), io::ErrorKind::TimedOut);
+        assert!(
+            elapsed < Duration::from_secs(5),
+            "withheld child lasted {elapsed:?}"
+        );
+        let pid = pid
+            .expect("child reached its withheld response")
+            .parse::<i32>()
+            .unwrap();
+        let mut status = 0;
+        assert_eq!(
+            unsafe { libc::waitpid(pid, &mut status, libc::WNOHANG) },
+            -1
+        );
+        assert_eq!(
+            io::Error::last_os_error().raw_os_error(),
+            Some(libc::ECHILD)
+        );
+    }
+
+    #[cfg(target_os = "linux")]
+    #[test]
+    fn bounded_command_output_drains_raw_bytes_and_inherited_pipes_on_rejection() {
+        let mut command = Command::new("/bin/sh");
+        command.args([
+            "-c",
+            "printf '\\000\\377partial'; printf 'rejected\\n' >&2; sleep 30 & exit 7",
+        ]);
+        let started = Instant::now();
+        let output = command_output_until(&mut command, started + Duration::from_secs(2)).unwrap();
+
+        assert_eq!(output.status.code(), Some(7));
+        assert_eq!(output.stdout, b"\0\xffpartial");
+        assert_eq!(output.stderr, b"rejected\n");
+        assert!(started.elapsed() < Duration::from_secs(5));
+    }
+
+    #[cfg(target_os = "linux")]
+    #[test]
+    fn bounded_command_output_does_not_spawn_after_owner_expiry() {
+        // A nonexistent executable distinguishes pre-spawn expiry from a
+        // freshly minted command lifetime that attempts execution anyway.
+        let error = command_output_until(
+            &mut Command::new("/definitely/not/a/real/bounded-command"),
+            Instant::now(),
+        )
+        .unwrap_err();
+        assert_eq!(error.kind(), io::ErrorKind::TimedOut);
+    }
+
     #[cfg(unix)]
     #[test]
     fn unix_listener_unlinks_its_bound_path_when_engine_stops() {

crates/process/src/supervisor.rs

@@ -1,6 +1,10 @@
 use std::io::{self, Read};
 use std::mem;
-use std::os::fd::RawFd;
+#[cfg(target_os = "linux")]
+use std::os::fd::FromRawFd;
+use std::os::fd::{AsRawFd, OwnedFd, RawFd};
+#[cfg(unix)]
+use std::os::unix::process::CommandExt;
 use std::process::{Child, Command, Stdio};
 use std::sync::Arc;
 use std::thread::{self, JoinHandle};
@@ -152,21 +156,32 @@ impl Drop for WakeFdInner {
     }
 }
 
-fn poll_command_wake(wake: &WakeFd, timeout: Duration) -> io::Result<bool> {
+fn poll_command_wake(
+    wake: &WakeFd,
+    child_exit: Option<RawFd>,
+    timeout: Duration,
+) -> io::Result<bool> {
     let timeout_ms = poll_timeout_ms(timeout);
-    let mut pollfd = libc::pollfd {
-        fd: wake.fd(),
-        events: libc::POLLIN,
-        revents: 0,
-    };
+    let mut pollfds = [
+        libc::pollfd {
+            fd: wake.fd(),
+            events: libc::POLLIN,
+            revents: 0,
+        },
+        libc::pollfd {
+            fd: child_exit.unwrap_or(-1),
+            events: libc::POLLIN,
+            revents: 0,
+        },
+    ];
 
     loop {
-        let result = unsafe { libc::poll(&mut pollfd, 1, timeout_ms) };
+        let result = unsafe { libc::poll(pollfds.as_mut_ptr(), pollfds.len() as _, timeout_ms) };
         if result == 0 {
             return Ok(false);
         }
         if result > 0 {
-            let revents = pollfd.revents;
+            let revents = pollfds[0].revents;
             if revents & (libc::POLLERR | libc::POLLNVAL | libc::POLLHUP) != 0 {
                 return Err(io::Error::other(format!(
                     "process supervisor wake fd poll failed: revents={revents}"
@@ -296,6 +311,21 @@ impl Drop for ProcessThreadHandle {
 
 const WAITPID_POLL_INTERVAL: Duration = Duration::from_millis(10);
 
+/// A pidfd is tied to this exact child, including an exit before poll starts.
+/// Kernels/platforms without pidfds retain the existing waitpid reconciliation.
+fn child_exit_fd(pid: u32) -> Option<OwnedFd> {
+    #[cfg(target_os = "linux")]
+    {
+        let fd = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, 0) };
+        if fd >= 0 {
+            return Some(unsafe { OwnedFd::from_raw_fd(fd as RawFd) });
+        }
+    }
+    #[cfg(not(target_os = "linux"))]
+    let _ = pid;
+    None
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 enum SupervisorState {
     Spawning,
@@ -337,6 +367,8 @@ fn supervisor_thread_main(
         .stdout(Stdio::piped())
         .stderr(Stdio::piped());
     #[cfg(unix)]
+    cmd.process_group(0);
+    #[cfg(unix)]
     resources.configure_command(&mut cmd);
 
     match cmd.spawn() {
@@ -381,9 +413,16 @@ fn supervisor_thread_main(
     }
 
     let child_pid = pid.expect("supervisor pid stored after successful spawn");
+    let child_exit = child_exit_fd(child_pid);
     loop {
-        let timeout = command_poll_timeout(kill_deadline);
-        match poll_command_wake(&wake, timeout) {
+        let timeout = if child_exit.is_some() {
+            kill_deadline.map_or(Duration::from_millis(i32::MAX as u64), |deadline| {
+                deadline.saturating_duration_since(Instant::now())
+            })
+        } else {
+            command_poll_timeout(kill_deadline)
+        };
+        match poll_command_wake(&wake, child_exit.as_ref().map(AsRawFd::as_raw_fd), timeout) {
             Ok(true) => {
                 if let Err(err) = wake.drain() {
                     finish_with_error(
@@ -646,14 +685,16 @@ fn signal_to_libc(signal: Signal) -> libc::c_int {
 
 fn send_signal(pid: u32, signal: Signal) -> Result<(), String> {
     let sig = signal_to_libc(signal);
-    let ret = unsafe { libc::kill(pid as libc::pid_t, sig) };
+    // The supervised child leads a private process group. Signalling the group
+    // prevents descendants from retaining its output pipes after the leader
+    // exits, which would otherwise indefinitely delay terminal observation.
+    let target = -(pid as libc::pid_t);
+    let ret = unsafe { libc::kill(target, sig) };
     if ret == 0 {
         Ok(())
     } else {
         Err(format!(
-            "kill({}, {}) failed: {}",
-            pid,
-            sig,
+            "kill({target}, {sig}) failed: {}",
             std::io::Error::last_os_error()
         ))
     }
@@ -885,7 +926,7 @@ mod tests {
             .expect("shutdown command should queue");
 
         assert!(
-            poll_command_wake(&wake, Duration::ZERO).expect("wake poll should succeed"),
+            poll_command_wake(&wake, None, Duration::ZERO).expect("wake poll should succeed"),
             "wake fd should be readable after queued commands"
         );
         wake.drain().expect("wake fd should drain");
@@ -995,7 +1036,7 @@ mod tests {
     fn supervisor_stop_escalates_to_kill_after_deadline() {
         let (sink, receiver) = thread_event_channel(|| {});
         let mut handle = ProcessSupervisorThread::start(
-            shell_spec("trap '' TERM; while true; do sleep 1; done"),
+            shell_spec("trap '' TERM; printf 'ready\\n'; while true; do sleep 1; done"),
             ProcessSpawnResources::new(),
             sink,
         )
@@ -1007,9 +1048,9 @@ mod tests {
         while Instant::now() < deadline {
             events.extend(receiver.drain());
             if !stop_sent
-                && events
-                    .iter()
-                    .any(|event| matches!(event, ThreadEvent::Started { pid } if *pid > 0))
+                && events.iter().any(
+                    |event| matches!(event, ThreadEvent::Output { stderr: false, bytes } if bytes == b"ready\n"),
+                )
             {
                 handle
                     .stop(Some(Duration::from_millis(20)))
@@ -1030,7 +1071,7 @@ mod tests {
 
         assert!(
             stop_sent,
-            "supervisor did not report Started before timeout"
+            "supervisor process did not report readiness before timeout"
         );
         assert_eq!(events.last(), Some(&ThreadEvent::ThreadFinished));
         assert!(events.iter().any(|event| {

crates/process/tests/public_api_stage2.rs

@@ -518,7 +518,7 @@ fn stop_before_spawn_success_reports_started_then_exited() {
         &rt,
         spawner,
         &sender,
-        shell_spec("sh", vec!["-c", "sleep 60"], Some("stop-before-start")),
+        shell_spec("sleep", vec!["60"], Some("stop-before-start")),
         ProcessOutputConfig::disabled(*upstream.addr()),
         &reply,
     );

crates/provisioning/Cargo.toml

@@ -6,13 +6,14 @@ license = "AGPL-3.0-only"
 publish = false
 
 [dependencies]
+myelin-control-contract = { path = "../myelin-control-contract" }
 serde = { version = "1", features = ["derive"] }
+serde_json = "1"
 swactor = { path = "../.." }
 swactor-engine = { path = "../engine" }
 
 [dev-dependencies]
 parking_lot = "0.12"
-serde_json = "1"
 swactor-process = { path = "../process" }
 
 [lints]

crates/provisioning/src/lib.rs

@@ -8,10 +8,12 @@
 pub mod bootstrap;
 pub mod executor;
 pub mod node;
+pub mod paid_admission;
 pub mod plugin;
 pub mod reconciler;
 pub use bootstrap::*;
 
 pub use executor::*;
 pub use node::*;
+pub use paid_admission::*;
 pub use reconciler::*;

crates/provisioning/src/plugin.rs

@@ -2,6 +2,7 @@ use std::sync::Arc;
 
 use serde::{Deserialize, Serialize};
 
+pub use myelin_control_contract::DeploymentIdentity;
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub struct NodeProvisionSpec {
     pub run_id: u64,
@@ -17,6 +18,10 @@ pub struct NodeProvisionSpec {
     /// explicitly selected marketplace offer.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub offer_criteria_json: Option<String>,
+    /// Desired deployment identity for this attempt. Providers that ship
+    /// artifacts must install exactly this identity before runtime ready.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub deployment: Option<DeploymentIdentity>,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub mounts: Vec<ProviderMount>,
 }
@@ -233,6 +238,7 @@ mod tests {
     #[test]
     fn mounted_node_specs_round_trip_without_losing_readonly_intent() {
         let spec = NodeProvisionSpec {
+            deployment: None,
             run_id: 17,
             node_id: 23,
             attempt_id: 7,

crates/provisioning/tests/common/mod.rs

@@ -177,6 +177,7 @@ pub fn null_sink() -> PluginSink {
 /// A `NodeProvisionSpec` for a concrete attempt, for direct plugin calls.
 pub fn plugin_spec(attempt: u64) -> NodeProvisionSpec {
     NodeProvisionSpec {
+        deployment: None,
         run_id: 7,
         node_id: attempt,
         attempt_id: attempt,
@@ -408,6 +409,7 @@ impl<P: TestablePlugin + 'static> EffectBackend for PluginBackendAdapter<P> {
                     ))));
                 }
                 let spec = NodeProvisionSpec {
+                    deployment: None,
                     run_id: request.spec.run_id.0,
                     node_id: attempt,
                     attempt_id: attempt,

crates/telemetry/Cargo.toml

@@ -11,7 +11,8 @@ swactor-engine = { path = "../engine" }
 futures-channel = "0.3"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
-iroh = "0.98"
+rmp-serde = "1"
+iroh-base.workspace = true
 crossbeam-channel = "0.5"
 
 [target.'cfg(target_os = "linux")'.dependencies]

crates/telemetry/TELEMETRY_SPEC.md

@@ -1,7 +1,7 @@
 # The Telemetry — Specification
 
 Id: 7
-Last modified: b887e941cbe6f1e209339abd0375507aca9bfe52
+Last modified: 74ba89c0cedae04b05a4b8b4af9a3856adfd5875
 Last reviewed:
 > Any edit to this spec must update `Last modified` above to the current `git HEAD` commit.
 
@@ -162,12 +162,18 @@ The human-readable channel name and display/routing metadata live in a channel
 descriptor:
 
 ```rust
-pub enum ChannelContentKind { Bytes, TextStream, JsonRecord }
+pub enum ChannelContentKind {
+    Bytes,
+    TextStream,
+    JsonRecord,
+    MessagePackRecord,
+}
 
 pub enum ChannelContent {
     Bytes,
     TextStream,
     JsonRecord { schema: Option<String> },
+    MessagePackRecord { schema: Option<String> },
 }
 
 pub struct ChannelDescriptor {
@@ -335,7 +341,7 @@ Examples:
 | `{ sources: All, channels: Prefix("proc.") }` | all process-output channels with descriptors visible to the subscriber |
 | `{ sources: Node(X), channels: Prefix("proc.trainer.") }` | trainer process output from every life of node `X` |
 | `{ sources: Stream(S), channels: Name("host.cpu") }` | exact host CPU channel in one stream |
-| `{ sources: Origin(Bootstrap), channels: Content(JsonRecord) }` | JSON-record channels from bootstrap streams |
+| `{ sources: Origin(Bootstrap), channels: Content(MessagePackRecord) }` | named-field MessagePack record channels from bootstrap streams |
 
 ---
 
@@ -516,6 +522,7 @@ pub enum ChannelContent {
     Bytes,
     TextStream,
     JsonRecord { schema: Option<String> },
+    MessagePackRecord { schema: Option<String> },
 }
 ```
 
@@ -525,17 +532,17 @@ transport catalog. It is not payload inspection.
 View classification remains caller-owned:
 
 ```rust
-pub enum ChannelKind { Typed, Text, Opaque }
+pub enum ChannelKind { MessagePackRecord, JsonRecord, Text, Opaque }
 
 pub trait ChannelClassifier {
     fn classify(&self, channel_name: &str) -> ChannelKind;
 }
 ```
 
-A `ChannelRegistry` can classify exact typed/text names and text prefixes, but
-unknown names default to `Opaque`. The pipe may route by `ChannelContentKind`; a
-view decides how far to decode a payload by `ChannelKind` and the caller's
-registry.
+A `ChannelRegistry` can classify exact MessagePack/JSON/text names and text
+prefixes, but unknown names default to `Opaque`. The pipe may route by
+`ChannelContentKind`; a view decides how far to decode a payload by
+`ChannelKind` and the caller's registry.
 
 ### 5.3 Static channels and dynamic families
 
@@ -554,19 +561,24 @@ A caller-owned classifier can know a family shape without knowing every concrete
 instance: for example, `proc.` may classify as text while
 `proc.trainer.stdout` first appears only when the trainer emits and registers.
 
-### 5.4 Bytes / TextStream / JsonRecord, and the raw fallback
+### 5.4 Bytes / TextStream / structured records, and the raw fallback
 
 Catalog content classes are:
 
 - **Bytes** — arbitrary bytes. Display as raw unless a view knows more.
 - **TextStream** — UTF-8-ish stream chunks. A view may render valid UTF-8 as
   text and invalid bytes as raw.
-- **JsonRecord** — payloads encoded with serde JSON for a record type. The
-  optional `schema` string is catalog metadata, not a versioned wire envelope.
+- **JsonRecord** — payloads encoded with serde JSON.
+- **MessagePackRecord** — payloads encoded as named-field MessagePack. This is
+  the default for [`Record`] and preserves field names so generic views can
+  decode records without knowing their Rust type.
+
+The optional `schema` string on either record class is catalog metadata, not a
+versioned wire envelope.
 
 View decode results are:
 
-- **Record** — a typed/JSON channel decoded to a structured JSON value.
+- **Record** — a JSON or MessagePack channel decoded to a structured JSON value.
 - **Text** — a text channel decoded as UTF-8.
 - **Raw** — unknown, invalid, or intentionally opaque bytes.
 
@@ -584,8 +596,8 @@ pub trait Record: Serialize + for<'de> Deserialize<'de> + Sized {
     const CHANNEL: &'static str;
 
     fn channel_name() -> &'static str { Self::CHANNEL }
-    fn encode(&self) -> Vec<u8> { serde_json::to_vec(self).unwrap() }
-    fn decode(payload: &[u8]) -> Result<Self, serde_json::Error>;
+    fn encode(&self) -> Vec<u8> { rmp_serde::to_vec_named(self).unwrap() }
+    fn decode(payload: &[u8]) -> Result<Self, rmp_serde::decode::Error>;
 }
 ```
 
@@ -601,15 +613,17 @@ Schema evolution is still serde discipline:
 - **Add fields freely.** New fields are emitted by new producers.
 - **Tolerate missing.** `#[serde(default)]` fills fields an older producer did
   not send.
+- **Omit absent optionals.** Optional failure context such as hardware
+  `error` fields is absent on success rather than encoded as an explicit null.
 - **Ignore unknown.** A consumer drops fields it does not recognize.
 - **Never remove or repurpose.** Retire a field by leaving it unused; introduce
   new meaning as a new field.
 
-Current `JsonRecord` descriptors carry `schema: Option<String>`. In the current
-implementation, `register_record<R>()` uses `Some(R::CHANNEL.to_owned())` as the
-schema string. Treat this as catalog metadata for display/subscription tooling,
-not as a frame-level schema version. There is still no schema version field in
-`Frame` itself.
+Current `JsonRecord` and `MessagePackRecord` descriptors carry
+`schema: Option<String>`. `register_record<R>()` uses
+`MessagePackRecord { schema: Some(R::CHANNEL.to_owned()) }`. Treat this as
+catalog metadata for display/subscription tooling, not as a frame-level schema
+version. There is still no schema version field in `Frame` itself.
 
 ### 5.7 Lifecycle and liveness
 

crates/telemetry/src/endpoint.rs

@@ -4,9 +4,9 @@
 //! stream/channel metadata, orders producer frames through the mux, and fans
 //! catalog-aware events to subscribers.
 
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, VecDeque};
 use std::sync::atomic::{AtomicU64, Ordering};
-use std::sync::{Arc, Mutex};
+use std::sync::{Arc, Mutex, Weak};
 use std::time::Instant;
 
 use crossbeam_channel::{
@@ -23,7 +23,7 @@ use crate::frame::{
     SourceFilter, StreamDescriptor, StreamId, StreamOrigin, SubscriptionRequest, TelemetryEvent,
 };
 use crate::mux::Mux;
-use crate::record::Record;
+use crate::record::{Record, encode_record};
 use crate::transport::Delivery;
 
 const DEFAULT_MUX_CAPACITY: usize = 4096;
@@ -76,7 +76,9 @@ pub struct TelemetrySubscription {
     name: String,
     request: SubscriptionRequest,
     snapshot: TelemetrySnapshot,
+    replay: Option<Mutex<VecDeque<Arc<TelemetryEvent>>>>,
     rx: Receiver<TelemetryEvent>,
+    fanout: Weak<Mutex<FanoutState>>,
 }
 
 impl TelemetrySubscription {
@@ -96,30 +98,66 @@ impl TelemetrySubscription {
         &self.snapshot
     }
 
+    pub fn dropped(&self) -> u64 {
+        self.fanout
+            .upgrade()
+            .and_then(|state| {
+                state
+                    .lock()
+                    .expect("telemetry fanout poisoned")
+                    .subscribers
+                    .get(&self.id)
+                    .map(|slot| slot.dropped)
+            })
+            .unwrap_or(0)
+    }
+
+    fn pop_replay(&self) -> Option<TelemetryEvent> {
+        self.replay
+            .as_ref()?
+            .lock()
+            .expect("telemetry replay poisoned")
+            .pop_front()
+            .map(|event| (*event).clone())
+    }
+
     pub fn try_recv(&self) -> Result<TelemetryEvent, TryRecvError> {
-        self.rx.try_recv()
+        self.pop_replay().map_or_else(|| self.rx.try_recv(), Ok)
     }
 
     pub fn recv(&self) -> Result<TelemetryEvent, RecvError> {
-        self.rx.recv()
+        self.pop_replay().map_or_else(|| self.rx.recv(), Ok)
     }
 
     pub fn recv_timeout(
         &self,
         timeout: std::time::Duration,
     ) -> Result<TelemetryEvent, RecvTimeoutError> {
-        self.rx.recv_timeout(timeout)
+        self.pop_replay()
+            .map_or_else(|| self.rx.recv_timeout(timeout), Ok)
     }
 
     pub fn drain_available(&self) -> Vec<TelemetryEvent> {
         let mut out = Vec::with_capacity(self.rx.len());
-        while let Ok(event) = self.rx.try_recv() {
+        while let Ok(event) = self.try_recv() {
             out.push(event);
         }
         out
     }
 }
 
+impl Drop for TelemetrySubscription {
+    fn drop(&mut self) {
+        if let Some(fanout) = self.fanout.upgrade() {
+            fanout
+                .lock()
+                .expect("telemetry fanout poisoned")
+                .subscribers
+                .remove(&self.id);
+        }
+    }
+}
+
 struct SubscriberSlot {
     name: String,
     tx: Sender<TelemetryEvent>,
@@ -170,17 +208,17 @@ struct FanoutState {
 /// Local fanout; future events are broadcast to every subscriber without request filtering.
 pub struct DeliveryFanout {
     default_capacity: usize,
-    state: Mutex<FanoutState>,
+    state: Arc<Mutex<FanoutState>>,
 }
 
 impl DeliveryFanout {
     pub fn new(default_capacity: usize) -> Self {
         Self {
             default_capacity: default_capacity.max(1),
-            state: Mutex::new(FanoutState {
+            state: Arc::new(Mutex::new(FanoutState {
                 next_id: 1,
                 subscribers: BTreeMap::new(),
-            }),
+            })),
         }
     }
 
@@ -226,7 +264,9 @@ impl DeliveryFanout {
             name,
             request,
             snapshot,
+            replay: None,
             rx,
+            fanout: Arc::downgrade(&self.state),
         }
     }
 
@@ -438,12 +478,51 @@ impl ChannelCatalogState {
     }
 }
 
+#[derive(Default)]
+struct EndpointDelivery {
+    frame_capacity: usize,
+    byte_capacity: usize,
+    retained_bytes: usize,
+    retained: VecDeque<Arc<TelemetryEvent>>,
+}
+
+impl EndpointDelivery {
+    fn retain(&mut self, events: &[TelemetryEvent]) {
+        if self.frame_capacity == 0 || self.byte_capacity == 0 {
+            return;
+        }
+        for event in events {
+            let TelemetryEvent::Frame(frame) = event else {
+                continue;
+            };
+            let bytes = frame.payload.len();
+            while !self.retained.is_empty()
+                && (self.retained.len() >= self.frame_capacity
+                    || bytes > self.byte_capacity.saturating_sub(self.retained_bytes))
+            {
+                if let Some(event) = self.retained.pop_front()
+                    && let TelemetryEvent::Frame(frame) = event.as_ref()
+                {
+                    self.retained_bytes -= frame.payload.len();
+                }
+            }
+            // An oversize frame cannot fit. Keep its original missing position,
+            // rather than renumbering or fabricating an event during replay.
+            if bytes <= self.byte_capacity {
+                self.retained_bytes += bytes;
+                self.retained.push_back(Arc::new(event.clone()));
+            }
+        }
+    }
+}
+
 /// Process-local telemetry endpoint.
 pub struct TelemetryEndpoint {
     stream: StreamId,
     mux: Arc<Mux>,
     catalog: Arc<Mutex<ChannelCatalogState>>,
     fanout: Arc<DeliveryFanout>,
+    delivery: Arc<Mutex<EndpointDelivery>>,
     drained: AtomicU64,
     bitbucketed: AtomicU64,
 }
@@ -481,11 +560,25 @@ impl TelemetryEndpoint {
             mux,
             catalog: Arc::new(Mutex::new(ChannelCatalogState::new(descriptor))),
             fanout: Arc::new(DeliveryFanout::new(subscriber_capacity)),
+            delivery: Arc::new(Mutex::new(EndpointDelivery::default())),
             drained: AtomicU64::new(0),
             bitbucketed: AtomicU64::new(0),
         }
     }
 
+    /// Retain a bounded suffix for explicitly replaying subscriptions.
+    ///
+    /// Configure this before creating producers. Evicted frames and rejected
+    /// mux submissions are not reconstructed; their original gaps remain visible.
+    pub fn with_retention(self, frame_capacity: usize, byte_capacity: usize) -> Self {
+        {
+            let mut delivery = self.delivery.lock().expect("telemetry delivery poisoned");
+            delivery.frame_capacity = frame_capacity;
+            delivery.byte_capacity = byte_capacity;
+        }
+        self
+    }
+
     pub fn stream_id(&self) -> &StreamId {
         &self.stream
     }
@@ -506,6 +599,7 @@ impl TelemetryEndpoint {
             mux: Arc::clone(&self.mux),
             catalog: Arc::clone(&self.catalog),
             fanout: Arc::clone(&self.fanout),
+            delivery: Arc::clone(&self.delivery),
         }
     }
 
@@ -514,7 +608,13 @@ impl TelemetryEndpoint {
         name: impl Into<String>,
         content: ChannelContent,
     ) -> Result<ChannelId, ChannelRegistrationError> {
-        register_channel(&self.catalog, &self.fanout, name.into(), content)
+        register_channel(
+            &self.catalog,
+            &self.fanout,
+            &self.delivery,
+            name.into(),
+            content,
+        )
     }
 
     pub fn register_channel(&self, name: impl Into<String>, content: ChannelContent) -> ChannelId {
@@ -529,7 +629,7 @@ impl TelemetryEndpoint {
     pub fn register_record<R: Record>(&self) -> ChannelId {
         self.register_channel(
             R::CHANNEL,
-            ChannelContent::JsonRecord {
+            ChannelContent::MessagePackRecord {
                 schema: Some(R::CHANNEL.to_owned()),
             },
         )
@@ -540,6 +640,7 @@ impl TelemetryEndpoint {
         name: impl Into<String>,
         request: SubscriptionRequest,
     ) -> TelemetrySubscription {
+        let _delivery = self.delivery.lock().expect("telemetry delivery poisoned");
         let snapshot = self.catalog_snapshot().telemetry_snapshot(&request);
         self.fanout.subscribe(name, request, snapshot)
     }
@@ -548,11 +649,31 @@ impl TelemetryEndpoint {
         self.subscribe(name, SubscriptionRequest::all())
     }
 
+    /// Replay retained frames in position order, then receive live events.
+    ///
+    /// Replay has its own bounded snapshot, so it cannot fill the live queue
+    /// before the subscription is returned. The delivery lock makes the
+    /// retained/live boundary atomic with ticks and channel declarations.
+    pub fn subscribe_retained(
+        &self,
+        name: impl Into<String>,
+        request: SubscriptionRequest,
+    ) -> TelemetrySubscription {
+        let delivery = self.delivery.lock().expect("telemetry delivery poisoned");
+        let snapshot = self.catalog_snapshot().telemetry_snapshot(&request);
+        let mut subscription = self.fanout.subscribe(name, request, snapshot);
+        if !delivery.retained.is_empty() {
+            subscription.replay = Some(Mutex::new(delivery.retained.clone()));
+        }
+        subscription
+    }
+
     pub fn subscribe_all_with_capacity(
         &self,
         name: impl Into<String>,
         capacity: usize,
     ) -> TelemetrySubscription {
+        let _delivery = self.delivery.lock().expect("telemetry delivery poisoned");
         let request = SubscriptionRequest::all();
         let snapshot = self.catalog_snapshot().telemetry_snapshot(&request);
         self.fanout
@@ -567,13 +688,16 @@ impl TelemetryEndpoint {
         self.fanout.subscriber_snapshots()
     }
 
-    /// Drain the mux once and broadcast future frame events; with no subscribers, drained frames are bitbucketed.
+    /// Drain the mux, retain the configured suffix, and broadcast live events.
     pub fn tick(&self) -> EndpointTick {
+        let mut delivery = self.delivery.lock().expect("telemetry delivery poisoned");
         let events = self.drain_events();
         if events.is_empty() {
             return EndpointTick::default();
         }
-        self.publish_events(events)
+        delivery.retain(&events);
+        let retained = delivery.retained.len().min(events.len());
+        self.publish_events(events, retained)
     }
 
     fn drain_events(&self) -> Vec<TelemetryEvent> {
@@ -593,13 +717,13 @@ impl TelemetryEndpoint {
             .collect()
     }
 
-    fn publish_events(&self, events: Vec<TelemetryEvent>) -> EndpointTick {
+    fn publish_events(&self, events: Vec<TelemetryEvent>, retained: usize) -> EndpointTick {
         let drained = events.len();
         self.drained.fetch_add(drained as u64, Ordering::Relaxed);
         let tick = self.fanout.publish_batch(events);
         if tick.subscribers == 0 {
             self.bitbucketed
-                .fetch_add(drained as u64, Ordering::Relaxed);
+                .fetch_add((drained - retained) as u64, Ordering::Relaxed);
         }
         tick
     }
@@ -627,6 +751,7 @@ pub struct TelemetryProducer {
     mux: Arc<Mux>,
     catalog: Arc<Mutex<ChannelCatalogState>>,
     fanout: Arc<DeliveryFanout>,
+    delivery: Arc<Mutex<EndpointDelivery>>,
 }
 
 impl TelemetryProducer {
@@ -639,7 +764,13 @@ impl TelemetryProducer {
         name: impl Into<String>,
         content: ChannelContent,
     ) -> Result<ChannelId, ChannelRegistrationError> {
-        register_channel(&self.catalog, &self.fanout, name.into(), content)
+        register_channel(
+            &self.catalog,
+            &self.fanout,
+            &self.delivery,
+            name.into(),
+            content,
+        )
     }
 
     pub fn register_channel(&self, name: impl Into<String>, content: ChannelContent) -> ChannelId {
@@ -654,7 +785,7 @@ impl TelemetryProducer {
     pub fn register_record<R: Record>(&self) -> ChannelId {
         self.register_channel(
             R::CHANNEL,
-            ChannelContent::JsonRecord {
+            ChannelContent::MessagePackRecord {
                 schema: Some(R::CHANNEL.to_owned()),
             },
         )
@@ -696,7 +827,7 @@ impl TelemetryProducer {
     pub fn stats_hook(&self) -> Arc<dyn StatsHook> {
         let channel = self.register_channel(
             DEFAULT_STATS_CHANNEL,
-            ChannelContent::JsonRecord {
+            ChannelContent::MessagePackRecord {
                 schema: Some(DEFAULT_STATS_CHANNEL.to_owned()),
             },
         );
@@ -715,9 +846,11 @@ impl TelemetryProducer {
 fn register_channel(
     catalog: &Arc<Mutex<ChannelCatalogState>>,
     fanout: &Arc<DeliveryFanout>,
+    delivery: &Arc<Mutex<EndpointDelivery>>,
     name: String,
     content: ChannelContent,
 ) -> Result<ChannelId, ChannelRegistrationError> {
+    let _delivery = delivery.lock().expect("telemetry delivery poisoned");
     // New channel declarations publish a future event immediately; existing-name
     // reuse only returns the prior id.
     let (id, event) = {
@@ -980,7 +1113,7 @@ impl StatsHook for TelemetryStatsHook {
                 .expect("actor telemetry record is an object");
             object.insert("generation".to_owned(), generation.into());
             object.insert("sequence".to_owned(), state.sequence.into());
-            let bytes = serde_json::to_vec(&payload).expect("actor telemetry record serializes");
+            let bytes = encode_record(&payload).expect("actor telemetry record serializes");
             self.producer.submit_bytes(self.channel, bytes);
         }
     }

crates/telemetry/src/frame.rs

@@ -45,6 +45,7 @@ pub enum ChannelContentKind {
     Bytes,
     TextStream,
     JsonRecord,
+    MessagePackRecord,
 }
 
 /// How consumers should decode/display payload bytes for a channel.
@@ -53,6 +54,7 @@ pub enum ChannelContent {
     Bytes,
     TextStream,
     JsonRecord { schema: Option<String> },
+    MessagePackRecord { schema: Option<String> },
 }
 
 impl ChannelContent {
@@ -61,6 +63,7 @@ impl ChannelContent {
             ChannelContent::Bytes => ChannelContentKind::Bytes,
             ChannelContent::TextStream => ChannelContentKind::TextStream,
             ChannelContent::JsonRecord { .. } => ChannelContentKind::JsonRecord,
+            ChannelContent::MessagePackRecord { .. } => ChannelContentKind::MessagePackRecord,
         }
     }
 }

crates/telemetry/src/hardware/cpu.rs

@@ -20,6 +20,7 @@ pub struct HostCpuSample {
     pub host: Option<CpuHostSample>,
     pub cores: Vec<CpuCoreSample>,
     pub processes: Vec<CpuProcessSample>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub error: Option<String>,
 }
 
@@ -71,6 +72,7 @@ pub struct CpuProcessSample {
     pub rss_bytes: Option<u64>,
     pub vms_bytes: Option<u64>,
     pub thread_count: Option<u64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub error: Option<String>,
 }
 

crates/telemetry/src/hardware/gpu.rs

@@ -26,6 +26,7 @@ pub struct HostGpuSample {
     pub query_elapsed_ms: Option<u64>,
     pub gpus: Vec<GpuDeviceSample>,
     pub processes: Vec<GpuProcessSample>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub error: Option<String>,
 }
 
@@ -298,4 +299,28 @@ mod tests {
         assert_eq!(sample.seq, 9);
         assert_eq!(sample.error, Some("no gpu".to_string()));
     }
+
+    #[test]
+    fn successful_sample_omits_error_but_failure_preserves_it() {
+        let sample = HostGpuSample {
+            schema: SCHEMA.to_owned(),
+            seq: 1,
+            sample_unix_ms: 2,
+            query_elapsed_ms: Some(3),
+            gpus: Vec::new(),
+            processes: Vec::new(),
+            error: None,
+        };
+        let value = crate::record::decode_record_value(&sample.encode()).expect("decode sample");
+        assert!(
+            !value
+                .as_object()
+                .expect("sample object")
+                .contains_key("error")
+        );
+
+        let failure = HostGpuSample::error(2, "nvidia-smi unavailable");
+        let value = crate::record::decode_record_value(&failure.encode()).expect("decode failure");
+        assert_eq!(value["error"], "nvidia-smi unavailable");
+    }
 }

crates/telemetry/src/hardware/memory.rs

@@ -26,6 +26,7 @@ pub struct HostMemorySample {
     pub swap_total_bytes: Option<u64>,
     pub swap_used_bytes: Option<u64>,
     pub pressure: Option<PressureSample>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub error: Option<String>,
 }
 

crates/telemetry/src/hardware/net.rs

@@ -16,6 +16,7 @@ pub struct HostNetSample {
     pub seq: u64,
     pub sample_unix_ms: u64,
     pub interfaces: Vec<NetInterfaceSample>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub error: Option<String>,
 }