Compare commits

...

3 commits

51 changed files with 4395 additions and 4936 deletions

5
Cargo.lock generated
View file

@ -937,6 +937,7 @@ dependencies = [
name = "datastream"
version = "0.1.0"
dependencies = [
"crossbeam-channel",
"iroh",
"libc",
"serde",
@ -2104,6 +2105,7 @@ dependencies = [
name = "iroh-driver"
version = "0.1.0"
dependencies = [
"crossbeam-channel",
"datastream",
"distribution",
"iroh",
@ -4342,9 +4344,8 @@ name = "swactor-process"
version = "0.1.0"
dependencies = [
"crossbeam-queue",
"datastream",
"libc",
"proptest",
"proptest-state-machine",
"serde",
"serde_json",
"serde_yaml",

View file

@ -8,7 +8,6 @@ pub mod view;
use std::sync::Arc;
use datastream::frame::{ChannelId, Frame, Lifetime, NodeId, Position, StreamId};
use parking_lot::Mutex;
use serde::Serialize;
use tokio::sync::broadcast;
@ -86,10 +85,32 @@ pub struct DashboardHandle {
store: Arc<DashboardStore>,
views: Arc<ViewRegistry>,
shutdown_notify: Arc<tokio::sync::Notify>,
standalone_rt: Mutex<Option<tokio::runtime::Runtime>>,
}
impl DashboardHandle {
/// Create the datastream dashboard state.
///
/// The HTTP server is not started until `start_http` or `start_http_standalone`
/// is called.
pub fn new(config: DashboardConfig) -> Self {
let views = Arc::new(ViewRegistry::new());
views.register(Arc::new(live_explorer::LiveDatastreamExplorer::default()));
views.register(Arc::new(hardware_view::HardwareDashboardView::default()));
views.register(swactor::worker_view());
let store = Arc::new(DashboardStore::new(
config.raw_frame_history,
Arc::clone(&views),
));
let (frames, _) = broadcast::channel(config.frame_buffer.max(1));
Self {
port: config.port,
frames,
store,
views,
shutdown_notify: Arc::new(tokio::sync::Notify::new()),
}
}
/// Register a read-only view. External crates can keep their interpretation
/// code beside their component and plug it into this registry.
pub fn register_view(&self, view: Arc<dyn DashboardView>) {
@ -113,8 +134,8 @@ impl DashboardHandle {
self.shutdown_notify.notify_waiters();
}
/// Start the HTTP server on an existing Tokio runtime.
pub fn start_http(&self, handle: tokio::runtime::Handle) {
/// Build the HTTP server future for an embedding runtime to poll directly.
pub fn http_server(&self) -> impl Future<Output = ()> + Send + 'static {
let state = server::AppState {
frames: self.frames.clone(),
store: Arc::clone(&self.store),
@ -122,44 +143,13 @@ impl DashboardHandle {
shutdown_notify: Arc::clone(&self.shutdown_notify),
};
let port = self.port;
handle.spawn(async move {
async move {
server::run_server(state, port).await;
});
}
}
/// Start the HTTP server on a standalone Tokio runtime.
pub fn start_http_standalone(&self) {
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(1)
.enable_all()
.build()
.expect("failed to create tokio runtime for dashboard HTTP");
let handle = rt.handle().clone();
*self.standalone_rt.lock() = Some(rt);
self.start_http(handle);
}
}
/// Create the datastream dashboard state.
///
/// The HTTP server is not started until `start_http` or `start_http_standalone`
/// is called.
pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle {
let views = Arc::new(ViewRegistry::new());
views.register(Arc::new(live_explorer::LiveDatastreamExplorer::default()));
views.register(Arc::new(hardware_view::HardwareDashboardView::default()));
views.register(swactor::worker_view());
let store = Arc::new(DashboardStore::new(
config.raw_frame_history,
Arc::clone(&views),
));
let (frames, _) = broadcast::channel(config.frame_buffer.max(1));
DashboardHandle {
port: config.port,
frames,
store,
views,
shutdown_notify: Arc::new(tokio::sync::Notify::new()),
standalone_rt: Mutex::new(None),
/// Spawn the HTTP server on an existing Tokio runtime and return its task handle.
pub fn spawn_http(&self, handle: &tokio::runtime::Handle) -> tokio::task::JoinHandle<()> {
handle.spawn(self.http_server())
}
}

View file

@ -9,6 +9,7 @@ swactor-transport = { path = "../transport" }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
iroh = "0.98"
crossbeam-channel = "0.5"
[target.'cfg(target_os = "linux")'.dependencies]
libc = "0.2"

File diff suppressed because it is too large Load diff

View file

@ -7,14 +7,14 @@ use swactor::actor::ActorAddress;
use swactor::process_observer::ProcessOutputObserver;
use swactor::runtime::Runtime;
use super::frame::{ChannelId, Frame, Lifetime, NodeId, Position, StreamId};
use super::frame::{ChannelId, Frame, Lifetime, NodeId, StreamId};
use super::mux::Mux;
use super::record::Record;
use super::wire::{DatastreamFrame, encode_delivery};
/// Where assembled frames go once the mux has ordered them.
/// Legacy sink for frames after mux drain has assigned positions.
pub trait FrameSink: Send {
/// Ship one ordered frame for `stream`. Best-effort: a sink may drop.
/// Ship one positioned frame for `stream`. Best-effort: a sink may drop.
fn ship(&mut self, stream: &StreamId, frame: &Frame);
}
@ -73,26 +73,23 @@ impl DatastreamEmitter {
self.mux.dropped()
}
pub fn set_frame_timing_enabled(&self, enabled: bool) {
self.mux.set_frame_timing_enabled(enabled);
}
pub fn frame_timing_enabled(&self) -> bool {
self.mux.frame_timing_enabled()
}
pub fn submit_record<R: Record>(&self, channel: ChannelId, record: &R) -> Position {
pub fn submit_record<R: Record>(&self, channel: ChannelId, record: &R) -> bool {
self.mux.submit(channel, record.encode())
}
pub fn submit_text(&self, channel: ChannelId, text: impl AsRef<[u8]>) -> Position {
pub fn submit_text(&self, channel: ChannelId, text: impl AsRef<[u8]>) -> bool {
self.mux.submit(channel, text.as_ref().to_vec())
}
pub fn submit_bytes(&self, channel: ChannelId, bytes: Vec<u8>) -> Position {
pub fn submit_bytes(&self, channel: ChannelId, bytes: Vec<u8>) -> bool {
self.mux.submit(channel, bytes)
}
pub fn submit_text_owned(&self, channel: ChannelId, text: String) -> bool {
self.mux.submit(channel, text.into_bytes())
}
/// Build a legacy/custom process-output observer that writes chunks into this mux.
pub fn process_observer_with<F>(&self, channel_for: F) -> Arc<dyn ProcessOutputObserver>
where
F: Fn(&str, bool) -> ChannelId + Send + Sync + 'static,
@ -123,17 +120,21 @@ pub struct DatastreamEventSink {
}
impl DatastreamEventSink {
pub fn submit_record<R: Record>(&self, channel: ChannelId, record: &R) -> Position {
pub fn submit_record<R: Record>(&self, channel: ChannelId, record: &R) -> bool {
self.mux.submit(channel, record.encode())
}
pub fn submit_text(&self, channel: ChannelId, text: impl AsRef<[u8]>) -> Position {
pub fn submit_text(&self, channel: ChannelId, text: impl AsRef<[u8]>) -> bool {
self.mux.submit(channel, text.as_ref().to_vec())
}
pub fn submit_bytes(&self, channel: ChannelId, bytes: Vec<u8>) -> Position {
pub fn submit_bytes(&self, channel: ChannelId, bytes: Vec<u8>) -> bool {
self.mux.submit(channel, bytes)
}
pub fn submit_text_owned(&self, channel: ChannelId, text: String) -> bool {
self.mux.submit(channel, text.into_bytes())
}
}
/// A sink that drops everything.

View file

@ -6,7 +6,11 @@
use std::collections::BTreeMap;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex, mpsc};
use std::sync::{Arc, Mutex};
use crossbeam_channel::{
Receiver, RecvError, RecvTimeoutError, Sender, TryRecvError, TrySendError, bounded,
};
use serde::{Deserialize, Serialize};
use swactor::process_observer::ProcessOutputObserver;
@ -14,12 +18,10 @@ use swactor::stats::{ActorSnapshot, StatsHook};
use crate::frame::{
ChannelContent, ChannelDescriptor, ChannelFilter, ChannelId, ChannelRef, DatastreamEvent,
FrameDelivery, Position, SourceFilter, StreamDescriptor, StreamId, StreamOrigin,
SubscriptionRequest,
FrameDelivery, SourceFilter, StreamDescriptor, StreamId, StreamOrigin, SubscriptionRequest,
};
use crate::mux::Mux;
use crate::record::Record;
use crate::timing::{FRAME_TIME_CHANNEL, FRAME_TIME_CHANNEL_ID};
use crate::transport::Delivery;
const DEFAULT_MUX_CAPACITY: usize = 4096;
@ -51,7 +53,7 @@ pub struct SubscriberSnapshot {
pub dropped: u64,
}
/// Current catalog snapshot delivered at subscription time.
/// Current catalog snapshot delivered at subscription time, filtered by request.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct DatastreamSnapshot {
pub streams: Vec<StreamDescriptor>,
@ -70,7 +72,7 @@ pub struct DatastreamSubscription {
name: String,
request: SubscriptionRequest,
snapshot: DatastreamSnapshot,
rx: mpsc::Receiver<DatastreamEvent>,
rx: Receiver<DatastreamEvent>,
}
impl DatastreamSubscription {
@ -90,18 +92,18 @@ impl DatastreamSubscription {
&self.snapshot
}
pub fn try_recv(&self) -> Result<DatastreamEvent, mpsc::TryRecvError> {
pub fn try_recv(&self) -> Result<DatastreamEvent, TryRecvError> {
self.rx.try_recv()
}
pub fn recv(&self) -> Result<DatastreamEvent, mpsc::RecvError> {
pub fn recv(&self) -> Result<DatastreamEvent, RecvError> {
self.rx.recv()
}
pub fn recv_timeout(
&self,
timeout: std::time::Duration,
) -> Result<DatastreamEvent, mpsc::RecvTimeoutError> {
) -> Result<DatastreamEvent, RecvTimeoutError> {
self.rx.recv_timeout(timeout)
}
@ -116,17 +118,27 @@ impl DatastreamSubscription {
struct SubscriberSlot {
name: String,
request: SubscriptionRequest,
tx: mpsc::SyncSender<DatastreamEvent>,
tx: Sender<DatastreamEvent>,
dropped: u64,
}
struct FanoutTarget {
id: SubscriptionId,
tx: Sender<DatastreamEvent>,
}
struct FanoutReport {
id: SubscriptionId,
dropped: u64,
disconnected: bool,
}
struct FanoutState {
next_id: u64,
subscribers: BTreeMap<SubscriptionId, SubscriberSlot>,
}
/// Local event fanout used by endpoints and collectors.
/// Local fanout; future events are broadcast to every subscriber without request filtering.
pub struct DeliveryFanout {
default_capacity: usize,
state: Mutex<FanoutState>,
@ -168,7 +180,7 @@ impl DeliveryFanout {
capacity: usize,
) -> DatastreamSubscription {
let name = name.into();
let (tx, rx) = mpsc::sync_channel(capacity.max(1));
let (tx, rx) = bounded(capacity.max(1));
let mut state = self.state.lock().expect("datastream fanout poisoned");
let id = SubscriptionId(state.next_id);
state.next_id = state.next_id.wrapping_add(1).max(1);
@ -176,7 +188,6 @@ impl DeliveryFanout {
id,
SubscriberSlot {
name: name.clone(),
request: request.clone(),
tx,
dropped: 0,
},
@ -212,23 +223,34 @@ impl DeliveryFanout {
.collect()
}
pub fn publish(&self, event: DatastreamEvent, catalog: &CatalogSnapshot) -> EndpointTick {
self.publish_batch(std::iter::once(event), catalog)
pub fn publish(&self, event: DatastreamEvent) -> EndpointTick {
self.publish_batch(std::iter::once(event))
}
pub fn publish_batch(
&self,
events: impl IntoIterator<Item = DatastreamEvent>,
catalog: &CatalogSnapshot,
) -> EndpointTick {
pub fn publish_batch(&self, events: impl IntoIterator<Item = DatastreamEvent>) -> EndpointTick {
let events: Vec<DatastreamEvent> = events.into_iter().collect();
if events.is_empty() {
return EndpointTick::default();
}
let mut state = self.state.lock().expect("datastream fanout poisoned");
let subscribers = state.subscribers.len();
if subscribers == 0 {
// Snapshot sender handles while holding the subscriber map lock, then
// deliver outside the lock so large batches or slow subscribers do not
// block subscribe/snapshot control-plane operations.
let (targets, subscribers) = {
let state = self.state.lock().expect("datastream fanout poisoned");
let subscribers = state.subscribers.len();
let targets = state
.subscribers
.iter()
.map(|(id, slot)| FanoutTarget {
id: *id,
tx: slot.tx.clone(),
})
.collect::<Vec<_>>();
(targets, subscribers)
};
if targets.is_empty() {
return EndpointTick {
drained: events.len(),
subscribers: 0,
@ -237,36 +259,48 @@ impl DeliveryFanout {
}
let mut delivered = 0;
let mut dropped = 0;
let mut disconnected = Vec::new();
for (id, slot) in state.subscribers.iter_mut() {
let mut reports = Vec::new();
for target in targets {
let mut dropped = 0;
let mut disconnected = false;
for event in &events {
if !event_matches_request(event, &slot.request, catalog) {
continue;
}
match slot.tx.try_send(event.clone()) {
match target.tx.try_send(event.clone()) {
Ok(()) => delivered += 1,
Err(mpsc::TrySendError::Full(_)) => {
slot.dropped = slot.dropped.saturating_add(1);
Err(TrySendError::Full(_)) => dropped += 1,
Err(TrySendError::Disconnected(_)) => {
dropped += 1;
}
Err(mpsc::TrySendError::Disconnected(_)) => {
slot.dropped = slot.dropped.saturating_add(1);
dropped += 1;
disconnected.push(*id);
break;
disconnected = true;
}
}
}
if dropped > 0 || disconnected {
reports.push(FanoutReport {
id: target.id,
dropped,
disconnected,
});
}
}
for id in disconnected {
state.subscribers.remove(&id);
let dropped_for_subscribers = reports.iter().map(|report| report.dropped).sum::<u64>();
if !reports.is_empty() {
let mut state = self.state.lock().expect("datastream fanout poisoned");
for report in &reports {
if let Some(slot) = state.subscribers.get_mut(&report.id) {
slot.dropped = slot.dropped.saturating_add(report.dropped);
}
}
for report in &reports {
if report.disconnected {
state.subscribers.remove(&report.id);
}
}
}
EndpointTick {
drained: events.len(),
delivered,
dropped_for_subscribers: dropped,
dropped_for_subscribers: usize::try_from(dropped_for_subscribers).unwrap_or(usize::MAX),
subscribers,
}
}
@ -279,6 +313,7 @@ pub struct CatalogSnapshot {
}
impl CatalogSnapshot {
/// Apply a subscription request to the initial metadata snapshot only.
pub fn datastream_snapshot(&self, request: &SubscriptionRequest) -> DatastreamSnapshot {
let channels: Vec<ChannelDescriptor> = self
.channels
@ -320,24 +355,12 @@ struct ChannelCatalogState {
impl ChannelCatalogState {
fn new(stream: StreamDescriptor) -> Self {
let mut state = Self {
stream: stream.clone(),
Self {
stream,
by_id: BTreeMap::new(),
by_name: BTreeMap::new(),
next_channel: 1,
};
let timing = ChannelDescriptor {
stream: stream.stream.clone(),
id: FRAME_TIME_CHANNEL_ID,
name: FRAME_TIME_CHANNEL.to_owned(),
label: Some("frame construction time".to_owned()),
content: ChannelContent::JsonRecord {
schema: Some("datastream.frame_time.v1".to_owned()),
},
};
state.by_name.insert(timing.name.clone(), timing.id);
state.by_id.insert(timing.id, timing);
state
}
}
fn snapshot(&self) -> CatalogSnapshot {
@ -456,14 +479,6 @@ impl DatastreamEndpoint {
.snapshot()
}
pub fn set_frame_timing_enabled(&self, enabled: bool) {
self.mux.set_frame_timing_enabled(enabled);
}
pub fn frame_timing_enabled(&self) -> bool {
self.mux.frame_timing_enabled()
}
pub fn producer(&self) -> DatastreamProducer {
DatastreamProducer {
mux: Arc::clone(&self.mux),
@ -530,26 +545,36 @@ impl DatastreamEndpoint {
self.fanout.subscriber_snapshots()
}
/// Drain the mux and fan out catalog-aware future events.
/// Drain the mux once and broadcast future frame events; with no subscribers, drained frames are bitbucketed.
pub fn tick(&self) -> EndpointTick {
let frames = self.mux.drain();
if frames.is_empty() {
let events = self.drain_events();
if events.is_empty() {
return EndpointTick::default();
}
let drained = frames.len();
self.drained.fetch_add(drained as u64, Ordering::Relaxed);
let events = frames.into_iter().map(|frame| {
DatastreamEvent::Frame(FrameDelivery {
channel: ChannelRef {
stream: self.stream.clone(),
channel: frame.channel,
},
position: frame.position,
payload: frame.payload,
self.publish_events(events)
}
fn drain_events(&self) -> Vec<DatastreamEvent> {
self.mux
.drain()
.into_iter()
.map(|frame| {
DatastreamEvent::Frame(FrameDelivery {
channel: ChannelRef {
stream: self.stream.clone(),
channel: frame.channel,
},
position: frame.position,
payload: frame.payload,
})
})
});
let catalog = self.catalog_snapshot();
let tick = self.fanout.publish_batch(events, &catalog);
.collect()
}
fn publish_events(&self, events: Vec<DatastreamEvent>) -> EndpointTick {
let drained = events.len();
self.drained.fetch_add(drained as u64, Ordering::Relaxed);
let tick = self.fanout.publish_batch(events);
if tick.subscribers == 0 {
self.bitbucketed
.fetch_add(drained as u64, Ordering::Relaxed);
@ -620,24 +645,20 @@ impl DatastreamProducer {
.id_for_name(name)
}
pub fn submit_record<R: Record>(&self, channel: ChannelId, record: &R) -> Position {
pub fn submit_record<R: Record>(&self, channel: ChannelId, record: &R) -> bool {
self.mux.submit(channel, record.encode())
}
pub fn submit_text(&self, channel: ChannelId, text: impl AsRef<[u8]>) -> Position {
pub fn submit_text(&self, channel: ChannelId, text: impl AsRef<[u8]>) -> bool {
self.mux.submit(channel, text.as_ref().to_vec())
}
pub fn submit_bytes(&self, channel: ChannelId, bytes: Vec<u8>) -> Position {
pub fn submit_bytes(&self, channel: ChannelId, bytes: Vec<u8>) -> bool {
self.mux.submit(channel, bytes)
}
pub fn set_frame_timing_enabled(&self, enabled: bool) {
self.mux.set_frame_timing_enabled(enabled);
}
pub fn frame_timing_enabled(&self) -> bool {
self.mux.frame_timing_enabled()
pub fn submit_text_owned(&self, channel: ChannelId, text: String) -> bool {
self.mux.submit(channel, text.into_bytes())
}
pub fn process_observer_with<F>(&self, channel_for: F) -> Arc<dyn ProcessOutputObserver>
@ -674,34 +695,30 @@ fn register_channel(
name: String,
content: ChannelContent,
) -> Result<ChannelId, ChannelRegistrationError> {
let (id, event, snapshot) = {
// New channel declarations publish a future event immediately; existing-name
// reuse only returns the prior id.
let (id, event) = {
let mut catalog = catalog.lock().expect("datastream catalog poisoned");
match catalog.try_register_channel(name.clone(), content)? {
Some(descriptor) => {
let id = descriptor.id;
let snapshot = catalog.snapshot();
(
id,
Some(DatastreamEvent::ChannelDeclared(descriptor)),
snapshot,
)
}
Some(descriptor) => (
descriptor.id,
Some(DatastreamEvent::ChannelDeclared(descriptor)),
),
None => {
let id = catalog
.id_for_name(&name)
.expect("duplicate channel name remains registered");
let snapshot = catalog.snapshot();
(id, None, snapshot)
(id, None)
}
}
};
if let Some(event) = event {
let _ = fanout.publish(event, &snapshot);
let _ = fanout.publish(event);
}
Ok(id)
}
/// Managed-process output observer that submits stdout/stderr chunks as frames.
/// Legacy/custom process-output observer adapter that submits stdout/stderr chunks as frames.
pub struct DatastreamProcessObserver {
producer: DatastreamProducer,
channel_for: Arc<dyn Fn(&str, bool) -> ChannelId + Send + Sync>,
@ -773,33 +790,6 @@ struct RuntimeMessageTypeCount<'a> {
count: u64,
}
fn event_matches_request(
event: &DatastreamEvent,
request: &SubscriptionRequest,
catalog: &CatalogSnapshot,
) -> bool {
match event {
DatastreamEvent::StreamDeclared(descriptor) => {
source_matches(&descriptor.stream, Some(descriptor), &request.sources)
&& (matches!(request.channels, ChannelFilter::All)
|| catalog.channels.values().any(|channel| {
channel.stream == descriptor.stream
&& descriptor_matches_request(channel, request, catalog)
}))
}
DatastreamEvent::ChannelDeclared(descriptor) => {
descriptor_matches_request(descriptor, request, catalog)
}
DatastreamEvent::Frame(delivery) => catalog
.descriptor_for(&delivery.channel)
.map(|descriptor| descriptor_matches_request(descriptor, request, catalog))
.unwrap_or_else(|| matches!(request.channels, ChannelFilter::All)),
DatastreamEvent::StreamEnded(stream) => {
source_matches(stream, catalog.stream_descriptor(stream), &request.sources)
}
}
}
fn descriptor_matches_request(
descriptor: &ChannelDescriptor,
request: &SubscriptionRequest,

View file

@ -1,4 +1,4 @@
//! Core data model: the framed, channel-multiplexed stream (spec §4).
//! Core data model: the framed, channel-multiplexed stream (spec §2).
//!
//! A stream is identified by the producing node and lifetime. Frames carry a
//! stream-local numeric channel id plus the mux-assigned position and opaque
@ -10,12 +10,12 @@ use std::sync::Arc;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
/// A position assigned by a node's mux (spec §5.2).
/// A position assigned by a node's mux during drain (spec §2.3).
///
/// Positions are **monotonic** and **gap-free** within a single node's stream:
/// the mux never reuses one and never skips one in its numbering. A position
/// that is assigned but never delivered surfaces downstream as a missing
/// position — a detectable gap (spec §5.3, §7.5).
/// Assignment is monotonic and gap-free for accepted frames: the mux never
/// reuses one and never skips one while draining. A frame assigned a position
/// can still be lost by transport and later surface downstream as a detectable
/// gap.
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Serialize, Deserialize)]
pub struct Position(pub u64);
@ -27,9 +27,9 @@ impl fmt::Display for Position {
/// Stream-local numeric channel id.
///
/// `ChannelId(0)` is reserved for the datastream frame-timing sidecar. Every
/// other id is allocated by the stream owner and is meaningful only with the
/// corresponding [`StreamId`]. Consumers resolve frames by `(stream, channel)`.
/// A raw `ChannelId` is meaningful only together with its [`StreamId`]. The
/// public endpoint allocator currently starts at `ChannelId(1)`, leaving
/// `ChannelId(0)` unallocated by normal registration.
#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)]
pub struct ChannelId(pub u32);
@ -73,7 +73,7 @@ pub enum StreamOrigin {
RemoteNode,
}
/// The stable identity of a node that produces a stream (spec §4.4, §8.1).
/// The stable identity of a node that produces a stream (spec §2.2, §7.1).
#[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct NodeId(Arc<str>);
@ -131,7 +131,7 @@ impl<'de> Deserialize<'de> for NodeId {
}
}
/// A lifetime discriminator distinguishing a node's incarnations (spec §8.4).
/// A lifetime discriminator distinguishing a node's incarnations (spec §2.2).
#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)]
pub struct Lifetime(pub u64);
@ -168,7 +168,7 @@ pub struct StreamDescriptor {
pub origin: StreamOrigin,
}
/// Channel metadata declared by the stream owner.
/// Catalog metadata declared by the stream owner; frames store only `id` and payload.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ChannelDescriptor {
pub stream: StreamId,
@ -236,7 +236,7 @@ impl SubscriptionRequest {
}
}
/// The unit the mux emits (spec §4.1): bytes tagged with a channel and a position.
/// The unit the mux emits (spec §2.1): bytes tagged with a channel and a position.
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Frame {
/// The stream-local lane these bytes belong to.

View file

@ -1,24 +1,23 @@
//! Consumer ingest: reconstruct each node's stream from deliveries (spec
//! §8.1).
//! Consumer ingest: reconstruct each stream from deliveries (spec §7.1).
//!
//! The consumer receives frames from many nodes, in any order, some never
//! arriving, and reconstructs each node's stream — keyed by stream id,
//! ordered by position — into the [`Store`]. Ingest is deliberately thin:
//! it routes a delivery to its stream and records the frame whole. It MUST
//! NOT thin, aggregate, decode-and-discard, or truncate (spec §8.2), and it
//! never inspects a channel or a payload, so a channel it cannot decode is
//! retained exactly like any other (spec §8.3).
//! arriving, and reconstructs each stream — keyed by stream id, ordered by
//! position — into the [`Store`]. Ingest is deliberately thin: it routes a
//! delivery to its stream and records the frame whole. It MUST NOT thin,
//! aggregate, decode-and-discard, or truncate (spec §7.1, §7.2), and it never
//! inspects a channel or payload, so an undecoded channel is retained exactly
//! like any other (spec §7.2, §8.3).
//!
//! Reconstruction is by position, not arrival: out-of-order deliveries land
//! in order in the store, and a position delivered twice collapses to one
//! (spec §7.5). Two lives of one node are different stream ids and never
//! merge (spec §8.4).
//! Reconstruction is by position, not arrival: out-of-order deliveries land in
//! order in the store, and a position delivered twice collapses to one (spec
//! §7.2). Two lives of one node are different stream ids and never merge (spec
//! §2.2).
use super::store::Store;
use super::transport::Delivery;
/// The single consumer toward which all telemetry flows (spec §2). It owns
/// the stored streams and grows them as deliveries arrive.
/// A store-owning ingest fold for deliveries. It grows stored streams as
/// deliveries arrive.
#[derive(Debug, Default)]
pub struct Consumer {
store: Store,
@ -33,7 +32,7 @@ impl Consumer {
/// Accept one delivery: route it to its stream and record the frame.
/// Returns `true` if the frame was new (a duplicate position is
/// ignored, keeping the first — the carrier cannot fabricate content,
/// spec §9).
/// spec §9.1).
pub fn accept(&mut self, delivery: Delivery) -> bool {
let Delivery { stream, frame } = delivery;
self.store.stream_mut(&stream).record(frame)
@ -46,7 +45,7 @@ impl Consumer {
}
}
/// The stored streams — the source of truth for every view (spec §8.2).
/// The stored streams — frame truth for raw storage and view projections (spec §7.3, §8.1).
pub fn store(&self) -> &Store {
&self.store
}

View file

@ -1,34 +1,35 @@
//! The per-node telemetry **datastream** (see `DATASTREAM_SPEC.md`).
//!
//! A deliberately dumb pipe: producers dump bytes tagged by channel, a
//! single per-node mux interleaves them into one ordered stream, a
//! best-effort transport carries that stream to the one consumer, ingest
//! reconstructs each node's stream by position, and views are read-time
//! projections over the stored stream. Nothing between a producer and a
//! A deliberately dumb pipe: producers dump bytes tagged by stream-local
//! channel id, a single per-node mux accepts those bytes and assigns canonical
//! positions during drain, the endpoint broadcasts catalog-aware events to
//! subscribers, ingest reconstructs streams by position, and views are
//! read-time projections over stored frames. Nothing between a producer and a
//! view interprets the payload.
//!
//! ```text
//! producers (caller-owned records + text)
//! │ bytes tagged by channel → [`record::Record`]
//! │ bytes tagged by registered ChannelId
//! ▼
//! per-node MUX → [`mux::Mux`]
//! │ one ordered stream of [`Frame`]s
//! endpoint / catalog → [`endpoint`]
//! │ channel metadata + producer handles
//! ▼
//! best-effort transport → [`transport`]
//! │ delivery: frames, maybe dropped/reordered/delayed
//! per-node MUX → [`mux::Mux`]
//! │ positioned [`frame::Frame`]s
//! ▼
//! consumer INGEST → [`ingest::Consumer`]
//! │ complete stream, stored whole
//! endpoint fanout → [`endpoint::DeliveryFanout`]
//! │ catalog-aware events, maybe dropped per subscriber
//! ▼
//! stored STREAM (truth) → [`store`]
//! │ read-time only
//! ingest / store → [`ingest`], [`store`]
//! │ position-keyed frame truth
//! ▼
//! VIEWS → [`views`]
//! views → [`views`]
//! ```
//!
//! The data model ([`frame`]), extension contract ([`record`]), and wire
//! envelope ([`wire`]) are the seams a test observes. Channel meanings live in
//! producer/consumer crates, not in a datastream-wide catalog.
//! The data model ([`frame`]), extension contract ([`record`]), endpoint/fanout
//! seam ([`endpoint`]), and compatibility wire helpers ([`wire`]) are the seams
//! tests observe. Channel meanings live in producer/consumer crates, not in a
//! datastream-wide global registry.
pub mod emit;
pub mod endpoint;
@ -41,7 +42,6 @@ pub mod publisher_actor;
pub mod record;
pub mod sink_actor;
pub mod store;
pub mod timing;
pub mod transport;
pub mod views;
pub mod wire;
@ -65,6 +65,5 @@ pub use publisher_actor::{
pub use record::{ChannelKind, ChannelRegistry, Record};
pub use sink_actor::{DATASTREAM_SINK_NAME, DatastreamSink};
pub use store::{GapSpan, Store, StoredStream};
pub use timing::{FRAME_TIME_CHANNEL, FRAME_TIME_CHANNEL_ID, FrameTimeSample};
pub use transport::{Delivery, Reorder, ScriptedTransport, StreamScript};
pub use views::{Body, LogEntry, MergedFrame};

View file

@ -1,42 +1,39 @@
//! The per-node mux: the single ordering authority (spec §5).
//! The per-stream mux: the single ordering authority (spec §4.4).
//!
//! Every producer on a node submits bytes tagged with a stream-local channel id
//! to one mux, and the mux assigns a single monotonic position sequence across
//! all channels. A drop consumes a position and is therefore visible downstream
//! as a gap.
//! Producers submit bytes tagged with a stream-local channel id. The mux accepts
//! payloads into a bounded queue first, then assigns a single monotonic position
//! sequence while draining accepted payloads.
use std::sync::Mutex;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::mpsc::{Receiver, SyncSender, TryRecvError, TrySendError, sync_channel};
use std::time::{SystemTime, UNIX_EPOCH};
use crossbeam_channel::{Receiver, Sender, TryRecvError, TrySendError, bounded};
use std::sync::atomic::{AtomicU64, Ordering};
use super::frame::{ChannelId, Frame, Position, StreamId};
use super::record::Record;
use super::timing::FRAME_TIME_CHANNEL_ID;
use super::timing::FrameTimeSample;
struct PendingFrame {
channel: ChannelId,
payload: Vec<u8>,
}
/// A node's single position authority and outgoing telemetry queue.
pub struct Mux {
stream: StreamId,
next: AtomicU64,
dropped: AtomicU64,
frame_timing_enabled: AtomicBool,
tx: SyncSender<Frame>,
rx: Mutex<Receiver<Frame>>,
tx: Sender<PendingFrame>,
rx: Receiver<PendingFrame>,
}
impl Mux {
/// Create a mux for `stream` with a bounded outgoing queue.
pub fn new(stream: StreamId, capacity: usize) -> Self {
let capacity = capacity.max(1).min(1_048_576);
let (tx, rx) = sync_channel(capacity);
let (tx, rx) = bounded(capacity);
Mux {
stream,
next: AtomicU64::new(0),
dropped: AtomicU64::new(0),
frame_timing_enabled: AtomicBool::new(true),
tx,
rx: Mutex::new(rx),
rx,
}
}
@ -45,88 +42,50 @@ impl Mux {
Mux::new(stream, usize::MAX)
}
/// The stream this mux produces (spec §8.4 ingest key).
/// The stream this mux produces (spec §2.2, §7.1 ingest key).
pub fn stream_id(&self) -> &StreamId {
&self.stream
}
/// Submit opaque bytes on a registered channel id.
pub fn submit(&self, channel: ChannelId, payload: Vec<u8>) -> Position {
let position = Position(self.next.fetch_add(1, Ordering::Relaxed));
let timing_sample = self
.frame_timing_enabled
.load(Ordering::Relaxed)
.then(|| now_unix_ns())
.filter(|_| channel != FRAME_TIME_CHANNEL_ID)
.map(|created_at_unix_ns| FrameTimeSample::new(position, created_at_unix_ns));
let frame = Frame {
channel,
position,
payload,
};
match self.tx.try_send(frame) {
Ok(()) => {
if let Some(sample) = timing_sample {
self.push_timing_sample_if_room(sample);
}
}
pub fn submit(&self, channel: ChannelId, payload: Vec<u8>) -> bool {
match self.tx.try_send(PendingFrame { channel, payload }) {
Ok(()) => true,
Err(TrySendError::Full(_)) | Err(TrySendError::Disconnected(_)) => {
self.dropped.fetch_add(1, Ordering::Relaxed);
false
}
}
position
}
/// Pull all currently queued frames, sorted by mux position.
/// Pull all currently queued frames in mux queue order.
pub fn drain(&self) -> Vec<Frame> {
let rx = self.rx.lock().expect("mux receiver poisoned");
let mut frames = Vec::new();
loop {
match rx.try_recv() {
Ok(frame) => frames.push(frame),
Err(TryRecvError::Empty) => break,
Err(TryRecvError::Disconnected) => break,
match self.rx.try_recv() {
Ok(pending) => {
// Position is consumed only after a pending frame has left
// the queue; failed submit never reaches this point.
let position = Position(self.next.fetch_add(1, Ordering::Relaxed));
frames.push(Frame {
channel: pending.channel,
position,
payload: pending.payload,
});
}
Err(TryRecvError::Empty) | Err(TryRecvError::Disconnected) => break,
}
}
frames.sort_by_key(|frame| frame.position);
frames
}
/// Enable or disable optional sidecar timing samples for newly submitted frames.
pub fn set_frame_timing_enabled(&self, enabled: bool) {
self.frame_timing_enabled.store(enabled, Ordering::Relaxed);
}
/// Whether this mux currently emits sidecar frame timing samples.
pub fn frame_timing_enabled(&self) -> bool {
self.frame_timing_enabled.load(Ordering::Relaxed)
}
fn push_timing_sample_if_room(&self, sample: FrameTimeSample) {
let position = Position(self.next.fetch_add(1, Ordering::Relaxed));
let frame = Frame {
channel: FRAME_TIME_CHANNEL_ID,
position,
payload: sample.encode(),
};
let _ = self.tx.try_send(frame);
}
/// How many positions have been assigned — the gap-free high-water mark.
/// How many positions have been assigned while draining accepted frames.
pub fn assigned(&self) -> u64 {
self.next.load(Ordering::Relaxed)
}
/// How many data frames have been dropped on overflow.
/// How many submissions have been dropped before entering the mux.
pub fn dropped(&self) -> u64 {
self.dropped.load(Ordering::Relaxed)
}
}
fn now_unix_ns() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX))
.unwrap_or(0)
}

View file

@ -1,15 +1,15 @@
//! `DatastreamSink` — the cluster-side consumer of [`DatastreamFrame`] messages.
//! Legacy `DatastreamSink` for [`DatastreamFrame`] actor messages.
//!
//! This is the counterpart to [`ClusterFrameSink`](super::emit::ClusterFrameSink):
//! a node ships its ordered telemetry as `DatastreamFrame` actor messages over the
//! regular swactor transport, and this actor — registered under a well-known name
//! on the collector (e.g. the orchestrator) — receives them, decodes each back
//! into a `(StreamId, Frame)` delivery, and hands it to a caller-supplied fold.
//! New live transports should prefer catalog-aware [`crate::DatastreamEvent`]
//! streams. This actor remains as the counterpart to
//! [`ClusterFrameSink`](super::emit::ClusterFrameSink): a node ships positioned
//! telemetry as legacy `DatastreamFrame` actor messages over regular Swactor
//! transport, and this actor receives them, decodes each back into a
//! `(StreamId, Frame)` delivery, and hands it to a caller-supplied fold.
//!
//! It deliberately knows nothing about any view layer. The actor owns an opaque
//! callback so binaries can wire the decoded deliveries into whichever fold they
//! need. Malformed payloads are dropped silently — the same best-effort tolerance
//! the UDP ingest had.
//! callback so binaries can wire decoded deliveries into whichever fold they
//! need. Malformed payloads are dropped silently.
use swactor::actor::ActorInterface;
use swactor::runtime::Ctx;
@ -17,9 +17,9 @@ use swactor::runtime::Ctx;
use super::frame::{Frame, StreamId};
use super::wire::{DatastreamFrame, decode_delivery};
/// Receives [`DatastreamFrame`] cluster messages and folds each decoded delivery
/// through `on_frame`. Spawn it, then publish its address under
/// [`DATASTREAM_SINK_NAME`] so emitters can resolve and ship to it.
/// Receives legacy [`DatastreamFrame`] cluster messages and folds each decoded
/// delivery through `on_frame`. Spawn it, then publish its address under
/// [`DATASTREAM_SINK_NAME`] so legacy emitters can resolve and ship to it.
pub struct DatastreamSink {
on_frame: Box<dyn FnMut(StreamId, Frame) + Send>,
}

View file

@ -1,15 +1,15 @@
//! The stored stream — the consumer's source of truth (spec §8).
//! The stored stream — the consumer's frame source of truth (spec §7).
//!
//! Storage holds each node's **complete** stream, whole and append-only.
//! Nothing is thinned, aggregated, decoded-and-discarded, or truncated at
//! ingest (spec §8.2); everything a view ever shows is derived from here
//! (spec §9.1). Frames on channels the consumer cannot decode are kept as
//! opaque bytes, in order, alongside the rest (spec §8.3) — the store never
//! looks at a channel or a payload.
//! Storage holds each node's complete stream, whole and append-only. Nothing is
//! thinned, aggregated, decoded-and-discarded, or truncated at ingest (spec
//! §7.1, §7.2); everything a view shows is derived from here (spec §8.1).
//! Frames on channels the consumer cannot decode are kept as opaque bytes,
//! alongside the rest (spec §7.2, §8.3) — the store never looks at a channel or
//! payload.
//!
//! A [`StoredStream`] is keyed in the [`Store`] by [`StreamId`] — node plus
//! lifetime — so a re-incarnated node does not append to its prior life
//! (spec §8.4).
//! lifetime — so a re-incarnated node does not append to its prior life (spec
//! §2.2, §7.1).
use std::collections::BTreeMap;
@ -19,8 +19,8 @@ use super::frame::{Frame, Position, StreamId};
///
/// Backed by a position-keyed map so out-of-order arrivals land in order
/// and a position seen twice collapses to one (the carrier may not
/// fabricate content, spec §9). Gaps are not stored — they are *derived*
/// at read time from the positions that are present (spec §9.1).
/// fabricate content, spec §9.1). Gaps are not stored — they are *derived*
/// at read time from the positions that are present (spec §7.3, §8.1).
#[derive(Debug, Clone, Default)]
pub struct StoredStream {
frames: BTreeMap<u64, Frame>,
@ -34,7 +34,7 @@ impl StoredStream {
/// Record a delivered frame. Idempotent by position: the first frame
/// seen for a position wins and is never mutated (append-only,
/// spec §8.2). Returns `true` if this was the first time the position
/// spec §7.2). Returns `true` if this was the first time the position
/// was seen.
pub fn record(&mut self, frame: Frame) -> bool {
match self.frames.entry(frame.position.0) {
@ -73,19 +73,19 @@ impl StoredStream {
}
/// The **interior** gaps — runs of positions assigned between the first
/// and last delivered frame but never delivered (spec §7.5, §8), each as
/// one [`GapSpan`].
/// and last delivered frame but never delivered (spec §7.3), each as one
/// [`GapSpan`].
///
/// Cost is O(stored frames), never O(gap size): it walks adjacent stored
/// positions and reads each span's endpoints from them, rather than
/// enumerating the (possibly enormous) range in between. A stream that
/// brackets a huge interior gap — what a long consumer outage produces
/// (spec §7.4), or a single wild position from a corrupt datagram — still
/// surfaces in work proportional to the frames held, not to `u64::MAX`.
/// (spec §6.3, §7.3), or a single wild position from a corrupt datagram —
/// still surfaces in work proportional to the frames held, not to `u64::MAX`.
///
/// Only interior gaps are knowable: a position lost *after* the last
/// delivered frame leaves no bracketing frame to reveal it, so it shows
/// up as the stream simply ending (spec §7.4 node death), not a gap.
/// Leading and trailing losses are not derivable from stored frames because
/// no bracketing position exists. A position lost after the last delivered
/// frame shows up as the stream simply ending (spec §5.7, §7.3), not a gap.
pub fn gap_spans(&self) -> Vec<GapSpan> {
let mut spans = Vec::new();
let mut prev: Option<u64> = None;
@ -105,7 +105,7 @@ impl StoredStream {
}
/// A contiguous run of missing positions surfaced in a stored stream
/// (spec §7.5). Inclusive on both ends.
/// (spec §7.3). Inclusive on both ends.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct GapSpan {
/// First missing position.
@ -121,7 +121,7 @@ impl GapSpan {
}
}
/// All stored streams at the consumer, keyed by [`StreamId`] (spec §8.4).
/// All stored streams at the consumer, keyed by [`StreamId`] (spec §2.2, §7.1).
///
/// Two streams with the same node but different lifetime are distinct keys
/// and never merge.

View file

@ -1,32 +0,0 @@
//! Optional frame-construction timing sidecar records.
use serde::{Deserialize, Serialize};
use crate::frame::ChannelId;
use crate::frame::Position;
use crate::record::Record;
/// Reserved channel carrying optional timing samples for frames in the same stream.
pub const FRAME_TIME_CHANNEL: &str = "datastream.frame_time";
pub const FRAME_TIME_CHANNEL_ID: ChannelId = ChannelId(0);
/// Sidecar timing sample keyed by the target frame's stream-local position.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct FrameTimeSample {
pub target_position: u64,
pub created_at_unix_ns: u64,
}
impl FrameTimeSample {
pub fn new(target_position: Position, created_at_unix_ns: u64) -> Self {
Self {
target_position: target_position.0,
created_at_unix_ns,
}
}
}
impl Record for FrameTimeSample {
const CHANNEL: &'static str = FRAME_TIME_CHANNEL;
}

View file

@ -1,28 +1,26 @@
//! Transport: best-effort carriage of a node's stream to the one consumer
//! (spec §7), and a scripted in-process carrier for offline tests (testing
//! spec §2, §9).
//! Legacy transport/test seam for carrying positioned frames into ingest.
//!
//! A [`Delivery`] is the value on the transport→ingest seam: which stream a
//! frame belongs to, and the frame. A real carrier rides the connections
//! the system already maintains (spec §7.1); a test replaces it with the
//! [`ScriptedTransport`] here, whose faults are chosen by the scenario and
//! stay inside the **envelope** (testing spec §9): a carrier may *deliver*,
//! *drop*, *reorder*, or *delay*, and it MUST NOT corrupt a payload,
//! fabricate a frame, or alter a position.
//! The live endpoint path now fans out catalog-aware [`DatastreamEvent`] values;
//! this module keeps the older [`Delivery`] shape used by ingest, storage tests,
//! and scripted conformance checks.
//!
//! Under position-ordering a *delay* is indistinguishable from a *reorder*
//! (a delayed frame simply arrives later), so the envelope's delay is
//! covered by [`Reorder`]. Everything the scripted carrier produces is a
//! reordered subsequence of what was sent — never a superset, never a
//! mutation — which is exactly the property the real-transport conformance
//! check pins (testing spec §9).
//! A [`Delivery`] pairs the producing stream id with one frame. A real carrier
//! rides connections the system already maintains; tests can replace it with
//! [`ScriptedTransport`], whose faults stay inside the transport envelope: it
//! may *deliver*, *drop*, *reorder*, or *delay*, and it MUST NOT corrupt a
//! payload, fabricate a frame, or alter a position.
//!
//! Under position-ordering a *delay* is indistinguishable from a *reorder*:
//! a delayed frame simply arrives later. Everything the scripted carrier
//! produces is a reordered subsequence of what was sent — never a superset and
//! never a mutation.
use std::collections::BTreeSet;
use super::frame::{Frame, StreamId};
/// A frame as the consumer receives it from the transport (testing spec §2
/// seam): tagged with the stream it belongs to.
/// A frame as the legacy transport/ingest seam receives it: tagged with the
/// stream it belongs to.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Delivery {
/// Which node's life produced the frame (spec §8.1 ingest key).
@ -38,9 +36,9 @@ impl Delivery {
}
}
/// How the surviving frames of a stream are reordered on arrival. This is
/// the envelope's *reorder* (and *delay*) axis (testing spec §9); each
/// variant is a permutation of the survivors, never adding or dropping.
/// How the surviving frames of a stream are reordered on arrival. This is the
/// envelope's *reorder* and *delay* axis (spec §6.3, §9.1); each variant is a
/// permutation of the survivors, never adding or dropping.
#[derive(Debug, Clone, Default)]
pub enum Reorder {
/// Delivered in the order sent.
@ -57,7 +55,7 @@ pub enum Reorder {
Permutation(Vec<usize>),
}
/// The faults a scripted carrier applies to one stream (testing spec §9
/// The faults a scripted carrier applies to one stream (spec §6.3, §9.1
/// envelope). Drops and reorders only — payloads and positions are never
/// touched.
#[derive(Debug, Clone, Default)]
@ -91,10 +89,9 @@ impl StreamScript {
}
}
/// A scripted, in-process transport (testing spec §2). It is a pure,
/// deterministic transform from what a node *sent* to what the consumer is
/// *delivered* — the entanglement of real wires replaced by a script so a
/// run completes in microseconds and returns the same result every time.
/// A scripted, in-process transport for conformance scenarios (spec §9.1). It
/// is a pure, deterministic transform from what a node sent to what the
/// consumer receives, replacing real wire behavior with a fast test script.
pub struct ScriptedTransport;
impl ScriptedTransport {

View file

@ -1,4 +1,4 @@
//! Views: read-time projections over a stored stream (spec §9).
//! Views: read-time projections over a stored stream (spec §8).
use std::fmt;
@ -70,6 +70,7 @@ where
C: ChannelClassifier + ?Sized,
R: Fn(ChannelId) -> Option<String>,
{
// Capacity covers stored frames; surfaced gaps may add extra log entries.
let mut out = Vec::with_capacity(stream.len());
let mut prev: Option<u64> = None;
for frame in stream.frames() {
@ -114,7 +115,8 @@ where
timeline_with_resolver(stream, classifier, resolve_name)
}
/// Transitional merged log using numeric channel ids as strings for classifier lookup.
/// Transitional helper for callers that still classify by rendered numeric ids;
/// named decoding should use [`merged_log_with_names`].
pub fn merged_log_with<C: ChannelClassifier + ?Sized>(
stream: &StoredStream,
classifier: &C,

View file

@ -59,7 +59,7 @@ pub fn decode_delivery(buf: &[u8]) -> Result<(StreamId, Frame), WireError> {
Ok((stream, frame))
}
/// One datastream event, addressed to a local/cluster datastream actor.
/// Legacy actor-message payload wrapper for datastream bytes.
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct DatastreamFrame {
pub payload: Vec<u8>,

View file

@ -1,567 +0,0 @@
//! Shared support for the datastream tests (testing spec §2, §3).
//!
//! Two things live here, both deliberately separate from the system under
//! test so a test never asserts the code against itself:
//!
//! * the **payload library** — realistic record shapes and log lines, the
//! bytes the pipe will actually carry (testing spec §5: "never
//! placeholder text"); and
//! * the **reference model** — the spec's rules restated as small, total
//! functions over sequences (testing spec §3). It is the trusted oracle:
//! every test's `expected` is derived from it, never captured from a run.
//! It is written naively on purpose (collect, sort, scan) so a reader can
//! confirm it against the spec by eye, while the pipe computes the same
//! answers the long way.
//!
//! This module is `#[path]`-included into more than one test binary, so
//! some items are unused in some of them.
#![allow(dead_code)]
use datastream::mux::Mux;
use datastream::store::GapSpan;
pub use datastream::transport::Delivery;
use datastream::{ChannelId, Frame, Position, Record, StreamId};
pub mod schema {
use datastream::{ChannelId, Record};
use serde::{Deserialize, Serialize};
pub const IDENTITY: &str = "identity";
pub const HOST_RESOURCE: &str = "host.resource";
pub const TRANSPORT_INTERNALS: &str = "transport.internals";
pub const MEMBERSHIP: &str = "membership";
pub const RUNTIME_STATS: &str = "runtime.stats";
pub const DIST_STATE: &str = "dist.state";
pub const RUNTIME_ACTORS: &str = "runtime.actors";
pub const RUNTIME_WORKERS: &str = "runtime.workers";
pub const DATASTREAM_HEALTH: &str = "datastream.health";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ProcStream {
Stdout,
Stderr,
}
impl ProcStream {
pub fn as_str(self) -> &'static str {
match self {
ProcStream::Stdout => "stdout",
ProcStream::Stderr => "stderr",
}
}
}
pub fn process_output(label: &str, stream: ProcStream) -> ChannelId {
ChannelId::new(format!("proc.{label}.{}", stream.as_str()))
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct IdentityRecord {
pub node: String,
#[serde(default)]
pub life: u64,
#[serde(default)]
pub node_name: String,
#[serde(default)]
pub listen_addr: String,
#[serde(default)]
pub relay_url: String,
#[serde(default)]
pub version: String,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ResourceSample {
#[serde(default)]
pub cpu_pct: f32,
#[serde(default)]
pub mem_used_mb: u32,
#[serde(default)]
pub mem_total_mb: u32,
#[serde(default)]
pub gpu_pct: f32,
#[serde(default)]
pub disk_used_gb: u32,
#[serde(default)]
pub net_rx_kbps: u32,
#[serde(default)]
pub net_tx_kbps: u32,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct TransportInternals {
#[serde(default)]
pub relay_connected: bool,
#[serde(default)]
pub direct_peers: u32,
#[serde(default)]
pub relay_peers: u32,
#[serde(default)]
pub rtt_ms_p50: u32,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct MembershipTransition {
pub peer: String,
pub from: String,
pub to: String,
#[serde(default)]
pub reason: String,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct RuntimeStats {
#[serde(default)]
pub actors_live: u32,
#[serde(default)]
pub mailbox_depth: u32,
#[serde(default)]
pub scheduled_tasks: u32,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct DistributionState {
#[serde(default)]
pub cache_size: u32,
#[serde(default)]
pub cache_entries: Vec<CacheEntryRec>,
#[serde(default)]
pub directory_route_count: u32,
#[serde(default)]
pub registry_size: u32,
#[serde(default)]
pub registry_tombstones: u32,
#[serde(default)]
pub registry_entries: Vec<RegistryEntryRec>,
#[serde(default)]
pub recent_probe_targets: Vec<String>,
#[serde(default)]
pub peer_auth_mode: String,
#[serde(default)]
pub authorized_peer_count: u32,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct CacheEntryRec {
#[serde(default)]
pub actor_addr: String,
#[serde(default)]
pub node_id: String,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct RegistryEntryRec {
#[serde(default)]
pub name: String,
#[serde(default)]
pub actor_addr: String,
#[serde(default)]
pub node_id: String,
#[serde(default)]
pub tombstone: bool,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct WorkerCounters {
#[serde(default)]
pub num_workers: u32,
#[serde(default)]
pub scheduled_tasks: u32,
#[serde(default)]
pub local_sends: u64,
#[serde(default)]
pub cross_sends: u64,
#[serde(default)]
pub inbox_sends: u64,
#[serde(default)]
pub type_mismatches: u64,
#[serde(default)]
pub panics: u64,
#[serde(default)]
pub messages_dropped: u64,
#[serde(default)]
pub restarts: u64,
#[serde(default)]
pub stops: u64,
#[serde(default)]
pub messages_processed: u64,
#[serde(default)]
pub tick_p50_us: u64,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct DatastreamHealth {
#[serde(default)]
pub assigned: u64,
#[serde(default)]
pub dropped: u64,
#[serde(default)]
pub loss_rate_ppm: u32,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct ActorRuntimeDetail {
#[serde(default)]
pub actors: Vec<ActorRec>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct ActorRec {
#[serde(default)]
pub address: String,
#[serde(default)]
pub name: String,
#[serde(default)]
pub mailbox_depth: u32,
#[serde(default)]
pub messages_processed: u64,
#[serde(default)]
pub last_msg_type: String,
#[serde(default)]
pub poisoned: bool,
#[serde(default)]
pub message_type_counts: Vec<(String, u64)>,
}
impl Record for IdentityRecord {
const CHANNEL: &'static str = IDENTITY;
}
impl Record for ResourceSample {
const CHANNEL: &'static str = HOST_RESOURCE;
}
impl Record for TransportInternals {
const CHANNEL: &'static str = TRANSPORT_INTERNALS;
}
impl Record for MembershipTransition {
const CHANNEL: &'static str = MEMBERSHIP;
}
impl Record for RuntimeStats {
const CHANNEL: &'static str = RUNTIME_STATS;
}
impl Record for DistributionState {
const CHANNEL: &'static str = DIST_STATE;
}
impl Record for ActorRuntimeDetail {
const CHANNEL: &'static str = RUNTIME_ACTORS;
}
impl Record for WorkerCounters {
const CHANNEL: &'static str = RUNTIME_WORKERS;
}
impl Record for DatastreamHealth {
const CHANNEL: &'static str = DATASTREAM_HEALTH;
}
}
use schema::*;
/// An in-process node (testing spec §2 — the faked machine boundary): a
/// *real* mux plus the producers that feed it. Producers push realistic
/// bytes in at the producer seam; [`Node::sent`] takes the mux's ordered
/// output stream — the value on the mux→transport seam. Only the machine
/// boundary is faked; the mux is the production code.
pub struct Node {
stream: StreamId,
mux: Mux,
}
impl Node {
/// Stand up a node for a given stream (node identity + lifetime).
pub fn new(stream: StreamId) -> Self {
Node {
mux: Mux::unbounded(stream.clone()),
stream,
}
}
/// The stream this node produces.
pub fn stream_id(&self) -> &StreamId {
&self.stream
}
/// A producer emits a typed record on its own channel (spec §6.1).
pub fn emit<R: Record>(&self, record: &R) -> Position {
self.mux.submit(R::channel(), record.encode())
}
/// A producer emits a line of raw process output (spec §6.2).
pub fn emit_text(&self, label: &str, stream: ProcStream, line: &str) -> Position {
self.mux
.submit(process_output(label, stream), line.as_bytes().to_vec())
}
/// A producer emits bytes on a channel the consumer may not know
/// (spec §6.3) — opaque to everything until a view learns the channel.
pub fn emit_opaque(&self, channel: &str, bytes: &[u8]) -> Position {
self.mux.submit(ChannelId::new(channel), bytes.to_vec())
}
/// Take the node's ordered output stream (drains the mux).
pub fn sent(&self) -> Vec<Frame> {
self.mux.drain()
}
}
/// Realistic payloads, drawn on by both the verified vectors (testing spec
/// §5) and the deployment scenario (testing spec §8). Nothing here is
/// placeholder text.
pub mod payloads {
use super::*;
/// A boot/identity record for a node, including the descriptive fields a
/// node fills once known (name, listen addr, embedded relay, build version).
pub fn identity(node: &str, life: u64) -> IdentityRecord {
IdentityRecord {
node: node.to_string(),
life,
node_name: format!("swift-{node}"),
listen_addr: format!("{node}.iroh:4242"),
relay_url: "https://relay.example:4443/".to_string(),
version: "ds-inference @ abc1234".to_string(),
}
}
/// A consolidated distribution-subsystem state; `tick` nudges the values so
/// a series is not constant.
pub fn dist_state(tick: u64) -> DistributionState {
DistributionState {
cache_size: 3 + (tick % 4) as u32,
cache_entries: vec![CacheEntryRec {
actor_addr: format!("actor-{}", tick % 5),
node_id: format!("node-{}", tick % 3),
}],
directory_route_count: 5 + (tick % 7) as u32,
registry_size: 8 + (tick % 3) as u32,
registry_tombstones: (tick % 2) as u32,
registry_entries: vec![RegistryEntryRec {
name: format!("svc-{}", tick % 4),
actor_addr: format!("actor-{}", tick % 5),
node_id: format!("node-{}", tick % 3),
tombstone: tick.is_multiple_of(2),
}],
recent_probe_targets: vec![format!("peer-{}", tick % 6)],
peer_auth_mode: if tick.is_multiple_of(2) {
"open".into()
} else {
"allow-list".into()
},
authorized_peer_count: (tick % 5) as u32,
}
}
/// A per-actor runtime-detail record (the real actor table).
pub fn actor_detail(tick: u64) -> ActorRuntimeDetail {
ActorRuntimeDetail {
actors: vec![
ActorRec {
address: format!("{:064x}", tick),
name: "SwimActor".to_string(),
mailbox_depth: (tick % 5) as u32,
messages_processed: 100 + tick,
last_msg_type: "swactor_dist::Ping".to_string(),
poisoned: false,
message_type_counts: vec![("swactor_dist::Ping".to_string(), 40 + tick)],
},
ActorRec {
address: format!("{:064x}", tick + 1),
name: "RegistryActor".to_string(),
mailbox_depth: 0,
messages_processed: 10 + tick % 3,
last_msg_type: "Tick".to_string(),
poisoned: false,
message_type_counts: vec![("Tick".to_string(), 10 + tick % 3)],
},
],
}
}
/// A plausible resource sample; `tick` nudges the values so a series is
/// not constant.
pub fn resource(tick: u64) -> ResourceSample {
ResourceSample {
cpu_pct: 12.5 + (tick % 7) as f32 * 3.0,
mem_used_mb: 2048 + (tick % 5) as u32 * 128,
mem_total_mb: 16384,
gpu_pct: (tick % 4) as f32 * 25.0,
disk_used_gb: 40 + (tick % 3) as u32,
net_rx_kbps: 900 + (tick % 11) as u32 * 30,
net_tx_kbps: 300 + (tick % 13) as u32 * 20,
}
}
/// A transport-internals snapshot.
pub fn transport(tick: u64) -> TransportInternals {
TransportInternals {
relay_connected: !tick.is_multiple_of(9),
direct_peers: 2 + (tick % 3) as u32,
relay_peers: 1,
rtt_ms_p50: 18 + (tick % 5) as u32 * 4,
}
}
/// A membership transition between two peers' states.
pub fn membership(peer: &str, from: &str, to: &str) -> MembershipTransition {
MembershipTransition {
peer: peer.to_string(),
from: from.to_string(),
to: to.to_string(),
reason: "probe timeout".to_string(),
}
}
/// A runtime-stats record.
pub fn runtime(tick: u64) -> RuntimeStats {
RuntimeStats {
actors_live: 30 + (tick % 6) as u32,
mailbox_depth: (tick % 17) as u32,
scheduled_tasks: 4 + (tick % 3) as u32,
}
}
/// Aggregated worker-runtime counters (the `runtime.workers` channel).
pub fn worker_counters(tick: u64) -> WorkerCounters {
WorkerCounters {
num_workers: 4,
scheduled_tasks: 4 + (tick % 3) as u32,
local_sends: 100 + tick,
cross_sends: 20 + tick,
inbox_sends: tick,
messages_processed: 1000 + tick * 7,
tick_p50_us: 50 + tick,
..Default::default()
}
}
/// Datastream self-health (the `datastream.health` channel). `assigned`
/// tracks the seed directly so a scenario can pick a frame out by its value.
pub fn datastream_health(tick: u64) -> DatastreamHealth {
DatastreamHealth {
assigned: tick,
dropped: tick % 4,
loss_rate_ppm: (tick % 4) as u32,
}
}
/// A realistic line of process output (without trailing newline).
pub fn log_line(label: &str, tick: u64) -> String {
format!(
"[{label}] step {tick} loss=0.{:03} lr=3e-4",
250 - (tick % 200)
)
}
}
/// The reference model: the spec's rules as plain total functions over
/// sequences. No transport, no storage, no concurrency, no time.
pub mod reference {
use super::*;
/// The frames a consumer was delivered for one stream, in arrival
/// order — the raw material reconstruction works over.
pub fn delivered_frames(stream: &StreamId, deliveries: &[Delivery]) -> Vec<Frame> {
deliveries
.iter()
.filter(|d| &d.stream == stream)
.map(|d| d.frame.clone())
.collect()
}
/// Reconstruction (spec §8.1, testing spec §3): "keep the frames that
/// were delivered, in position order." Duplicates of a position
/// collapse to one (the carrier may not fabricate content, spec §9, so
/// a repeat carries identical bytes).
pub fn reconstruct(delivered: &[Frame]) -> Vec<Frame> {
let mut frames: Vec<Frame> = Vec::new();
for f in delivered {
if !frames.iter().any(|seen| seen.position == f.position) {
frames.push(f.clone());
}
}
frames.sort_by_key(|f| f.position);
frames
}
/// The surfaced gaps as spans: the **interior** runs of positions missing
/// between the first and last delivered position (spec §7.5, §8). A
/// consumer can only detect gaps it has bracketing frames for; positions
/// lost after the last delivered frame are invisible and manifest as the
/// stream ending (spec §7.4 node death = truncation, not a gap).
///
/// Walks the sorted delivered positions — O(frames), never the gap size —
/// so the oracle agrees with the store on the cheap path even across a
/// near-`u64::MAX` gap.
pub fn gap_spans(delivered: &[Frame]) -> Vec<GapSpan> {
let recon = reconstruct(delivered);
let mut spans = Vec::new();
let mut prev: Option<u64> = None;
for f in &recon {
let p = f.position.0;
if let Some(q) = prev
&& p > q + 1
{
spans.push(GapSpan {
start: q + 1,
end: p - 1,
});
}
prev = Some(p);
}
spans
}
/// One structural item on the merged timeline (testing spec §3: "all
/// stored frames in position order, channels interleaved"). This is the
/// *structure* of the merged log — order and surfaced gaps — decoupled
/// from how each payload is rendered for display, which is a separate
/// §9.3 concern the tests assert on its own.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum TimelineItem {
Frame { position: u64, channel: String },
Gap { start: u64, end: u64 },
}
/// The merged log oracle (spec §9.2): every delivered frame in position
/// order, channels interleaved, with an interior gap surfaced wherever a
/// position is missing. Written naively so it is obviously the spec.
pub fn merged_log(delivered: &[Frame]) -> Vec<TimelineItem> {
let frames = reconstruct(delivered);
let mut out = Vec::new();
let mut prev: Option<u64> = None;
for f in &frames {
let pos = f.position.0;
if let Some(p) = prev
&& pos > p + 1
{
out.push(TimelineItem::Gap {
start: p + 1,
end: pos - 1,
});
}
out.push(TimelineItem::Frame {
position: pos,
channel: f.channel.to_string(),
});
prev = Some(pos);
}
out
}
}
/// Convenience: build a frame on a typed channel from a record.
pub fn typed_frame<R: Record>(record: &R, position: u64) -> Frame {
Frame::new(R::channel(), Position(position), record.encode())
}
/// Convenience: build a frame on a raw-text process-output channel.
pub fn text_frame(label: &str, stream: ProcStream, line: &str, position: u64) -> Frame {
Frame::new(
process_output(label, stream),
Position(position),
line.as_bytes().to_vec(),
)
}
/// Convenience: a frame on a channel id the consumer does not know — an
/// opaque channel (spec §6.3).
pub fn opaque_frame(id: &str, payload: &[u8], position: u64) -> Frame {
Frame::new(ChannelId::new(id), Position(position), payload.to_vec())
}

View file

@ -7,8 +7,7 @@ use datastream::transport::{Delivery, Reorder, ScriptedTransport, StreamScript};
use datastream::views::{self, Body, LogEntry};
use datastream::wire::{decode_delivery, encode_delivery};
use datastream::{
ChannelId, ChannelKind, ChannelRegistry, FRAME_TIME_CHANNEL_ID, Frame, FrameTimeSample,
Lifetime, NodeId, Position, Record, StreamId,
ChannelId, ChannelKind, ChannelRegistry, Frame, Lifetime, NodeId, Position, Record, StreamId,
};
use serde::{Deserialize, Serialize};
@ -58,67 +57,81 @@ fn record_codecs_round_trip_without_global_catalog() {
}
#[test]
fn mux_numbers_monotonic_and_gap_free() {
fn mux_assigns_positions_when_drained() {
let mux = Mux::unbounded(stream());
mux.set_frame_timing_enabled(false);
for i in 0..64u64 {
let position = mux.submit(RESOURCE_CHANNEL, resource(i as u32).encode());
assert_eq!(position, Position(i));
assert!(mux.submit(RESOURCE_CHANNEL, resource(i as u32).encode()));
}
assert_eq!(mux.assigned(), 64);
assert_eq!(mux.assigned(), 0);
assert_eq!(mux.dropped(), 0);
let positions: Vec<u64> = mux.drain().iter().map(|frame| frame.position.0).collect();
let mut positions: Vec<u64> = mux.drain().iter().map(|frame| frame.position.0).collect();
positions.sort_unstable();
assert_eq!(positions, (0..64).collect::<Vec<_>>());
assert_eq!(mux.assigned(), 64);
}
#[test]
fn mux_queue_preserves_position_order_across_producers() {
fn mux_concurrent_producers_assign_unique_positions_on_drain() {
let mux = Arc::new(Mux::unbounded(stream()));
mux.set_frame_timing_enabled(false);
let mut threads = Vec::new();
for producer in 0..4u8 {
let mux = Arc::clone(&mux);
threads.push(thread::spawn(move || {
let mut accepted = 0;
for seq in 0..32u8 {
mux.submit(LOG_CHANNEL, vec![producer, seq]);
if mux.submit(LOG_CHANNEL, vec![producer, seq]) {
accepted += 1;
}
}
accepted
}));
}
for thread in threads {
thread.join().expect("producer thread completes");
}
let accepted: usize = threads
.into_iter()
.map(|thread| thread.join().expect("producer thread completes"))
.sum();
assert_eq!(accepted, 128);
assert_eq!(mux.assigned(), 0);
let frames = mux.drain();
assert_eq!(frames.len(), 128);
for (expected, frame) in frames.iter().enumerate() {
assert_eq!(frame.position, Position(expected as u64));
}
let mut positions: Vec<u64> = frames.iter().map(|frame| frame.position.0).collect();
positions.sort_unstable();
assert_eq!(positions, (0..128).collect::<Vec<_>>());
assert_eq!(mux.assigned(), 128);
}
#[test]
fn mux_timing_sidecar_uses_reserved_numeric_channel_and_does_not_recurse() {
let mux = Mux::unbounded(stream());
assert!(mux.frame_timing_enabled());
fn mux_full_queue_drops_without_consuming_position() {
let mux = Mux::new(stream(), 1);
let data_position = mux.submit(RESOURCE_CHANNEL, resource(0).encode());
let timing_position = mux.submit(
FRAME_TIME_CHANNEL_ID,
FrameTimeSample::new(Position(42), 123).encode(),
);
assert!(mux.submit(LOG_CHANNEL, b"first".to_vec()));
assert!(!mux.submit(LOG_CHANNEL, b"second".to_vec()));
assert_eq!(mux.dropped(), 1);
assert_eq!(mux.assigned(), 0);
let frames = mux.drain();
assert_eq!(frames.len(), 1);
assert_eq!(frames[0].channel, LOG_CHANNEL);
assert_eq!(frames[0].position, Position(0));
assert_eq!(frames[0].payload, b"first");
assert_eq!(mux.assigned(), 1);
}
#[test]
fn mux_one_submit_one_drained_frame_without_timing_sidecar() {
let mux = Mux::unbounded(stream());
assert!(mux.submit(RESOURCE_CHANNEL, resource(0).encode()));
let frames = mux.drain();
assert_eq!(data_position, Position(0));
assert_eq!(timing_position, Position(2));
assert_eq!(mux.assigned(), 3);
assert_eq!(frames.len(), 3);
assert_eq!(frames.len(), 1);
assert_eq!(frames[0].channel, RESOURCE_CHANNEL);
assert_eq!(frames[1].channel, FRAME_TIME_CHANNEL_ID);
assert_eq!(frames[2].channel, FRAME_TIME_CHANNEL_ID);
let sample = FrameTimeSample::decode(&frames[1].payload).expect("timing decodes");
assert_eq!(sample.target_position, data_position.0);
assert_eq!(frames[0].position, Position(0));
assert_eq!(frames[0].payload, resource(0).encode());
}
#[test]

View file

@ -2,8 +2,8 @@ use std::time::Duration;
use datastream::{
ChannelContent, ChannelContentKind, ChannelFilter, ChannelId, DatastreamEndpoint,
DatastreamEvent, FRAME_TIME_CHANNEL_ID, FrameDelivery, FrameTimeSample, Lifetime, NodeId,
Position, Record, SourceFilter, StreamId, SubscriptionRequest,
DatastreamEvent, FrameDelivery, Lifetime, NodeId, Position, Record, SourceFilter, StreamId,
SubscriptionRequest,
};
use serde_json::Value;
use swactor::actor::ActorAddress;
@ -37,7 +37,6 @@ fn frame_event(event: &DatastreamEvent) -> &FrameDelivery {
fn endpoint_without_subscribers_drains_to_bitbucket() {
let endpoint = endpoint();
let producer = endpoint.producer();
producer.set_frame_timing_enabled(false);
let log = producer.register_channel("runtime.log", ChannelContent::TextStream);
producer.submit_text(log, "before");
@ -63,12 +62,12 @@ fn channel_registration_allocates_numeric_ids() {
assert_eq!(stderr, ChannelId(2));
assert_eq!(runtime, ChannelId(3));
let catalog = endpoint.catalog_snapshot();
let removed_timing_name = ["datastream", "frame_time"].join(".");
assert!(
catalog
!catalog
.channels
.values()
.any(|descriptor| descriptor.id == FRAME_TIME_CHANNEL_ID
&& descriptor.name == "datastream.frame_time")
.any(|descriptor| descriptor.name == removed_timing_name)
);
}
@ -118,7 +117,6 @@ fn subscription_snapshot_contains_stream_and_channel_metadata() {
fn subscription_receives_only_future_matching_frames() {
let endpoint = endpoint();
let producer = endpoint.producer();
producer.set_frame_timing_enabled(false);
let log = producer.register_channel("runtime.log", ChannelContent::TextStream);
producer.submit_text(log, "pre-subscription");
@ -138,10 +136,9 @@ fn subscription_receives_only_future_matching_frames() {
}
#[test]
fn subscription_filters_textstream_channels() {
fn subscription_snapshot_filters_but_future_fanout_broadcasts() {
let endpoint = endpoint();
let producer = endpoint.producer();
producer.set_frame_timing_enabled(false);
let stdout = producer.register_channel("stdout", ChannelContent::TextStream);
let json = producer.register_record::<RuntimeRecord>();
let text_subscription = endpoint.subscribe(
@ -152,22 +149,26 @@ fn subscription_filters_textstream_channels() {
},
);
assert_eq!(text_subscription.snapshot().channels.len(), 1);
assert_eq!(text_subscription.snapshot().channels[0].id, stdout);
producer.submit_text(stdout, "line");
producer.submit_record(json, &RuntimeRecord { value: 5 });
endpoint.tick();
let events = text_subscription.drain_available();
assert_eq!(events.len(), 1);
let delivery = frame_event(&events[0]);
assert_eq!(delivery.channel.channel, stdout);
assert_eq!(delivery.payload, b"line");
assert_eq!(events.len(), 2);
let channels: Vec<ChannelId> = events
.iter()
.map(|event| frame_event(event).channel.channel)
.collect();
assert_eq!(channels, vec![stdout, json]);
}
#[test]
fn endpoint_fans_out_ordered_frames_to_multiple_subscribers() {
let endpoint = endpoint();
let producer = endpoint.producer();
producer.set_frame_timing_enabled(false);
let log = producer.register_channel("runtime.log", ChannelContent::TextStream);
let left = endpoint.subscribe_all("left");
let right = endpoint.subscribe_all("right");
@ -188,7 +189,6 @@ fn endpoint_fans_out_ordered_frames_to_multiple_subscribers() {
fn slow_subscriber_drops_without_blocking_fast_subscriber() {
let endpoint = endpoint();
let producer = endpoint.producer();
producer.set_frame_timing_enabled(false);
let log = producer.register_channel("runtime.log", ChannelContent::TextStream);
let slow = endpoint.subscribe_all_with_capacity("slow", 1);
let fast = endpoint.subscribe_all_with_capacity("fast", 8);
@ -212,35 +212,56 @@ fn slow_subscriber_drops_without_blocking_fast_subscriber() {
}
#[test]
fn endpoint_producer_timing_sidecars_are_fanned_out_when_enabled() {
fn channel_declared_is_broadcast_to_filtered_subscribers() {
let endpoint = endpoint();
let subscription = endpoint.subscribe(
"text-only",
SubscriptionRequest {
sources: SourceFilter::All,
channels: ChannelFilter::Content(ChannelContentKind::TextStream),
},
);
let channel = endpoint.register_channel(
"runtime.json",
ChannelContent::JsonRecord {
schema: Some("runtime.json".to_owned()),
},
);
let events = subscription.drain_available();
assert_eq!(events.len(), 1);
match &events[0] {
DatastreamEvent::ChannelDeclared(descriptor) => {
assert_eq!(descriptor.id, channel);
assert_eq!(descriptor.name, "runtime.json");
}
other => panic!("expected channel declaration, got {other:?}"),
}
}
#[test]
fn submit_text_owned_queues_owned_string() {
let endpoint = endpoint();
let producer = endpoint.producer();
let log = producer.register_channel("runtime.log", ChannelContent::TextStream);
let sub = endpoint.subscribe_all("test");
producer.set_frame_timing_enabled(true);
let data_position = producer.submit_text(log, "visible");
assert!(producer.submit_text_owned(log, String::from("hello")));
let tick = endpoint.tick();
assert_eq!(data_position, Position(0));
assert_eq!(tick.drained, 2);
let events = sub.drain_available();
assert_eq!(events.len(), 2);
assert_eq!(frame_event(&events[0]).channel.channel, log);
assert_eq!(frame_event(&events[0]).payload, b"visible");
assert_eq!(
frame_event(&events[1]).channel.channel,
FRAME_TIME_CHANNEL_ID
);
let sample = FrameTimeSample::decode(&frame_event(&events[1]).payload).expect("timing decodes");
assert_eq!(sample.target_position, data_position.0);
assert_eq!(tick.drained, 1);
assert_eq!(tick.delivered, 1);
let event = sub.recv_timeout(Duration::from_millis(50)).unwrap();
let delivery = frame_event(&event);
assert_eq!(delivery.channel.channel, log);
assert_eq!(delivery.payload, b"hello");
}
#[test]
fn process_observer_adapter_submits_configured_channels() {
let endpoint = endpoint();
let producer = endpoint.producer();
producer.set_frame_timing_enabled(false);
let stdout = producer.register_channel("proc.trainer.stdout", ChannelContent::TextStream);
let stderr = producer.register_channel("proc.trainer.stderr", ChannelContent::TextStream);
let observer = producer.process_observer_with(
@ -300,8 +321,10 @@ fn stats_hook_adapter_submits_worker_snapshot_json() {
}
fn positions(events: &[DatastreamEvent]) -> Vec<u64> {
events
let mut positions: Vec<u64> = events
.iter()
.map(|event| frame_event(event).position.0)
.collect()
.collect();
positions.sort_unstable();
positions
}

View file

@ -28,7 +28,6 @@ impl Record for ResourceSample {
fn build_stream() -> (StreamId, Vec<Frame>) {
let id = StreamId::new(NodeId::new("node-real"), Lifetime(1));
let mux = Mux::unbounded(id.clone());
mux.set_frame_timing_enabled(false);
for tick in 0..20 {
mux.submit(
RESOURCE_CHANNEL,

View file

@ -14,7 +14,7 @@ mod datastream_records {
//! the mux.
use datastream::frame::{Lifetime, NodeId, StreamId};
use datastream::{Mux, Record};
use datastream::{ChannelId, Mux, Position, Record};
use distribution::telemetry::{
CacheEntryRec, DIST_STATE, DistributionState, MembershipTransition, RegistryEntryRec,
};
@ -49,18 +49,18 @@ mod datastream_records {
fn distribution_emits_owned_channel_through_datastream_mux() {
let stream = StreamId::new(NodeId::new("dist-node"), Lifetime(1));
let mux = Mux::unbounded(stream);
mux.set_frame_timing_enabled(false);
let state = DistributionState {
registry_size: 9,
..Default::default()
};
let pos = mux.submit(DistributionState::channel(), state.encode());
let channel = ChannelId(1);
assert!(mux.submit(channel, state.encode()));
let frames = mux.drain();
assert_eq!(pos.0, 0);
assert_eq!(frames.len(), 1);
assert_eq!(frames[0].channel.as_str(), DIST_STATE);
assert_eq!(frames[0].channel, channel);
assert_eq!(frames[0].position, Position(0));
assert_eq!(
DistributionState::decode(&frames[0].payload)
.unwrap()

View file

@ -7,11 +7,10 @@ autobins = false
[features]
default = []
local-e2e = ["dep:dashboard"]
[dependencies]
datastream = { path = "../datastream" }
dashboard = { path = "../dashboard", optional = true }
dashboard = { path = "../dashboard" }
serde_json = "1"
serde = { version = "1", features = ["derive"] }
swactor = { path = "../..", features = ["serde", "transport"] }
@ -19,7 +18,7 @@ swactor-transport = { path = "../transport" }
distribution = { path = "../distribution" }
iroh-driver = { path = "../iroh-driver" }
iroh = "0.98"
tokio = { version = "1", features = ["rt-multi-thread", "macros", "process", "io-util", "sync", "time", "net"] }
tokio = { version = "1", features = ["rt-multi-thread", "macros", "process", "io-util", "sync", "time", "net", "signal"] }
swactor-vastai = { path = "../../tools/vastai" }
parking_lot = "0.12"
blake3 = "1"
@ -60,3 +59,8 @@ name = "local-e2e-cluster"
path = "tests/local_e2e_cluster.rs"
harness = false
required-features = ["local-e2e"]
[[test]]
name = "mvp_chat_mock"
path = "tests/mvp_chat_mock.rs"
harness = false

View file

@ -1430,23 +1430,28 @@ enum GpuWorkerCtlMsg {
ExecuteStep(ExecuteStep),
ReleaseDeviceObject { device_handle: DeviceObjectHandle },
ShutdownWorker(ShutdownWorker),
Process(ProcessNotification),
Process(ProcessOutput),
WorkerAdapter(WorkerAdapterEvent),
}
```
`Process(ProcessNotification)` is delivered by a small `ProcessBridge` actor.
`GpuWorkerCtl` sends process input through `ProcessCommand::WriteStdin`.
`Process(ProcessOutput)` is delivered by the configured upstream process owner;
there is no process-local notification subscription bridge. If the
production worker still uses a subprocess stdin/stdout protocol, `GpuWorkerCtl`
talks to a separate worker I/O adapter. That adapter owns the child stdio handles
and is outside managed-process core.
The process adapter may use newline-delimited JSON for worker commands/events.
This is an adapter, not a second distributed protocol.
The optional worker I/O adapter may use newline-delimited JSON for worker
commands/events. This is an adapter-local protocol, not part of
`crates/process` and not a second distributed protocol.
Adapter rules:
- one command/event JSON object per line
- stdout is reserved for worker events
- stderr is reserved for logs and diagnostics
- payload bytes are forbidden in JSON
- invalid JSON or unknown event shape is a worker/process fault
- one command/event JSON object per line;
- adapter-owned stdout may carry worker events;
- adapter-owned stderr may carry logs and diagnostics;
- payload bytes are forbidden in JSON;
- invalid JSON or unknown event shape is a worker/adapter fault.
Worker environment:
@ -1675,7 +1680,7 @@ Wake hints emitted by worker:
- egress rings: `RingReadable` after advancing `commit`
`GpuWorkerCtl` may synthesize `WorkerCrashed` and `RingFault` after process exit,
process error, or stdout control-stream failure.
process error, or worker I/O adapter control-stream failure.
---
@ -1688,11 +1693,11 @@ NotStarted
on StartWorker -> Spawning
Spawning
spawn ProcessActor with ProcessSpec
spawn ProcessBridge
subscribe bridge to ProcessActor
wait for ProcessNotification::Started
send InitializeWorker through ProcessCommand::WriteStdin
spawn ProcessActor with ProcessSpec and upstream = GpuWorkerCtl/process owner
wait for ProcessOutput::Started
if subprocess worker protocol is enabled:
start/connect worker I/O adapter
send InitializeWorker through worker I/O adapter
-> Initializing
Initializing

View file

@ -160,7 +160,7 @@ Accepted runtime fields:
- `[runtime].max_tokens`
`[runtime].max_tokens` sets the maximum number of tokens requested for each
prompt submission. It must be a positive integer.
prompt submission. A value of `0` implies no specified limit.
Accepted observability fields:
@ -352,16 +352,13 @@ Default values:
- provider: `process`;
- pipeline stages: `1`;
- max tokens: `64`;
- max tokens: `0`;
- dump logs: disabled;
- cached model: disabled unless `--cached-model` is supplied;
- rebuild: enabled unless `--skip-rebuild` is supplied.
Invalid values must fail before runtime preparation begins.
`[runtime].max_tokens` must be greater than zero. Zero and invalid values are
configuration errors detected before runtime preparation.
For provider `process`, no node image is required.
For provider `docker`, an image reference is required. It may be local or remote.

View file

@ -5,8 +5,6 @@ pub mod actors;
pub mod arena_manager;
pub mod bootstrap_datastream;
pub mod config;
#[cfg(feature = "local-e2e")]
pub mod dashboard_view;
pub mod device_bridge;
pub mod distribution_stack;
pub mod docker_cluster_provisioning;
@ -19,6 +17,7 @@ pub mod gpu_worker_egress_producer;
pub mod gpu_worker_ingress_parser;
pub mod gpu_worker_process_adapter;
pub mod membership_pool_readiness;
pub mod mvp_chat;
pub mod node_boot_lifecycle;
pub mod node_image;
pub mod node_provisioning;

View file

@ -0,0 +1,317 @@
use std::io::{self, BufRead, Write};
use std::path::PathBuf;
use std::sync::Arc;
use datastream::{DatastreamEndpoint, Lifetime, StreamDescriptor, StreamId, StreamOrigin};
use swactor::Error;
use tokio::sync::Notify;
type Result<T> = std::result::Result<T, swactor::Error>;
#[derive(Clone)]
enum ChatSignals {}
const MVP_CHAT_DATASTREAM_NODE: &str = "mvp-chat";
const MVP_CHAT_DATASTREAM_LABEL: &str = "mvp chat";
struct MvpChatDatastream {
endpoint: Arc<DatastreamEndpoint>,
wake: Arc<Notify>,
}
impl MvpChatDatastream {
fn new(run_id: u64) -> Self {
let stream = StreamId::new(MVP_CHAT_DATASTREAM_NODE, Lifetime(run_id));
let endpoint = DatastreamEndpoint::with_descriptor(
StreamDescriptor {
stream,
label: Some(MVP_CHAT_DATASTREAM_LABEL.to_owned()),
origin: StreamOrigin::Orchestrator,
},
4096,
1024,
);
Self {
endpoint: Arc::new(endpoint),
wake: Arc::new(Notify::new()),
}
}
fn spawn_handler(&self, handle: &tokio::runtime::Handle) -> tokio::task::JoinHandle<()> {
let endpoint = Arc::clone(&self.endpoint);
let wake = Arc::clone(&self.wake);
handle.spawn(async move {
loop {
wake.notified().await;
while endpoint.tick().drained != 0 {}
}
})
}
}
#[derive(Debug, PartialEq, Eq)]
enum PromptStep {
Response(String),
Ignore,
Exit,
}
fn read_prompt_step<R>(input: &mut R) -> PromptStep
where
R: BufRead,
{
let mut line = String::new();
match input.read_line(&mut line) {
Ok(0) => return PromptStep::Exit,
Ok(_) => {}
Err(_) => return PromptStep::Exit,
}
let prompt = line.trim_end().to_owned();
if prompt.trim().is_empty() {
return PromptStep::Ignore;
}
PromptStep::Response(prompt_response(&prompt))
}
fn prompt_response(prompt: &str) -> String {
format!("Hello, {prompt}!")
}
fn run_prompt_loop() -> Result<()> {
let stdin = io::stdin();
let mut input = stdin.lock();
let stdout = io::stdout();
let mut output = stdout.lock();
run_prompt_loop_with_io(&mut input, &mut output)
}
fn run_prompt_loop_with_io<R, W>(input: &mut R, output: &mut W) -> Result<()>
where
R: BufRead,
W: Write,
{
loop {
write!(output, "prompt:> ")
.map_err(|error| Error::from(format!("write prompt marker: {error}")))?;
output
.flush()
.map_err(|error| Error::from(format!("flush prompt marker: {error}")))?;
match read_prompt_step(input) {
PromptStep::Response(response) => {
writeln!(output, "{response}")
.map_err(|error| Error::from(format!("write prompt response: {error}")))?;
}
PromptStep::Ignore => continue,
PromptStep::Exit => return Ok(()),
}
}
}
enum ChatRuntimeEvent {
PromptExited(Result<()>),
PromptPanicked,
CtrlC(std::io::Result<()>),
}
struct PromptLoop {
join: Option<std::thread::JoinHandle<()>>,
}
impl PromptLoop {
fn spawn(events: tokio::sync::mpsc::UnboundedSender<ChatRuntimeEvent>) -> Result<Self> {
let join = std::thread::Builder::new()
.name("mvp-chat-prompt".to_owned())
.spawn(move || {
let event = match std::panic::catch_unwind(run_prompt_loop) {
Ok(result) => ChatRuntimeEvent::PromptExited(result),
Err(_) => ChatRuntimeEvent::PromptPanicked,
};
let _ = events.send(event);
})
.map_err(|error| Error::from(format!("spawn prompt loop: {error}")))?;
Ok(Self { join: Some(join) })
}
fn join_finished(&mut self) -> Result<()> {
let Some(join) = self.join.take() else {
return Ok(());
};
join.join()
.map_err(|_| Error::from("prompt loop thread panicked".to_owned()))
}
fn detach(mut self) {
let _ = self.join.take();
}
}
fn spawn_ctrl_c_reporter(
handle: &tokio::runtime::Handle,
events: tokio::sync::mpsc::UnboundedSender<ChatRuntimeEvent>,
) {
handle.spawn(async move {
let result = tokio::signal::ctrl_c().await;
let _ = events.send(ChatRuntimeEvent::CtrlC(result));
});
}
fn join_dashboard_http(result: std::result::Result<(), tokio::task::JoinError>) -> Result<()> {
result.map_err(|error| Error::from(format!("dashboard HTTP task failed: {error}")))
}
fn join_datastream_handler(result: std::result::Result<(), tokio::task::JoinError>) -> Result<()> {
result.map_err(|error| Error::from(format!("datastream handler task failed: {error}")))
}
async fn supervise_chat_runtime(
dashboard: &dashboard::DashboardHandle,
mut dashboard_http: tokio::task::JoinHandle<()>,
mut datastream_handler: tokio::task::JoinHandle<()>,
prompt_loop: &mut PromptLoop,
events: &mut tokio::sync::mpsc::UnboundedReceiver<ChatRuntimeEvent>,
) -> Result<()> {
tokio::select! {
result = &mut dashboard_http => {
join_dashboard_http(result)?;
Err("dashboard HTTP server exited before shutdown"
.to_owned()
.into())
}
result = &mut datastream_handler => {
join_datastream_handler(result)?;
Err("datastream handler exited before shutdown"
.to_owned()
.into())
}
event = events.recv() => {
let event = event.ok_or_else(|| Error::from("runtime event channel closed".to_owned()))?;
let run_result = match event {
ChatRuntimeEvent::PromptExited(prompt_result) => {
match prompt_loop.join_finished() {
Ok(()) => prompt_result,
Err(error) => Err(error),
}
}
ChatRuntimeEvent::PromptPanicked => {
match prompt_loop.join_finished() {
Ok(()) => Err("prompt loop panicked".to_owned().into()),
Err(error) => Err(error),
}
}
ChatRuntimeEvent::CtrlC(result) => {
result
.map_err(|error| Error::from(format!("ctrl-c handler failed: {error}")))
.map(|_| ())
}
};
dashboard.shutdown();
join_dashboard_http(dashboard_http.await)?;
run_result
}
}
}
pub fn run_from_args<I>(args: I) -> Result<()>
where
I: IntoIterator<Item = String>,
{
let mut config_path: Option<PathBuf> = None;
let mut provider_selector: Option<&'static str> = None;
let mut args = args.into_iter();
while let Some(arg) = args.next() {
match arg.as_str() {
"--process" | "--docker" | "--vastai" => {
let selected = match arg.as_str() {
"--process" => "process",
"--docker" => "docker",
"--vastai" => "vastai",
_ => unreachable!(),
};
if provider_selector.replace(selected).is_some() {
return Err("conflicting provider selectors; use exactly one of --process, --docker, or --vastai".to_owned().into());
}
}
"--config" => {
let value = args
.next()
.ok_or_else(|| "--config requires a path".to_owned())?;
config_path = Some(PathBuf::from(value));
}
"--yes" | "-y" | "--dump-logs" | "--cached-model" | "--skip-rebuild" => {}
"--pipeline-stages" => {
let value = args
.next()
.ok_or_else(|| "--pipeline-stages requires a value".to_owned())?;
let stages = value
.parse::<u32>()
.map_err(|error| format!("parse --pipeline-stages: {error}"))?;
if stages == 0 {
return Err("--pipeline-stages must be greater than 0".into());
}
}
value if value.starts_with("--dump-logs=") => {
if value["--dump-logs=".len()..].is_empty() {
return Err("--dump-logs= requires a path".into());
}
}
value if value.starts_with("--cached-model=") => {
if value["--cached-model=".len()..].is_empty() {
return Err("--cached-model= requires a path".into());
}
}
value => return Err(format!("unknown mvp-chat argument: {value}").into()),
}
}
let loaded_config = crate::config::TomlConfigOverlay::load(config_path.as_deref())?;
let run_id = loaded_config.overlay.runtime.run_id.unwrap_or(0);
let rt = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.map_err(|e| Error::from(e.to_string()))?;
let chat_datastream = MvpChatDatastream::new(run_id);
let datastream_handler = chat_datastream.spawn_handler(rt.handle());
let _swactor = swactor::runtime::Runtime::new(swactor::runtime::RuntimeConfig {
num_threads: 1,
..Default::default()
});
let _chat_inbox = _swactor.new_inbox::<ChatSignals>()?;
let dashboard = dashboard::DashboardHandle::new(dashboard::DashboardConfig::default());
let dashboard_http = dashboard.spawn_http(rt.handle());
let (events_tx, mut events_rx) = tokio::sync::mpsc::unbounded_channel();
spawn_ctrl_c_reporter(rt.handle(), events_tx.clone());
let mut prompt_loop = PromptLoop::spawn(events_tx)?;
let result = rt.block_on(supervise_chat_runtime(
&dashboard,
dashboard_http,
datastream_handler,
&mut prompt_loop,
&mut events_rx,
));
if result.is_err() {
prompt_loop.detach();
}
result
}

View file

@ -0,0 +1,3 @@
fn main() {
mvp_system::mvp_chat::run_from_args(std::env::args().skip(1)).expect("failed");
}

View file

@ -5,12 +5,9 @@ edition = "2024"
[dependencies]
swactor = { path = "../..", default-features = false, features = ["no_random"] }
datastream = { path = "../datastream" }
crossbeam-queue = "0.3.12"
libc = "0.2"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
serde_yaml = "0.9"
[dev-dependencies]
proptest = "1"
proptest-state-machine = "0.3"

View file

@ -0,0 +1,327 @@
# Swactor Managed Process Specification
**Status:** current contract for `swactor-process`.
`swactor-process` provides a Swactor actor interface for launching, supervising,
stopping, and observing one operating-system child process per process actor.
The crate owns process lifecycle/control only. Child stdin/stdout/stderr are not
managed or observed by this crate.
---
## 1. Public API
The crate exports:
```text
ProcessSpec
ProcessLifecycleObservability
ProcessOutputConfig
ProcessCommand
ProcessOutput
ExitStatus
spawn_local_process
send_process_command
```
Pipeline and YAML exports are separate crate features and are not part of the
managed-process protocol described here.
### 1.1 ProcessSpec
```text
ProcessSpec {
command: String,
args: Vec<String>,
env: HashMap<String, String>,
working_dir: Option<PathBuf>,
label: Option<String>,
}
```
`command` is passed directly to `std::process::Command::new`.
`args` are passed as direct argv entries. The crate does not split, quote,
unquote, expand, or shell-parse argument strings.
`env` contains child environment overrides.
`working_dir`, when present, is passed as the child current working directory.
`label`, when present, is the lifecycle datastream label source. When `label` is
absent, the label source is the basename of `command`.
### 1.2 ProcessOutputConfig
```text
ProcessOutputConfig::disabled(upstream: ActorAddress) -> ProcessOutputConfig
ProcessOutputConfig::datastream_mirror(
upstream: ActorAddress,
producer: DatastreamProducer,
) -> ProcessOutputConfig
ProcessOutputConfig::upstream(&self) -> ActorAddress
ProcessOutputConfig::observability(&self) -> ProcessLifecycleObservability
```
`upstream` is the actor address that receives every public `ProcessOutput`.
`disabled` sends only upstream `ProcessOutput`.
`datastream_mirror` sends upstream `ProcessOutput` and also mirrors each
lifecycle/control output to one datastream channel.
### 1.3 ProcessLifecycleObservability
```text
ProcessLifecycleObservability::Disabled
ProcessLifecycleObservability::DatastreamMirror
```
This setting controls lifecycle/control mirroring only. It does not enable child
stdin/stdout/stderr handling.
### 1.4 ProcessCommand
```text
ProcessCommand::Stop {
kill_after: Option<Duration>,
}
```
`Stop` asks the supervisor to terminate the child process. `kill_after`, when
present, is the grace duration before kill escalation.
### 1.5 ProcessOutput
```text
ProcessOutput::Started { pid: u32 }
ProcessOutput::SpawnFailed { error: String }
ProcessOutput::Exited { status: ExitStatus }
ProcessOutput::Error { error: String }
```
`Started` means the OS child spawned and `pid` is the child process id.
`SpawnFailed` means the process actor was created but the OS child did not spawn.
`Exited` means the OS child reached a terminal status.
`Error` means the process supervisor or process actor hit an operational failure
other than OS spawn failure.
### 1.6 ExitStatus
```text
ExitStatus::Code(i32)
ExitStatus::Signal(i32)
ExitStatus::Unknown
```
### 1.7 Spawn and command helpers
```text
spawn_local_process(
ctx: &Ctx,
sender: &ExternalSender,
spec: ProcessSpec,
output: ProcessOutputConfig,
) -> Result<ActorAddress, Error>
```
`spawn_local_process` validates lifecycle output configuration, creates one
process actor, wires its private supervisor wake path, and returns the process
actor address.
Success from `spawn_local_process` means the process actor was created. It does
not mean the OS child spawned successfully. OS spawn success or failure is
reported later as `ProcessOutput`.
```text
send_process_command(
sender: &ExternalSender,
process: ActorAddress,
command: ProcessCommand,
) -> Result<(), Error>
```
`send_process_command` is the public helper for sending process commands. It
wraps the public `ProcessCommand` in the actor's private mailbox type.
---
## 2. Runtime topology
One process actor owns one OS child process lifecycle.
The actor owns:
- lifecycle state;
- the configured upstream output address;
- optional lifecycle datastream mirror state;
- a private supervisor thread handle.
The private supervisor thread owns:
- the child process handle and pid;
- blocking-prone child exit polling;
- terminate/kill signal delivery;
- the stop kill deadline.
The child process owns its own execution.
The supervisor thread is not a public actor and not a public extension point.
---
## 3. Child spawn behavior
The supervisor starts the child with:
```text
Command::new(&spec.command)
cmd.args(&spec.args)
cmd.env(key, value) for each spec.env entry
cmd.current_dir(dir) when spec.working_dir is Some(dir)
cmd.stdin(Stdio::null())
cmd.stdout(Stdio::null())
cmd.stderr(Stdio::null())
cmd.spawn()
```
The crate does not invoke a shell unless the caller explicitly sets `command` to
a shell executable and supplies shell arguments.
Child stdin/stdout/stderr are connected to null handles. The managed-process
protocol does not expose stdin writes, stdout/stderr output events, PTY resize,
or arbitrary signal commands.
If `cmd.spawn()` fails, the actor emits exactly one terminal
`ProcessOutput::SpawnFailed { error }` and does not emit `Started`, `Exited`, or
`Error` for that spawn failure.
---
## 4. Lifecycle output delivery
The actor sends each public `ProcessOutput` to the configured upstream actor.
When `ProcessOutputConfig::datastream_mirror` is used, the actor also mirrors
each output to the configured datastream producer. Datastream submit failure is
ignored and does not suppress upstream output or emit `ProcessOutput::Error`.
Public lifecycle/control output order follows observed lifecycle:
- successful spawn emits `Started` before any terminal `Exited`;
- spawn failure emits `SpawnFailed` without `Started` or `Exited`;
- supervisor failure emits `Error`;
- after a terminal output, later public stop attempts emit no additional
`ProcessOutput`.
---
## 5. Lifecycle datastream labels and records
The label source is:
1. `ProcessSpec::label`, when present;
2. otherwise, the final non-empty path segment of `ProcessSpec::command`;
3. otherwise, the full `command` string.
The label sanitizer:
- trims source whitespace;
- lowercases ASCII alphanumeric characters;
- preserves `_` and `-`;
- replaces every other character with `_`;
- collapses repeated `_`;
- trims leading and trailing `_`;
- rejects an empty sanitized result with:
```text
invalid process lifecycle label: empty segment
```
The lifecycle channel name is:
```text
proc.<label>.lifecycle
```
When lifecycle mirroring is enabled, the channel is registered as:
```text
ChannelContent::JsonRecord {
schema: Some("swactor_process.lifecycle.v1")
}
```
Duplicate lifecycle labels on the same datastream stream are rejected before the
process actor is spawned with an error containing:
```text
duplicate process lifecycle datastream channel: proc.<label>.lifecycle on stream <stream>
```
Lifecycle JSON records are:
```json
{"event":"started","pid":123}
{"event":"spawn_failed","error":"..."}
{"event":"exited","status":{"kind":"code","value":0}}
{"event":"exited","status":{"kind":"signal","value":9}}
{"event":"exited","status":{"kind":"unknown"}}
{"event":"error","error":"..."}
```
The managed-process core registers no `proc.<label>.stdout` or
`proc.<label>.stderr` channels.
---
## 6. Stop semantics
Use `send_process_command` with `ProcessCommand::Stop { kill_after }` to request
shutdown.
If stop is requested before the child reports successful spawn, the actor queues
the stop intent. When the child later starts, the actor still emits `Started`
first, asks the supervisor to terminate the child, and later emits terminal
`Exited` unless a supervisor failure occurs.
If stop is requested before an OS spawn failure is reported, the actor still
emits only `SpawnFailed` for that failed spawn.
If the child is running, stop sends terminate to the child process.
If `kill_after` is `Some(duration)`, the supervisor sends kill after that
deadline if the child has not exited.
If `kill_after` is `None`, the supervisor does not schedule kill escalation.
Duplicate stop while already stopping is a no-op. It does not tighten, extend,
or replace the original kill deadline.
Stop after terminal output is a no-op if the actor is still alive. If the actor
has already stopped, sending the command may fail at the runtime address layer;
that failure does not produce process output.
---
## 7. Actor and supervisor cleanup
The actor uses a private mailbox wrapper so supervisor wake messages and public
process commands share one actor input type without changing the Swactor runtime.
The actor drains supervisor events when it starts, when it receives a supervisor
wake, and before/after applying a public stop command.
The supervisor sends a private `ThreadFinished` event when its thread reaches the
end of process supervision. The actor stops itself only after a terminal state is
recorded and the supervisor handle has been cleared.
If the actor stops while the supervisor is still present, it sends best-effort
shutdown to the supervisor without blocking for a waiting join.
Dropping the supervisor handle sends best-effort shutdown and joins only when
the thread is already finished.

View file

@ -1,53 +0,0 @@
use std::time::Duration;
use crate::types::{ExitStatus, ProcessError, ProcessSpec, PtySize, Signal};
use swactor::actor::ActorAddress;
/// Which output stream produced data.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum OutputStream {
Stdout,
Stderr,
}
/// Actions emitted by ProcessSession for the driver or actor layer to execute.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ProcessAction {
// --- Driver commands ---
/// Spawn the process described by the spec.
SpawnProcess { spec: ProcessSpec },
/// Write bytes to the process's stdin.
WriteStdin { data: Vec<u8> },
/// Send a signal to the process.
SendSignal { signal: Signal },
/// Resize the process's PTY.
ResizePty { size: PtySize },
/// Close the process's stdin pipe.
CloseStdin,
/// Schedule a kill timeout that fires KillTimeout after the given duration.
ScheduleKillTimeout { duration: Duration },
// --- Subscriber notifications ---
/// Notify subscribers that the process started.
NotifyStarted { subscribers: Vec<ActorAddress> },
/// Notify subscribers of output.
NotifyOutput {
subscribers: Vec<ActorAddress>,
data: Vec<u8>,
stream: OutputStream,
},
/// Notify subscribers that the process exited.
NotifyExited {
subscribers: Vec<ActorAddress>,
status: ExitStatus,
},
/// Notify subscribers of an error.
NotifyError {
subscribers: Vec<ActorAddress>,
error: ProcessError,
},
// --- Lifecycle ---
/// The session is done; the owning actor should stop itself.
SelfTerminate,
}

View file

@ -1,174 +1,283 @@
use std::sync::{Arc, OnceLock};
use swactor::actor::{ActorAddress, ActorInterface, Ctx};
use swactor::process_observer::ProcessOutputObserver;
use swactor::runtime::ExternalSender;
use crate::action::{OutputStream, ProcessAction};
use crate::event::ProcessEvent;
use crate::message::{ProcessCommand, ProcessNotification};
use crate::session::ProcessSession;
use crate::types::{ProcessDriver, ProcessWaker};
use crate::lifecycle::PreparedProcessOutput;
use crate::message::{ProcessActorCommand, ProcessCommand, ProcessOutput};
use crate::supervisor::{
ProcessSupervisorThread, ProcessThreadHandle, ThreadEvent, ThreadEventReceiver,
thread_event_channel,
};
use crate::types::{ExitStatus, ProcessSpec};
/// Actor wrapper around a `ProcessSession` and its driver.
///
/// Generic over `D: ProcessDriver` so that tests can use `MockDriver` or
/// `TestDriver` while production uses `LocalDriver`.
pub struct ProcessActor<D: ProcessDriver> {
session: ProcessSession,
driver: D,
self_addr: Option<ActorAddress>,
/// Actions from `ProcessSession::new()`, executed in `on_start`.
deferred_actions: Option<Vec<ProcessAction>>,
/// Shared slot for the waker — filled after the actor address is known.
pub waker_slot: Arc<OnceLock<ProcessWaker>>,
/// Per-node observer that taps this process's output (command-basename
/// `label`) onto the node's telemetry stream. `None` when the runtime has
/// no observer installed.
output_observer: Option<Arc<dyn ProcessOutputObserver>>,
/// Command basename used to label this process's output to the observer.
label: String,
enum ProcessActorState {
Spawning { stop_requested: bool },
Running,
Stopping,
Done(ProcessDoneState),
}
impl<D: ProcessDriver> ProcessActor<D> {
pub fn new(
session: ProcessSession,
driver: D,
initial_actions: Vec<ProcessAction>,
waker_slot: Arc<OnceLock<ProcessWaker>>,
output_observer: Option<Arc<dyn ProcessOutputObserver>>,
label: String,
enum ProcessDoneState {
Exited(ExitStatus),
SpawnFailed(String),
SupervisorFailed(String),
}
impl ProcessDoneState {
fn observe(&self) {
match self {
Self::Exited(status) => {
let _ = *status;
}
Self::SpawnFailed(error) | Self::SupervisorFailed(error) => {
let _ = error.as_str();
}
}
}
}
pub(crate) struct ProcessActor {
spec: Option<ProcessSpec>,
output: PreparedProcessOutput,
sender: ExternalSender,
addr_slot: Arc<OnceLock<ActorAddress>>,
state: ProcessActorState,
pid: Option<u32>,
supervisor: Option<ProcessThreadHandle>,
supervisor_events: Option<ThreadEventReceiver>,
}
impl ProcessActor {
pub(crate) fn new(
spec: ProcessSpec,
output: PreparedProcessOutput,
sender: ExternalSender,
addr_slot: Arc<OnceLock<ActorAddress>>,
) -> Self {
Self {
session,
driver,
self_addr: None,
deferred_actions: Some(initial_actions),
waker_slot,
output_observer,
label,
spec: Some(spec),
output,
sender,
addr_slot,
state: ProcessActorState::Spawning {
stop_requested: false,
},
pid: None,
supervisor: None,
supervisor_events: None,
}
}
/// Drain events from the driver, apply each to the session, and dispatch
/// all resulting actions.
fn drain_and_dispatch(&mut self, ctx: &Ctx) {
let events = self.driver.poll();
fn emit_process_output(&self, ctx: &Ctx, output: ProcessOutput) {
let _ = ctx.send(self.output.upstream, output.clone());
if let Some(mirror) = &self.output.mirror {
mirror.submit(&output);
}
}
fn apply_stop(&mut self, ctx: &Ctx, kill_after: Option<std::time::Duration>) {
let mut next_state = None;
let mut terminal_error = None;
match &mut self.state {
ProcessActorState::Spawning { stop_requested } => {
if *stop_requested {
return;
}
match self
.supervisor
.as_ref()
.expect("supervisor started before commands")
.stop(kill_after)
{
Ok(()) => *stop_requested = true,
Err(error) => terminal_error = Some(error),
}
}
ProcessActorState::Running => {
match self
.supervisor
.as_ref()
.expect("supervisor started before commands")
.stop(kill_after)
{
Ok(()) => next_state = Some(ProcessActorState::Stopping),
Err(error) => terminal_error = Some(error),
}
}
ProcessActorState::Stopping | ProcessActorState::Done(_) => {}
}
if let Some(state) = next_state {
self.state = state;
}
if let Some(error) = terminal_error {
self.emit_terminal_error(ctx, error);
}
}
fn drain_supervisor_events(&mut self, ctx: &Ctx) {
let events = self
.supervisor_events
.as_ref()
.map(ThreadEventReceiver::drain)
.unwrap_or_default();
for event in events {
let actions = self.session.apply(event);
self.dispatch_actions(ctx, actions);
self.handle_thread_event(ctx, event);
self.finish_if_safe(ctx);
}
self.finish_if_safe(ctx);
}
/// Execute actions produced by the session state machine.
fn dispatch_actions(&mut self, ctx: &Ctx, actions: Vec<ProcessAction>) {
let self_addr = self.self_addr.expect("self_addr not set");
for action in actions {
match action {
// Driver commands — forward to the driver
ProcessAction::SpawnProcess { .. }
| ProcessAction::WriteStdin { .. }
| ProcessAction::SendSignal { .. }
| ProcessAction::ResizePty { .. }
| ProcessAction::CloseStdin
| ProcessAction::ScheduleKillTimeout { .. } => {
self.driver.execute(action);
}
fn handle_thread_event(&mut self, ctx: &Ctx, event: ThreadEvent) {
match event {
ThreadEvent::Started { pid } => {
let stop_requested = match &self.state {
ProcessActorState::Spawning { stop_requested } => *stop_requested,
ProcessActorState::Running
| ProcessActorState::Stopping
| ProcessActorState::Done(_) => return,
};
// Subscriber notifications — send to each subscriber
ProcessAction::NotifyStarted { subscribers } => {
let notif = ProcessNotification::Started {
process: self_addr,
pid: self.driver.pid(),
};
for sub in subscribers {
let _ = ctx.send(sub, notif.clone());
self.pid = Some(pid);
self.emit_process_output(ctx, ProcessOutput::Started { pid });
self.state = if stop_requested {
ProcessActorState::Stopping
} else {
ProcessActorState::Running
};
}
ThreadEvent::SpawnFailed { error } => {
if !matches!(self.state, ProcessActorState::Done(_)) {
self.emit_process_output(
ctx,
ProcessOutput::SpawnFailed {
error: error.clone(),
},
);
self.state = ProcessActorState::Done(ProcessDoneState::SpawnFailed(error));
}
}
ThreadEvent::Exited { status } => {
if !matches!(self.state, ProcessActorState::Done(_)) {
self.emit_process_output(ctx, ProcessOutput::Exited { status });
self.state = ProcessActorState::Done(ProcessDoneState::Exited(status));
}
}
ThreadEvent::Error { error } => {
if !matches!(self.state, ProcessActorState::Done(_)) {
self.emit_process_output(
ctx,
ProcessOutput::Error {
error: error.clone(),
},
);
self.state = ProcessActorState::Done(ProcessDoneState::SupervisorFailed(error));
}
}
ThreadEvent::ThreadFinished => {
if let Some(supervisor) = self.supervisor.as_mut() {
match supervisor.join_if_finished() {
Ok(_) => {}
Err(_) => {
if !matches!(self.state, ProcessActorState::Done(_)) {
let error = "process supervisor thread failed".to_owned();
self.emit_process_output(
ctx,
ProcessOutput::Error {
error: error.clone(),
},
);
self.state = ProcessActorState::Done(
ProcessDoneState::SupervisorFailed(error),
);
}
}
}
}
ProcessAction::NotifyOutput {
subscribers,
data,
stream,
} => {
// Tap the node's telemetry observer before the data is moved
// into the subscriber notification. Per-node, auto-attached.
if let Some(obs) = &self.output_observer {
obs.on_output(&self.label, matches!(stream, OutputStream::Stderr), &data);
}
let notif = ProcessNotification::Output {
process: self_addr,
data,
stream,
};
for sub in subscribers {
let _ = ctx.send(sub, notif.clone());
}
}
ProcessAction::NotifyExited {
subscribers,
status,
} => {
let notif = ProcessNotification::Exited {
process: self_addr,
status,
};
for sub in subscribers {
let _ = ctx.send(sub, notif.clone());
}
}
ProcessAction::NotifyError { subscribers, error } => {
let notif = ProcessNotification::Error {
process: self_addr,
error,
};
for sub in subscribers {
let _ = ctx.send(sub, notif.clone());
}
}
// Lifecycle
ProcessAction::SelfTerminate => {
ctx.stop_self();
}
self.supervisor = None;
self.supervisor_events = None;
}
}
}
/// Map a `ProcessCommand` to the corresponding `ProcessEvent`.
fn command_to_event(cmd: ProcessCommand) -> Option<ProcessEvent> {
match cmd {
ProcessCommand::WriteStdin { data } => Some(ProcessEvent::WriteStdin { data }),
ProcessCommand::SendSignal { signal } => Some(ProcessEvent::SendSignal { signal }),
ProcessCommand::ResizePty { size } => Some(ProcessEvent::ResizePty { size }),
ProcessCommand::CloseStdin => Some(ProcessEvent::CloseStdin),
ProcessCommand::Close => Some(ProcessEvent::CloseRequested),
ProcessCommand::Subscribe { address } => Some(ProcessEvent::Subscribe { address }),
ProcessCommand::Unsubscribe { address } => Some(ProcessEvent::Unsubscribe { address }),
ProcessCommand::PollTick => None, // handled by drain
fn finish_if_safe(&mut self, ctx: &Ctx) {
if let ProcessActorState::Done(done) = &self.state {
done.observe();
let _ = self.pid;
if self.supervisor.is_none() {
ctx.stop_self();
}
}
}
fn emit_terminal_error(&mut self, ctx: &Ctx, error: String) {
if matches!(self.state, ProcessActorState::Done(_)) {
return;
}
self.emit_process_output(
ctx,
ProcessOutput::Error {
error: error.clone(),
},
);
self.state = ProcessActorState::Done(ProcessDoneState::SupervisorFailed(error));
ctx.stop_self();
}
}
impl<D: ProcessDriver + 'static> ActorInterface for ProcessActor<D> {
type Incoming = ProcessCommand;
impl ActorInterface for ProcessActor {
type Incoming = ProcessActorCommand;
type Response = ();
fn on_start(&mut self, ctx: &Ctx) {
self.self_addr = Some(ctx.self_addr());
if let Some(actions) = self.deferred_actions.take() {
self.dispatch_actions(ctx, actions);
let sender = self.sender.clone();
let addr_slot = self.addr_slot.clone();
let (event_sink, event_receiver) = thread_event_channel(move || {
if let Some(addr) = addr_slot.get() {
let _ = sender.send_to(*addr, ProcessActorCommand::SupervisorWake);
}
});
let spec = self.spec.take().expect("process spec already taken");
match ProcessSupervisorThread::start(spec, event_sink) {
Ok(supervisor) => {
self.supervisor = Some(supervisor);
self.supervisor_events = Some(event_receiver);
self.drain_supervisor_events(ctx);
}
Err(err) => {
let error = format!("process supervisor thread failed: {err}");
self.emit_process_output(
ctx,
ProcessOutput::Error {
error: error.clone(),
},
);
self.state = ProcessActorState::Done(ProcessDoneState::SupervisorFailed(error));
ctx.stop_self();
}
}
}
fn handle(&mut self, ctx: &Ctx, msg: ProcessCommand) {
// Process the incoming command first — this ensures Subscribe
// registers before drain dispatches notifications, and keeps
// user commands (Close, WriteStdin) responsive.
if let Some(event) = Self::command_to_event(msg) {
let actions = self.session.apply(event);
self.dispatch_actions(ctx, actions);
fn handle(&mut self, ctx: &Ctx, msg: ProcessActorCommand) {
match msg {
ProcessActorCommand::SupervisorWake => self.drain_supervisor_events(ctx),
ProcessActorCommand::Command(ProcessCommand::Stop { kill_after }) => {
self.drain_supervisor_events(ctx);
self.apply_stop(ctx, kill_after);
self.drain_supervisor_events(ctx);
}
}
}
// Then drain pending I/O events from background threads.
self.drain_and_dispatch(ctx);
fn on_stop(&mut self, _ctx: &Ctx) {
if let Some(supervisor) = &self.supervisor {
let _ = supervisor.shutdown_now();
}
}
}

View file

@ -1,71 +0,0 @@
use crate::types::{ExitStatus, PtySize, Signal};
use swactor::actor::ActorAddress;
/// Events that can be applied to a ProcessSession.
///
/// Some events come from the driver (Started, SpawnFailed, OutputReceived, etc.),
/// others come from the owning actor (WriteStdin, SendSignal, Subscribe, etc.).
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ProcessEvent {
// --- Driver-sourced events ---
/// The process spawned successfully.
Started,
/// The kill timeout fired (process didn't exit after SIGTERM).
KillTimeout,
/// The process failed to spawn.
SpawnFailed { reason: String },
/// Output received on stdout or stderr.
OutputReceived { data: Vec<u8>, is_stderr: bool },
/// The process exited.
Exited { status: ExitStatus },
/// Connection to the process was lost unexpectedly.
ConnectionLost { reason: String },
// --- Driver acknowledgement events ---
/// Stdin bytes were successfully written.
StdinWritten { byte_count: usize },
/// A signal was delivered.
SignalSent,
/// The PTY was resized.
PtyResized,
// --- Actor-sourced events ---
/// Write data to the process's stdin.
WriteStdin { data: Vec<u8> },
/// Send a signal to the process.
SendSignal { signal: Signal },
/// Resize the process's PTY.
ResizePty { size: PtySize },
/// Close the process's stdin.
CloseStdin,
/// Request a graceful close of the process.
CloseRequested,
/// Subscribe an actor to process notifications.
Subscribe { address: ActorAddress },
/// Unsubscribe an actor from process notifications.
Unsubscribe { address: ActorAddress },
}
impl ProcessEvent {
/// Human-readable name for error messages.
pub fn name(&self) -> &'static str {
match self {
Self::Started => "Started",
Self::KillTimeout => "KillTimeout",
Self::SpawnFailed { .. } => "SpawnFailed",
Self::OutputReceived { .. } => "OutputReceived",
Self::Exited { .. } => "Exited",
Self::ConnectionLost { .. } => "ConnectionLost",
Self::StdinWritten { .. } => "StdinWritten",
Self::SignalSent => "SignalSent",
Self::PtyResized => "PtyResized",
Self::WriteStdin { .. } => "WriteStdin",
Self::SendSignal { .. } => "SendSignal",
Self::ResizePty { .. } => "ResizePty",
Self::CloseStdin => "CloseStdin",
Self::CloseRequested => "CloseRequested",
Self::Subscribe { .. } => "Subscribe",
Self::Unsubscribe { .. } => "Unsubscribe",
}
}
}

View file

@ -1,28 +1,18 @@
pub mod action;
pub mod actor;
pub mod event;
pub mod local;
pub mod message;
pub mod mock;
mod actor;
mod lifecycle;
mod message;
mod spawn;
mod supervisor;
mod types;
pub mod pipeline;
pub mod session;
pub mod spawn;
pub mod types;
pub mod yaml;
pub use action::{OutputStream, ProcessAction};
pub use actor::ProcessActor;
pub use event::ProcessEvent;
pub use local::LocalDriver;
pub use message::{ProcessCommand, ProcessNotification};
pub use mock::MockDriver;
pub use lifecycle::{ProcessLifecycleObservability, ProcessOutputConfig};
pub use message::{ProcessCommand, ProcessOutput};
pub use pipeline::{
JobComplete, JobDefinition, JobFailure, JobId, JobProgress, JobStatus, JobSuccess,
LocalPipelineConfig, LocalStartJob, PipelineId, PipelineStatus,
};
pub use session::{ProcessSession, ProcessState};
pub use spawn::{spawn_local_process, spawn_process};
pub use types::{
EventQueue, ExitStatus, FlowControl, ProcessDriver, ProcessError, ProcessMode, ProcessSpec,
ProcessWaker, PtySize, Signal,
};
pub use spawn::{send_process_command, spawn_local_process};
pub use types::{ExitStatus, ProcessSpec};

View file

@ -0,0 +1,321 @@
use std::collections::HashSet;
use std::sync::{Mutex, OnceLock};
use datastream::{ChannelContent, DatastreamProducer, StreamId};
use serde_json::json;
use swactor::actor::ActorAddress;
use crate::message::ProcessOutput;
use crate::types::{ExitStatus, ProcessSpec};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ProcessLifecycleObservability {
Disabled,
DatastreamMirror,
}
#[derive(Clone)]
pub struct ProcessOutputConfig {
upstream: ActorAddress,
observability: ProcessLifecycleObservability,
datastream: Option<DatastreamProducer>,
}
impl ProcessOutputConfig {
pub fn disabled(upstream: ActorAddress) -> Self {
Self {
upstream,
observability: ProcessLifecycleObservability::Disabled,
datastream: None,
}
}
pub fn datastream_mirror(upstream: ActorAddress, producer: DatastreamProducer) -> Self {
Self {
upstream,
observability: ProcessLifecycleObservability::DatastreamMirror,
datastream: Some(producer),
}
}
pub fn upstream(&self) -> ActorAddress {
self.upstream
}
pub fn observability(&self) -> ProcessLifecycleObservability {
self.observability
}
}
pub(crate) struct PreparedProcessOutput {
pub(crate) upstream: ActorAddress,
pub(crate) mirror: Option<LifecycleDatastreamMirror>,
pub(crate) _label_reservation: Option<LifecycleLabelReservation>,
}
pub(crate) struct LifecycleDatastreamMirror {
channel: datastream::ChannelId,
producer: DatastreamProducer,
}
impl LifecycleDatastreamMirror {
pub(crate) fn submit(&self, output: &ProcessOutput) {
let payload = match output {
ProcessOutput::Started { pid } => json!({"event": "started", "pid": pid}),
ProcessOutput::SpawnFailed { error } => {
json!({"event": "spawn_failed", "error": error})
}
ProcessOutput::Exited { status } => match status {
ExitStatus::Code(value) => {
json!({"event": "exited", "status": {"kind": "code", "value": value}})
}
ExitStatus::Signal(value) => {
json!({"event": "exited", "status": {"kind": "signal", "value": value}})
}
ExitStatus::Unknown => json!({"event": "exited", "status": {"kind": "unknown"}}),
},
ProcessOutput::Error { error } => json!({"event": "error", "error": error}),
};
let bytes = serde_json::to_vec(&payload).expect("process lifecycle record serializes");
let _ = self.producer.submit_bytes(self.channel, bytes);
}
}
pub(crate) fn prepare_process_output(
spec: &ProcessSpec,
config: ProcessOutputConfig,
) -> Result<PreparedProcessOutput, swactor::Error> {
let label = derive_lifecycle_label(spec)?;
match config.observability {
ProcessLifecycleObservability::Disabled => Ok(PreparedProcessOutput {
upstream: config.upstream,
mirror: None,
_label_reservation: None,
}),
ProcessLifecycleObservability::DatastreamMirror => {
let producer = config
.datastream
.expect("datastream mirror config stores producer");
let reservation =
LifecycleLabelReservation::reserve(producer.stream_id().clone(), &label)?;
let channel_name = label.channel_name();
let channel = producer
.try_register_channel(
channel_name,
ChannelContent::JsonRecord {
schema: Some("swactor_process.lifecycle.v1".to_owned()),
},
)
.map_err(|err| match err {
datastream::ChannelRegistrationError::ConflictingName { name } => {
swactor::Error::from(format!(
"conflicting datastream channel registration for {name}"
))
}
})?;
Ok(PreparedProcessOutput {
upstream: config.upstream,
mirror: Some(LifecycleDatastreamMirror { channel, producer }),
_label_reservation: Some(reservation),
})
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub(crate) struct LifecycleLabel(String);
impl LifecycleLabel {
pub(crate) fn as_str(&self) -> &str {
&self.0
}
pub(crate) fn channel_name(&self) -> String {
format!("proc.{}.lifecycle", self.0)
}
}
pub(crate) fn derive_lifecycle_label(spec: &ProcessSpec) -> Result<LifecycleLabel, swactor::Error> {
let source = match spec.label.as_deref() {
Some(label) => label,
None => spec
.command
.rsplit(['/', '\\'])
.find(|segment| !segment.is_empty())
.unwrap_or(&spec.command),
};
sanitize_lifecycle_label(source)
}
fn sanitize_lifecycle_label(source: &str) -> Result<LifecycleLabel, swactor::Error> {
let mut sanitized = String::new();
let mut last_was_underscore = false;
for ch in source.trim().chars() {
let next = if ch.is_ascii_alphanumeric() {
ch.to_ascii_lowercase()
} else if ch == '_' || ch == '-' {
ch
} else {
'_'
};
if next == '_' {
if !last_was_underscore {
sanitized.push(next);
}
last_was_underscore = true;
} else {
sanitized.push(next);
last_was_underscore = false;
}
}
let sanitized = sanitized.trim_matches('_').to_owned();
if sanitized.is_empty() {
return Err(swactor::Error::from(
"invalid process lifecycle label: empty segment",
));
}
Ok(LifecycleLabel(sanitized))
}
static LIFECYCLE_LABELS: OnceLock<Mutex<HashSet<(StreamId, String)>>> = OnceLock::new();
pub(crate) struct LifecycleLabelReservation {
key: Option<(StreamId, String)>,
}
impl LifecycleLabelReservation {
fn reserve(stream: StreamId, label: &LifecycleLabel) -> Result<Self, swactor::Error> {
let key = (stream, label.as_str().to_owned());
let mut labels = LIFECYCLE_LABELS
.get_or_init(|| Mutex::new(HashSet::new()))
.lock()
.expect("process lifecycle label registry poisoned");
if !labels.insert(key.clone()) {
return Err(swactor::Error::from(format!(
"duplicate process lifecycle datastream channel: {} on stream {}",
label.channel_name(),
key.0
)));
}
Ok(Self { key: Some(key) })
}
}
impl Drop for LifecycleLabelReservation {
fn drop(&mut self) {
if let Some(key) = self.key.take() {
LIFECYCLE_LABELS
.get_or_init(|| Mutex::new(HashSet::new()))
.lock()
.expect("process lifecycle label registry poisoned")
.remove(&key);
}
}
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use datastream::{DatastreamEndpoint, Lifetime, NodeId, StreamId};
use swactor::actor::ActorAddress;
use super::*;
fn spec(command: &str, label: Option<&str>) -> ProcessSpec {
ProcessSpec {
command: command.to_owned(),
args: Vec::new(),
env: HashMap::new(),
working_dir: None,
label: label.map(str::to_owned),
}
}
#[test]
fn explicit_labels_are_sanitized() {
assert_eq!(
derive_lifecycle_label(&spec("ignored", Some("trainer.0/foo")))
.unwrap()
.as_str(),
"trainer_0_foo"
);
assert_eq!(
derive_lifecycle_label(&spec("ignored", Some(" GPU Worker ")))
.unwrap()
.as_str(),
"gpu_worker"
);
}
#[test]
fn command_basename_labels_are_sanitized() {
assert_eq!(
derive_lifecycle_label(&spec("/usr/bin/python3", None))
.unwrap()
.as_str(),
"python3"
);
assert_eq!(
derive_lifecycle_label(&spec("./bin/train.v2", None))
.unwrap()
.as_str(),
"train_v2"
);
}
#[test]
fn empty_lifecycle_labels_are_rejected() {
assert_eq!(
derive_lifecycle_label(&spec("ignored", Some("../")))
.unwrap_err()
.to_string(),
"invalid process lifecycle label: empty segment"
);
assert_eq!(
derive_lifecycle_label(&spec("", None))
.unwrap_err()
.to_string(),
"invalid process lifecycle label: empty segment"
);
}
#[test]
fn duplicate_datastream_label_reservations_are_released_on_drop() {
let endpoint = DatastreamEndpoint::new(StreamId::new(NodeId::new("stage2"), Lifetime(1)));
let upstream = ActorAddress::new_random();
let spec = spec("sh", Some("trainer.0/foo"));
let first = prepare_process_output(
&spec,
ProcessOutputConfig::datastream_mirror(upstream, endpoint.producer()),
)
.unwrap();
let err = match prepare_process_output(
&spec,
ProcessOutputConfig::datastream_mirror(upstream, endpoint.producer()),
) {
Ok(_) => panic!("duplicate lifecycle label should be rejected"),
Err(err) => err,
};
assert_eq!(
err.to_string(),
"duplicate process lifecycle datastream channel: proc.trainer_0_foo.lifecycle on stream stage2#1"
);
drop(first);
let second = prepare_process_output(
&spec,
ProcessOutputConfig::datastream_mirror(upstream, endpoint.producer()),
)
.unwrap();
drop(second);
}
}

View file

@ -1,291 +0,0 @@
use std::io::{Read, Write};
use std::process::{Child, ChildStdin, Command, Stdio};
use std::sync::{Arc, OnceLock};
use std::thread::{self, JoinHandle};
use crate::action::ProcessAction;
use crate::event::ProcessEvent;
use crate::types::EventQueue;
use crate::types::ProcessDriver;
use crate::types::ProcessWaker;
use crate::types::{ExitStatus, ProcessSpec, Signal};
// ─── Signal ────────────────────────────────────────────────────────────────
/// Map a `Signal` enum variant to the corresponding libc signal constant.
fn signal_to_libc(signal: Signal) -> libc::c_int {
match signal {
Signal::Terminate => libc::SIGTERM,
Signal::Kill => libc::SIGKILL,
Signal::Hangup => libc::SIGHUP,
Signal::Interrupt => libc::SIGINT,
Signal::Other(n) => n,
}
}
/// Send a signal to a process by PID. Returns `Ok(())` on success.
fn send_signal(pid: u32, signal: Signal) -> Result<(), String> {
let sig = signal_to_libc(signal);
// Safety: kill() is safe to call with any pid/signal combo;
// it returns -1 on error which we check.
let ret = unsafe { libc::kill(pid as libc::pid_t, sig) };
if ret == 0 {
Ok(())
} else {
Err(format!(
"kill({}, {}) failed: {}",
pid,
sig,
std::io::Error::last_os_error()
))
}
}
// ─── Pipes ─────────────────────────────────────────────────────────────────
/// Read from a pipe in a loop, pushing events to the queue and waking the actor.
///
/// Runs in a background thread. Exits when the pipe reaches EOF or errors.
fn read_pipe(
mut pipe: impl Read + Send + 'static,
is_stderr: bool,
queue: EventQueue,
waker: Arc<OnceLock<ProcessWaker>>,
) {
let mut buf = [0u8; 8192];
loop {
match pipe.read(&mut buf) {
Ok(0) => break, // EOF
Ok(n) => {
queue.push(ProcessEvent::OutputReceived {
data: buf[..n].to_vec(),
is_stderr,
});
if let Some(w) = waker.get() {
w.wake();
}
}
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
Err(_) => break,
}
}
}
// ─── Wait ──────────────────────────────────────────────────────────────────
/// Wait for a child process to exit, then push the appropriate event.
///
/// Runs in a background thread. Uses `libc::waitpid` for accurate exit status.
/// After waitpid returns, joins the pipe reader threads so all buffered
/// stdout/stderr is drained before the `Exited` event is enqueued.
fn wait_for_exit(
pid: u32,
reader_threads: Vec<JoinHandle<()>>,
queue: EventQueue,
waker: Arc<OnceLock<ProcessWaker>>,
) {
let mut status: libc::c_int = 0;
let ret = unsafe { libc::waitpid(pid as libc::pid_t, &mut status, 0) };
let exit_status = if ret < 0 {
ExitStatus::Unknown
} else {
decode_wait_status(status)
};
// Wait for pipe readers to finish draining all output before signaling exit.
// Once the process exits, its pipe ends close, so readers will hit EOF shortly.
for handle in reader_threads {
let _ = handle.join();
}
queue.push(ProcessEvent::Exited {
status: exit_status,
});
if let Some(w) = waker.get() {
w.wake();
}
}
fn decode_wait_status(status: libc::c_int) -> ExitStatus {
if libc::WIFEXITED(status) {
ExitStatus::Code(libc::WEXITSTATUS(status))
} else if libc::WIFSIGNALED(status) {
ExitStatus::Signal(libc::WTERMSIG(status))
} else {
ExitStatus::Unknown
}
}
// ─── LocalDriver ───────────────────────────────────────────────────────────
/// A `ProcessDriver` that spawns real OS subprocesses via `std::process::Command`.
///
/// Background threads read stdout/stderr and wait for process exit,
/// pushing events into a shared `EventQueue`. The actor polls via `poll()`.
pub struct LocalDriver {
queue: EventQueue,
waker_slot: Arc<OnceLock<ProcessWaker>>,
child: Option<Child>,
stdin: Option<ChildStdin>,
_wait_thread: Option<JoinHandle<()>>,
}
impl LocalDriver {
pub fn new(queue: EventQueue, waker_slot: Arc<OnceLock<ProcessWaker>>) -> Self {
Self {
queue,
waker_slot,
child: None,
stdin: None,
_wait_thread: None,
}
}
fn spawn_process(&mut self, spec: &ProcessSpec) {
let mut cmd = Command::new(&spec.command);
cmd.args(&spec.args);
for (k, v) in &spec.env {
cmd.env(k, v);
}
if let Some(ref dir) = spec.working_dir {
cmd.current_dir(dir);
}
cmd.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped());
match cmd.spawn() {
Ok(mut child) => {
let pid = child.id();
// Take the stdin handle
self.stdin = child.stdin.take();
// Spawn stdout reader thread
let mut reader_threads = Vec::new();
if let Some(stdout) = child.stdout.take() {
let queue = self.queue.clone();
let waker = self.waker_slot.clone();
reader_threads.push(
thread::Builder::new()
.name(format!("proc-{}-stdout", pid))
.spawn(move || read_pipe(stdout, false, queue, waker))
.expect("failed to spawn stdout reader"),
);
}
// Spawn stderr reader thread
if let Some(stderr) = child.stderr.take() {
let queue = self.queue.clone();
let waker = self.waker_slot.clone();
reader_threads.push(
thread::Builder::new()
.name(format!("proc-{}-stderr", pid))
.spawn(move || read_pipe(stderr, true, queue, waker))
.expect("failed to spawn stderr reader"),
);
}
// Spawn wait thread — it joins the reader threads before pushing Exited,
// ensuring all output is drained before the exit event.
let queue = self.queue.clone();
let waker = self.waker_slot.clone();
self._wait_thread = Some(
thread::Builder::new()
.name(format!("proc-{}-wait", pid))
.spawn(move || wait_for_exit(pid, reader_threads, queue, waker))
.expect("failed to spawn wait thread"),
);
self.child = Some(child);
self.queue.push(ProcessEvent::Started);
}
Err(e) => {
self.queue.push(ProcessEvent::SpawnFailed {
reason: e.to_string(),
});
}
}
}
}
impl ProcessDriver for LocalDriver {
fn execute(&mut self, action: ProcessAction) {
match action {
ProcessAction::SpawnProcess { spec } => {
self.spawn_process(&spec);
}
ProcessAction::WriteStdin { data } => {
if let Some(ref mut stdin) = self.stdin {
match stdin.write_all(&data) {
Ok(()) => {
self.queue.push(ProcessEvent::StdinWritten {
byte_count: data.len(),
});
}
Err(e) => {
self.queue.push(ProcessEvent::ConnectionLost {
reason: format!("stdin write failed: {}", e),
});
}
}
}
}
ProcessAction::SendSignal { signal } => {
if let Some(ref child) = self.child {
let pid = child.id();
match send_signal(pid, signal) {
Ok(()) => {
self.queue.push(ProcessEvent::SignalSent);
}
Err(reason) => {
self.queue.push(ProcessEvent::ConnectionLost { reason });
}
}
}
}
ProcessAction::ResizePty { .. } => {
// No-op for Phase 1 (pipes only, no PTY support)
self.queue.push(ProcessEvent::PtyResized);
}
ProcessAction::CloseStdin => {
// Drop the stdin handle to close the pipe
self.stdin.take();
}
ProcessAction::ScheduleKillTimeout { duration } => {
let queue = self.queue.clone();
let waker = self.waker_slot.clone();
thread::spawn(move || {
thread::sleep(duration);
queue.push(ProcessEvent::KillTimeout);
if let Some(w) = waker.get() {
w.wake();
}
});
}
// Notification actions are not driver commands
_ => {}
}
}
fn poll(&mut self) -> Vec<ProcessEvent> {
self.queue.drain()
}
fn pid(&self) -> Option<u32> {
self.child.as_ref().map(|c| c.id())
}
}
impl Drop for LocalDriver {
fn drop(&mut self) {
// Close stdin to let the process know we're done
self.stdin.take();
// Kill the process if still alive
if let Some(ref mut child) = self.child {
let _ = child.kill();
let _ = child.wait();
}
}
}

View file

@ -1,58 +1,24 @@
use swactor::actor::ActorAddress;
use crate::types::ExitStatus;
use crate::action::OutputStream;
use crate::types::{ExitStatus, ProcessError, PtySize, Signal};
/// Commands sent to a process actor.
#[derive(Debug, Clone)]
/// Public managed-process commands.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ProcessCommand {
/// Write data to the process's stdin.
WriteStdin { data: Vec<u8> },
/// Send a signal to the process.
SendSignal { signal: Signal },
/// Resize the process's PTY.
ResizePty { size: PtySize },
/// Close the process's stdin pipe.
CloseStdin,
/// Request a graceful close of the process.
Close,
/// Subscribe to process notifications.
Subscribe { address: ActorAddress },
/// Unsubscribe from process notifications.
Unsubscribe { address: ActorAddress },
/// Internal: sent by the waker to trigger event draining.
#[doc(hidden)]
PollTick,
Stop {
kill_after: Option<std::time::Duration>,
},
}
/// Notifications sent from a process actor to subscribers.
#[derive(Debug, Clone)]
pub enum ProcessNotification {
/// The process started successfully.
///
/// `pid` is `Some(u32)` when the underlying driver knows the OS
/// pid (real `LocalDriver`) and `None` when it doesn't
/// (mock drivers, future SSH-tunnel-style drivers). Telemetry hooks
/// can read this to attribute the subprocess in a per-node datastream.
Started {
process: ActorAddress,
#[doc(hidden)]
pid: Option<u32>,
},
/// Output was received from the process.
Output {
process: ActorAddress,
data: Vec<u8>,
stream: OutputStream,
},
/// The process exited.
Exited {
process: ActorAddress,
status: ExitStatus,
},
/// An error occurred.
Error {
process: ActorAddress,
error: ProcessError,
},
/// Public managed-process outputs.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ProcessOutput {
Started { pid: u32 },
SpawnFailed { error: String },
Exited { status: ExitStatus },
Error { error: String },
}
#[derive(Debug, Clone)]
pub(crate) enum ProcessActorCommand {
Command(ProcessCommand),
SupervisorWake,
}

View file

@ -1,61 +0,0 @@
use std::collections::VecDeque;
use crate::action::ProcessAction;
use crate::event::ProcessEvent;
use crate::types::ProcessDriver;
/// A test-oriented driver that records executed actions and lets you inject events.
pub struct MockDriver {
pending_events: VecDeque<ProcessEvent>,
executed_actions: Vec<ProcessAction>,
}
impl MockDriver {
pub fn new() -> Self {
Self {
pending_events: VecDeque::new(),
executed_actions: Vec::new(),
}
}
/// Queue a single event to be returned by the next `poll()`.
pub fn inject(&mut self, event: ProcessEvent) {
self.pending_events.push_back(event);
}
/// Queue multiple events to be returned by subsequent `poll()` calls.
pub fn inject_many(&mut self, events: impl IntoIterator<Item = ProcessEvent>) {
self.pending_events.extend(events);
}
/// View all actions that have been executed so far.
pub fn executed_actions(&self) -> &[ProcessAction] {
&self.executed_actions
}
/// Take all executed actions, clearing the internal log.
pub fn take_executed_actions(&mut self) -> Vec<ProcessAction> {
std::mem::take(&mut self.executed_actions)
}
/// Number of events waiting to be polled.
pub fn pending_event_count(&self) -> usize {
self.pending_events.len()
}
}
impl Default for MockDriver {
fn default() -> Self {
Self::new()
}
}
impl ProcessDriver for MockDriver {
fn execute(&mut self, action: ProcessAction) {
self.executed_actions.push(action);
}
fn poll(&mut self) -> Vec<ProcessEvent> {
self.pending_events.drain(..).collect()
}
}

View file

@ -1,398 +0,0 @@
use std::collections::VecDeque;
use swactor::actor::ActorAddress;
use crate::action::{OutputStream, ProcessAction};
use crate::event::ProcessEvent;
use crate::types::{ExitStatus, FlowControl, ProcessError, ProcessMode, ProcessSpec, Signal};
// ─── SubscriberSet ─────────────────────────────────────────────────────────
/// A deduplicated collection of subscriber addresses.
#[derive(Debug, Clone)]
pub(crate) struct SubscriberSet {
inner: Vec<ActorAddress>,
}
impl SubscriberSet {
pub(crate) fn new() -> Self {
Self { inner: Vec::new() }
}
/// Add an address. No-op if already present.
pub(crate) fn add(&mut self, address: ActorAddress) {
if !self.inner.contains(&address) {
self.inner.push(address);
}
}
/// Remove an address. No-op if not present.
pub(crate) fn remove(&mut self, address: &ActorAddress) {
self.inner.retain(|a| a != address);
}
/// Snapshot of current subscribers.
pub(crate) fn snapshot(&self) -> Vec<ActorAddress> {
self.inner.clone()
}
/// Number of subscribers.
pub(crate) fn count(&self) -> usize {
self.inner.len()
}
}
/// The lifecycle states of a process session.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ProcessState {
Starting,
Running,
Stopping,
Exited,
}
impl ProcessState {
pub fn name(&self) -> &'static str {
match self {
Self::Starting => "Starting",
Self::Running => "Running",
Self::Stopping => "Stopping",
Self::Exited => "Exited",
}
}
}
/// Pure-logic state machine for managing a process lifecycle.
///
/// Created via `new()` which returns the session plus initial actions (SpawnProcess).
/// Drive it forward by calling `apply(event)` which returns actions to execute.
pub struct ProcessSession {
spec: ProcessSpec,
state: ProcessState,
subscribers: SubscriberSet,
flow: FlowControl,
exit_status: Option<ExitStatus>,
stdin_closed: bool,
close_requested_before_start: bool,
stdin_buffer: VecDeque<Vec<u8>>,
stdin_buffer_bytes: usize,
}
impl ProcessSession {
/// Create a new session. Returns the session and the initial actions to execute
/// (always a single `SpawnProcess` action).
pub fn new(spec: ProcessSpec) -> (Self, Vec<ProcessAction>) {
let actions = vec![ProcessAction::SpawnProcess { spec: spec.clone() }];
let session = Self {
spec,
state: ProcessState::Starting,
subscribers: SubscriberSet::new(),
flow: FlowControl::default(),
exit_status: None,
stdin_closed: false,
close_requested_before_start: false,
stdin_buffer: VecDeque::new(),
stdin_buffer_bytes: 0,
};
(session, actions)
}
/// Apply an event and return the resulting actions.
pub fn apply(&mut self, event: ProcessEvent) -> Vec<ProcessAction> {
// Subscribe/Unsubscribe handled in all states
match &event {
ProcessEvent::Subscribe { address } => {
self.subscribers.add(*address);
return vec![];
}
ProcessEvent::Unsubscribe { address } => {
self.subscribers.remove(address);
return vec![];
}
_ => {}
}
// Driver acks — silently consumed in all states
match &event {
ProcessEvent::StdinWritten { byte_count } => {
self.flow.pending_stdin_bytes =
self.flow.pending_stdin_bytes.saturating_sub(*byte_count);
return self.drain_stdin_buffer();
}
ProcessEvent::SignalSent | ProcessEvent::PtyResized => {
return vec![];
}
_ => {}
}
// KillTimeout — handled in all states before per-state dispatch
if matches!(event, ProcessEvent::KillTimeout) {
return if self.state == ProcessState::Stopping {
vec![ProcessAction::SendSignal {
signal: Signal::Kill,
}]
} else {
vec![]
};
}
// Dispatch to per-state handler
match self.state {
ProcessState::Starting => self.handle_starting(event),
ProcessState::Running => self.handle_running(event),
ProcessState::Stopping => self.handle_stopping(event),
ProcessState::Exited => self.handle_exited(event),
}
}
// --- Per-state handlers ---
fn handle_starting(&mut self, event: ProcessEvent) -> Vec<ProcessAction> {
match event {
ProcessEvent::Started => {
self.state = ProcessState::Running;
let mut actions = vec![ProcessAction::NotifyStarted {
subscribers: self.subscribers.snapshot(),
}];
// If close was requested before the process started, transition to Stopping
if self.close_requested_before_start {
self.state = ProcessState::Stopping;
actions.push(ProcessAction::SendSignal {
signal: Signal::Terminate,
});
if let Some(duration) = self.spec.kill_timeout {
actions.push(ProcessAction::ScheduleKillTimeout { duration });
}
}
actions
}
ProcessEvent::SpawnFailed { reason } => {
self.state = ProcessState::Exited;
vec![
ProcessAction::NotifyError {
subscribers: self.subscribers.snapshot(),
error: ProcessError::SpawnFailed { reason },
},
ProcessAction::SelfTerminate,
]
}
ProcessEvent::CloseRequested => {
self.close_requested_before_start = true;
vec![]
}
_ => self.invalid_state_error(&event),
}
}
fn handle_running(&mut self, event: ProcessEvent) -> Vec<ProcessAction> {
match event {
ProcessEvent::OutputReceived { data, is_stderr } => {
let stream = if is_stderr {
OutputStream::Stderr
} else {
OutputStream::Stdout
};
vec![ProcessAction::NotifyOutput {
subscribers: self.subscribers.snapshot(),
data,
stream,
}]
}
ProcessEvent::Exited { status } => self.enter_exited(status),
ProcessEvent::ConnectionLost { reason } => {
self.state = ProcessState::Exited;
self.exit_status = Some(ExitStatus::Unknown);
self.clear_stdin_buffer();
vec![
ProcessAction::NotifyError {
subscribers: self.subscribers.snapshot(),
error: ProcessError::ConnectionLost { reason },
},
ProcessAction::SelfTerminate,
]
}
ProcessEvent::WriteStdin { data } => {
if self.stdin_closed {
return self.notify_error(ProcessError::InvalidState {
attempted: "WriteStdin",
current_state: "Running (stdin closed)",
});
}
// Backpressure: buffer if over limit
if let Some(limit) = self.spec.stdin_buffer_limit
&& self.flow.pending_stdin_bytes >= limit
{
self.stdin_buffer_bytes += data.len();
self.stdin_buffer.push_back(data);
return vec![];
}
self.flow.pending_stdin_bytes += data.len();
vec![ProcessAction::WriteStdin { data }]
}
ProcessEvent::SendSignal { signal } => {
vec![ProcessAction::SendSignal { signal }]
}
ProcessEvent::ResizePty { size } => {
vec![ProcessAction::ResizePty { size }]
}
ProcessEvent::CloseStdin => {
if self.stdin_closed {
return vec![];
}
self.stdin_closed = true;
self.clear_stdin_buffer();
vec![ProcessAction::CloseStdin]
}
ProcessEvent::CloseRequested => {
self.state = ProcessState::Stopping;
self.clear_stdin_buffer();
let mut actions = vec![ProcessAction::SendSignal {
signal: Signal::Terminate,
}];
if let Some(duration) = self.spec.kill_timeout {
actions.push(ProcessAction::ScheduleKillTimeout { duration });
}
actions
}
_ => self.invalid_state_error(&event),
}
}
fn handle_stopping(&mut self, event: ProcessEvent) -> Vec<ProcessAction> {
match event {
ProcessEvent::OutputReceived { data, is_stderr } => {
let stream = if is_stderr {
OutputStream::Stderr
} else {
OutputStream::Stdout
};
vec![ProcessAction::NotifyOutput {
subscribers: self.subscribers.snapshot(),
data,
stream,
}]
}
ProcessEvent::Exited { status } => self.enter_exited(status),
ProcessEvent::ConnectionLost { reason } => {
self.state = ProcessState::Exited;
self.exit_status = Some(ExitStatus::Unknown);
vec![
ProcessAction::NotifyError {
subscribers: self.subscribers.snapshot(),
error: ProcessError::ConnectionLost { reason },
},
ProcessAction::SelfTerminate,
]
}
ProcessEvent::SendSignal { signal } => {
// Escalation (e.g., Kill after Terminate) is allowed in Stopping
vec![ProcessAction::SendSignal { signal }]
}
ProcessEvent::CloseStdin => {
if self.stdin_closed {
return vec![];
}
self.stdin_closed = true;
vec![ProcessAction::CloseStdin]
}
ProcessEvent::CloseRequested => {
// Already stopping, no-op
vec![]
}
_ => self.invalid_state_error(&event),
}
}
fn handle_exited(&mut self, event: ProcessEvent) -> Vec<ProcessAction> {
// Everything in Exited is invalid — produce an error.
// (Acks and Subscribe/Unsubscribe are already handled before dispatch.)
self.invalid_state_error(&event)
}
// --- Helpers ---
fn enter_exited(&mut self, status: ExitStatus) -> Vec<ProcessAction> {
self.state = ProcessState::Exited;
self.exit_status = Some(status);
self.clear_stdin_buffer();
vec![
ProcessAction::NotifyExited {
subscribers: self.subscribers.snapshot(),
status,
},
ProcessAction::SelfTerminate,
]
}
fn clear_stdin_buffer(&mut self) {
self.stdin_buffer.clear();
self.stdin_buffer_bytes = 0;
}
fn drain_stdin_buffer(&mut self) -> Vec<ProcessAction> {
let limit = match self.spec.stdin_buffer_limit {
Some(limit) => limit,
None => return vec![],
};
let mut actions = Vec::new();
while self.flow.pending_stdin_bytes < limit {
match self.stdin_buffer.pop_front() {
Some(data) => {
self.stdin_buffer_bytes -= data.len();
self.flow.pending_stdin_bytes += data.len();
actions.push(ProcessAction::WriteStdin { data });
}
None => break,
}
}
actions
}
fn invalid_state_error(&self, event: &ProcessEvent) -> Vec<ProcessAction> {
self.notify_error(ProcessError::InvalidState {
attempted: event.name(),
current_state: self.state.name(),
})
}
fn notify_error(&self, error: ProcessError) -> Vec<ProcessAction> {
vec![ProcessAction::NotifyError {
subscribers: self.subscribers.snapshot(),
error,
}]
}
// --- Query methods ---
pub fn state(&self) -> ProcessState {
self.state
}
pub fn spec(&self) -> &ProcessSpec {
&self.spec
}
pub fn mode(&self) -> ProcessMode {
self.spec.mode
}
pub fn exit_status(&self) -> Option<ExitStatus> {
self.exit_status
}
pub fn flow_control(&self) -> &FlowControl {
&self.flow
}
pub fn subscriber_count(&self) -> usize {
self.subscribers.count()
}
pub fn stdin_closed(&self) -> bool {
self.stdin_closed
}
pub fn stdin_buffer_bytes(&self) -> usize {
self.stdin_buffer_bytes
}
}

View file

@ -5,89 +5,32 @@ use swactor::actor::{ActorAddress, Ctx};
use swactor::runtime::ExternalSender;
use crate::actor::ProcessActor;
use crate::local::LocalDriver;
use crate::message::ProcessCommand;
use crate::session::ProcessSession;
use crate::types::{EventQueue, ProcessDriver, ProcessSpec, ProcessWaker};
use crate::lifecycle::{ProcessOutputConfig, prepare_process_output};
use crate::message::{ProcessActorCommand, ProcessCommand};
use crate::types::ProcessSpec;
/// Spawn a process actor using the real `LocalDriver` (OS subprocess).
///
/// Creates a `ProcessActor<LocalDriver>`, spawns it in the runtime, and
/// wires up the waker so that I/O thread events automatically wake the actor.
///
/// Returns the actor's address. Send `ProcessCommand` messages to control it.
/// Spawn a process actor using the OS subprocess supervisor.
pub fn spawn_local_process(
ctx: &Ctx,
sender: &ExternalSender,
spec: ProcessSpec,
output: ProcessOutputConfig,
) -> Result<ActorAddress, Error> {
let waker_slot = Arc::new(OnceLock::new());
let queue = EventQueue::new();
let driver = LocalDriver::new(queue, waker_slot.clone());
spawn_process_inner(ctx, sender, spec, driver, waker_slot, None)
}
/// Spawn a process actor with a custom driver.
///
/// Useful for testing with `MockDriver` or other custom drivers while
/// still getting the full actor integration (waker, lifecycle, etc.).
pub fn spawn_process<D: ProcessDriver + 'static>(
ctx: &Ctx,
sender: &ExternalSender,
spec: ProcessSpec,
driver: D,
waker_slot: Arc<OnceLock<ProcessWaker>>,
) -> Result<ActorAddress, Error> {
spawn_process_inner(ctx, sender, spec, driver, waker_slot, None)
}
/// The basename of a command path, used as the process's telemetry label.
/// `/usr/bin/python3` → `python3`, `python` → `python`. Falls back to the whole
/// string when there is no path separator or trailing component.
fn command_basename(command: &str) -> String {
command
.rsplit(['/', '\\'])
.find(|s| !s.is_empty())
.unwrap_or(command)
.to_string()
}
fn spawn_process_inner<D: ProcessDriver + 'static>(
ctx: &Ctx,
sender: &ExternalSender,
spec: ProcessSpec,
driver: D,
waker_slot: Arc<OnceLock<ProcessWaker>>,
label_override: Option<String>,
) -> Result<ActorAddress, Error> {
// Label this process's output so the node's per-runtime observer (if any)
// taps it onto `proc.<label>.*` automatically. Default to the command
// basename; a caller can override (e.g. to keep several remote shells
// running the same command distinguishable).
let label = label_override.unwrap_or_else(|| command_basename(&spec.command));
let observer = ctx.process_output_observer();
let (session, initial_actions) = ProcessSession::new(spec);
let actor = ProcessActor::new(
session,
driver,
initial_actions,
waker_slot.clone(),
observer,
label,
);
let prepared_output = prepare_process_output(&spec, output)?;
let addr_slot = Arc::new(OnceLock::new());
let actor = ProcessActor::new(spec, prepared_output, sender.clone(), addr_slot.clone());
let addr = ctx.spawn(actor)?;
// Now that we have the address, fill the waker
let sender = sender.clone();
let waker = ProcessWaker::new(move || {
let _ = sender.send_to(addr, ProcessCommand::PollTick);
});
waker_slot
.set(waker.clone())
.expect("waker slot already set");
// Flush any events from the startup race window
waker.wake();
addr_slot
.set(addr)
.expect("process actor address already set");
let _ = sender.send_to(addr, ProcessActorCommand::SupervisorWake);
Ok(addr)
}
pub fn send_process_command(
sender: &ExternalSender,
process: ActorAddress,
command: ProcessCommand,
) -> Result<(), Error> {
sender.send_to(process, ProcessActorCommand::Command(command))
}

View file

@ -0,0 +1,955 @@
use std::io;
use std::mem;
use std::os::fd::RawFd;
use std::process::{Child, Command, Stdio};
use std::sync::Arc;
use std::thread::{self, JoinHandle};
use std::time::{Duration, Instant};
use crossbeam_queue::SegQueue;
use crate::types::{ExitStatus, ProcessSpec, Signal};
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum ThreadCommand {
Stop { kill_after: Option<Duration> },
ShutdownNow,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum ThreadEvent {
Started { pid: u32 },
SpawnFailed { error: String },
Exited { status: ExitStatus },
Error { error: String },
ThreadFinished,
}
#[derive(Clone)]
pub(crate) struct ThreadEventSink {
queue: Arc<SegQueue<ThreadEvent>>,
wake: Arc<dyn Fn() + Send + Sync>,
}
pub(crate) struct ThreadEventReceiver {
queue: Arc<SegQueue<ThreadEvent>>,
}
pub(crate) fn thread_event_channel(
wake: impl Fn() + Send + Sync + 'static,
) -> (ThreadEventSink, ThreadEventReceiver) {
let queue = Arc::new(SegQueue::new());
(
ThreadEventSink {
queue: queue.clone(),
wake: Arc::new(wake),
},
ThreadEventReceiver { queue },
)
}
impl ThreadEventSink {
pub(crate) fn push(&self, event: ThreadEvent) {
self.queue.push(event);
(self.wake)();
}
}
impl ThreadEventReceiver {
pub(crate) fn drain(&self) -> Vec<ThreadEvent> {
let mut events = Vec::new();
while let Some(event) = self.queue.pop() {
events.push(event);
}
events
}
}
#[derive(Clone)]
struct WakeFd(Arc<WakeFdInner>);
struct WakeFdInner {
fd: RawFd,
}
impl WakeFd {
fn new() -> io::Result<Self> {
let fd = unsafe { libc::eventfd(0, libc::EFD_NONBLOCK | libc::EFD_CLOEXEC) };
if fd < 0 {
return Err(io::Error::last_os_error());
}
Ok(Self(Arc::new(WakeFdInner { fd })))
}
fn fd(&self) -> RawFd {
self.0.fd
}
fn wake(&self) -> io::Result<()> {
let value: u64 = 1;
let ptr = (&value as *const u64).cast::<libc::c_void>();
let len = mem::size_of::<u64>();
loop {
let written = unsafe { libc::write(self.fd(), ptr, len) };
if written == len as libc::ssize_t {
return Ok(());
}
if written < 0 {
let err = io::Error::last_os_error();
if err.kind() == io::ErrorKind::Interrupted {
continue;
}
if is_would_block(&err) {
return Ok(());
}
return Err(err);
}
return Err(io::Error::new(
io::ErrorKind::WriteZero,
"short write to process supervisor wake fd",
));
}
}
fn drain(&self) -> io::Result<()> {
let mut value: u64 = 0;
let ptr = (&mut value as *mut u64).cast::<libc::c_void>();
let len = mem::size_of::<u64>();
loop {
let read = unsafe { libc::read(self.fd(), ptr, len) };
if read == len as libc::ssize_t {
continue;
}
if read < 0 {
let err = io::Error::last_os_error();
if err.kind() == io::ErrorKind::Interrupted {
continue;
}
if is_would_block(&err) {
return Ok(());
}
return Err(err);
}
return Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"short read from process supervisor wake fd",
));
}
}
}
impl Drop for WakeFdInner {
fn drop(&mut self) {
let _ = unsafe { libc::close(self.fd) };
}
}
fn poll_command_wake(wake: &WakeFd, timeout: Duration) -> io::Result<bool> {
let timeout_ms = poll_timeout_ms(timeout);
let mut pollfd = libc::pollfd {
fd: wake.fd(),
events: libc::POLLIN,
revents: 0,
};
loop {
let result = unsafe { libc::poll(&mut pollfd, 1, timeout_ms) };
if result == 0 {
return Ok(false);
}
if result > 0 {
let revents = pollfd.revents;
if revents & (libc::POLLERR | libc::POLLNVAL | libc::POLLHUP) != 0 {
return Err(io::Error::other(format!(
"process supervisor wake fd poll failed: revents={revents}"
)));
}
return Ok(revents & libc::POLLIN != 0);
}
let err = io::Error::last_os_error();
if err.kind() == io::ErrorKind::Interrupted {
continue;
}
return Err(err);
}
}
fn poll_timeout_ms(timeout: Duration) -> libc::c_int {
if timeout.is_zero() {
return 0;
}
let millis = timeout.as_millis();
if millis == 0 {
1
} else {
millis.min(libc::c_int::MAX as u128) as libc::c_int
}
}
fn is_would_block(err: &io::Error) -> bool {
matches!(
err.raw_os_error(),
Some(code) if code == libc::EAGAIN || code == libc::EWOULDBLOCK
)
}
pub(crate) struct ProcessThreadHandle {
commands: Arc<SegQueue<ThreadCommand>>,
wake: WakeFd,
join: Option<JoinHandle<()>>,
}
pub(crate) struct ProcessSupervisorThread;
impl ProcessSupervisorThread {
pub(crate) fn start(
spec: ProcessSpec,
events: ThreadEventSink,
) -> Result<ProcessThreadHandle, swactor::Error> {
let commands = Arc::new(SegQueue::new());
let wake = WakeFd::new().map_err(|err| {
swactor::Error::from(format!(
"failed to create process supervisor wake fd: {err}"
))
})?;
let thread_commands = commands.clone();
let thread_wake = wake.clone();
let join = thread::Builder::new()
.name("swactor-process-supervisor".to_owned())
.spawn(move || supervisor_thread_main(spec, events, thread_commands, thread_wake))
.map_err(|err| {
swactor::Error::from(format!("failed to start process supervisor thread: {err}"))
})?;
Ok(ProcessThreadHandle {
commands,
wake,
join: Some(join),
})
}
}
impl ProcessThreadHandle {
pub(crate) fn send(&self, command: ThreadCommand) -> Result<(), String> {
self.commands.push(command);
self.wake
.wake()
.map_err(|err| format!("failed to wake process supervisor: {err}"))
}
pub(crate) fn stop(&self, kill_after: Option<Duration>) -> Result<(), String> {
self.send(ThreadCommand::Stop { kill_after })
}
pub(crate) fn shutdown_now(&self) -> Result<(), String> {
self.send(ThreadCommand::ShutdownNow)
}
pub(crate) fn is_finished(&self) -> bool {
match &self.join {
Some(join) => join.is_finished(),
None => true,
}
}
pub(crate) fn join_if_finished(&mut self) -> Result<bool, String> {
if !self.is_finished() {
return Ok(false);
}
if let Some(join) = self.join.take() {
join.join()
.map_err(|_| "process supervisor thread panicked".to_owned())?;
}
Ok(true)
}
}
impl Drop for ProcessThreadHandle {
fn drop(&mut self) {
let _ = self.shutdown_now();
let _ = self.join_if_finished();
}
}
const WAITPID_POLL_INTERVAL: Duration = Duration::from_millis(10);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum SupervisorState {
Spawning,
Running,
Stopping,
Done,
}
enum CommandOutcome {
Continue,
Exited(ExitStatus),
Error(String),
}
fn supervisor_thread_main(
spec: ProcessSpec,
events: ThreadEventSink,
commands: Arc<SegQueue<ThreadCommand>>,
wake: WakeFd,
) {
let mut state = SupervisorState::Spawning;
let mut child: Option<Child>;
let pid: Option<u32>;
let mut kill_deadline: Option<Instant> = None;
let mut kill_sent = false;
debug_assert!(matches!(state, SupervisorState::Spawning));
let mut cmd = Command::new(&spec.command);
cmd.args(&spec.args);
for (key, value) in &spec.env {
cmd.env(key, value);
}
if let Some(dir) = &spec.working_dir {
cmd.current_dir(dir);
}
cmd.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null());
match cmd.spawn() {
Ok(spawned_child) => {
let child_pid = spawned_child.id();
pid = Some(child_pid);
child = Some(spawned_child);
events.push(ThreadEvent::Started { pid: child_pid });
state = SupervisorState::Running;
match drain_queued_commands(
child_pid,
&commands,
&mut state,
&mut kill_deadline,
&mut kill_sent,
) {
CommandOutcome::Continue => {}
CommandOutcome::Exited(status) => {
finish_with_exit(&events, &mut state, &mut child, status);
return;
}
CommandOutcome::Error(error) => {
finish_with_error(&events, &mut state, &mut child, error);
return;
}
}
}
Err(err) => {
events.push(ThreadEvent::SpawnFailed {
error: err.to_string(),
});
events.push(ThreadEvent::ThreadFinished);
return;
}
}
let child_pid = pid.expect("supervisor pid stored after successful spawn");
loop {
let timeout = command_poll_timeout(kill_deadline);
match poll_command_wake(&wake, timeout) {
Ok(true) => {
if let Err(err) = wake.drain() {
finish_with_error(
&events,
&mut state,
&mut child,
format!("process supervisor command wake failed: {err}"),
);
return;
}
match drain_queued_commands(
child_pid,
&commands,
&mut state,
&mut kill_deadline,
&mut kill_sent,
) {
CommandOutcome::Continue => {}
CommandOutcome::Exited(status) => {
finish_with_exit(&events, &mut state, &mut child, status);
return;
}
CommandOutcome::Error(error) => {
finish_with_error(&events, &mut state, &mut child, error);
return;
}
}
}
Ok(false) => {}
Err(err) => {
finish_with_error(
&events,
&mut state,
&mut child,
format!("process supervisor command wake failed: {err}"),
);
return;
}
}
match try_wait_pid(child_pid) {
Ok(Some(status)) => {
finish_with_exit(&events, &mut state, &mut child, status);
return;
}
Ok(None) => {}
Err(error) => {
finish_with_error(&events, &mut state, &mut child, error);
return;
}
}
if kill_deadline.is_some_and(|deadline| deadline <= Instant::now()) && !kill_sent {
match send_kill_or_observe_exit(child_pid) {
CommandOutcome::Continue => {
kill_sent = true;
kill_deadline = None;
continue;
}
CommandOutcome::Exited(status) => {
finish_with_exit(&events, &mut state, &mut child, status);
return;
}
CommandOutcome::Error(error) => {
finish_with_error(&events, &mut state, &mut child, error);
return;
}
}
}
}
}
fn command_poll_timeout(kill_deadline: Option<Instant>) -> Duration {
let Some(deadline) = kill_deadline else {
return WAITPID_POLL_INTERVAL;
};
let now = Instant::now();
if deadline <= now {
Duration::ZERO
} else {
(deadline - now).min(WAITPID_POLL_INTERVAL)
}
}
fn drain_queued_commands(
pid: u32,
commands: &SegQueue<ThreadCommand>,
state: &mut SupervisorState,
kill_deadline: &mut Option<Instant>,
kill_sent: &mut bool,
) -> CommandOutcome {
while let Some(command) = commands.pop() {
match apply_thread_command(pid, command, state, kill_deadline, kill_sent) {
CommandOutcome::Continue => {}
outcome => return outcome,
}
}
CommandOutcome::Continue
}
fn apply_thread_command(
pid: u32,
command: ThreadCommand,
state: &mut SupervisorState,
kill_deadline: &mut Option<Instant>,
kill_sent: &mut bool,
) -> CommandOutcome {
match command {
ThreadCommand::Stop { kill_after } => match state {
SupervisorState::Running => match send_terminate_or_observe_exit(pid) {
CommandOutcome::Continue => {
if let Some(duration) = kill_after {
*kill_deadline = Some(Instant::now() + duration);
} else {
*kill_deadline = None;
}
*state = SupervisorState::Stopping;
CommandOutcome::Continue
}
outcome => outcome,
},
SupervisorState::Spawning | SupervisorState::Stopping | SupervisorState::Done => {
CommandOutcome::Continue
}
},
ThreadCommand::ShutdownNow => match state {
SupervisorState::Done => CommandOutcome::Continue,
SupervisorState::Spawning | SupervisorState::Running | SupervisorState::Stopping => {
*state = SupervisorState::Stopping;
*kill_deadline = None;
match send_terminate_or_observe_exit(pid) {
CommandOutcome::Continue => match try_wait_pid(pid) {
Ok(Some(status)) => CommandOutcome::Exited(status),
Ok(None) => match send_kill_or_observe_exit(pid) {
CommandOutcome::Continue => {
*kill_sent = true;
CommandOutcome::Continue
}
outcome => outcome,
},
Err(error) => CommandOutcome::Error(error),
},
outcome => outcome,
}
}
},
}
}
fn send_terminate_or_observe_exit(pid: u32) -> CommandOutcome {
match send_signal(pid, Signal::Terminate) {
Ok(()) => CommandOutcome::Continue,
Err(kill_error) => match try_wait_pid(pid) {
Ok(Some(status)) => CommandOutcome::Exited(status),
Ok(None) => {
CommandOutcome::Error(format!("failed to terminate process {pid}: {kill_error}"))
}
Err(error) => CommandOutcome::Error(error),
},
}
}
fn send_kill_or_observe_exit(pid: u32) -> CommandOutcome {
match send_signal(pid, Signal::Kill) {
Ok(()) => CommandOutcome::Continue,
Err(kill_error) => match try_wait_pid(pid) {
Ok(Some(status)) => CommandOutcome::Exited(status),
Ok(None) => {
CommandOutcome::Error(format!("failed to kill process {pid}: {kill_error}"))
}
Err(error) => CommandOutcome::Error(error),
},
}
}
fn finish_with_exit(
events: &ThreadEventSink,
state: &mut SupervisorState,
child: &mut Option<Child>,
status: ExitStatus,
) {
*state = SupervisorState::Done;
debug_assert!(matches!(*state, SupervisorState::Done));
let _ = child.take();
events.push(ThreadEvent::Exited { status });
events.push(ThreadEvent::ThreadFinished);
}
fn finish_with_error(
events: &ThreadEventSink,
state: &mut SupervisorState,
child: &mut Option<Child>,
error: String,
) {
*state = SupervisorState::Done;
debug_assert!(matches!(*state, SupervisorState::Done));
let _ = child.take();
events.push(ThreadEvent::Error { error });
events.push(ThreadEvent::ThreadFinished);
}
fn signal_to_libc(signal: Signal) -> libc::c_int {
match signal {
Signal::Terminate => libc::SIGTERM,
Signal::Kill => libc::SIGKILL,
}
}
fn send_signal(pid: u32, signal: Signal) -> Result<(), String> {
let sig = signal_to_libc(signal);
let ret = unsafe { libc::kill(pid as libc::pid_t, sig) };
if ret == 0 {
Ok(())
} else {
Err(format!(
"kill({}, {}) failed: {}",
pid,
sig,
std::io::Error::last_os_error()
))
}
}
fn decode_wait_status(status: libc::c_int) -> ExitStatus {
if libc::WIFEXITED(status) {
ExitStatus::Code(libc::WEXITSTATUS(status))
} else if libc::WIFSIGNALED(status) {
ExitStatus::Signal(libc::WTERMSIG(status))
} else {
ExitStatus::Unknown
}
}
fn try_wait_pid(pid: u32) -> Result<Option<ExitStatus>, String> {
loop {
let mut status: libc::c_int = 0;
let result = unsafe { libc::waitpid(pid as libc::pid_t, &mut status, libc::WNOHANG) };
if result == 0 {
return Ok(None);
}
if result == pid as libc::pid_t {
return Ok(Some(decode_wait_status(status)));
}
if result > 0 {
return Ok(Some(ExitStatus::Unknown));
}
let err = std::io::Error::last_os_error();
if err.kind() == io::ErrorKind::Interrupted {
continue;
}
return Err(format!("waitpid({pid}) failed: {err}"));
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::HashMap;
use std::sync::atomic::{AtomicUsize, Ordering};
fn spec(command: &str, args: Vec<&str>) -> ProcessSpec {
ProcessSpec {
command: command.to_owned(),
args: args.into_iter().map(str::to_owned).collect(),
env: HashMap::new(),
working_dir: None,
label: None,
}
}
fn shell_spec(script: &str) -> ProcessSpec {
spec("sh", vec!["-c", script])
}
fn collect_until_finished(
receiver: &ThreadEventReceiver,
timeout: Duration,
) -> Vec<ThreadEvent> {
let deadline = Instant::now() + timeout;
let mut events = Vec::new();
while Instant::now() < deadline {
events.extend(receiver.drain());
if matches!(events.last(), Some(ThreadEvent::ThreadFinished)) {
return events;
}
thread::sleep(Duration::from_millis(5));
}
events.extend(receiver.drain());
events
}
fn join_finished(handle: &mut ProcessThreadHandle) {
let deadline = Instant::now() + Duration::from_secs(2);
loop {
if handle
.join_if_finished()
.expect("supervisor thread should join")
{
return;
}
assert!(
Instant::now() < deadline,
"supervisor thread did not finish before join timeout"
);
thread::sleep(Duration::from_millis(5));
}
}
#[test]
fn thread_command_and_event_shapes_match_target_contract() {
let stop_with_deadline = ThreadCommand::Stop {
kill_after: Some(Duration::from_millis(10)),
};
assert_eq!(
stop_with_deadline,
ThreadCommand::Stop {
kill_after: Some(Duration::from_millis(10))
}
);
match stop_with_deadline {
ThreadCommand::Stop {
kill_after: Some(duration),
} => assert_eq!(duration, Duration::from_millis(10)),
_ => panic!("expected stop command with deadline"),
}
let stop_without_deadline = ThreadCommand::Stop { kill_after: None };
assert_eq!(
stop_without_deadline,
ThreadCommand::Stop { kill_after: None }
);
match stop_without_deadline {
ThreadCommand::Stop { kill_after: None } => {}
_ => panic!("expected stop command without deadline"),
}
assert_eq!(ThreadCommand::ShutdownNow, ThreadCommand::ShutdownNow);
match ThreadCommand::ShutdownNow {
ThreadCommand::ShutdownNow => {}
_ => panic!("expected shutdown command"),
}
let started = ThreadEvent::Started { pid: 42 };
assert_eq!(started, ThreadEvent::Started { pid: 42 });
match started {
ThreadEvent::Started { pid } => assert_eq!(pid, 42),
_ => panic!("expected started event"),
}
let spawn_failed = ThreadEvent::SpawnFailed {
error: "spawn failed".to_owned(),
};
assert_eq!(
spawn_failed,
ThreadEvent::SpawnFailed {
error: "spawn failed".to_owned()
}
);
match spawn_failed {
ThreadEvent::SpawnFailed { error } => assert_eq!(error, "spawn failed"),
_ => panic!("expected spawn failed event"),
}
let exited = ThreadEvent::Exited {
status: ExitStatus::Code(7),
};
assert_eq!(
exited,
ThreadEvent::Exited {
status: ExitStatus::Code(7)
}
);
match exited {
ThreadEvent::Exited { status } => assert_eq!(status, ExitStatus::Code(7)),
_ => panic!("expected exited event"),
}
let error = ThreadEvent::Error {
error: "lost".to_owned(),
};
assert_eq!(
error,
ThreadEvent::Error {
error: "lost".to_owned()
}
);
match error {
ThreadEvent::Error { error } => assert_eq!(error, "lost"),
_ => panic!("expected error event"),
}
assert_eq!(ThreadEvent::ThreadFinished, ThreadEvent::ThreadFinished);
match ThreadEvent::ThreadFinished {
ThreadEvent::ThreadFinished => {}
_ => panic!("expected thread finished event"),
}
}
#[test]
fn thread_event_sink_enqueues_then_wakes() {
let wake_count = Arc::new(AtomicUsize::new(0));
let wake_count_for_callback = wake_count.clone();
let (sink, receiver) = thread_event_channel(move || {
wake_count_for_callback.fetch_add(1, Ordering::SeqCst);
});
sink.push(ThreadEvent::Started { pid: 1 });
sink.push(ThreadEvent::Exited {
status: ExitStatus::Code(0),
});
assert_eq!(wake_count.load(Ordering::SeqCst), 2);
assert_eq!(
receiver.drain(),
vec![
ThreadEvent::Started { pid: 1 },
ThreadEvent::Exited {
status: ExitStatus::Code(0)
}
]
);
assert!(receiver.drain().is_empty());
}
#[test]
fn process_thread_handle_queues_commands_and_wakes_supervisor() {
let commands = Arc::new(SegQueue::new());
let wake = WakeFd::new().expect("wake fd should be created");
let handle = ProcessThreadHandle {
commands: commands.clone(),
wake: wake.clone(),
join: None,
};
handle
.send(ThreadCommand::Stop {
kill_after: Some(Duration::from_millis(25)),
})
.expect("stop command should queue");
handle
.send(ThreadCommand::ShutdownNow)
.expect("shutdown command should queue");
assert!(
poll_command_wake(&wake, Duration::ZERO).expect("wake poll should succeed"),
"wake fd should be readable after queued commands"
);
wake.drain().expect("wake fd should drain");
assert_eq!(
commands.pop(),
Some(ThreadCommand::Stop {
kill_after: Some(Duration::from_millis(25))
})
);
assert_eq!(commands.pop(), Some(ThreadCommand::ShutdownNow));
assert_eq!(commands.pop(), None);
}
#[test]
fn supervisor_reports_started_exited_and_finished() {
let (sink, receiver) = thread_event_channel(|| {});
let mut handle = ProcessSupervisorThread::start(shell_spec("exit 7"), sink)
.expect("supervisor should start");
let events = collect_until_finished(&receiver, Duration::from_secs(2));
join_finished(&mut handle);
assert_eq!(events.len(), 3);
assert!(matches!(
events.first(),
Some(ThreadEvent::Started { pid }) if *pid > 0
));
assert_eq!(
events.get(1),
Some(&ThreadEvent::Exited {
status: ExitStatus::Code(7)
})
);
assert_eq!(events.last(), Some(&ThreadEvent::ThreadFinished));
}
#[test]
fn supervisor_reports_spawn_failed_and_finished() {
let (sink, receiver) = thread_event_channel(|| {});
let mut handle =
ProcessSupervisorThread::start(spec("/definitely/not/a/real/binary", vec![]), sink)
.expect("supervisor should start");
let events = collect_until_finished(&receiver, Duration::from_secs(2));
join_finished(&mut handle);
assert_eq!(events.len(), 2);
assert!(matches!(
events.first(),
Some(ThreadEvent::SpawnFailed { error }) if !error.is_empty()
));
assert_eq!(events.last(), Some(&ThreadEvent::ThreadFinished));
assert!(!events.iter().any(|event| {
matches!(
event,
ThreadEvent::Started { .. }
| ThreadEvent::Exited { .. }
| ThreadEvent::Error { .. }
)
}));
}
#[test]
fn supervisor_uses_null_stdio_and_reports_only_lifecycle() {
let (sink, receiver) = thread_event_channel(|| {});
let mut handle = ProcessSupervisorThread::start(
shell_spec("echo stdout; echo stderr >&2; exit 0"),
sink,
)
.expect("supervisor should start");
let events = collect_until_finished(&receiver, Duration::from_secs(2));
join_finished(&mut handle);
assert_eq!(events.len(), 3);
assert!(matches!(
events.first(),
Some(ThreadEvent::Started { pid }) if *pid > 0
));
assert_eq!(
events.get(1),
Some(&ThreadEvent::Exited {
status: ExitStatus::Code(0)
})
);
assert_eq!(events.last(), Some(&ThreadEvent::ThreadFinished));
}
#[test]
fn supervisor_stop_escalates_to_kill_after_deadline() {
let (sink, receiver) = thread_event_channel(|| {});
let mut handle = ProcessSupervisorThread::start(
shell_spec("trap '' TERM; while true; do sleep 1; done"),
sink,
)
.expect("supervisor should start");
let deadline = Instant::now() + Duration::from_secs(3);
let mut events = Vec::new();
let mut stop_sent = false;
while Instant::now() < deadline {
events.extend(receiver.drain());
if !stop_sent
&& events
.iter()
.any(|event| matches!(event, ThreadEvent::Started { pid } if *pid > 0))
{
handle
.stop(Some(Duration::from_millis(20)))
.expect("stop command should queue");
stop_sent = true;
}
if matches!(events.last(), Some(ThreadEvent::ThreadFinished)) {
break;
}
thread::sleep(Duration::from_millis(5));
}
if !matches!(events.last(), Some(ThreadEvent::ThreadFinished)) {
let _ = handle.shutdown_now();
events.extend(collect_until_finished(&receiver, Duration::from_secs(2)));
}
join_finished(&mut handle);
assert!(
stop_sent,
"supervisor did not report Started before timeout"
);
assert_eq!(events.last(), Some(&ThreadEvent::ThreadFinished));
assert!(events.iter().any(|event| {
matches!(
event,
ThreadEvent::Exited {
status: ExitStatus::Signal(9)
}
)
}));
}
}

View file

@ -1,41 +1,12 @@
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use crossbeam_queue::SegQueue;
use crate::action::ProcessAction;
use crate::event::ProcessEvent;
/// Describes how to spawn a process.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProcessSpec {
pub command: String,
pub args: Vec<String>,
pub env: HashMap<String, String>,
pub working_dir: Option<String>,
pub mode: ProcessMode,
pub initial_pty_size: Option<PtySize>,
/// If set, escalate to SIGKILL after this duration if the process hasn't exited
/// after SIGTERM. None = no escalation.
pub kill_timeout: Option<Duration>,
/// If set, buffer stdin writes when pending bytes exceed this limit.
/// None = unlimited (current behavior).
pub stdin_buffer_limit: Option<usize>,
}
/// Whether the process is interactive (PTY) or automated (pipes).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ProcessMode {
Interactive,
Automated,
}
/// Dimensions of a pseudo-terminal.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct PtySize {
pub cols: u16,
pub rows: u16,
pub working_dir: Option<std::path::PathBuf>,
pub label: Option<String>,
}
/// How a process exited.
@ -46,128 +17,8 @@ pub enum ExitStatus {
Unknown,
}
/// Signals that can be sent to a process.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Signal {
pub(crate) enum Signal {
Terminate,
Kill,
Hangup,
Interrupt,
Other(i32),
}
/// Errors produced by the process session.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ProcessError {
SpawnFailed {
reason: String,
},
ConnectionLost {
reason: String,
},
InvalidState {
attempted: &'static str,
current_state: &'static str,
},
}
/// Passive tracking of stdin backpressure.
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct FlowControl {
pub pending_stdin_bytes: usize,
}
// ─── ProcessDriver ─────────────────────────────────────────────────────────
/// Abstraction over the mechanism that actually runs a process.
///
/// Implementations translate `ProcessAction` commands into real I/O (or mock I/O)
/// and produce `ProcessEvent`s by polling for state changes.
pub trait ProcessDriver: Send {
/// Execute an action (spawn, write stdin, send signal, etc.).
fn execute(&mut self, action: ProcessAction);
/// Poll for new events from the underlying process.
fn poll(&mut self) -> Vec<ProcessEvent>;
/// PID of the underlying OS process when the driver knows one.
///
/// Returns `None` before the child has spawned, after it has been
/// reaped, or for drivers that do not run an OS process (mocks,
/// SSH-tunnel drivers that wrap a remote shell). The default
/// impl returns `None` so existing drivers compile unchanged.
///
/// Read by [`crate::actor::ProcessActor`] when it builds the
/// outbound `ProcessNotification::Started { pid }` — this is the
/// channel observability hooks use to learn the subprocess's PID
/// without coupling to a particular driver implementation
/// (`N3_OBSERVABILITY_UPGRADE_SPEC.md` §4 wiring contract).
fn pid(&self) -> Option<u32> {
None
}
}
// ─── ProcessWaker ──────────────────────────────────────────────────────────
/// A handle that I/O threads use to wake the owning actor.
///
/// Constructed with a closure that sends a `ProcessCommand::PollTick`
/// to the actor via `ExternalSender`. Thread-safe and cloneable.
#[derive(Clone)]
pub struct ProcessWaker(Arc<dyn Fn() + Send + Sync>);
impl std::fmt::Debug for ProcessWaker {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ProcessWaker").finish_non_exhaustive()
}
}
impl ProcessWaker {
pub fn new(f: impl Fn() + Send + Sync + 'static) -> Self {
Self(Arc::new(f))
}
/// Wake the owning actor so it drains pending events.
pub fn wake(&self) {
(self.0)();
}
}
// ─── EventQueue ────────────────────────────────────────────────────────────
/// Thread-safe queue for buffering process events from I/O threads.
///
/// Cloneable via inner `Arc` — I/O threads push events, the driver's
/// `poll()` drains them.
#[derive(Clone)]
pub struct EventQueue {
inner: Arc<SegQueue<ProcessEvent>>,
}
impl EventQueue {
pub fn new() -> Self {
Self {
inner: Arc::new(SegQueue::new()),
}
}
/// Push an event (called from I/O threads).
pub fn push(&self, event: ProcessEvent) {
self.inner.push(event);
}
/// Drain all pending events (called from driver's `poll()`).
pub fn drain(&self) -> Vec<ProcessEvent> {
let mut events = Vec::new();
while let Some(event) = self.inner.pop() {
events.push(event);
}
events
}
}
impl Default for EventQueue {
fn default() -> Self {
Self::new()
}
}

View file

@ -1,570 +0,0 @@
//! Layer 3 — Actor integration tests.
//!
//! Uses a TestDriver backed by a shared EventQueue so tests can inject
//! events and observe actions without real OS processes.
use std::collections::HashMap;
use std::sync::{Arc, Mutex, OnceLock};
use swactor::actor::{ActorAddress, ActorInterface, Ctx};
use swactor::runtime::{Inbox, Runtime, RuntimeConfig};
use swactor_process::*;
// ── TestDriver ──────────────────────────────────────────────────────────────
/// Shared harness for injecting events and inspecting driver actions.
#[derive(Clone)]
struct TestHarness {
queue: EventQueue,
actions: Arc<Mutex<Vec<ProcessAction>>>,
}
impl TestHarness {
fn new() -> Self {
Self {
queue: EventQueue::new(),
actions: Arc::new(Mutex::new(Vec::new())),
}
}
fn inject(&self, event: ProcessEvent) {
self.queue.push(event);
}
fn take_actions(&self) -> Vec<ProcessAction> {
std::mem::take(&mut self.actions.lock().unwrap())
}
}
/// A ProcessDriver that records actions and drains from a shared queue.
struct TestDriver {
queue: EventQueue,
actions: Arc<Mutex<Vec<ProcessAction>>>,
}
impl TestDriver {
fn from_harness(harness: &TestHarness) -> Self {
Self {
queue: harness.queue.clone(),
actions: harness.actions.clone(),
}
}
}
impl ProcessDriver for TestDriver {
fn execute(&mut self, action: ProcessAction) {
self.actions.lock().unwrap().push(action);
}
fn poll(&mut self) -> Vec<ProcessEvent> {
self.queue.drain()
}
}
// ── Helpers ─────────────────────────────────────────────────────────────────
fn automated_spec() -> ProcessSpec {
ProcessSpec {
command: "echo".into(),
args: vec!["hello".into()],
env: HashMap::new(),
working_dir: None,
mode: ProcessMode::Automated,
initial_pty_size: None,
kill_timeout: None,
stdin_buffer_limit: None,
}
}
fn setup() -> (Runtime, ExternalSender) {
let rt = Runtime::new(RuntimeConfig::default());
let sender = rt.create_sender();
(rt, sender)
}
/// Helper: tick until we receive N messages, returning them.
fn tick_collect<M: swactor::actor::Message>(
rt: &Runtime,
inbox: &Inbox<M>,
n: usize,
max_ticks: usize,
) -> Vec<M> {
let mut msgs = Vec::new();
for _ in 0..max_ticks {
rt.tick();
while let Some(m) = inbox.try_recv() {
msgs.push(m);
if msgs.len() >= n {
return msgs;
}
}
}
msgs
}
use swactor::runtime::ExternalSender;
// ── Spawner actor ───────────────────────────────────────────────────────────
// We can't call ctx.spawn from outside a handle(), so we use a small "spawner"
// actor that spawns the process actor and reports its address.
#[derive(Clone)]
struct SpawnRequest {
spec: ProcessSpec,
harness: TestHarness,
reply_to: ActorAddress,
sender: ExternalSender,
}
#[derive(Clone, Debug)]
struct SpawnedAddr(ActorAddress);
struct SpawnerActor;
impl ActorInterface for SpawnerActor {
type Incoming = SpawnRequest;
type Response = SpawnedAddr;
fn handle(&mut self, ctx: &Ctx, msg: SpawnRequest) {
let waker_slot = Arc::new(OnceLock::new());
let driver = TestDriver::from_harness(&msg.harness);
let addr = spawn_process(ctx, &msg.sender, msg.spec, driver, waker_slot)
.expect("spawn_process failed");
let _ = ctx.send(msg.reply_to, SpawnedAddr(addr));
}
}
// ── Tests ───────────────────────────────────────────────────────────────────
#[test]
fn happy_path_spawn_output_exit_notifies_subscriber() {
let (rt, sender) = setup();
let harness = TestHarness::new();
let notif_inbox = rt.new_inbox::<ProcessNotification>().unwrap();
// Spawn the spawner actor
let spawner_addr = rt.spawn(SpawnerActor).unwrap();
let reply_inbox = rt.new_inbox::<SpawnedAddr>().unwrap();
rt.tick();
// Ask spawner to create a process actor
rt.send_to(
spawner_addr,
SpawnRequest {
spec: automated_spec(),
harness: harness.clone(),
reply_to: *reply_inbox.addr(),
sender: sender.clone(),
},
)
.unwrap();
// Tick to process spawn request
for _ in 0..5 {
rt.tick();
}
let spawned = reply_inbox.try_recv().expect("should get spawned addr");
let proc_addr = spawned.0;
// Verify SpawnProcess action was sent to driver
let actions = harness.take_actions();
assert!(
actions
.iter()
.any(|a| matches!(a, ProcessAction::SpawnProcess { .. })),
"driver should receive SpawnProcess, got: {:?}",
actions
);
// Subscribe to notifications
rt.send_to(
proc_addr,
ProcessCommand::Subscribe {
address: *notif_inbox.addr(),
},
)
.unwrap();
rt.tick();
// Inject Started event from "driver"
harness.inject(ProcessEvent::Started);
// Send PollTick to trigger drain
rt.send_to(proc_addr, ProcessCommand::PollTick).unwrap();
for _ in 0..3 {
rt.tick();
}
let msgs = tick_collect(&rt, &notif_inbox, 1, 10);
assert!(
msgs.iter()
.any(|m| matches!(m, ProcessNotification::Started { .. })),
"subscriber should get Started notification, got: {:?}",
msgs
);
// Inject output
harness.inject(ProcessEvent::OutputReceived {
data: b"hello\n".to_vec(),
is_stderr: false,
});
rt.send_to(proc_addr, ProcessCommand::PollTick).unwrap();
let msgs = tick_collect(&rt, &notif_inbox, 1, 10);
assert!(
msgs.iter()
.any(|m| matches!(m, ProcessNotification::Output { .. })),
"subscriber should get Output notification"
);
// Inject exit
harness.inject(ProcessEvent::Exited {
status: ExitStatus::Code(0),
});
rt.send_to(proc_addr, ProcessCommand::PollTick).unwrap();
let msgs = tick_collect(&rt, &notif_inbox, 1, 10);
assert!(
msgs.iter().any(|m| matches!(
m,
ProcessNotification::Exited {
status: ExitStatus::Code(0),
..
}
)),
"subscriber should get Exited(0) notification"
);
}
#[test]
fn polltick_drains_queued_events() {
let (rt, sender) = setup();
let harness = TestHarness::new();
let notif_inbox = rt.new_inbox::<ProcessNotification>().unwrap();
let spawner_addr = rt.spawn(SpawnerActor).unwrap();
let reply_inbox = rt.new_inbox::<SpawnedAddr>().unwrap();
rt.tick();
rt.send_to(
spawner_addr,
SpawnRequest {
spec: automated_spec(),
harness: harness.clone(),
reply_to: *reply_inbox.addr(),
sender: sender.clone(),
},
)
.unwrap();
for _ in 0..5 {
rt.tick();
}
let proc_addr = reply_inbox.try_recv().unwrap().0;
// Subscribe
rt.send_to(
proc_addr,
ProcessCommand::Subscribe {
address: *notif_inbox.addr(),
},
)
.unwrap();
rt.tick();
// Queue multiple events before sending PollTick
harness.inject(ProcessEvent::Started);
harness.inject(ProcessEvent::OutputReceived {
data: b"line1\n".to_vec(),
is_stderr: false,
});
harness.inject(ProcessEvent::OutputReceived {
data: b"line2\n".to_vec(),
is_stderr: false,
});
// Single PollTick should drain all
rt.send_to(proc_addr, ProcessCommand::PollTick).unwrap();
let msgs = tick_collect(&rt, &notif_inbox, 3, 20);
assert_eq!(
msgs.len(),
3,
"all three events should produce notifications"
);
assert!(matches!(msgs[0], ProcessNotification::Started { .. }));
assert!(matches!(msgs[1], ProcessNotification::Output { .. }));
assert!(matches!(msgs[2], ProcessNotification::Output { .. }));
}
#[test]
fn close_command_triggers_graceful_shutdown() {
let (rt, sender) = setup();
let harness = TestHarness::new();
let notif_inbox = rt.new_inbox::<ProcessNotification>().unwrap();
let spawner_addr = rt.spawn(SpawnerActor).unwrap();
let reply_inbox = rt.new_inbox::<SpawnedAddr>().unwrap();
rt.tick();
rt.send_to(
spawner_addr,
SpawnRequest {
spec: automated_spec(),
harness: harness.clone(),
reply_to: *reply_inbox.addr(),
sender: sender.clone(),
},
)
.unwrap();
for _ in 0..5 {
rt.tick();
}
let proc_addr = reply_inbox.try_recv().unwrap().0;
// Subscribe and get to Running state
rt.send_to(
proc_addr,
ProcessCommand::Subscribe {
address: *notif_inbox.addr(),
},
)
.unwrap();
rt.tick();
harness.inject(ProcessEvent::Started);
rt.send_to(proc_addr, ProcessCommand::PollTick).unwrap();
let _ = tick_collect::<ProcessNotification>(&rt, &notif_inbox, 1, 10);
// Send Close
harness.take_actions(); // clear previous actions
rt.send_to(proc_addr, ProcessCommand::Close).unwrap();
for _ in 0..5 {
rt.tick();
}
let actions = harness.take_actions();
assert!(
actions.iter().any(|a| matches!(
a,
ProcessAction::SendSignal {
signal: Signal::Terminate
}
)),
"Close should trigger SIGTERM, got: {:?}",
actions
);
}
#[test]
fn write_stdin_and_signal_forwarded_to_driver() {
let (rt, sender) = setup();
let harness = TestHarness::new();
let spawner_addr = rt.spawn(SpawnerActor).unwrap();
let reply_inbox = rt.new_inbox::<SpawnedAddr>().unwrap();
rt.tick();
rt.send_to(
spawner_addr,
SpawnRequest {
spec: automated_spec(),
harness: harness.clone(),
reply_to: *reply_inbox.addr(),
sender: sender.clone(),
},
)
.unwrap();
for _ in 0..5 {
rt.tick();
}
let proc_addr = reply_inbox.try_recv().unwrap().0;
// Get to Running
harness.inject(ProcessEvent::Started);
rt.send_to(proc_addr, ProcessCommand::PollTick).unwrap();
for _ in 0..5 {
rt.tick();
}
harness.take_actions(); // clear SpawnProcess action
// Write stdin
rt.send_to(
proc_addr,
ProcessCommand::WriteStdin {
data: b"input\n".to_vec(),
},
)
.unwrap();
for _ in 0..3 {
rt.tick();
}
let actions = harness.take_actions();
assert!(
actions
.iter()
.any(|a| matches!(a, ProcessAction::WriteStdin { .. })),
"WriteStdin should be forwarded to driver, got: {:?}",
actions
);
// Send signal
rt.send_to(
proc_addr,
ProcessCommand::SendSignal {
signal: Signal::Interrupt,
},
)
.unwrap();
for _ in 0..3 {
rt.tick();
}
let actions = harness.take_actions();
assert!(
actions.iter().any(|a| matches!(
a,
ProcessAction::SendSignal {
signal: Signal::Interrupt
}
)),
"SendSignal should be forwarded to driver, got: {:?}",
actions
);
}
#[test]
fn spawn_failure_notifies_error_and_stops_actor() {
let (rt, sender) = setup();
let harness = TestHarness::new();
let notif_inbox = rt.new_inbox::<ProcessNotification>().unwrap();
let spawner_addr = rt.spawn(SpawnerActor).unwrap();
let reply_inbox = rt.new_inbox::<SpawnedAddr>().unwrap();
rt.tick();
rt.send_to(
spawner_addr,
SpawnRequest {
spec: automated_spec(),
harness: harness.clone(),
reply_to: *reply_inbox.addr(),
sender: sender.clone(),
},
)
.unwrap();
for _ in 0..5 {
rt.tick();
}
let proc_addr = reply_inbox.try_recv().unwrap().0;
// Subscribe
rt.send_to(
proc_addr,
ProcessCommand::Subscribe {
address: *notif_inbox.addr(),
},
)
.unwrap();
rt.tick();
// Inject spawn failure
harness.inject(ProcessEvent::SpawnFailed {
reason: "command not found".into(),
});
rt.send_to(proc_addr, ProcessCommand::PollTick).unwrap();
let msgs = tick_collect(&rt, &notif_inbox, 1, 20);
assert!(
msgs.iter()
.any(|m| matches!(m, ProcessNotification::Error { .. })),
"subscriber should get Error notification on spawn failure"
);
// Actor should have stopped — sending further messages should fail or be ignored
// (the address may still be in the map briefly, but the actor won't process)
for _ in 0..10 {
rt.tick();
}
}
#[test]
fn subscribe_and_unsubscribe_routing() {
let (rt, sender) = setup();
let harness = TestHarness::new();
let inbox_a = rt.new_inbox::<ProcessNotification>().unwrap();
let inbox_b = rt.new_inbox::<ProcessNotification>().unwrap();
let spawner_addr = rt.spawn(SpawnerActor).unwrap();
let reply_inbox = rt.new_inbox::<SpawnedAddr>().unwrap();
rt.tick();
rt.send_to(
spawner_addr,
SpawnRequest {
spec: automated_spec(),
harness: harness.clone(),
reply_to: *reply_inbox.addr(),
sender: sender.clone(),
},
)
.unwrap();
for _ in 0..5 {
rt.tick();
}
let proc_addr = reply_inbox.try_recv().unwrap().0;
// Subscribe both
rt.send_to(
proc_addr,
ProcessCommand::Subscribe {
address: *inbox_a.addr(),
},
)
.unwrap();
rt.send_to(
proc_addr,
ProcessCommand::Subscribe {
address: *inbox_b.addr(),
},
)
.unwrap();
rt.tick();
// Get to Running
harness.inject(ProcessEvent::Started);
rt.send_to(proc_addr, ProcessCommand::PollTick).unwrap();
for _ in 0..5 {
rt.tick();
}
// Both should have received Started
assert!(inbox_a.try_recv().is_some(), "inbox_a should get Started");
assert!(inbox_b.try_recv().is_some(), "inbox_b should get Started");
// Unsubscribe inbox_b
rt.send_to(
proc_addr,
ProcessCommand::Unsubscribe {
address: *inbox_b.addr(),
},
)
.unwrap();
rt.tick();
// Inject output — only inbox_a should receive it
harness.inject(ProcessEvent::OutputReceived {
data: b"data".to_vec(),
is_stderr: false,
});
rt.send_to(proc_addr, ProcessCommand::PollTick).unwrap();
for _ in 0..5 {
rt.tick();
}
assert!(inbox_a.try_recv().is_some(), "inbox_a should get Output");
assert!(
inbox_b.try_recv().is_none(),
"inbox_b should NOT get Output after unsubscribe"
);
}

View file

@ -1,269 +0,0 @@
//! End-to-end tests: full Runtime + ExternalSender + ProcessActor<LocalDriver>.
//!
//! Spawns real OS processes through the actor system and verifies the
//! complete notification flow.
use std::collections::HashMap;
use swactor::actor::{ActorAddress, ActorInterface, Ctx};
use swactor::runtime::{ExternalSender, Inbox, Runtime, RuntimeConfig};
use swactor_process::*;
fn automated_spec(cmd: &str, args: &[&str]) -> ProcessSpec {
ProcessSpec {
command: cmd.into(),
args: args.iter().map(|s| s.to_string()).collect(),
env: HashMap::new(),
working_dir: None,
mode: ProcessMode::Automated,
initial_pty_size: None,
kill_timeout: None,
stdin_buffer_limit: None,
}
}
/// Tick and collect up to `n` notifications, with a max tick budget.
fn tick_collect(
rt: &Runtime,
inbox: &Inbox<ProcessNotification>,
n: usize,
max_ticks: usize,
) -> Vec<ProcessNotification> {
let mut msgs = Vec::new();
for _ in 0..max_ticks {
rt.tick();
// Small sleep to let I/O threads produce events
std::thread::sleep(std::time::Duration::from_millis(5));
while let Some(m) = inbox.try_recv() {
msgs.push(m);
if msgs.len() >= n {
return msgs;
}
}
}
msgs
}
// ── Spawner actor (needed because spawn_local_process requires &Ctx) ────────
#[derive(Clone)]
struct E2eSpawnRequest {
spec: ProcessSpec,
subscriber: ActorAddress,
reply_to: ActorAddress,
sender: ExternalSender,
}
#[derive(Clone, Debug)]
struct E2eSpawned(ActorAddress);
struct E2eSpawnerActor;
impl ActorInterface for E2eSpawnerActor {
type Incoming = E2eSpawnRequest;
type Response = E2eSpawned;
fn handle(&mut self, ctx: &Ctx, msg: E2eSpawnRequest) {
let addr =
spawn_local_process(ctx, &msg.sender, msg.spec).expect("spawn_local_process failed");
// Subscribe the notification inbox
let _ = ctx.send(
addr,
ProcessCommand::Subscribe {
address: msg.subscriber,
},
);
let _ = ctx.send(msg.reply_to, E2eSpawned(addr));
}
}
// ── Tests ───────────────────────────────────────────────────────────────────
#[test]
fn echo_hello_full_lifecycle() {
let rt = Runtime::new(RuntimeConfig::default());
let sender = rt.create_sender();
let notif_inbox = rt.new_inbox::<ProcessNotification>().unwrap();
let spawner_addr = rt.spawn(E2eSpawnerActor).unwrap();
let reply_inbox = rt.new_inbox::<E2eSpawned>().unwrap();
rt.tick();
rt.send_to(
spawner_addr,
E2eSpawnRequest {
spec: automated_spec("echo", &["hello"]),
subscriber: *notif_inbox.addr(),
reply_to: *reply_inbox.addr(),
sender: sender.clone(),
},
)
.unwrap();
// Tick enough for the spawner to process + the process actor to start
for _ in 0..10 {
rt.tick();
std::thread::sleep(std::time::Duration::from_millis(5));
}
let spawned = reply_inbox.try_recv().expect("should get spawned address");
let _proc_addr = spawned.0;
// Collect notifications: Started, Output("hello\n"), Exited(0)
let msgs = tick_collect(&rt, &notif_inbox, 3, 200);
let has_started = msgs
.iter()
.any(|m| matches!(m, ProcessNotification::Started { .. }));
let has_output = msgs.iter().any(|m| {
if let ProcessNotification::Output { data, .. } = m {
String::from_utf8_lossy(data).contains("hello")
} else {
false
}
});
let has_exited = msgs.iter().any(|m| {
matches!(
m,
ProcessNotification::Exited {
status: ExitStatus::Code(0),
..
}
)
});
assert!(
has_started,
"should receive Started notification, got: {:?}",
msgs
);
assert!(
has_output,
"should receive Output with 'hello', got: {:?}",
msgs
);
assert!(
has_exited,
"should receive Exited(0) notification, got: {:?}",
msgs
);
// Verify ordering: Started before Output before Exited
let started_idx = msgs
.iter()
.position(|m| matches!(m, ProcessNotification::Started { .. }))
.unwrap();
let output_idx = msgs
.iter()
.position(|m| matches!(m, ProcessNotification::Output { .. }))
.unwrap();
let exited_idx = msgs
.iter()
.position(|m| matches!(m, ProcessNotification::Exited { .. }))
.unwrap();
assert!(
started_idx < output_idx,
"Started should come before Output"
);
assert!(output_idx < exited_idx, "Output should come before Exited");
}
/// The per-node process-output observer is the mechanism a telemetry node uses
/// to tap *every* managed process with no per-spawn wiring: it installs one
/// observer on its runtime, and any process spawned through the facility hands
/// its output there, labeled by command basename. This is the contract a node's
/// `proc.<label>.*` capture relies on, so it must hold for a real spawn driven
/// only through the public Runtime + `spawn_local_process` API.
#[test]
fn runtime_observer_taps_managed_process_output_by_basename() {
use std::sync::{Arc, Mutex};
use swactor::process_observer::ProcessOutputObserver;
#[derive(Default)]
struct Recorder {
// (label, is_stderr, text) for each chunk observed.
seen: Mutex<Vec<(String, bool, String)>>,
}
impl ProcessOutputObserver for Recorder {
fn on_output(&self, label: &str, is_stderr: bool, data: &[u8]) {
self.seen.lock().unwrap().push((
label.to_string(),
is_stderr,
String::from_utf8_lossy(data).into_owned(),
));
}
}
let rt = Runtime::new(RuntimeConfig::default());
let recorder = Arc::new(Recorder::default());
// Install the observer before the first managed process is spawned.
rt.set_process_output_observer(recorder.clone());
let sender = rt.create_sender();
let notif_inbox = rt.new_inbox::<ProcessNotification>().unwrap();
let spawner_addr = rt.spawn(E2eSpawnerActor).unwrap();
let reply_inbox = rt.new_inbox::<E2eSpawned>().unwrap();
rt.tick();
// A path command so the basename (`echo`) is what labels the output, not the
// full path — the node keys `proc.<label>.*` on the basename.
rt.send_to(
spawner_addr,
E2eSpawnRequest {
spec: automated_spec("/bin/echo", &["telemetry-line"]),
subscriber: *notif_inbox.addr(),
reply_to: *reply_inbox.addr(),
sender: sender.clone(),
},
)
.unwrap();
// Drive until the process has produced output (also drains the inbox).
let _ = tick_collect(&rt, &notif_inbox, 3, 200);
let seen = recorder.seen.lock().unwrap().clone();
let captured = seen.iter().any(|(label, is_stderr, text)| {
label == "echo" && !*is_stderr && text.contains("telemetry-line")
});
assert!(
captured,
"observer should capture stdout of the managed process under its command basename, got: {seen:?}"
);
}
#[test]
fn bad_command_reports_error_e2e() {
let rt = Runtime::new(RuntimeConfig::default());
let sender = rt.create_sender();
let notif_inbox = rt.new_inbox::<ProcessNotification>().unwrap();
let spawner_addr = rt.spawn(E2eSpawnerActor).unwrap();
let reply_inbox = rt.new_inbox::<E2eSpawned>().unwrap();
rt.tick();
rt.send_to(
spawner_addr,
E2eSpawnRequest {
spec: automated_spec("/nonexistent/binary/xyz", &[]),
subscriber: *notif_inbox.addr(),
reply_to: *reply_inbox.addr(),
sender: sender.clone(),
},
)
.unwrap();
for _ in 0..10 {
rt.tick();
std::thread::sleep(std::time::Duration::from_millis(5));
}
let msgs = tick_collect(&rt, &notif_inbox, 1, 200);
assert!(
msgs.iter()
.any(|m| matches!(m, ProcessNotification::Error { .. })),
"should receive Error notification for bad command, got: {:?}",
msgs
);
}

View file

@ -1,342 +0,0 @@
//! Layer 4 — LocalDriver integration tests.
//!
//! Real OS processes, no actor layer. Tests LocalDriver in isolation.
use std::collections::HashMap;
use std::sync::{Arc, OnceLock};
use std::thread;
use std::time::Duration;
use swactor_process::*;
fn automated_spec(cmd: &str, args: &[&str]) -> ProcessSpec {
ProcessSpec {
command: cmd.into(),
args: args.iter().map(|s| s.to_string()).collect(),
env: HashMap::new(),
working_dir: None,
mode: ProcessMode::Automated,
initial_pty_size: None,
kill_timeout: None,
stdin_buffer_limit: None,
}
}
/// Poll the driver until `pred` matches at least one collected event, or timeout.
fn poll_until_match(
driver: &mut LocalDriver,
timeout: Duration,
pred: impl Fn(&ProcessEvent) -> bool,
) -> Vec<ProcessEvent> {
let start = std::time::Instant::now();
let mut all_events = Vec::new();
loop {
let events = driver.poll();
if events.is_empty() {
if start.elapsed() >= timeout {
break;
}
thread::sleep(Duration::from_millis(10));
}
all_events.extend(events);
if all_events.iter().any(&pred) {
break;
}
}
all_events
}
fn has_event(events: &[ProcessEvent], pred: impl Fn(&ProcessEvent) -> bool) -> bool {
events.iter().any(pred)
}
#[test]
fn echo_produces_started_output_and_exit_zero() {
let queue = EventQueue::new();
let waker_slot = Arc::new(OnceLock::new());
let mut driver = LocalDriver::new(queue, waker_slot);
let spec = automated_spec("echo", &["hello"]);
driver.execute(ProcessAction::SpawnProcess { spec });
// Wait for Exited (which means Started + output + exit are all in)
let events = poll_until_match(&mut driver, Duration::from_secs(5), |e| {
matches!(e, ProcessEvent::Exited { .. })
});
assert!(
has_event(&events, |e| matches!(e, ProcessEvent::Started)),
"should have Started event, got: {:?}",
events
);
assert!(
has_event(&events, |e| matches!(
e,
ProcessEvent::OutputReceived {
is_stderr: false,
..
}
)),
"should have stdout OutputReceived"
);
// Check the output contains "hello"
let output: Vec<u8> = events
.iter()
.filter_map(|e| match e {
ProcessEvent::OutputReceived {
data,
is_stderr: false,
} => Some(data.clone()),
_ => None,
})
.flatten()
.collect();
let output_str = String::from_utf8_lossy(&output);
assert!(
output_str.contains("hello"),
"output should contain 'hello', got: {:?}",
output_str
);
assert!(
has_event(&events, |e| matches!(
e,
ProcessEvent::Exited {
status: ExitStatus::Code(0)
}
)),
"should have Exited(0)"
);
}
#[test]
fn cat_stdin_echo_and_close() {
let queue = EventQueue::new();
let waker_slot = Arc::new(OnceLock::new());
let mut driver = LocalDriver::new(queue, waker_slot);
let spec = automated_spec("cat", &[]);
driver.execute(ProcessAction::SpawnProcess { spec });
// Wait for Started
let events = poll_until_match(&mut driver, Duration::from_secs(5), |e| {
matches!(e, ProcessEvent::Started)
});
assert!(has_event(&events, |e| matches!(e, ProcessEvent::Started)));
// Write to stdin
driver.execute(ProcessAction::WriteStdin {
data: b"ping\n".to_vec(),
});
// Wait until we see actual output (not just the StdinWritten ack)
let events = poll_until_match(&mut driver, Duration::from_secs(5), |e| {
matches!(e, ProcessEvent::OutputReceived { .. })
});
let output: Vec<u8> = events
.iter()
.filter_map(|e| match e {
ProcessEvent::OutputReceived { data, .. } => Some(data.clone()),
_ => None,
})
.flatten()
.collect();
let output_str = String::from_utf8_lossy(&output);
assert!(
output_str.contains("ping"),
"cat should echo back 'ping', got: {:?}",
output_str
);
// Close stdin — cat should exit
driver.execute(ProcessAction::CloseStdin);
let events = poll_until_match(&mut driver, Duration::from_secs(5), |e| {
matches!(e, ProcessEvent::Exited { .. })
});
assert!(
has_event(&events, |e| matches!(
e,
ProcessEvent::Exited {
status: ExitStatus::Code(0)
}
)),
"cat should exit cleanly after stdin close, got: {:?}",
events
);
}
#[test]
fn signal_terminates_long_running_process() {
let queue = EventQueue::new();
let waker_slot = Arc::new(OnceLock::new());
let mut driver = LocalDriver::new(queue, waker_slot);
let spec = automated_spec("sleep", &["60"]);
driver.execute(ProcessAction::SpawnProcess { spec });
// Wait for Started
let events = poll_until_match(&mut driver, Duration::from_secs(5), |e| {
matches!(e, ProcessEvent::Started)
});
assert!(has_event(&events, |e| matches!(e, ProcessEvent::Started)));
// Send SIGTERM
driver.execute(ProcessAction::SendSignal {
signal: Signal::Terminate,
});
// Wait for Exited (may also see SignalSent ack first)
let events = poll_until_match(&mut driver, Duration::from_secs(5), |e| {
matches!(e, ProcessEvent::Exited { .. })
});
assert!(
has_event(&events, |e| matches!(
e,
ProcessEvent::Exited {
status: ExitStatus::Signal(_)
}
)),
"sleep should exit with signal status after SIGTERM, got: {:?}",
events
);
}
#[test]
fn bad_command_produces_spawn_failed() {
let queue = EventQueue::new();
let waker_slot = Arc::new(OnceLock::new());
let mut driver = LocalDriver::new(queue, waker_slot);
let spec = automated_spec("/nonexistent/binary/that/does/not/exist", &[]);
driver.execute(ProcessAction::SpawnProcess { spec });
let events = poll_until_match(&mut driver, Duration::from_secs(5), |e| {
matches!(e, ProcessEvent::SpawnFailed { .. })
});
assert!(
has_event(&events, |e| matches!(e, ProcessEvent::SpawnFailed { .. })),
"nonexistent binary should produce SpawnFailed, got: {:?}",
events
);
}
#[test]
fn large_output_no_data_loss() {
let queue = EventQueue::new();
let waker_slot = Arc::new(OnceLock::new());
let mut driver = LocalDriver::new(queue, waker_slot);
// Generate a large amount of output: seq 1 10000
let spec = automated_spec("seq", &["1", "10000"]);
driver.execute(ProcessAction::SpawnProcess { spec });
// Collect all events until exit
let events = poll_until_match(&mut driver, Duration::from_secs(10), |e| {
matches!(e, ProcessEvent::Exited { .. })
});
// Gather all output
let output: Vec<u8> = events
.iter()
.filter_map(|e| match e {
ProcessEvent::OutputReceived { data, .. } => Some(data.clone()),
_ => None,
})
.flatten()
.collect();
let output_str = String::from_utf8_lossy(&output);
// seq 1 10000 should end with "10000\n"
assert!(
output_str.contains("10000"),
"large output should contain '10000'"
);
// Check that it starts with "1\n"
assert!(
output_str.starts_with("1\n"),
"large output should start with '1\\n'"
);
assert!(
has_event(&events, |e| matches!(
e,
ProcessEvent::Exited {
status: ExitStatus::Code(0)
}
)),
"seq should exit cleanly"
);
}
#[test]
fn kill_timeout_escalates_to_sigkill() {
let queue = EventQueue::new();
let waker_slot = Arc::new(OnceLock::new());
let mut driver = LocalDriver::new(queue, waker_slot);
// Spawn a process that traps SIGTERM. Use exec to replace the shell so
// SIGTERM goes directly to the perl process (avoids shell vs child races).
let spec = automated_spec("perl", &["-e", "$SIG{TERM} = 'IGNORE'; sleep 300"]);
driver.execute(ProcessAction::SpawnProcess { spec });
// Wait for Started
let events = poll_until_match(&mut driver, Duration::from_secs(5), |e| {
matches!(e, ProcessEvent::Started)
});
assert!(has_event(&events, |e| matches!(e, ProcessEvent::Started)));
// Give the process a moment to set up the trap
thread::sleep(Duration::from_millis(100));
// Send SIGTERM (the process ignores it)
driver.execute(ProcessAction::SendSignal {
signal: Signal::Terminate,
});
poll_until_match(&mut driver, Duration::from_secs(1), |e| {
matches!(e, ProcessEvent::SignalSent)
});
// Verify the process is still alive after a short wait (SIGTERM was ignored)
thread::sleep(Duration::from_millis(200));
let events = driver.poll();
assert!(
!has_event(&events, |e| matches!(e, ProcessEvent::Exited { .. })),
"process should still be alive after SIGTERM (trap should ignore it)"
);
// Schedule a short kill timeout
driver.execute(ProcessAction::ScheduleKillTimeout {
duration: Duration::from_millis(200),
});
// Wait for KillTimeout event
let events = poll_until_match(&mut driver, Duration::from_secs(3), |e| {
matches!(e, ProcessEvent::KillTimeout)
});
assert!(
has_event(&events, |e| matches!(e, ProcessEvent::KillTimeout)),
"should receive KillTimeout, got: {:?}",
events
);
// Now send SIGKILL
driver.execute(ProcessAction::SendSignal {
signal: Signal::Kill,
});
// Wait for exit
let events = poll_until_match(&mut driver, Duration::from_secs(5), |e| {
matches!(e, ProcessEvent::Exited { .. })
});
assert!(
has_event(&events, |e| matches!(
e,
ProcessEvent::Exited {
status: ExitStatus::Signal(_)
}
)),
"process should exit with signal after SIGKILL, got: {:?}",
events
);
}

View file

@ -1,203 +0,0 @@
use std::collections::HashMap;
use proptest::prelude::*;
use swactor::actor::ActorAddress;
use swactor_process::*;
fn automated_spec() -> ProcessSpec {
ProcessSpec {
command: "test".into(),
args: vec![],
env: HashMap::new(),
working_dir: None,
mode: ProcessMode::Automated,
initial_pty_size: None,
kill_timeout: None,
stdin_buffer_limit: None,
}
}
fn addr(n: u8) -> ActorAddress {
let mut bytes = [0u8; 32];
bytes[0] = n;
ActorAddress(bytes)
}
fn arb_signal() -> impl Strategy<Value = Signal> {
prop_oneof![
Just(Signal::Terminate),
Just(Signal::Kill),
Just(Signal::Hangup),
Just(Signal::Interrupt),
(0..32i32).prop_map(Signal::Other),
]
}
fn arb_event() -> impl Strategy<Value = ProcessEvent> {
prop_oneof![
Just(ProcessEvent::Started),
Just(ProcessEvent::KillTimeout),
".*".prop_map(|reason| ProcessEvent::SpawnFailed { reason }),
proptest::collection::vec(any::<u8>(), 0..64).prop_map(|data| {
ProcessEvent::OutputReceived {
data,
is_stderr: false,
}
}),
proptest::collection::vec(any::<u8>(), 0..64).prop_map(|data| {
ProcessEvent::OutputReceived {
data,
is_stderr: true,
}
}),
prop_oneof![
any::<i32>().prop_map(ExitStatus::Code),
any::<i32>().prop_map(ExitStatus::Signal),
Just(ExitStatus::Unknown),
]
.prop_map(|status| ProcessEvent::Exited { status }),
".*".prop_map(|reason| ProcessEvent::ConnectionLost { reason }),
(0..5usize).prop_map(|n| ProcessEvent::StdinWritten { byte_count: n * 10 }),
Just(ProcessEvent::SignalSent),
Just(ProcessEvent::PtyResized),
proptest::collection::vec(any::<u8>(), 0..64)
.prop_map(|data| ProcessEvent::WriteStdin { data }),
arb_signal().prop_map(|signal| ProcessEvent::SendSignal { signal }),
Just(ProcessEvent::ResizePty {
size: PtySize { cols: 80, rows: 24 },
}),
Just(ProcessEvent::CloseStdin),
Just(ProcessEvent::CloseRequested),
(0..4u8).prop_map(|n| ProcessEvent::Subscribe { address: addr(n) }),
(0..4u8).prop_map(|n| ProcessEvent::Unsubscribe { address: addr(n) }),
]
}
// ──────────────────────────────────────────────
// 1. No panics for arbitrary event sequences
// ──────────────────────────────────────────────
proptest! {
#[test]
fn no_panics_on_arbitrary_events(events in proptest::collection::vec(arb_event(), 0..50)) {
let (mut session, _) = ProcessSession::new(automated_spec());
for event in events {
let _ = session.apply(event);
}
}
}
// ──────────────────────────────────────────────
// 2. Exited is terminal
// ──────────────────────────────────────────────
proptest! {
#[test]
fn exited_is_terminal(events in proptest::collection::vec(arb_event(), 0..50)) {
let (mut session, _) = ProcessSession::new(automated_spec());
let mut reached_exited = false;
for event in events {
let _ = session.apply(event);
if session.state() == ProcessState::Exited {
reached_exited = true;
}
if reached_exited {
prop_assert_eq!(session.state(), ProcessState::Exited);
}
}
}
}
// ──────────────────────────────────────────────
// 3. SelfTerminate always last action when entering Exited
// ──────────────────────────────────────────────
proptest! {
#[test]
fn self_terminate_is_last_when_entering_exited(events in proptest::collection::vec(arb_event(), 0..50)) {
let (mut session, _) = ProcessSession::new(automated_spec());
let mut was_exited = false;
for event in events {
let prev_state = session.state();
let actions = session.apply(event);
// If we just transitioned into Exited
if session.state() == ProcessState::Exited && !was_exited && prev_state != ProcessState::Exited {
prop_assert!(
matches!(actions.last(), Some(ProcessAction::SelfTerminate)),
"SelfTerminate must be last action when entering Exited, got: {:?}", actions
);
}
if session.state() == ProcessState::Exited {
was_exited = true;
}
}
}
}
// ──────────────────────────────────────────────
// 4. Subscriber count matches add/remove operations
// ──────────────────────────────────────────────
proptest! {
#[test]
fn subscriber_count_is_consistent(
ops in proptest::collection::vec(
prop_oneof![
(0..8u8).prop_map(|n| (true, n)),
(0..8u8).prop_map(|n| (false, n)),
],
0..30
)
) {
let (mut session, _) = ProcessSession::new(automated_spec());
let mut expected: Vec<u8> = Vec::new();
for (is_add, n) in ops {
if is_add {
session.apply(ProcessEvent::Subscribe { address: addr(n) });
if !expected.contains(&n) {
expected.push(n);
}
} else {
session.apply(ProcessEvent::Unsubscribe { address: addr(n) });
expected.retain(|&x| x != n);
}
prop_assert_eq!(session.subscriber_count(), expected.len());
}
}
}
// ──────────────────────────────────────────────
// 5. State monotonicity (never goes backward)
// ──────────────────────────────────────────────
fn state_ordinal(s: ProcessState) -> u8 {
match s {
ProcessState::Starting => 0,
ProcessState::Running => 1,
ProcessState::Stopping => 2,
ProcessState::Exited => 3,
}
}
proptest! {
#[test]
fn state_never_goes_backward(events in proptest::collection::vec(arb_event(), 0..50)) {
let (mut session, _) = ProcessSession::new(automated_spec());
let mut max_ordinal = state_ordinal(session.state());
for event in events {
let _ = session.apply(event);
let current = state_ordinal(session.state());
prop_assert!(
current >= max_ordinal,
"State went backward: ordinal {} -> {}", max_ordinal, current
);
max_ordinal = current;
}
}
}

View file

@ -0,0 +1,54 @@
use std::collections::HashMap;
use std::time::Duration;
use swactor_process::{
ExitStatus, ProcessCommand, ProcessOutput, ProcessSpec, send_process_command,
};
#[test]
fn stage1_public_api_exposes_only_target_process_shapes() {
let spec = ProcessSpec {
command: "echo".to_string(),
args: vec!["ok".to_string()],
env: HashMap::new(),
working_dir: None,
label: Some("echo_ok".to_string()),
};
assert_eq!(spec.label.as_deref(), Some("echo_ok"));
let stop = ProcessCommand::Stop {
kill_after: Some(Duration::from_millis(10)),
};
assert!(matches!(
stop,
ProcessCommand::Stop {
kill_after: Some(_)
}
));
let outputs = [
ProcessOutput::Started { pid: 1 },
ProcessOutput::SpawnFailed {
error: "spawn failed".to_string(),
},
ProcessOutput::Exited {
status: ExitStatus::Code(0),
},
ProcessOutput::Error {
error: "supervisor failed".to_string(),
},
];
assert!(matches!(outputs[0], ProcessOutput::Started { pid: 1 }));
assert!(matches!(
outputs[2],
ProcessOutput::Exited {
status: ExitStatus::Code(0)
}
));
let _send: fn(
&swactor::runtime::ExternalSender,
swactor::actor::ActorAddress,
ProcessCommand,
) -> Result<(), swactor::Error> = send_process_command;
}

View file

@ -0,0 +1,974 @@
use std::collections::HashMap;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use datastream::{DatastreamEndpoint, DatastreamEvent, Lifetime, NodeId, StreamId};
use serde_json::{Value, json};
use swactor::actor::{ActorAddress, ActorInterface, Ctx};
use swactor::runtime::{ExternalSender, Inbox, Runtime, RuntimeConfig};
use swactor_process::{
ExitStatus, ProcessCommand, ProcessOutput, ProcessOutputConfig, ProcessSpec,
send_process_command, spawn_local_process,
};
#[derive(Clone)]
struct SpawnRequest {
spec: ProcessSpec,
output: ProcessOutputConfig,
sender: ExternalSender,
reply_to: ActorAddress,
}
#[derive(Clone, Debug)]
enum SpawnReply {
Spawned(ActorAddress),
Failed(String),
}
struct SpawnerActor;
impl ActorInterface for SpawnerActor {
type Incoming = SpawnRequest;
type Response = ();
fn handle(&mut self, ctx: &Ctx, msg: SpawnRequest) {
let reply = match spawn_local_process(ctx, &msg.sender, msg.spec, msg.output) {
Ok(addr) => SpawnReply::Spawned(addr),
Err(err) => SpawnReply::Failed(err.to_string()),
};
let _ = ctx.send(msg.reply_to, reply);
}
}
static NEXT_STREAM: AtomicU64 = AtomicU64::new(1);
fn stage2_stream() -> StreamId {
let id = NEXT_STREAM.fetch_add(1, Ordering::Relaxed);
StreamId::new(NodeId::new(format!("process-stage2-{id}")), Lifetime(2))
}
fn shell_spec(command: &str, args: Vec<&str>, label: Option<&str>) -> ProcessSpec {
ProcessSpec {
command: command.to_owned(),
args: args.into_iter().map(str::to_owned).collect(),
env: HashMap::new(),
working_dir: None,
label: label.map(str::to_owned),
}
}
fn drain_outputs(inbox: &Inbox<ProcessOutput>, outputs: &mut Vec<ProcessOutput>) {
while let Some(output) = inbox.try_recv() {
outputs.push(output);
}
}
fn drive_once(
rt: &Runtime,
endpoint: Option<&DatastreamEndpoint>,
upstream: &Inbox<ProcessOutput>,
outputs: &mut Vec<ProcessOutput>,
) {
rt.tick();
std::thread::sleep(Duration::from_millis(5));
if let Some(endpoint) = endpoint {
endpoint.tick();
}
drain_outputs(upstream, outputs);
}
fn send_spawn(
rt: &Runtime,
spawner: ActorAddress,
sender: &ExternalSender,
spec: ProcessSpec,
output: ProcessOutputConfig,
reply: &Inbox<SpawnReply>,
) {
rt.send_to(
spawner,
SpawnRequest {
spec,
output,
sender: sender.clone(),
reply_to: *reply.addr(),
},
)
.unwrap();
}
fn drive_until_spawn_reply(
rt: &Runtime,
endpoint: Option<&DatastreamEndpoint>,
upstream: &Inbox<ProcessOutput>,
outputs: &mut Vec<ProcessOutput>,
reply: &Inbox<SpawnReply>,
) -> SpawnReply {
for _ in 0..400 {
drive_once(rt, endpoint, upstream, outputs);
if let Some(reply) = reply.try_recv() {
return reply;
}
}
panic!("spawner did not reply; outputs={outputs:?}");
}
fn drive_until(
rt: &Runtime,
endpoint: Option<&DatastreamEndpoint>,
upstream: &Inbox<ProcessOutput>,
outputs: &mut Vec<ProcessOutput>,
mut done: impl FnMut(&[ProcessOutput]) -> bool,
) {
for _ in 0..800 {
drive_once(rt, endpoint, upstream, outputs);
if done(outputs) {
return;
}
}
panic!("runtime condition was not reached; outputs={outputs:?}");
}
fn expect_spawned(reply: SpawnReply) -> ActorAddress {
match reply {
SpawnReply::Spawned(addr) => {
assert_ne!(addr, ActorAddress::default());
addr
}
SpawnReply::Failed(error) => panic!("spawn helper failed: {error}"),
}
}
fn expect_failed(reply: SpawnReply) -> String {
match reply {
SpawnReply::Spawned(addr) => panic!("spawn helper unexpectedly spawned {addr:?}"),
SpawnReply::Failed(error) => error,
}
}
fn is_terminal(output: &ProcessOutput) -> bool {
matches!(
output,
ProcessOutput::SpawnFailed { .. }
| ProcessOutput::Exited { .. }
| ProcessOutput::Error { .. }
)
}
fn terminal_count(outputs: &[ProcessOutput]) -> usize {
outputs.iter().filter(|output| is_terminal(output)).count()
}
fn has_started(outputs: &[ProcessOutput]) -> bool {
outputs
.iter()
.any(|output| matches!(output, ProcessOutput::Started { pid } if *pid > 0))
}
fn has_exited(outputs: &[ProcessOutput], status: ExitStatus) -> bool {
outputs.iter().any(|output| {
matches!(
output,
ProcessOutput::Exited { status: observed } if *observed == status
)
})
}
fn has_spawn_failed(outputs: &[ProcessOutput]) -> bool {
outputs
.iter()
.any(|output| matches!(output, ProcessOutput::SpawnFailed { .. }))
}
fn assert_no_errors(outputs: &[ProcessOutput]) {
assert!(
!outputs
.iter()
.any(|output| matches!(output, ProcessOutput::Error { .. })),
"unexpected process error output: {outputs:?}"
);
}
fn assert_no_spawn_failed(outputs: &[ProcessOutput]) {
assert!(
!has_spawn_failed(outputs),
"unexpected spawn failure output: {outputs:?}"
);
}
fn started_index(outputs: &[ProcessOutput]) -> usize {
outputs
.iter()
.position(|output| matches!(output, ProcessOutput::Started { pid } if *pid > 0))
.unwrap_or_else(|| panic!("missing Started output: {outputs:?}"))
}
fn exited_index(outputs: &[ProcessOutput]) -> usize {
outputs
.iter()
.position(|output| matches!(output, ProcessOutput::Exited { .. }))
.unwrap_or_else(|| panic!("missing Exited output: {outputs:?}"))
}
fn channel_names(endpoint: &DatastreamEndpoint) -> Vec<String> {
endpoint
.catalog_snapshot()
.channels
.values()
.map(|descriptor| descriptor.name.clone())
.collect()
}
#[test]
fn lifecycle_outputs_are_sent_upstream_and_mirrored_to_datastream() {
let rt = Runtime::new(RuntimeConfig::default());
let sender = rt.create_sender();
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
let reply = rt.new_inbox::<SpawnReply>().unwrap();
let spawner = rt.spawn(SpawnerActor).unwrap();
let endpoint = DatastreamEndpoint::new(stage2_stream());
let subscription = endpoint.subscribe_all("process-stage2-lifecycle");
send_spawn(
&rt,
spawner,
&sender,
shell_spec(
"sh",
vec!["-c", "echo stdout; echo stderr >&2; exit 0"],
Some("trainer.0/foo"),
),
ProcessOutputConfig::datastream_mirror(*upstream.addr(), endpoint.producer()),
&reply,
);
let mut outputs = Vec::new();
let process_addr = expect_spawned(drive_until_spawn_reply(
&rt,
Some(&endpoint),
&upstream,
&mut outputs,
&reply,
));
assert_ne!(process_addr, ActorAddress::default());
let mut datastream_events = subscription.drain_available();
drive_until(&rt, Some(&endpoint), &upstream, &mut outputs, |outputs| {
has_exited(outputs, ExitStatus::Code(0))
});
for _ in 0..5 {
drive_once(&rt, Some(&endpoint), &upstream, &mut outputs);
datastream_events.extend(subscription.drain_available());
}
let started = started_index(&outputs);
let exited = exited_index(&outputs);
assert!(
started < exited,
"Started should be delivered before Exited: {outputs:?}"
);
assert_no_spawn_failed(&outputs);
assert_no_errors(&outputs);
let names = channel_names(&endpoint);
assert!(
names
.iter()
.any(|name| name == "proc.trainer_0_foo.lifecycle"),
"catalog should contain lifecycle channel, got {names:?}"
);
assert!(
names
.iter()
.all(|name| !name.contains("stdout") && !name.contains("stderr")),
"process core should not register stdout/stderr channels: {names:?}"
);
let payloads: Vec<Value> = datastream_events
.iter()
.filter_map(|event| match event {
DatastreamEvent::Frame(delivery) => serde_json::from_slice(&delivery.payload).ok(),
_ => None,
})
.collect();
assert!(
payloads.iter().any(|payload| {
payload.get("event") == Some(&Value::String("started".to_owned()))
&& payload
.get("pid")
.and_then(Value::as_u64)
.is_some_and(|pid| pid > 0)
}),
"lifecycle mirror should include started JSON, got {payloads:?}"
);
assert!(
payloads.iter().any(|payload| {
payload.get("event") == Some(&Value::String("exited".to_owned()))
&& payload.get("status") == Some(&json!({"kind": "code", "value": 0}))
}),
"lifecycle mirror should include exited JSON, got {payloads:?}"
);
assert!(
payloads.iter().all(|payload| {
payload.get("event") != Some(&Value::String("stdout".to_owned()))
&& payload.get("event") != Some(&Value::String("stderr".to_owned()))
}),
"lifecycle mirror should not include child output events: {payloads:?}"
);
}
#[test]
fn spawn_failure_maps_to_public_spawn_failed_output() {
let rt = Runtime::new(RuntimeConfig::default());
let sender = rt.create_sender();
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
let reply = rt.new_inbox::<SpawnReply>().unwrap();
let spawner = rt.spawn(SpawnerActor).unwrap();
send_spawn(
&rt,
spawner,
&sender,
shell_spec(
"/definitely/not/a/real/binary",
Vec::new(),
Some("missing-binary"),
),
ProcessOutputConfig::disabled(*upstream.addr()),
&reply,
);
let mut outputs = Vec::new();
let _process_addr = expect_spawned(drive_until_spawn_reply(
&rt,
None,
&upstream,
&mut outputs,
&reply,
));
drive_until(&rt, None, &upstream, &mut outputs, has_spawn_failed);
let spawn_failures: Vec<&String> = outputs
.iter()
.filter_map(|output| match output {
ProcessOutput::SpawnFailed { error } => Some(error),
_ => None,
})
.collect();
assert_eq!(
spawn_failures.len(),
1,
"expected exactly one spawn failure output, got {outputs:?}"
);
assert!(
!spawn_failures[0].is_empty(),
"spawn failure error should not be empty"
);
assert!(
!outputs
.iter()
.any(|output| matches!(output, ProcessOutput::Started { .. })),
"spawn failure should not emit Started: {outputs:?}"
);
assert!(
!outputs
.iter()
.any(|output| matches!(output, ProcessOutput::Exited { .. })),
"spawn failure should not emit Exited: {outputs:?}"
);
assert_no_errors(&outputs);
}
#[test]
fn command_basename_is_default_lifecycle_label_source() {
let rt = Runtime::new(RuntimeConfig::default());
let sender = rt.create_sender();
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
let reply = rt.new_inbox::<SpawnReply>().unwrap();
let spawner = rt.spawn(SpawnerActor).unwrap();
let endpoint = DatastreamEndpoint::new(stage2_stream());
send_spawn(
&rt,
spawner,
&sender,
shell_spec("/bin/sh", vec!["-c", "exit 0"], None),
ProcessOutputConfig::datastream_mirror(*upstream.addr(), endpoint.producer()),
&reply,
);
let mut outputs = Vec::new();
let _process_addr = expect_spawned(drive_until_spawn_reply(
&rt,
Some(&endpoint),
&upstream,
&mut outputs,
&reply,
));
drive_until(&rt, Some(&endpoint), &upstream, &mut outputs, |outputs| {
has_exited(outputs, ExitStatus::Code(0))
});
let names = channel_names(&endpoint);
assert!(
names.iter().any(|name| name == "proc.sh.lifecycle"),
"catalog should contain basename lifecycle channel, got {names:?}"
);
assert!(started_index(&outputs) < exited_index(&outputs));
assert_no_spawn_failed(&outputs);
assert_no_errors(&outputs);
}
#[test]
fn explicit_label_overrides_basename_and_duplicate_labels_are_rejected() {
let rt = Runtime::new(RuntimeConfig::default());
let sender = rt.create_sender();
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
let reply = rt.new_inbox::<SpawnReply>().unwrap();
let spawner = rt.spawn(SpawnerActor).unwrap();
let endpoint = DatastreamEndpoint::new(stage2_stream());
send_spawn(
&rt,
spawner,
&sender,
shell_spec("/bin/sh", vec!["-c", "sleep 2"], Some("trainer.0/foo")),
ProcessOutputConfig::datastream_mirror(*upstream.addr(), endpoint.producer()),
&reply,
);
let mut outputs = Vec::new();
let first_addr = expect_spawned(drive_until_spawn_reply(
&rt,
Some(&endpoint),
&upstream,
&mut outputs,
&reply,
));
send_spawn(
&rt,
spawner,
&sender,
shell_spec("/bin/echo", vec!["unused"], Some("trainer.0/foo")),
ProcessOutputConfig::datastream_mirror(*upstream.addr(), endpoint.producer()),
&reply,
);
let error = expect_failed(drive_until_spawn_reply(
&rt,
Some(&endpoint),
&upstream,
&mut outputs,
&reply,
));
assert!(
error.contains(
"duplicate process lifecycle datastream channel: proc.trainer_0_foo.lifecycle"
),
"duplicate label error should name lifecycle channel, got {error}"
);
send_process_command(
&sender,
first_addr,
ProcessCommand::Stop {
kill_after: Some(Duration::from_millis(20)),
},
)
.unwrap();
drive_until(&rt, Some(&endpoint), &upstream, &mut outputs, |outputs| {
outputs
.iter()
.any(|output| matches!(output, ProcessOutput::Exited { .. }))
});
assert_no_errors(&outputs);
}
#[test]
fn stop_before_spawn_success_reports_started_then_exited() {
let rt = Runtime::new(RuntimeConfig::default());
let sender = rt.create_sender();
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
let reply = rt.new_inbox::<SpawnReply>().unwrap();
let spawner = rt.spawn(SpawnerActor).unwrap();
send_spawn(
&rt,
spawner,
&sender,
shell_spec("sh", vec!["-c", "sleep 60"], Some("stop-before-start")),
ProcessOutputConfig::disabled(*upstream.addr()),
&reply,
);
let mut outputs = Vec::new();
let process_addr = expect_spawned(drive_until_spawn_reply(
&rt,
None,
&upstream,
&mut outputs,
&reply,
));
send_process_command(
&sender,
process_addr,
ProcessCommand::Stop {
kill_after: Some(Duration::from_millis(20)),
},
)
.unwrap();
drive_until(&rt, None, &upstream, &mut outputs, |outputs| {
outputs
.iter()
.any(|output| matches!(output, ProcessOutput::Exited { .. }))
});
assert!(started_index(&outputs) < exited_index(&outputs));
assert_eq!(
outputs
.iter()
.filter(|output| matches!(output, ProcessOutput::Exited { .. }))
.count(),
1,
"expected exactly one Exited output, got {outputs:?}"
);
assert_eq!(
terminal_count(&outputs),
1,
"unexpected terminal outputs: {outputs:?}"
);
assert_no_spawn_failed(&outputs);
assert_no_errors(&outputs);
}
#[test]
fn stop_before_spawn_failure_reports_only_spawn_failed() {
let rt = Runtime::new(RuntimeConfig::default());
let sender = rt.create_sender();
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
let reply = rt.new_inbox::<SpawnReply>().unwrap();
let spawner = rt.spawn(SpawnerActor).unwrap();
send_spawn(
&rt,
spawner,
&sender,
shell_spec(
"/definitely/not/a/real/binary",
Vec::new(),
Some("stop-before-spawn-failure"),
),
ProcessOutputConfig::disabled(*upstream.addr()),
&reply,
);
let mut outputs = Vec::new();
let process_addr = expect_spawned(drive_until_spawn_reply(
&rt,
None,
&upstream,
&mut outputs,
&reply,
));
send_process_command(
&sender,
process_addr,
ProcessCommand::Stop {
kill_after: Some(Duration::from_millis(20)),
},
)
.unwrap();
drive_until(&rt, None, &upstream, &mut outputs, has_spawn_failed);
assert_eq!(
terminal_count(&outputs),
1,
"unexpected terminal outputs: {outputs:?}"
);
assert!(
!has_started(&outputs),
"spawn failure should not emit Started: {outputs:?}"
);
assert!(
!outputs
.iter()
.any(|output| matches!(output, ProcessOutput::Exited { .. })),
"spawn failure should not emit Exited: {outputs:?}"
);
assert_no_errors(&outputs);
}
#[test]
fn stop_running_with_kill_after_escalates_to_kill() {
let rt = Runtime::new(RuntimeConfig::default());
let sender = rt.create_sender();
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
let reply = rt.new_inbox::<SpawnReply>().unwrap();
let spawner = rt.spawn(SpawnerActor).unwrap();
send_spawn(
&rt,
spawner,
&sender,
shell_spec(
"sh",
vec!["-c", "trap '' TERM; while true; do sleep 1; done"],
Some("kill-escalates"),
),
ProcessOutputConfig::disabled(*upstream.addr()),
&reply,
);
let mut outputs = Vec::new();
let process_addr = expect_spawned(drive_until_spawn_reply(
&rt,
None,
&upstream,
&mut outputs,
&reply,
));
drive_until(&rt, None, &upstream, &mut outputs, has_started);
send_process_command(
&sender,
process_addr,
ProcessCommand::Stop {
kill_after: Some(Duration::from_millis(20)),
},
)
.unwrap();
drive_until(&rt, None, &upstream, &mut outputs, |outputs| {
has_exited(outputs, ExitStatus::Signal(9))
});
assert!(
has_exited(&outputs, ExitStatus::Signal(9)),
"expected SIGKILL exit: {outputs:?}"
);
assert_no_errors(&outputs);
}
#[test]
fn stop_running_without_kill_after_terminates_without_kill_escalation() {
let rt = Runtime::new(RuntimeConfig::default());
let sender = rt.create_sender();
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
let reply = rt.new_inbox::<SpawnReply>().unwrap();
let spawner = rt.spawn(SpawnerActor).unwrap();
send_spawn(
&rt,
spawner,
&sender,
shell_spec(
"sh",
vec!["-c", "trap 'exit 0' TERM; while true; do sleep 1; done"],
Some("terminate-without-kill"),
),
ProcessOutputConfig::disabled(*upstream.addr()),
&reply,
);
let mut outputs = Vec::new();
let process_addr = expect_spawned(drive_until_spawn_reply(
&rt,
None,
&upstream,
&mut outputs,
&reply,
));
drive_until(&rt, None, &upstream, &mut outputs, has_started);
send_process_command(
&sender,
process_addr,
ProcessCommand::Stop { kill_after: None },
)
.unwrap();
drive_until(&rt, None, &upstream, &mut outputs, |outputs| {
has_exited(outputs, ExitStatus::Code(0))
});
assert!(
has_exited(&outputs, ExitStatus::Code(0)),
"expected graceful exit: {outputs:?}"
);
assert!(
!has_exited(&outputs, ExitStatus::Signal(9)),
"stop without deadline should not escalate to SIGKILL: {outputs:?}"
);
assert_no_errors(&outputs);
}
#[test]
fn child_exit_before_kill_deadline_suppresses_kill_escalation() {
let rt = Runtime::new(RuntimeConfig::default());
let sender = rt.create_sender();
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
let reply = rt.new_inbox::<SpawnReply>().unwrap();
let spawner = rt.spawn(SpawnerActor).unwrap();
send_spawn(
&rt,
spawner,
&sender,
shell_spec(
"sh",
vec!["-c", "trap 'exit 0' TERM; while true; do sleep 1; done"],
Some("deadline-suppresses-kill"),
),
ProcessOutputConfig::disabled(*upstream.addr()),
&reply,
);
let mut outputs = Vec::new();
let process_addr = expect_spawned(drive_until_spawn_reply(
&rt,
None,
&upstream,
&mut outputs,
&reply,
));
drive_until(&rt, None, &upstream, &mut outputs, has_started);
send_process_command(
&sender,
process_addr,
ProcessCommand::Stop {
kill_after: Some(Duration::from_secs(1)),
},
)
.unwrap();
drive_until(&rt, None, &upstream, &mut outputs, |outputs| {
has_exited(outputs, ExitStatus::Code(0))
});
for _ in 0..5 {
drive_once(&rt, None, &upstream, &mut outputs);
}
assert!(
has_exited(&outputs, ExitStatus::Code(0)),
"expected graceful exit: {outputs:?}"
);
assert!(
!has_exited(&outputs, ExitStatus::Signal(9)),
"child exit before deadline should suppress SIGKILL: {outputs:?}"
);
assert_no_errors(&outputs);
}
#[test]
fn duplicate_stop_while_stopping_is_noop_and_keeps_original_deadline() {
let rt = Runtime::new(RuntimeConfig::default());
let sender = rt.create_sender();
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
let reply = rt.new_inbox::<SpawnReply>().unwrap();
let spawner = rt.spawn(SpawnerActor).unwrap();
send_spawn(
&rt,
spawner,
&sender,
shell_spec(
"sh",
vec!["-c", "trap '' TERM; while true; do sleep 1; done"],
Some("duplicate-stop"),
),
ProcessOutputConfig::disabled(*upstream.addr()),
&reply,
);
let mut outputs = Vec::new();
let process_addr = expect_spawned(drive_until_spawn_reply(
&rt,
None,
&upstream,
&mut outputs,
&reply,
));
drive_until(&rt, None, &upstream, &mut outputs, has_started);
let first_stop = Instant::now();
send_process_command(
&sender,
process_addr,
ProcessCommand::Stop {
kill_after: Some(Duration::from_millis(250)),
},
)
.unwrap();
send_process_command(
&sender,
process_addr,
ProcessCommand::Stop {
kill_after: Some(Duration::from_secs(5)),
},
)
.unwrap();
while first_stop.elapsed() < Duration::from_secs(2) {
drive_once(&rt, None, &upstream, &mut outputs);
if has_exited(&outputs, ExitStatus::Signal(9)) {
break;
}
}
assert!(
has_exited(&outputs, ExitStatus::Signal(9)),
"duplicate stop should keep the original kill deadline: {outputs:?}"
);
assert!(
first_stop.elapsed() < Duration::from_secs(2),
"SIGKILL arrived too late after duplicate stop: {:?}",
first_stop.elapsed()
);
assert_eq!(
terminal_count(&outputs),
1,
"unexpected terminal outputs: {outputs:?}"
);
assert_no_errors(&outputs);
}
#[test]
fn stop_after_terminal_output_emits_no_additional_output() {
let rt = Runtime::new(RuntimeConfig::default());
let sender = rt.create_sender();
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
let reply = rt.new_inbox::<SpawnReply>().unwrap();
let spawner = rt.spawn(SpawnerActor).unwrap();
send_spawn(
&rt,
spawner,
&sender,
shell_spec("sh", vec!["-c", "exit 0"], Some("post-terminal-stop")),
ProcessOutputConfig::disabled(*upstream.addr()),
&reply,
);
let mut outputs = Vec::new();
let process_addr = expect_spawned(drive_until_spawn_reply(
&rt,
None,
&upstream,
&mut outputs,
&reply,
));
drive_until(&rt, None, &upstream, &mut outputs, |outputs| {
has_exited(outputs, ExitStatus::Code(0))
});
let output_len = outputs.len();
let _ = send_process_command(
&sender,
process_addr,
ProcessCommand::Stop {
kill_after: Some(Duration::from_millis(10)),
},
);
for _ in 0..10 {
drive_once(&rt, None, &upstream, &mut outputs);
}
assert_eq!(
outputs.len(),
output_len,
"post-terminal stop should not emit more output: {outputs:?}"
);
assert_no_errors(&outputs);
}
#[test]
fn lifecycle_mirror_submit_failure_does_not_suppress_upstream_or_emit_error() {
let rt = Runtime::new(RuntimeConfig::default());
let sender = rt.create_sender();
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
let reply = rt.new_inbox::<SpawnReply>().unwrap();
let spawner = rt.spawn(SpawnerActor).unwrap();
let endpoint = DatastreamEndpoint::with_capacity(stage2_stream(), 1, 16);
send_spawn(
&rt,
spawner,
&sender,
shell_spec("sh", vec!["-c", "exit 0"], Some("mirror-drop")),
ProcessOutputConfig::datastream_mirror(*upstream.addr(), endpoint.producer()),
&reply,
);
let mut outputs = Vec::new();
let _process_addr = expect_spawned(drive_until_spawn_reply(
&rt,
None,
&upstream,
&mut outputs,
&reply,
));
drive_until(&rt, None, &upstream, &mut outputs, |outputs| {
has_exited(outputs, ExitStatus::Code(0))
});
let dropped = endpoint.mux().dropped();
endpoint.tick();
assert!(
has_started(&outputs),
"upstream should receive Started: {outputs:?}"
);
assert!(
has_exited(&outputs, ExitStatus::Code(0)),
"upstream should receive Exited: {outputs:?}"
);
assert_no_errors(&outputs);
assert!(
dropped > 0,
"full lifecycle mirror mux should drop at least one frame"
);
}
#[test]
fn stdout_and_stderr_writes_do_not_affect_lifecycle() {
let rt = Runtime::new(RuntimeConfig::default());
let sender = rt.create_sender();
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
let reply = rt.new_inbox::<SpawnReply>().unwrap();
let spawner = rt.spawn(SpawnerActor).unwrap();
send_spawn(
&rt,
spawner,
&sender,
shell_spec(
"sh",
vec![
"-c",
"for i in $(seq 1 1000); do echo out; echo err >&2; done; exit 0",
],
Some("child-output-is-null"),
),
ProcessOutputConfig::disabled(*upstream.addr()),
&reply,
);
let mut outputs = Vec::new();
let _process_addr = expect_spawned(drive_until_spawn_reply(
&rt,
None,
&upstream,
&mut outputs,
&reply,
));
drive_until(&rt, None, &upstream, &mut outputs, |outputs| {
has_exited(outputs, ExitStatus::Code(0))
});
assert_eq!(
outputs.len(),
2,
"process core should emit only lifecycle outputs: {outputs:?}"
);
assert!(started_index(&outputs) < exited_index(&outputs));
assert!(
has_exited(&outputs, ExitStatus::Code(0)),
"expected clean exit: {outputs:?}"
);
assert_no_spawn_failed(&outputs);
assert_no_errors(&outputs);
}

View file

@ -1,787 +0,0 @@
use std::collections::HashMap;
use std::time::Duration;
use swactor::actor::ActorAddress;
use swactor_process::*;
fn automated_spec() -> ProcessSpec {
ProcessSpec {
command: "echo".into(),
args: vec!["hello".into()],
env: HashMap::new(),
working_dir: None,
mode: ProcessMode::Automated,
initial_pty_size: None,
kill_timeout: None,
stdin_buffer_limit: None,
}
}
fn spec_with_kill_timeout(timeout: Duration) -> ProcessSpec {
ProcessSpec {
kill_timeout: Some(timeout),
..automated_spec()
}
}
fn spec_with_stdin_limit(limit: usize) -> ProcessSpec {
ProcessSpec {
stdin_buffer_limit: Some(limit),
..automated_spec()
}
}
fn interactive_spec() -> ProcessSpec {
ProcessSpec {
command: "/bin/bash".into(),
args: vec![],
env: HashMap::new(),
working_dir: None,
mode: ProcessMode::Interactive,
initial_pty_size: Some(PtySize { cols: 80, rows: 24 }),
kill_timeout: None,
stdin_buffer_limit: None,
}
}
fn addr(n: u8) -> ActorAddress {
let mut bytes = [0u8; 32];
bytes[0] = n;
ActorAddress(bytes)
}
/// Verify that a SelfTerminate is present and is the last action.
fn assert_self_terminate_is_last(actions: &[ProcessAction]) {
assert!(
matches!(actions.last(), Some(ProcessAction::SelfTerminate)),
"SelfTerminate must be the last action, got: {actions:?}"
);
}
// ──────────────────────────────────────────────
// 1. Happy path — automated process
// ──────────────────────────────────────────────
#[test]
fn automated_process_runs_produces_output_and_exits_cleanly() {
let (mut session, init) = ProcessSession::new(automated_spec());
assert_eq!(session.state(), ProcessState::Starting);
assert!(matches!(&init[0], ProcessAction::SpawnProcess { .. }));
// Process starts
let actions = session.apply(ProcessEvent::Started);
assert_eq!(session.state(), ProcessState::Running);
assert!(matches!(&actions[0], ProcessAction::NotifyStarted { .. }));
// Some output arrives
let actions = session.apply(ProcessEvent::OutputReceived {
data: b"hello\n".to_vec(),
is_stderr: false,
});
assert!(matches!(
&actions[0],
ProcessAction::NotifyOutput {
stream: OutputStream::Stdout,
..
}
));
// More output on stderr
let actions = session.apply(ProcessEvent::OutputReceived {
data: b"warn\n".to_vec(),
is_stderr: true,
});
assert!(matches!(
&actions[0],
ProcessAction::NotifyOutput {
stream: OutputStream::Stderr,
..
}
));
// Process exits
let actions = session.apply(ProcessEvent::Exited {
status: ExitStatus::Code(0),
});
assert_eq!(session.state(), ProcessState::Exited);
assert_eq!(session.exit_status(), Some(ExitStatus::Code(0)));
assert_self_terminate_is_last(&actions);
}
// ──────────────────────────────────────────────
// 2. Interactive process with subscriber lifecycle
// ──────────────────────────────────────────────
#[test]
fn interactive_session_manages_subscribers_correctly() {
let (mut session, _) = ProcessSession::new(interactive_spec());
// Add two subscribers before start
session.apply(ProcessEvent::Subscribe { address: addr(1) });
session.apply(ProcessEvent::Subscribe { address: addr(2) });
assert_eq!(session.subscriber_count(), 2);
// Duplicate add is a no-op
session.apply(ProcessEvent::Subscribe { address: addr(1) });
assert_eq!(session.subscriber_count(), 2);
// Start — both subscribers notified
let actions = session.apply(ProcessEvent::Started);
match &actions[0] {
ProcessAction::NotifyStarted { subscribers } => {
assert_eq!(subscribers.len(), 2);
}
other => panic!("expected NotifyStarted, got {other:?}"),
}
// Remove one subscriber
session.apply(ProcessEvent::Unsubscribe { address: addr(1) });
assert_eq!(session.subscriber_count(), 1);
// Output only goes to remaining subscriber
let actions = session.apply(ProcessEvent::OutputReceived {
data: b"data".to_vec(),
is_stderr: false,
});
match &actions[0] {
ProcessAction::NotifyOutput { subscribers, .. } => {
assert_eq!(subscribers, &vec![addr(2)]);
}
other => panic!("expected NotifyOutput, got {other:?}"),
}
// Exit
let actions = session.apply(ProcessEvent::Exited {
status: ExitStatus::Code(0),
});
match &actions[0] {
ProcessAction::NotifyExited { subscribers, .. } => {
assert_eq!(subscribers, &vec![addr(2)]);
}
other => panic!("expected NotifyExited, got {other:?}"),
}
assert_self_terminate_is_last(&actions);
}
// ──────────────────────────────────────────────
// 3. Spawn failure
// ──────────────────────────────────────────────
#[test]
fn spawn_failure_notifies_and_self_terminates() {
let (mut session, _) = ProcessSession::new(automated_spec());
session.apply(ProcessEvent::Subscribe { address: addr(1) });
let actions = session.apply(ProcessEvent::SpawnFailed {
reason: "command not found".into(),
});
assert_eq!(session.state(), ProcessState::Exited);
assert!(matches!(
&actions[0],
ProcessAction::NotifyError {
error: ProcessError::SpawnFailed { .. },
..
}
));
assert_self_terminate_is_last(&actions);
}
// ──────────────────────────────────────────────
// 4. Connection loss mid-run
// ──────────────────────────────────────────────
#[test]
fn connection_loss_during_running_transitions_to_exited() {
let (mut session, _) = ProcessSession::new(automated_spec());
session.apply(ProcessEvent::Started);
let actions = session.apply(ProcessEvent::ConnectionLost {
reason: "pipe broken".into(),
});
assert_eq!(session.state(), ProcessState::Exited);
assert_eq!(session.exit_status(), Some(ExitStatus::Unknown));
assert!(matches!(
&actions[0],
ProcessAction::NotifyError {
error: ProcessError::ConnectionLost { .. },
..
}
));
assert_self_terminate_is_last(&actions);
}
// ──────────────────────────────────────────────
// 5. Close requested before start
// ──────────────────────────────────────────────
#[test]
fn close_before_start_sends_signal_on_belated_start() {
let (mut session, _) = ProcessSession::new(automated_spec());
// Close requested while still Starting
let actions = session.apply(ProcessEvent::CloseRequested);
assert!(actions.is_empty());
assert_eq!(session.state(), ProcessState::Starting);
// Process starts belatedly — should immediately get SIGTERM
let actions = session.apply(ProcessEvent::Started);
assert_eq!(session.state(), ProcessState::Stopping);
assert!(matches!(&actions[0], ProcessAction::NotifyStarted { .. }));
assert!(matches!(
&actions[1],
ProcessAction::SendSignal {
signal: Signal::Terminate
}
));
}
// ──────────────────────────────────────────────
// 6. Invalid operations produce errors, not panics
// ──────────────────────────────────────────────
#[test]
fn invalid_event_in_starting_produces_error() {
let (mut session, _) = ProcessSession::new(automated_spec());
let actions = session.apply(ProcessEvent::WriteStdin {
data: b"hi".to_vec(),
});
assert!(matches!(
&actions[0],
ProcessAction::NotifyError {
error: ProcessError::InvalidState {
attempted: "WriteStdin",
current_state: "Starting"
},
..
}
));
// State unchanged
assert_eq!(session.state(), ProcessState::Starting);
}
#[test]
fn invalid_event_in_exited_produces_error() {
let (mut session, _) = ProcessSession::new(automated_spec());
session.apply(ProcessEvent::SpawnFailed {
reason: "no".into(),
});
assert_eq!(session.state(), ProcessState::Exited);
let actions = session.apply(ProcessEvent::WriteStdin {
data: b"hi".to_vec(),
});
assert!(matches!(
&actions[0],
ProcessAction::NotifyError {
error: ProcessError::InvalidState {
attempted: "WriteStdin",
current_state: "Exited"
},
..
}
));
}
// ──────────────────────────────────────────────
// 7. Stdin closed then write → error
// ──────────────────────────────────────────────
#[test]
fn write_after_stdin_closed_produces_error() {
let (mut session, _) = ProcessSession::new(automated_spec());
session.apply(ProcessEvent::Started);
let actions = session.apply(ProcessEvent::CloseStdin);
assert!(matches!(&actions[0], ProcessAction::CloseStdin));
assert!(session.stdin_closed());
// Duplicate close is a no-op
let actions = session.apply(ProcessEvent::CloseStdin);
assert!(actions.is_empty());
// Write after close → error
let actions = session.apply(ProcessEvent::WriteStdin {
data: b"too late".to_vec(),
});
assert!(matches!(
&actions[0],
ProcessAction::NotifyError {
error: ProcessError::InvalidState { .. },
..
}
));
}
// ──────────────────────────────────────────────
// 8. MockDriver round-trip (driver + session tick loop)
// ──────────────────────────────────────────────
#[test]
fn mock_driver_round_trip() {
let (mut session, init_actions) = ProcessSession::new(automated_spec());
let mut driver = MockDriver::new();
// Execute initial actions (SpawnProcess)
for action in init_actions {
driver.execute(action);
}
assert!(matches!(
&driver.executed_actions()[0],
ProcessAction::SpawnProcess { .. }
));
// Simulate: driver produces Started
driver.inject(ProcessEvent::Started);
// Tick loop: poll → apply → execute
let events = driver.poll();
for event in events {
let actions = session.apply(event);
for action in actions {
driver.execute(action);
}
}
assert_eq!(session.state(), ProcessState::Running);
// Simulate output and exit
driver.inject(ProcessEvent::OutputReceived {
data: b"done".to_vec(),
is_stderr: false,
});
driver.inject(ProcessEvent::Exited {
status: ExitStatus::Code(0),
});
let events = driver.poll();
for event in events {
let actions = session.apply(event);
for action in actions {
driver.execute(action);
}
}
assert_eq!(session.state(), ProcessState::Exited);
// Verify the driver saw the expected sequence
let all_actions = driver.take_executed_actions();
assert!(matches!(
&all_actions[0],
ProcessAction::SpawnProcess { .. }
));
assert!(matches!(
&all_actions[1],
ProcessAction::NotifyStarted { .. }
));
assert!(matches!(
&all_actions[2],
ProcessAction::NotifyOutput { .. }
));
assert!(matches!(
&all_actions[3],
ProcessAction::NotifyExited { .. }
));
assert!(matches!(&all_actions[4], ProcessAction::SelfTerminate));
}
// ──────────────────────────────────────────────
// 9. Signal escalation in Stopping
// ──────────────────────────────────────────────
#[test]
fn signal_escalation_allowed_in_stopping() {
let (mut session, _) = ProcessSession::new(automated_spec());
session.apply(ProcessEvent::Started);
session.apply(ProcessEvent::CloseRequested);
assert_eq!(session.state(), ProcessState::Stopping);
// Escalate to Kill
let actions = session.apply(ProcessEvent::SendSignal {
signal: Signal::Kill,
});
assert!(matches!(
&actions[0],
ProcessAction::SendSignal {
signal: Signal::Kill
}
));
// Can still receive output while stopping
let actions = session.apply(ProcessEvent::OutputReceived {
data: b"final".to_vec(),
is_stderr: false,
});
assert!(matches!(&actions[0], ProcessAction::NotifyOutput { .. }));
// Finally exits
let actions = session.apply(ProcessEvent::Exited {
status: ExitStatus::Signal(9),
});
assert_eq!(session.exit_status(), Some(ExitStatus::Signal(9)));
assert_self_terminate_is_last(&actions);
}
// ──────────────────────────────────────────────
// 10. Late acks in Exited silently consumed
// ──────────────────────────────────────────────
#[test]
fn late_acks_in_exited_are_silently_consumed() {
let (mut session, _) = ProcessSession::new(automated_spec());
session.apply(ProcessEvent::Started);
session.apply(ProcessEvent::Exited {
status: ExitStatus::Code(0),
});
assert_eq!(session.state(), ProcessState::Exited);
// Acks should produce no actions, no errors
assert!(
session
.apply(ProcessEvent::StdinWritten { byte_count: 10 })
.is_empty()
);
assert!(session.apply(ProcessEvent::SignalSent).is_empty());
assert!(session.apply(ProcessEvent::PtyResized).is_empty());
// Subscribe/Unsubscribe also still works in Exited
assert!(
session
.apply(ProcessEvent::Subscribe { address: addr(1) })
.is_empty()
);
assert_eq!(session.subscriber_count(), 1);
assert!(
session
.apply(ProcessEvent::Unsubscribe { address: addr(1) })
.is_empty()
);
assert_eq!(session.subscriber_count(), 0);
}
// ──────────────────────────────────────────────
// Flow control tracking
// ──────────────────────────────────────────────
#[test]
fn flow_control_tracks_pending_stdin_bytes() {
let (mut session, _) = ProcessSession::new(automated_spec());
session.apply(ProcessEvent::Started);
session.apply(ProcessEvent::WriteStdin {
data: vec![0u8; 100],
});
assert_eq!(session.flow_control().pending_stdin_bytes, 100);
session.apply(ProcessEvent::WriteStdin {
data: vec![0u8; 50],
});
assert_eq!(session.flow_control().pending_stdin_bytes, 150);
session.apply(ProcessEvent::StdinWritten { byte_count: 80 });
assert_eq!(session.flow_control().pending_stdin_bytes, 70);
// Ack more than pending → saturates at 0
session.apply(ProcessEvent::StdinWritten { byte_count: 200 });
assert_eq!(session.flow_control().pending_stdin_bytes, 0);
}
// ──────────────────────────────────────────────
// CloseStdin in Stopping
// ──────────────────────────────────────────────
#[test]
fn close_stdin_allowed_in_stopping() {
let (mut session, _) = ProcessSession::new(automated_spec());
session.apply(ProcessEvent::Started);
session.apply(ProcessEvent::CloseRequested);
assert_eq!(session.state(), ProcessState::Stopping);
let actions = session.apply(ProcessEvent::CloseStdin);
assert!(matches!(&actions[0], ProcessAction::CloseStdin));
assert!(session.stdin_closed());
}
// ──────────────────────────────────────────────
// Connection loss in Stopping
// ──────────────────────────────────────────────
#[test]
fn connection_loss_in_stopping_transitions_to_exited() {
let (mut session, _) = ProcessSession::new(automated_spec());
session.apply(ProcessEvent::Started);
session.apply(ProcessEvent::CloseRequested);
assert_eq!(session.state(), ProcessState::Stopping);
let actions = session.apply(ProcessEvent::ConnectionLost {
reason: "gone".into(),
});
assert_eq!(session.state(), ProcessState::Exited);
assert_self_terminate_is_last(&actions);
}
// ──────────────────────────────────────────────
// Redundant CloseRequested in Stopping is no-op
// ──────────────────────────────────────────────
#[test]
fn duplicate_close_requested_in_stopping_is_noop() {
let (mut session, _) = ProcessSession::new(automated_spec());
session.apply(ProcessEvent::Started);
session.apply(ProcessEvent::CloseRequested);
assert_eq!(session.state(), ProcessState::Stopping);
let actions = session.apply(ProcessEvent::CloseRequested);
assert!(actions.is_empty());
assert_eq!(session.state(), ProcessState::Stopping);
}
// ──────────────────────────────────────────────
// Kill timeout — A1–A6
// ──────────────────────────────────────────────
#[test]
fn close_requested_with_kill_timeout_schedules_timer() {
let (mut session, _) = ProcessSession::new(spec_with_kill_timeout(Duration::from_secs(5)));
session.apply(ProcessEvent::Started);
let actions = session.apply(ProcessEvent::CloseRequested);
assert_eq!(session.state(), ProcessState::Stopping);
assert!(matches!(
&actions[0],
ProcessAction::SendSignal {
signal: Signal::Terminate
}
));
assert!(matches!(
&actions[1],
ProcessAction::ScheduleKillTimeout { duration } if *duration == Duration::from_secs(5)
));
}
#[test]
fn close_before_start_with_kill_timeout_schedules_timer_on_belated_start() {
let (mut session, _) = ProcessSession::new(spec_with_kill_timeout(Duration::from_secs(3)));
session.apply(ProcessEvent::CloseRequested);
let actions = session.apply(ProcessEvent::Started);
assert_eq!(session.state(), ProcessState::Stopping);
assert!(matches!(&actions[0], ProcessAction::NotifyStarted { .. }));
assert!(matches!(
&actions[1],
ProcessAction::SendSignal {
signal: Signal::Terminate
}
));
assert!(matches!(
&actions[2],
ProcessAction::ScheduleKillTimeout { duration } if *duration == Duration::from_secs(3)
));
}
#[test]
fn kill_timeout_in_stopping_sends_sigkill() {
let (mut session, _) = ProcessSession::new(spec_with_kill_timeout(Duration::from_secs(5)));
session.apply(ProcessEvent::Started);
session.apply(ProcessEvent::CloseRequested);
assert_eq!(session.state(), ProcessState::Stopping);
let actions = session.apply(ProcessEvent::KillTimeout);
assert!(matches!(
&actions[0],
ProcessAction::SendSignal {
signal: Signal::Kill
}
));
assert_eq!(session.state(), ProcessState::Stopping);
}
#[test]
fn kill_timeout_silently_consumed_outside_stopping() {
// Starting
let (mut session, _) = ProcessSession::new(automated_spec());
assert!(session.apply(ProcessEvent::KillTimeout).is_empty());
assert_eq!(session.state(), ProcessState::Starting);
// Running
session.apply(ProcessEvent::Started);
assert!(session.apply(ProcessEvent::KillTimeout).is_empty());
assert_eq!(session.state(), ProcessState::Running);
// Exited
session.apply(ProcessEvent::Exited {
status: ExitStatus::Code(0),
});
assert!(session.apply(ProcessEvent::KillTimeout).is_empty());
assert_eq!(session.state(), ProcessState::Exited);
}
#[test]
fn close_requested_without_kill_timeout_no_schedule_action() {
let (mut session, _) = ProcessSession::new(automated_spec());
session.apply(ProcessEvent::Started);
let actions = session.apply(ProcessEvent::CloseRequested);
assert_eq!(actions.len(), 1);
assert!(matches!(
&actions[0],
ProcessAction::SendSignal {
signal: Signal::Terminate
}
));
}
#[test]
fn kill_timeout_full_escalation_to_sigkill_then_exit() {
let (mut session, _) = ProcessSession::new(spec_with_kill_timeout(Duration::from_secs(1)));
session.apply(ProcessEvent::Started);
// CloseRequested → SIGTERM + schedule
let actions = session.apply(ProcessEvent::CloseRequested);
assert_eq!(session.state(), ProcessState::Stopping);
assert!(matches!(
&actions[0],
ProcessAction::SendSignal {
signal: Signal::Terminate
}
));
assert!(matches!(
&actions[1],
ProcessAction::ScheduleKillTimeout { .. }
));
// KillTimeout fires → SIGKILL
let actions = session.apply(ProcessEvent::KillTimeout);
assert!(matches!(
&actions[0],
ProcessAction::SendSignal {
signal: Signal::Kill
}
));
// Process finally exits via signal 9
let actions = session.apply(ProcessEvent::Exited {
status: ExitStatus::Signal(9),
});
assert_eq!(session.state(), ProcessState::Exited);
assert_eq!(session.exit_status(), Some(ExitStatus::Signal(9)));
assert_self_terminate_is_last(&actions);
}
// ──────────────────────────────────────────────
// Backpressure — B1–B5
// ──────────────────────────────────────────────
#[test]
fn backpressure_buffers_when_over_limit() {
let (mut session, _) = ProcessSession::new(spec_with_stdin_limit(100));
session.apply(ProcessEvent::Started);
// First write (50 bytes) — under limit, passes through
let actions = session.apply(ProcessEvent::WriteStdin {
data: vec![1u8; 50],
});
assert_eq!(actions.len(), 1);
assert!(matches!(&actions[0], ProcessAction::WriteStdin { .. }));
assert_eq!(session.flow_control().pending_stdin_bytes, 50);
// Second write (60 bytes) — still under limit (50 < 100), passes through
let actions = session.apply(ProcessEvent::WriteStdin {
data: vec![2u8; 60],
});
assert_eq!(actions.len(), 1);
assert_eq!(session.flow_control().pending_stdin_bytes, 110);
// Third write (30 bytes) — now at 110 >= 100, buffered
let actions = session.apply(ProcessEvent::WriteStdin {
data: vec![3u8; 30],
});
assert!(actions.is_empty());
assert_eq!(session.stdin_buffer_bytes(), 30);
// pending_stdin_bytes unchanged (buffered data not counted as pending)
assert_eq!(session.flow_control().pending_stdin_bytes, 110);
}
#[test]
fn stdin_written_ack_drains_buffer() {
let (mut session, _) = ProcessSession::new(spec_with_stdin_limit(100));
session.apply(ProcessEvent::Started);
// Fill up: 100 bytes pending
session.apply(ProcessEvent::WriteStdin {
data: vec![1u8; 100],
});
assert_eq!(session.flow_control().pending_stdin_bytes, 100);
// Buffer two chunks
session.apply(ProcessEvent::WriteStdin {
data: vec![2u8; 40],
});
session.apply(ProcessEvent::WriteStdin {
data: vec![3u8; 30],
});
assert_eq!(session.stdin_buffer_bytes(), 70);
// Ack 80 bytes → pending drops to 20, buffer should drain in FIFO order
let actions = session.apply(ProcessEvent::StdinWritten { byte_count: 80 });
// pending was 100, now 20. Drain first chunk (40 bytes) → pending = 60.
// 60 < 100, drain second chunk (30 bytes) → pending = 90.
// 90 < 100, buffer empty.
assert_eq!(actions.len(), 2);
assert!(matches!(&actions[0], ProcessAction::WriteStdin { data } if data.len() == 40));
assert!(matches!(&actions[1], ProcessAction::WriteStdin { data } if data.len() == 30));
assert_eq!(session.flow_control().pending_stdin_bytes, 90);
assert_eq!(session.stdin_buffer_bytes(), 0);
}
#[test]
fn close_requested_clears_stdin_buffer() {
let (mut session, _) = ProcessSession::new(spec_with_stdin_limit(50));
session.apply(ProcessEvent::Started);
session.apply(ProcessEvent::WriteStdin {
data: vec![1u8; 60],
});
session.apply(ProcessEvent::WriteStdin {
data: vec![2u8; 30],
});
assert_eq!(session.stdin_buffer_bytes(), 30);
session.apply(ProcessEvent::CloseRequested);
assert_eq!(session.stdin_buffer_bytes(), 0);
}
#[test]
fn no_backpressure_when_limit_is_none() {
let (mut session, _) = ProcessSession::new(automated_spec());
session.apply(ProcessEvent::Started);
// All writes pass through regardless of pending bytes
for _ in 0..10 {
let actions = session.apply(ProcessEvent::WriteStdin {
data: vec![0u8; 1000],
});
assert_eq!(actions.len(), 1);
assert!(matches!(&actions[0], ProcessAction::WriteStdin { .. }));
}
assert_eq!(session.flow_control().pending_stdin_bytes, 10_000);
assert_eq!(session.stdin_buffer_bytes(), 0);
}
#[test]
fn exit_clears_stdin_buffer() {
let (mut session, _) = ProcessSession::new(spec_with_stdin_limit(50));
session.apply(ProcessEvent::Started);
session.apply(ProcessEvent::WriteStdin {
data: vec![1u8; 60],
});
session.apply(ProcessEvent::WriteStdin {
data: vec![2u8; 30],
});
assert_eq!(session.stdin_buffer_bytes(), 30);
session.apply(ProcessEvent::Exited {
status: ExitStatus::Code(0),
});
assert_eq!(session.stdin_buffer_bytes(), 0);
}

View file

@ -681,14 +681,6 @@ impl<'a> Ctx<'a> {
self.inner.extension()
}
/// Access the per-runtime process-output observer (if installed). The
/// process facility reads this when spawning so every managed process's
/// output is taped automatically.
pub fn process_output_observer(
&self,
) -> Option<std::sync::Arc<dyn crate::process_observer::ProcessOutputObserver>> {
self.inner.process_output_observer()
}
/// Return system-level information (worker count, actor count, uptime).
pub fn system_info(&self) -> SystemInfo {

View file

@ -1,20 +1,16 @@
//! Per-runtime hook for observing output from processes spawned through the
//! process facility.
//! Legacy/custom hook for adapters that choose to observe stdout/stderr bytes.
//!
//! This trait lives in swactor rather than `swactor-process` so the runtime can
//! store an observer without a dependency cycle: `swactor-process` depends on
//! swactor, and a telemetry emitter (which lives even higher up) implements this
//! trait. The runtime is merely the carrier — it never interprets the bytes.
//! Current `swactor-process` managed-process core does not call this hook.
//! Managed-process lifecycle/control output is delivered as `ProcessOutput` to
//! the configured upstream owner and may be mirrored to datastream through
//! `ProcessOutputConfig::datastream_mirror`.
//!
//! The observer is per-node (per-runtime), installed via
//! [`Runtime::set_process_output_observer`](crate::runtime::Runtime::set_process_output_observer).
//! The process facility hands every managed process's output to it automatically,
//! labeled by the process's command basename, so a node taps all of its managed
//! processes with no per-spawn wiring.
//! The runtime is only the storage location for callers that still wire this
//! hook themselves; it does not interpret the bytes.
/// Observes every chunk of stdout/stderr from processes the runtime spawns.
/// Observes chunks supplied by legacy/custom process-output adapters.
pub trait ProcessOutputObserver: Send + Sync {
/// `label` identifies the process (its command basename); `is_stderr`
/// selects the stream; `data` is one raw output chunk.
/// `label` identifies the adapter-defined source; `is_stderr` selects the
/// stream; `data` is one raw output chunk.
fn on_output(&self, label: &str, is_stderr: bool, data: &[u8]);
}

View file

@ -598,17 +598,6 @@ impl Runtime {
self.stats_hook = Some(hook);
}
/// Install the per-node process-output observer. Every process spawned
/// through the process facility on this runtime hands its stdout/stderr to
/// `obs`, labeled by command basename. Takes `&self` (the slot is a
/// `OnceLock`) so it can be installed on an already-shared `Arc<Runtime>`,
/// before the first managed process is spawned. Subsequent calls are no-ops.
pub fn set_process_output_observer(
&self,
obs: Arc<dyn crate::process_observer::ProcessOutputObserver>,
) {
let _ = self.process_output_observer.set(obs);
}
/// Set the sink for non-local (remote) message delivery.
///