Compare commits
3 commits
17491c5247
...
808435ce2a
| Author | SHA1 | Date | |
|---|---|---|---|
| 808435ce2a | |||
| f3900401fb | |||
| dfe4507eb1 |
51 changed files with 4395 additions and 4936 deletions
5
Cargo.lock
generated
5
Cargo.lock
generated
|
|
@ -937,6 +937,7 @@ dependencies = [
|
||||||
name = "datastream"
|
name = "datastream"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"crossbeam-channel",
|
||||||
"iroh",
|
"iroh",
|
||||||
"libc",
|
"libc",
|
||||||
"serde",
|
"serde",
|
||||||
|
|
@ -2104,6 +2105,7 @@ dependencies = [
|
||||||
name = "iroh-driver"
|
name = "iroh-driver"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"crossbeam-channel",
|
||||||
"datastream",
|
"datastream",
|
||||||
"distribution",
|
"distribution",
|
||||||
"iroh",
|
"iroh",
|
||||||
|
|
@ -4342,9 +4344,8 @@ name = "swactor-process"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"crossbeam-queue",
|
"crossbeam-queue",
|
||||||
|
"datastream",
|
||||||
"libc",
|
"libc",
|
||||||
"proptest",
|
|
||||||
"proptest-state-machine",
|
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"serde_yaml",
|
"serde_yaml",
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,6 @@ pub mod view;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use datastream::frame::{ChannelId, Frame, Lifetime, NodeId, Position, StreamId};
|
use datastream::frame::{ChannelId, Frame, Lifetime, NodeId, Position, StreamId};
|
||||||
use parking_lot::Mutex;
|
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use tokio::sync::broadcast;
|
use tokio::sync::broadcast;
|
||||||
|
|
||||||
|
|
@ -86,10 +85,32 @@ pub struct DashboardHandle {
|
||||||
store: Arc<DashboardStore>,
|
store: Arc<DashboardStore>,
|
||||||
views: Arc<ViewRegistry>,
|
views: Arc<ViewRegistry>,
|
||||||
shutdown_notify: Arc<tokio::sync::Notify>,
|
shutdown_notify: Arc<tokio::sync::Notify>,
|
||||||
standalone_rt: Mutex<Option<tokio::runtime::Runtime>>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl DashboardHandle {
|
impl DashboardHandle {
|
||||||
|
/// Create the datastream dashboard state.
|
||||||
|
///
|
||||||
|
/// The HTTP server is not started until `start_http` or `start_http_standalone`
|
||||||
|
/// is called.
|
||||||
|
pub fn new(config: DashboardConfig) -> Self {
|
||||||
|
let views = Arc::new(ViewRegistry::new());
|
||||||
|
views.register(Arc::new(live_explorer::LiveDatastreamExplorer::default()));
|
||||||
|
views.register(Arc::new(hardware_view::HardwareDashboardView::default()));
|
||||||
|
views.register(swactor::worker_view());
|
||||||
|
let store = Arc::new(DashboardStore::new(
|
||||||
|
config.raw_frame_history,
|
||||||
|
Arc::clone(&views),
|
||||||
|
));
|
||||||
|
let (frames, _) = broadcast::channel(config.frame_buffer.max(1));
|
||||||
|
Self {
|
||||||
|
port: config.port,
|
||||||
|
frames,
|
||||||
|
store,
|
||||||
|
views,
|
||||||
|
shutdown_notify: Arc::new(tokio::sync::Notify::new()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Register a read-only view. External crates can keep their interpretation
|
/// Register a read-only view. External crates can keep their interpretation
|
||||||
/// code beside their component and plug it into this registry.
|
/// code beside their component and plug it into this registry.
|
||||||
pub fn register_view(&self, view: Arc<dyn DashboardView>) {
|
pub fn register_view(&self, view: Arc<dyn DashboardView>) {
|
||||||
|
|
@ -113,8 +134,8 @@ impl DashboardHandle {
|
||||||
self.shutdown_notify.notify_waiters();
|
self.shutdown_notify.notify_waiters();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Start the HTTP server on an existing Tokio runtime.
|
/// Build the HTTP server future for an embedding runtime to poll directly.
|
||||||
pub fn start_http(&self, handle: tokio::runtime::Handle) {
|
pub fn http_server(&self) -> impl Future<Output = ()> + Send + 'static {
|
||||||
let state = server::AppState {
|
let state = server::AppState {
|
||||||
frames: self.frames.clone(),
|
frames: self.frames.clone(),
|
||||||
store: Arc::clone(&self.store),
|
store: Arc::clone(&self.store),
|
||||||
|
|
@ -122,44 +143,13 @@ impl DashboardHandle {
|
||||||
shutdown_notify: Arc::clone(&self.shutdown_notify),
|
shutdown_notify: Arc::clone(&self.shutdown_notify),
|
||||||
};
|
};
|
||||||
let port = self.port;
|
let port = self.port;
|
||||||
handle.spawn(async move {
|
async move {
|
||||||
server::run_server(state, port).await;
|
server::run_server(state, port).await;
|
||||||
});
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Start the HTTP server on a standalone Tokio runtime.
|
/// Spawn the HTTP server on an existing Tokio runtime and return its task handle.
|
||||||
pub fn start_http_standalone(&self) {
|
pub fn spawn_http(&self, handle: &tokio::runtime::Handle) -> tokio::task::JoinHandle<()> {
|
||||||
let rt = tokio::runtime::Builder::new_multi_thread()
|
handle.spawn(self.http_server())
|
||||||
.worker_threads(1)
|
|
||||||
.enable_all()
|
|
||||||
.build()
|
|
||||||
.expect("failed to create tokio runtime for dashboard HTTP");
|
|
||||||
let handle = rt.handle().clone();
|
|
||||||
*self.standalone_rt.lock() = Some(rt);
|
|
||||||
self.start_http(handle);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Create the datastream dashboard state.
|
|
||||||
///
|
|
||||||
/// The HTTP server is not started until `start_http` or `start_http_standalone`
|
|
||||||
/// is called.
|
|
||||||
pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle {
|
|
||||||
let views = Arc::new(ViewRegistry::new());
|
|
||||||
views.register(Arc::new(live_explorer::LiveDatastreamExplorer::default()));
|
|
||||||
views.register(Arc::new(hardware_view::HardwareDashboardView::default()));
|
|
||||||
views.register(swactor::worker_view());
|
|
||||||
let store = Arc::new(DashboardStore::new(
|
|
||||||
config.raw_frame_history,
|
|
||||||
Arc::clone(&views),
|
|
||||||
));
|
|
||||||
let (frames, _) = broadcast::channel(config.frame_buffer.max(1));
|
|
||||||
DashboardHandle {
|
|
||||||
port: config.port,
|
|
||||||
frames,
|
|
||||||
store,
|
|
||||||
views,
|
|
||||||
shutdown_notify: Arc::new(tokio::sync::Notify::new()),
|
|
||||||
standalone_rt: Mutex::new(None),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,7 @@ swactor-transport = { path = "../transport" }
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
iroh = "0.98"
|
iroh = "0.98"
|
||||||
|
crossbeam-channel = "0.5"
|
||||||
|
|
||||||
[target.'cfg(target_os = "linux")'.dependencies]
|
[target.'cfg(target_os = "linux")'.dependencies]
|
||||||
libc = "0.2"
|
libc = "0.2"
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -7,14 +7,14 @@ use swactor::actor::ActorAddress;
|
||||||
use swactor::process_observer::ProcessOutputObserver;
|
use swactor::process_observer::ProcessOutputObserver;
|
||||||
use swactor::runtime::Runtime;
|
use swactor::runtime::Runtime;
|
||||||
|
|
||||||
use super::frame::{ChannelId, Frame, Lifetime, NodeId, Position, StreamId};
|
use super::frame::{ChannelId, Frame, Lifetime, NodeId, StreamId};
|
||||||
use super::mux::Mux;
|
use super::mux::Mux;
|
||||||
use super::record::Record;
|
use super::record::Record;
|
||||||
use super::wire::{DatastreamFrame, encode_delivery};
|
use super::wire::{DatastreamFrame, encode_delivery};
|
||||||
|
|
||||||
/// Where assembled frames go once the mux has ordered them.
|
/// Legacy sink for frames after mux drain has assigned positions.
|
||||||
pub trait FrameSink: Send {
|
pub trait FrameSink: Send {
|
||||||
/// Ship one ordered frame for `stream`. Best-effort: a sink may drop.
|
/// Ship one positioned frame for `stream`. Best-effort: a sink may drop.
|
||||||
fn ship(&mut self, stream: &StreamId, frame: &Frame);
|
fn ship(&mut self, stream: &StreamId, frame: &Frame);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -73,26 +73,23 @@ impl DatastreamEmitter {
|
||||||
self.mux.dropped()
|
self.mux.dropped()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn set_frame_timing_enabled(&self, enabled: bool) {
|
pub fn submit_record<R: Record>(&self, channel: ChannelId, record: &R) -> bool {
|
||||||
self.mux.set_frame_timing_enabled(enabled);
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn frame_timing_enabled(&self) -> bool {
|
|
||||||
self.mux.frame_timing_enabled()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn submit_record<R: Record>(&self, channel: ChannelId, record: &R) -> Position {
|
|
||||||
self.mux.submit(channel, record.encode())
|
self.mux.submit(channel, record.encode())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn submit_text(&self, channel: ChannelId, text: impl AsRef<[u8]>) -> Position {
|
pub fn submit_text(&self, channel: ChannelId, text: impl AsRef<[u8]>) -> bool {
|
||||||
self.mux.submit(channel, text.as_ref().to_vec())
|
self.mux.submit(channel, text.as_ref().to_vec())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn submit_bytes(&self, channel: ChannelId, bytes: Vec<u8>) -> Position {
|
pub fn submit_bytes(&self, channel: ChannelId, bytes: Vec<u8>) -> bool {
|
||||||
self.mux.submit(channel, bytes)
|
self.mux.submit(channel, bytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn submit_text_owned(&self, channel: ChannelId, text: String) -> bool {
|
||||||
|
self.mux.submit(channel, text.into_bytes())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build a legacy/custom process-output observer that writes chunks into this mux.
|
||||||
pub fn process_observer_with<F>(&self, channel_for: F) -> Arc<dyn ProcessOutputObserver>
|
pub fn process_observer_with<F>(&self, channel_for: F) -> Arc<dyn ProcessOutputObserver>
|
||||||
where
|
where
|
||||||
F: Fn(&str, bool) -> ChannelId + Send + Sync + 'static,
|
F: Fn(&str, bool) -> ChannelId + Send + Sync + 'static,
|
||||||
|
|
@ -123,17 +120,21 @@ pub struct DatastreamEventSink {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl DatastreamEventSink {
|
impl DatastreamEventSink {
|
||||||
pub fn submit_record<R: Record>(&self, channel: ChannelId, record: &R) -> Position {
|
pub fn submit_record<R: Record>(&self, channel: ChannelId, record: &R) -> bool {
|
||||||
self.mux.submit(channel, record.encode())
|
self.mux.submit(channel, record.encode())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn submit_text(&self, channel: ChannelId, text: impl AsRef<[u8]>) -> Position {
|
pub fn submit_text(&self, channel: ChannelId, text: impl AsRef<[u8]>) -> bool {
|
||||||
self.mux.submit(channel, text.as_ref().to_vec())
|
self.mux.submit(channel, text.as_ref().to_vec())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn submit_bytes(&self, channel: ChannelId, bytes: Vec<u8>) -> Position {
|
pub fn submit_bytes(&self, channel: ChannelId, bytes: Vec<u8>) -> bool {
|
||||||
self.mux.submit(channel, bytes)
|
self.mux.submit(channel, bytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn submit_text_owned(&self, channel: ChannelId, text: String) -> bool {
|
||||||
|
self.mux.submit(channel, text.into_bytes())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A sink that drops everything.
|
/// A sink that drops everything.
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,11 @@
|
||||||
|
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
use std::sync::atomic::{AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
use std::sync::{Arc, Mutex, mpsc};
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
|
use crossbeam_channel::{
|
||||||
|
Receiver, RecvError, RecvTimeoutError, Sender, TryRecvError, TrySendError, bounded,
|
||||||
|
};
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use swactor::process_observer::ProcessOutputObserver;
|
use swactor::process_observer::ProcessOutputObserver;
|
||||||
|
|
@ -14,12 +18,10 @@ use swactor::stats::{ActorSnapshot, StatsHook};
|
||||||
|
|
||||||
use crate::frame::{
|
use crate::frame::{
|
||||||
ChannelContent, ChannelDescriptor, ChannelFilter, ChannelId, ChannelRef, DatastreamEvent,
|
ChannelContent, ChannelDescriptor, ChannelFilter, ChannelId, ChannelRef, DatastreamEvent,
|
||||||
FrameDelivery, Position, SourceFilter, StreamDescriptor, StreamId, StreamOrigin,
|
FrameDelivery, SourceFilter, StreamDescriptor, StreamId, StreamOrigin, SubscriptionRequest,
|
||||||
SubscriptionRequest,
|
|
||||||
};
|
};
|
||||||
use crate::mux::Mux;
|
use crate::mux::Mux;
|
||||||
use crate::record::Record;
|
use crate::record::Record;
|
||||||
use crate::timing::{FRAME_TIME_CHANNEL, FRAME_TIME_CHANNEL_ID};
|
|
||||||
use crate::transport::Delivery;
|
use crate::transport::Delivery;
|
||||||
|
|
||||||
const DEFAULT_MUX_CAPACITY: usize = 4096;
|
const DEFAULT_MUX_CAPACITY: usize = 4096;
|
||||||
|
|
@ -51,7 +53,7 @@ pub struct SubscriberSnapshot {
|
||||||
pub dropped: u64,
|
pub dropped: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Current catalog snapshot delivered at subscription time.
|
/// Current catalog snapshot delivered at subscription time, filtered by request.
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct DatastreamSnapshot {
|
pub struct DatastreamSnapshot {
|
||||||
pub streams: Vec<StreamDescriptor>,
|
pub streams: Vec<StreamDescriptor>,
|
||||||
|
|
@ -70,7 +72,7 @@ pub struct DatastreamSubscription {
|
||||||
name: String,
|
name: String,
|
||||||
request: SubscriptionRequest,
|
request: SubscriptionRequest,
|
||||||
snapshot: DatastreamSnapshot,
|
snapshot: DatastreamSnapshot,
|
||||||
rx: mpsc::Receiver<DatastreamEvent>,
|
rx: Receiver<DatastreamEvent>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl DatastreamSubscription {
|
impl DatastreamSubscription {
|
||||||
|
|
@ -90,18 +92,18 @@ impl DatastreamSubscription {
|
||||||
&self.snapshot
|
&self.snapshot
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn try_recv(&self) -> Result<DatastreamEvent, mpsc::TryRecvError> {
|
pub fn try_recv(&self) -> Result<DatastreamEvent, TryRecvError> {
|
||||||
self.rx.try_recv()
|
self.rx.try_recv()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn recv(&self) -> Result<DatastreamEvent, mpsc::RecvError> {
|
pub fn recv(&self) -> Result<DatastreamEvent, RecvError> {
|
||||||
self.rx.recv()
|
self.rx.recv()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn recv_timeout(
|
pub fn recv_timeout(
|
||||||
&self,
|
&self,
|
||||||
timeout: std::time::Duration,
|
timeout: std::time::Duration,
|
||||||
) -> Result<DatastreamEvent, mpsc::RecvTimeoutError> {
|
) -> Result<DatastreamEvent, RecvTimeoutError> {
|
||||||
self.rx.recv_timeout(timeout)
|
self.rx.recv_timeout(timeout)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -116,17 +118,27 @@ impl DatastreamSubscription {
|
||||||
|
|
||||||
struct SubscriberSlot {
|
struct SubscriberSlot {
|
||||||
name: String,
|
name: String,
|
||||||
request: SubscriptionRequest,
|
tx: Sender<DatastreamEvent>,
|
||||||
tx: mpsc::SyncSender<DatastreamEvent>,
|
|
||||||
dropped: u64,
|
dropped: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct FanoutTarget {
|
||||||
|
id: SubscriptionId,
|
||||||
|
tx: Sender<DatastreamEvent>,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct FanoutReport {
|
||||||
|
id: SubscriptionId,
|
||||||
|
dropped: u64,
|
||||||
|
disconnected: bool,
|
||||||
|
}
|
||||||
|
|
||||||
struct FanoutState {
|
struct FanoutState {
|
||||||
next_id: u64,
|
next_id: u64,
|
||||||
subscribers: BTreeMap<SubscriptionId, SubscriberSlot>,
|
subscribers: BTreeMap<SubscriptionId, SubscriberSlot>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Local event fanout used by endpoints and collectors.
|
/// Local fanout; future events are broadcast to every subscriber without request filtering.
|
||||||
pub struct DeliveryFanout {
|
pub struct DeliveryFanout {
|
||||||
default_capacity: usize,
|
default_capacity: usize,
|
||||||
state: Mutex<FanoutState>,
|
state: Mutex<FanoutState>,
|
||||||
|
|
@ -168,7 +180,7 @@ impl DeliveryFanout {
|
||||||
capacity: usize,
|
capacity: usize,
|
||||||
) -> DatastreamSubscription {
|
) -> DatastreamSubscription {
|
||||||
let name = name.into();
|
let name = name.into();
|
||||||
let (tx, rx) = mpsc::sync_channel(capacity.max(1));
|
let (tx, rx) = bounded(capacity.max(1));
|
||||||
let mut state = self.state.lock().expect("datastream fanout poisoned");
|
let mut state = self.state.lock().expect("datastream fanout poisoned");
|
||||||
let id = SubscriptionId(state.next_id);
|
let id = SubscriptionId(state.next_id);
|
||||||
state.next_id = state.next_id.wrapping_add(1).max(1);
|
state.next_id = state.next_id.wrapping_add(1).max(1);
|
||||||
|
|
@ -176,7 +188,6 @@ impl DeliveryFanout {
|
||||||
id,
|
id,
|
||||||
SubscriberSlot {
|
SubscriberSlot {
|
||||||
name: name.clone(),
|
name: name.clone(),
|
||||||
request: request.clone(),
|
|
||||||
tx,
|
tx,
|
||||||
dropped: 0,
|
dropped: 0,
|
||||||
},
|
},
|
||||||
|
|
@ -212,23 +223,34 @@ impl DeliveryFanout {
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn publish(&self, event: DatastreamEvent, catalog: &CatalogSnapshot) -> EndpointTick {
|
pub fn publish(&self, event: DatastreamEvent) -> EndpointTick {
|
||||||
self.publish_batch(std::iter::once(event), catalog)
|
self.publish_batch(std::iter::once(event))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn publish_batch(
|
pub fn publish_batch(&self, events: impl IntoIterator<Item = DatastreamEvent>) -> EndpointTick {
|
||||||
&self,
|
|
||||||
events: impl IntoIterator<Item = DatastreamEvent>,
|
|
||||||
catalog: &CatalogSnapshot,
|
|
||||||
) -> EndpointTick {
|
|
||||||
let events: Vec<DatastreamEvent> = events.into_iter().collect();
|
let events: Vec<DatastreamEvent> = events.into_iter().collect();
|
||||||
if events.is_empty() {
|
if events.is_empty() {
|
||||||
return EndpointTick::default();
|
return EndpointTick::default();
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut state = self.state.lock().expect("datastream fanout poisoned");
|
// Snapshot sender handles while holding the subscriber map lock, then
|
||||||
let subscribers = state.subscribers.len();
|
// deliver outside the lock so large batches or slow subscribers do not
|
||||||
if subscribers == 0 {
|
// block subscribe/snapshot control-plane operations.
|
||||||
|
let (targets, subscribers) = {
|
||||||
|
let state = self.state.lock().expect("datastream fanout poisoned");
|
||||||
|
let subscribers = state.subscribers.len();
|
||||||
|
let targets = state
|
||||||
|
.subscribers
|
||||||
|
.iter()
|
||||||
|
.map(|(id, slot)| FanoutTarget {
|
||||||
|
id: *id,
|
||||||
|
tx: slot.tx.clone(),
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
(targets, subscribers)
|
||||||
|
};
|
||||||
|
|
||||||
|
if targets.is_empty() {
|
||||||
return EndpointTick {
|
return EndpointTick {
|
||||||
drained: events.len(),
|
drained: events.len(),
|
||||||
subscribers: 0,
|
subscribers: 0,
|
||||||
|
|
@ -237,36 +259,48 @@ impl DeliveryFanout {
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut delivered = 0;
|
let mut delivered = 0;
|
||||||
let mut dropped = 0;
|
let mut reports = Vec::new();
|
||||||
let mut disconnected = Vec::new();
|
for target in targets {
|
||||||
for (id, slot) in state.subscribers.iter_mut() {
|
let mut dropped = 0;
|
||||||
|
let mut disconnected = false;
|
||||||
for event in &events {
|
for event in &events {
|
||||||
if !event_matches_request(event, &slot.request, catalog) {
|
match target.tx.try_send(event.clone()) {
|
||||||
continue;
|
|
||||||
}
|
|
||||||
match slot.tx.try_send(event.clone()) {
|
|
||||||
Ok(()) => delivered += 1,
|
Ok(()) => delivered += 1,
|
||||||
Err(mpsc::TrySendError::Full(_)) => {
|
Err(TrySendError::Full(_)) => dropped += 1,
|
||||||
slot.dropped = slot.dropped.saturating_add(1);
|
Err(TrySendError::Disconnected(_)) => {
|
||||||
dropped += 1;
|
dropped += 1;
|
||||||
}
|
disconnected = true;
|
||||||
Err(mpsc::TrySendError::Disconnected(_)) => {
|
|
||||||
slot.dropped = slot.dropped.saturating_add(1);
|
|
||||||
dropped += 1;
|
|
||||||
disconnected.push(*id);
|
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if dropped > 0 || disconnected {
|
||||||
|
reports.push(FanoutReport {
|
||||||
|
id: target.id,
|
||||||
|
dropped,
|
||||||
|
disconnected,
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
for id in disconnected {
|
|
||||||
state.subscribers.remove(&id);
|
let dropped_for_subscribers = reports.iter().map(|report| report.dropped).sum::<u64>();
|
||||||
|
if !reports.is_empty() {
|
||||||
|
let mut state = self.state.lock().expect("datastream fanout poisoned");
|
||||||
|
for report in &reports {
|
||||||
|
if let Some(slot) = state.subscribers.get_mut(&report.id) {
|
||||||
|
slot.dropped = slot.dropped.saturating_add(report.dropped);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for report in &reports {
|
||||||
|
if report.disconnected {
|
||||||
|
state.subscribers.remove(&report.id);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
EndpointTick {
|
EndpointTick {
|
||||||
drained: events.len(),
|
drained: events.len(),
|
||||||
delivered,
|
delivered,
|
||||||
dropped_for_subscribers: dropped,
|
dropped_for_subscribers: usize::try_from(dropped_for_subscribers).unwrap_or(usize::MAX),
|
||||||
subscribers,
|
subscribers,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -279,6 +313,7 @@ pub struct CatalogSnapshot {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CatalogSnapshot {
|
impl CatalogSnapshot {
|
||||||
|
/// Apply a subscription request to the initial metadata snapshot only.
|
||||||
pub fn datastream_snapshot(&self, request: &SubscriptionRequest) -> DatastreamSnapshot {
|
pub fn datastream_snapshot(&self, request: &SubscriptionRequest) -> DatastreamSnapshot {
|
||||||
let channels: Vec<ChannelDescriptor> = self
|
let channels: Vec<ChannelDescriptor> = self
|
||||||
.channels
|
.channels
|
||||||
|
|
@ -320,24 +355,12 @@ struct ChannelCatalogState {
|
||||||
|
|
||||||
impl ChannelCatalogState {
|
impl ChannelCatalogState {
|
||||||
fn new(stream: StreamDescriptor) -> Self {
|
fn new(stream: StreamDescriptor) -> Self {
|
||||||
let mut state = Self {
|
Self {
|
||||||
stream: stream.clone(),
|
stream,
|
||||||
by_id: BTreeMap::new(),
|
by_id: BTreeMap::new(),
|
||||||
by_name: BTreeMap::new(),
|
by_name: BTreeMap::new(),
|
||||||
next_channel: 1,
|
next_channel: 1,
|
||||||
};
|
}
|
||||||
let timing = ChannelDescriptor {
|
|
||||||
stream: stream.stream.clone(),
|
|
||||||
id: FRAME_TIME_CHANNEL_ID,
|
|
||||||
name: FRAME_TIME_CHANNEL.to_owned(),
|
|
||||||
label: Some("frame construction time".to_owned()),
|
|
||||||
content: ChannelContent::JsonRecord {
|
|
||||||
schema: Some("datastream.frame_time.v1".to_owned()),
|
|
||||||
},
|
|
||||||
};
|
|
||||||
state.by_name.insert(timing.name.clone(), timing.id);
|
|
||||||
state.by_id.insert(timing.id, timing);
|
|
||||||
state
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn snapshot(&self) -> CatalogSnapshot {
|
fn snapshot(&self) -> CatalogSnapshot {
|
||||||
|
|
@ -456,14 +479,6 @@ impl DatastreamEndpoint {
|
||||||
.snapshot()
|
.snapshot()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn set_frame_timing_enabled(&self, enabled: bool) {
|
|
||||||
self.mux.set_frame_timing_enabled(enabled);
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn frame_timing_enabled(&self) -> bool {
|
|
||||||
self.mux.frame_timing_enabled()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn producer(&self) -> DatastreamProducer {
|
pub fn producer(&self) -> DatastreamProducer {
|
||||||
DatastreamProducer {
|
DatastreamProducer {
|
||||||
mux: Arc::clone(&self.mux),
|
mux: Arc::clone(&self.mux),
|
||||||
|
|
@ -530,26 +545,36 @@ impl DatastreamEndpoint {
|
||||||
self.fanout.subscriber_snapshots()
|
self.fanout.subscriber_snapshots()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Drain the mux and fan out catalog-aware future events.
|
/// Drain the mux once and broadcast future frame events; with no subscribers, drained frames are bitbucketed.
|
||||||
pub fn tick(&self) -> EndpointTick {
|
pub fn tick(&self) -> EndpointTick {
|
||||||
let frames = self.mux.drain();
|
let events = self.drain_events();
|
||||||
if frames.is_empty() {
|
if events.is_empty() {
|
||||||
return EndpointTick::default();
|
return EndpointTick::default();
|
||||||
}
|
}
|
||||||
let drained = frames.len();
|
self.publish_events(events)
|
||||||
self.drained.fetch_add(drained as u64, Ordering::Relaxed);
|
}
|
||||||
let events = frames.into_iter().map(|frame| {
|
|
||||||
DatastreamEvent::Frame(FrameDelivery {
|
fn drain_events(&self) -> Vec<DatastreamEvent> {
|
||||||
channel: ChannelRef {
|
self.mux
|
||||||
stream: self.stream.clone(),
|
.drain()
|
||||||
channel: frame.channel,
|
.into_iter()
|
||||||
},
|
.map(|frame| {
|
||||||
position: frame.position,
|
DatastreamEvent::Frame(FrameDelivery {
|
||||||
payload: frame.payload,
|
channel: ChannelRef {
|
||||||
|
stream: self.stream.clone(),
|
||||||
|
channel: frame.channel,
|
||||||
|
},
|
||||||
|
position: frame.position,
|
||||||
|
payload: frame.payload,
|
||||||
|
})
|
||||||
})
|
})
|
||||||
});
|
.collect()
|
||||||
let catalog = self.catalog_snapshot();
|
}
|
||||||
let tick = self.fanout.publish_batch(events, &catalog);
|
|
||||||
|
fn publish_events(&self, events: Vec<DatastreamEvent>) -> EndpointTick {
|
||||||
|
let drained = events.len();
|
||||||
|
self.drained.fetch_add(drained as u64, Ordering::Relaxed);
|
||||||
|
let tick = self.fanout.publish_batch(events);
|
||||||
if tick.subscribers == 0 {
|
if tick.subscribers == 0 {
|
||||||
self.bitbucketed
|
self.bitbucketed
|
||||||
.fetch_add(drained as u64, Ordering::Relaxed);
|
.fetch_add(drained as u64, Ordering::Relaxed);
|
||||||
|
|
@ -620,24 +645,20 @@ impl DatastreamProducer {
|
||||||
.id_for_name(name)
|
.id_for_name(name)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn submit_record<R: Record>(&self, channel: ChannelId, record: &R) -> Position {
|
pub fn submit_record<R: Record>(&self, channel: ChannelId, record: &R) -> bool {
|
||||||
self.mux.submit(channel, record.encode())
|
self.mux.submit(channel, record.encode())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn submit_text(&self, channel: ChannelId, text: impl AsRef<[u8]>) -> Position {
|
pub fn submit_text(&self, channel: ChannelId, text: impl AsRef<[u8]>) -> bool {
|
||||||
self.mux.submit(channel, text.as_ref().to_vec())
|
self.mux.submit(channel, text.as_ref().to_vec())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn submit_bytes(&self, channel: ChannelId, bytes: Vec<u8>) -> Position {
|
pub fn submit_bytes(&self, channel: ChannelId, bytes: Vec<u8>) -> bool {
|
||||||
self.mux.submit(channel, bytes)
|
self.mux.submit(channel, bytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn set_frame_timing_enabled(&self, enabled: bool) {
|
pub fn submit_text_owned(&self, channel: ChannelId, text: String) -> bool {
|
||||||
self.mux.set_frame_timing_enabled(enabled);
|
self.mux.submit(channel, text.into_bytes())
|
||||||
}
|
|
||||||
|
|
||||||
pub fn frame_timing_enabled(&self) -> bool {
|
|
||||||
self.mux.frame_timing_enabled()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn process_observer_with<F>(&self, channel_for: F) -> Arc<dyn ProcessOutputObserver>
|
pub fn process_observer_with<F>(&self, channel_for: F) -> Arc<dyn ProcessOutputObserver>
|
||||||
|
|
@ -674,34 +695,30 @@ fn register_channel(
|
||||||
name: String,
|
name: String,
|
||||||
content: ChannelContent,
|
content: ChannelContent,
|
||||||
) -> Result<ChannelId, ChannelRegistrationError> {
|
) -> Result<ChannelId, ChannelRegistrationError> {
|
||||||
let (id, event, snapshot) = {
|
// New channel declarations publish a future event immediately; existing-name
|
||||||
|
// reuse only returns the prior id.
|
||||||
|
let (id, event) = {
|
||||||
let mut catalog = catalog.lock().expect("datastream catalog poisoned");
|
let mut catalog = catalog.lock().expect("datastream catalog poisoned");
|
||||||
match catalog.try_register_channel(name.clone(), content)? {
|
match catalog.try_register_channel(name.clone(), content)? {
|
||||||
Some(descriptor) => {
|
Some(descriptor) => (
|
||||||
let id = descriptor.id;
|
descriptor.id,
|
||||||
let snapshot = catalog.snapshot();
|
Some(DatastreamEvent::ChannelDeclared(descriptor)),
|
||||||
(
|
),
|
||||||
id,
|
|
||||||
Some(DatastreamEvent::ChannelDeclared(descriptor)),
|
|
||||||
snapshot,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
None => {
|
None => {
|
||||||
let id = catalog
|
let id = catalog
|
||||||
.id_for_name(&name)
|
.id_for_name(&name)
|
||||||
.expect("duplicate channel name remains registered");
|
.expect("duplicate channel name remains registered");
|
||||||
let snapshot = catalog.snapshot();
|
(id, None)
|
||||||
(id, None, snapshot)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
if let Some(event) = event {
|
if let Some(event) = event {
|
||||||
let _ = fanout.publish(event, &snapshot);
|
let _ = fanout.publish(event);
|
||||||
}
|
}
|
||||||
Ok(id)
|
Ok(id)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Managed-process output observer that submits stdout/stderr chunks as frames.
|
/// Legacy/custom process-output observer adapter that submits stdout/stderr chunks as frames.
|
||||||
pub struct DatastreamProcessObserver {
|
pub struct DatastreamProcessObserver {
|
||||||
producer: DatastreamProducer,
|
producer: DatastreamProducer,
|
||||||
channel_for: Arc<dyn Fn(&str, bool) -> ChannelId + Send + Sync>,
|
channel_for: Arc<dyn Fn(&str, bool) -> ChannelId + Send + Sync>,
|
||||||
|
|
@ -773,33 +790,6 @@ struct RuntimeMessageTypeCount<'a> {
|
||||||
count: u64,
|
count: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn event_matches_request(
|
|
||||||
event: &DatastreamEvent,
|
|
||||||
request: &SubscriptionRequest,
|
|
||||||
catalog: &CatalogSnapshot,
|
|
||||||
) -> bool {
|
|
||||||
match event {
|
|
||||||
DatastreamEvent::StreamDeclared(descriptor) => {
|
|
||||||
source_matches(&descriptor.stream, Some(descriptor), &request.sources)
|
|
||||||
&& (matches!(request.channels, ChannelFilter::All)
|
|
||||||
|| catalog.channels.values().any(|channel| {
|
|
||||||
channel.stream == descriptor.stream
|
|
||||||
&& descriptor_matches_request(channel, request, catalog)
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
DatastreamEvent::ChannelDeclared(descriptor) => {
|
|
||||||
descriptor_matches_request(descriptor, request, catalog)
|
|
||||||
}
|
|
||||||
DatastreamEvent::Frame(delivery) => catalog
|
|
||||||
.descriptor_for(&delivery.channel)
|
|
||||||
.map(|descriptor| descriptor_matches_request(descriptor, request, catalog))
|
|
||||||
.unwrap_or_else(|| matches!(request.channels, ChannelFilter::All)),
|
|
||||||
DatastreamEvent::StreamEnded(stream) => {
|
|
||||||
source_matches(stream, catalog.stream_descriptor(stream), &request.sources)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn descriptor_matches_request(
|
fn descriptor_matches_request(
|
||||||
descriptor: &ChannelDescriptor,
|
descriptor: &ChannelDescriptor,
|
||||||
request: &SubscriptionRequest,
|
request: &SubscriptionRequest,
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
//! Core data model: the framed, channel-multiplexed stream (spec §4).
|
//! Core data model: the framed, channel-multiplexed stream (spec §2).
|
||||||
//!
|
//!
|
||||||
//! A stream is identified by the producing node and lifetime. Frames carry a
|
//! A stream is identified by the producing node and lifetime. Frames carry a
|
||||||
//! stream-local numeric channel id plus the mux-assigned position and opaque
|
//! stream-local numeric channel id plus the mux-assigned position and opaque
|
||||||
|
|
@ -10,12 +10,12 @@ use std::sync::Arc;
|
||||||
|
|
||||||
use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
||||||
|
|
||||||
/// A position assigned by a node's mux (spec §5.2).
|
/// A position assigned by a node's mux during drain (spec §2.3).
|
||||||
///
|
///
|
||||||
/// Positions are **monotonic** and **gap-free** within a single node's stream:
|
/// Assignment is monotonic and gap-free for accepted frames: the mux never
|
||||||
/// the mux never reuses one and never skips one in its numbering. A position
|
/// reuses one and never skips one while draining. A frame assigned a position
|
||||||
/// that is assigned but never delivered surfaces downstream as a missing
|
/// can still be lost by transport and later surface downstream as a detectable
|
||||||
/// position — a detectable gap (spec §5.3, §7.5).
|
/// gap.
|
||||||
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Serialize, Deserialize)]
|
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Serialize, Deserialize)]
|
||||||
pub struct Position(pub u64);
|
pub struct Position(pub u64);
|
||||||
|
|
||||||
|
|
@ -27,9 +27,9 @@ impl fmt::Display for Position {
|
||||||
|
|
||||||
/// Stream-local numeric channel id.
|
/// Stream-local numeric channel id.
|
||||||
///
|
///
|
||||||
/// `ChannelId(0)` is reserved for the datastream frame-timing sidecar. Every
|
/// A raw `ChannelId` is meaningful only together with its [`StreamId`]. The
|
||||||
/// other id is allocated by the stream owner and is meaningful only with the
|
/// public endpoint allocator currently starts at `ChannelId(1)`, leaving
|
||||||
/// corresponding [`StreamId`]. Consumers resolve frames by `(stream, channel)`.
|
/// `ChannelId(0)` unallocated by normal registration.
|
||||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)]
|
#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)]
|
||||||
pub struct ChannelId(pub u32);
|
pub struct ChannelId(pub u32);
|
||||||
|
|
||||||
|
|
@ -73,7 +73,7 @@ pub enum StreamOrigin {
|
||||||
RemoteNode,
|
RemoteNode,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The stable identity of a node that produces a stream (spec §4.4, §8.1).
|
/// The stable identity of a node that produces a stream (spec §2.2, §7.1).
|
||||||
#[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
|
#[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
|
||||||
pub struct NodeId(Arc<str>);
|
pub struct NodeId(Arc<str>);
|
||||||
|
|
||||||
|
|
@ -131,7 +131,7 @@ impl<'de> Deserialize<'de> for NodeId {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A lifetime discriminator distinguishing a node's incarnations (spec §8.4).
|
/// A lifetime discriminator distinguishing a node's incarnations (spec §2.2).
|
||||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)]
|
#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)]
|
||||||
pub struct Lifetime(pub u64);
|
pub struct Lifetime(pub u64);
|
||||||
|
|
||||||
|
|
@ -168,7 +168,7 @@ pub struct StreamDescriptor {
|
||||||
pub origin: StreamOrigin,
|
pub origin: StreamOrigin,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Channel metadata declared by the stream owner.
|
/// Catalog metadata declared by the stream owner; frames store only `id` and payload.
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct ChannelDescriptor {
|
pub struct ChannelDescriptor {
|
||||||
pub stream: StreamId,
|
pub stream: StreamId,
|
||||||
|
|
@ -236,7 +236,7 @@ impl SubscriptionRequest {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The unit the mux emits (spec §4.1): bytes tagged with a channel and a position.
|
/// The unit the mux emits (spec §2.1): bytes tagged with a channel and a position.
|
||||||
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct Frame {
|
pub struct Frame {
|
||||||
/// The stream-local lane these bytes belong to.
|
/// The stream-local lane these bytes belong to.
|
||||||
|
|
|
||||||
|
|
@ -1,24 +1,23 @@
|
||||||
//! Consumer ingest: reconstruct each node's stream from deliveries (spec
|
//! Consumer ingest: reconstruct each stream from deliveries (spec §7.1).
|
||||||
//! §8.1).
|
|
||||||
//!
|
//!
|
||||||
//! The consumer receives frames from many nodes, in any order, some never
|
//! The consumer receives frames from many nodes, in any order, some never
|
||||||
//! arriving, and reconstructs each node's stream — keyed by stream id,
|
//! arriving, and reconstructs each stream — keyed by stream id, ordered by
|
||||||
//! ordered by position — into the [`Store`]. Ingest is deliberately thin:
|
//! position — into the [`Store`]. Ingest is deliberately thin: it routes a
|
||||||
//! it routes a delivery to its stream and records the frame whole. It MUST
|
//! delivery to its stream and records the frame whole. It MUST NOT thin,
|
||||||
//! NOT thin, aggregate, decode-and-discard, or truncate (spec §8.2), and it
|
//! aggregate, decode-and-discard, or truncate (spec §7.1, §7.2), and it never
|
||||||
//! never inspects a channel or a payload, so a channel it cannot decode is
|
//! inspects a channel or payload, so an undecoded channel is retained exactly
|
||||||
//! retained exactly like any other (spec §8.3).
|
//! like any other (spec §7.2, §8.3).
|
||||||
//!
|
//!
|
||||||
//! Reconstruction is by position, not arrival: out-of-order deliveries land
|
//! Reconstruction is by position, not arrival: out-of-order deliveries land in
|
||||||
//! in order in the store, and a position delivered twice collapses to one
|
//! order in the store, and a position delivered twice collapses to one (spec
|
||||||
//! (spec §7.5). Two lives of one node are different stream ids and never
|
//! §7.2). Two lives of one node are different stream ids and never merge (spec
|
||||||
//! merge (spec §8.4).
|
//! §2.2).
|
||||||
|
|
||||||
use super::store::Store;
|
use super::store::Store;
|
||||||
use super::transport::Delivery;
|
use super::transport::Delivery;
|
||||||
|
|
||||||
/// The single consumer toward which all telemetry flows (spec §2). It owns
|
/// A store-owning ingest fold for deliveries. It grows stored streams as
|
||||||
/// the stored streams and grows them as deliveries arrive.
|
/// deliveries arrive.
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
pub struct Consumer {
|
pub struct Consumer {
|
||||||
store: Store,
|
store: Store,
|
||||||
|
|
@ -33,7 +32,7 @@ impl Consumer {
|
||||||
/// Accept one delivery: route it to its stream and record the frame.
|
/// Accept one delivery: route it to its stream and record the frame.
|
||||||
/// Returns `true` if the frame was new (a duplicate position is
|
/// Returns `true` if the frame was new (a duplicate position is
|
||||||
/// ignored, keeping the first — the carrier cannot fabricate content,
|
/// ignored, keeping the first — the carrier cannot fabricate content,
|
||||||
/// spec §9).
|
/// spec §9.1).
|
||||||
pub fn accept(&mut self, delivery: Delivery) -> bool {
|
pub fn accept(&mut self, delivery: Delivery) -> bool {
|
||||||
let Delivery { stream, frame } = delivery;
|
let Delivery { stream, frame } = delivery;
|
||||||
self.store.stream_mut(&stream).record(frame)
|
self.store.stream_mut(&stream).record(frame)
|
||||||
|
|
@ -46,7 +45,7 @@ impl Consumer {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The stored streams — the source of truth for every view (spec §8.2).
|
/// The stored streams — frame truth for raw storage and view projections (spec §7.3, §8.1).
|
||||||
pub fn store(&self) -> &Store {
|
pub fn store(&self) -> &Store {
|
||||||
&self.store
|
&self.store
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,34 +1,35 @@
|
||||||
//! The per-node telemetry **datastream** (see `DATASTREAM_SPEC.md`).
|
//! The per-node telemetry **datastream** (see `DATASTREAM_SPEC.md`).
|
||||||
//!
|
//!
|
||||||
//! A deliberately dumb pipe: producers dump bytes tagged by channel, a
|
//! A deliberately dumb pipe: producers dump bytes tagged by stream-local
|
||||||
//! single per-node mux interleaves them into one ordered stream, a
|
//! channel id, a single per-node mux accepts those bytes and assigns canonical
|
||||||
//! best-effort transport carries that stream to the one consumer, ingest
|
//! positions during drain, the endpoint broadcasts catalog-aware events to
|
||||||
//! reconstructs each node's stream by position, and views are read-time
|
//! subscribers, ingest reconstructs streams by position, and views are
|
||||||
//! projections over the stored stream. Nothing between a producer and a
|
//! read-time projections over stored frames. Nothing between a producer and a
|
||||||
//! view interprets the payload.
|
//! view interprets the payload.
|
||||||
//!
|
//!
|
||||||
//! ```text
|
//! ```text
|
||||||
//! producers (caller-owned records + text)
|
//! producers (caller-owned records + text)
|
||||||
//! │ bytes tagged by channel → [`record::Record`]
|
//! │ bytes tagged by registered ChannelId
|
||||||
//! ▼
|
//! ▼
|
||||||
//! per-node MUX → [`mux::Mux`]
|
//! endpoint / catalog → [`endpoint`]
|
||||||
//! │ one ordered stream of [`Frame`]s
|
//! │ channel metadata + producer handles
|
||||||
//! ▼
|
//! ▼
|
||||||
//! best-effort transport → [`transport`]
|
//! per-node MUX → [`mux::Mux`]
|
||||||
//! │ delivery: frames, maybe dropped/reordered/delayed
|
//! │ positioned [`frame::Frame`]s
|
||||||
//! ▼
|
//! ▼
|
||||||
//! consumer INGEST → [`ingest::Consumer`]
|
//! endpoint fanout → [`endpoint::DeliveryFanout`]
|
||||||
//! │ complete stream, stored whole
|
//! │ catalog-aware events, maybe dropped per subscriber
|
||||||
//! ▼
|
//! ▼
|
||||||
//! stored STREAM (truth) → [`store`]
|
//! ingest / store → [`ingest`], [`store`]
|
||||||
//! │ read-time only
|
//! │ position-keyed frame truth
|
||||||
//! ▼
|
//! ▼
|
||||||
//! VIEWS → [`views`]
|
//! views → [`views`]
|
||||||
//! ```
|
//! ```
|
||||||
//!
|
//!
|
||||||
//! The data model ([`frame`]), extension contract ([`record`]), and wire
|
//! The data model ([`frame`]), extension contract ([`record`]), endpoint/fanout
|
||||||
//! envelope ([`wire`]) are the seams a test observes. Channel meanings live in
|
//! seam ([`endpoint`]), and compatibility wire helpers ([`wire`]) are the seams
|
||||||
//! producer/consumer crates, not in a datastream-wide catalog.
|
//! tests observe. Channel meanings live in producer/consumer crates, not in a
|
||||||
|
//! datastream-wide global registry.
|
||||||
|
|
||||||
pub mod emit;
|
pub mod emit;
|
||||||
pub mod endpoint;
|
pub mod endpoint;
|
||||||
|
|
@ -41,7 +42,6 @@ pub mod publisher_actor;
|
||||||
pub mod record;
|
pub mod record;
|
||||||
pub mod sink_actor;
|
pub mod sink_actor;
|
||||||
pub mod store;
|
pub mod store;
|
||||||
pub mod timing;
|
|
||||||
pub mod transport;
|
pub mod transport;
|
||||||
pub mod views;
|
pub mod views;
|
||||||
pub mod wire;
|
pub mod wire;
|
||||||
|
|
@ -65,6 +65,5 @@ pub use publisher_actor::{
|
||||||
pub use record::{ChannelKind, ChannelRegistry, Record};
|
pub use record::{ChannelKind, ChannelRegistry, Record};
|
||||||
pub use sink_actor::{DATASTREAM_SINK_NAME, DatastreamSink};
|
pub use sink_actor::{DATASTREAM_SINK_NAME, DatastreamSink};
|
||||||
pub use store::{GapSpan, Store, StoredStream};
|
pub use store::{GapSpan, Store, StoredStream};
|
||||||
pub use timing::{FRAME_TIME_CHANNEL, FRAME_TIME_CHANNEL_ID, FrameTimeSample};
|
|
||||||
pub use transport::{Delivery, Reorder, ScriptedTransport, StreamScript};
|
pub use transport::{Delivery, Reorder, ScriptedTransport, StreamScript};
|
||||||
pub use views::{Body, LogEntry, MergedFrame};
|
pub use views::{Body, LogEntry, MergedFrame};
|
||||||
|
|
|
||||||
|
|
@ -1,42 +1,39 @@
|
||||||
//! The per-node mux: the single ordering authority (spec §5).
|
//! The per-stream mux: the single ordering authority (spec §4.4).
|
||||||
//!
|
//!
|
||||||
//! Every producer on a node submits bytes tagged with a stream-local channel id
|
//! Producers submit bytes tagged with a stream-local channel id. The mux accepts
|
||||||
//! to one mux, and the mux assigns a single monotonic position sequence across
|
//! payloads into a bounded queue first, then assigns a single monotonic position
|
||||||
//! all channels. A drop consumes a position and is therefore visible downstream
|
//! sequence while draining accepted payloads.
|
||||||
//! as a gap.
|
|
||||||
|
|
||||||
use std::sync::Mutex;
|
use crossbeam_channel::{Receiver, Sender, TryRecvError, TrySendError, bounded};
|
||||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
use std::sync::mpsc::{Receiver, SyncSender, TryRecvError, TrySendError, sync_channel};
|
|
||||||
use std::time::{SystemTime, UNIX_EPOCH};
|
|
||||||
|
|
||||||
use super::frame::{ChannelId, Frame, Position, StreamId};
|
use super::frame::{ChannelId, Frame, Position, StreamId};
|
||||||
use super::record::Record;
|
|
||||||
use super::timing::FRAME_TIME_CHANNEL_ID;
|
struct PendingFrame {
|
||||||
use super::timing::FrameTimeSample;
|
channel: ChannelId,
|
||||||
|
payload: Vec<u8>,
|
||||||
|
}
|
||||||
|
|
||||||
/// A node's single position authority and outgoing telemetry queue.
|
/// A node's single position authority and outgoing telemetry queue.
|
||||||
pub struct Mux {
|
pub struct Mux {
|
||||||
stream: StreamId,
|
stream: StreamId,
|
||||||
next: AtomicU64,
|
next: AtomicU64,
|
||||||
dropped: AtomicU64,
|
dropped: AtomicU64,
|
||||||
frame_timing_enabled: AtomicBool,
|
tx: Sender<PendingFrame>,
|
||||||
tx: SyncSender<Frame>,
|
rx: Receiver<PendingFrame>,
|
||||||
rx: Mutex<Receiver<Frame>>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Mux {
|
impl Mux {
|
||||||
/// Create a mux for `stream` with a bounded outgoing queue.
|
/// Create a mux for `stream` with a bounded outgoing queue.
|
||||||
pub fn new(stream: StreamId, capacity: usize) -> Self {
|
pub fn new(stream: StreamId, capacity: usize) -> Self {
|
||||||
let capacity = capacity.max(1).min(1_048_576);
|
let capacity = capacity.max(1).min(1_048_576);
|
||||||
let (tx, rx) = sync_channel(capacity);
|
let (tx, rx) = bounded(capacity);
|
||||||
Mux {
|
Mux {
|
||||||
stream,
|
stream,
|
||||||
next: AtomicU64::new(0),
|
next: AtomicU64::new(0),
|
||||||
dropped: AtomicU64::new(0),
|
dropped: AtomicU64::new(0),
|
||||||
frame_timing_enabled: AtomicBool::new(true),
|
|
||||||
tx,
|
tx,
|
||||||
rx: Mutex::new(rx),
|
rx,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -45,88 +42,50 @@ impl Mux {
|
||||||
Mux::new(stream, usize::MAX)
|
Mux::new(stream, usize::MAX)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The stream this mux produces (spec §8.4 ingest key).
|
/// The stream this mux produces (spec §2.2, §7.1 ingest key).
|
||||||
pub fn stream_id(&self) -> &StreamId {
|
pub fn stream_id(&self) -> &StreamId {
|
||||||
&self.stream
|
&self.stream
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Submit opaque bytes on a registered channel id.
|
/// Submit opaque bytes on a registered channel id.
|
||||||
pub fn submit(&self, channel: ChannelId, payload: Vec<u8>) -> Position {
|
pub fn submit(&self, channel: ChannelId, payload: Vec<u8>) -> bool {
|
||||||
let position = Position(self.next.fetch_add(1, Ordering::Relaxed));
|
match self.tx.try_send(PendingFrame { channel, payload }) {
|
||||||
let timing_sample = self
|
Ok(()) => true,
|
||||||
.frame_timing_enabled
|
|
||||||
.load(Ordering::Relaxed)
|
|
||||||
.then(|| now_unix_ns())
|
|
||||||
.filter(|_| channel != FRAME_TIME_CHANNEL_ID)
|
|
||||||
.map(|created_at_unix_ns| FrameTimeSample::new(position, created_at_unix_ns));
|
|
||||||
let frame = Frame {
|
|
||||||
channel,
|
|
||||||
position,
|
|
||||||
payload,
|
|
||||||
};
|
|
||||||
|
|
||||||
match self.tx.try_send(frame) {
|
|
||||||
Ok(()) => {
|
|
||||||
if let Some(sample) = timing_sample {
|
|
||||||
self.push_timing_sample_if_room(sample);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(TrySendError::Full(_)) | Err(TrySendError::Disconnected(_)) => {
|
Err(TrySendError::Full(_)) | Err(TrySendError::Disconnected(_)) => {
|
||||||
self.dropped.fetch_add(1, Ordering::Relaxed);
|
self.dropped.fetch_add(1, Ordering::Relaxed);
|
||||||
|
false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
position
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Pull all currently queued frames, sorted by mux position.
|
/// Pull all currently queued frames in mux queue order.
|
||||||
pub fn drain(&self) -> Vec<Frame> {
|
pub fn drain(&self) -> Vec<Frame> {
|
||||||
let rx = self.rx.lock().expect("mux receiver poisoned");
|
|
||||||
let mut frames = Vec::new();
|
let mut frames = Vec::new();
|
||||||
loop {
|
loop {
|
||||||
match rx.try_recv() {
|
match self.rx.try_recv() {
|
||||||
Ok(frame) => frames.push(frame),
|
Ok(pending) => {
|
||||||
Err(TryRecvError::Empty) => break,
|
// Position is consumed only after a pending frame has left
|
||||||
Err(TryRecvError::Disconnected) => break,
|
// the queue; failed submit never reaches this point.
|
||||||
|
let position = Position(self.next.fetch_add(1, Ordering::Relaxed));
|
||||||
|
frames.push(Frame {
|
||||||
|
channel: pending.channel,
|
||||||
|
position,
|
||||||
|
payload: pending.payload,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Err(TryRecvError::Empty) | Err(TryRecvError::Disconnected) => break,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
frames.sort_by_key(|frame| frame.position);
|
|
||||||
frames
|
frames
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Enable or disable optional sidecar timing samples for newly submitted frames.
|
/// How many positions have been assigned while draining accepted frames.
|
||||||
pub fn set_frame_timing_enabled(&self, enabled: bool) {
|
|
||||||
self.frame_timing_enabled.store(enabled, Ordering::Relaxed);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Whether this mux currently emits sidecar frame timing samples.
|
|
||||||
pub fn frame_timing_enabled(&self) -> bool {
|
|
||||||
self.frame_timing_enabled.load(Ordering::Relaxed)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn push_timing_sample_if_room(&self, sample: FrameTimeSample) {
|
|
||||||
let position = Position(self.next.fetch_add(1, Ordering::Relaxed));
|
|
||||||
let frame = Frame {
|
|
||||||
channel: FRAME_TIME_CHANNEL_ID,
|
|
||||||
position,
|
|
||||||
payload: sample.encode(),
|
|
||||||
};
|
|
||||||
let _ = self.tx.try_send(frame);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// How many positions have been assigned — the gap-free high-water mark.
|
|
||||||
pub fn assigned(&self) -> u64 {
|
pub fn assigned(&self) -> u64 {
|
||||||
self.next.load(Ordering::Relaxed)
|
self.next.load(Ordering::Relaxed)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// How many data frames have been dropped on overflow.
|
/// How many submissions have been dropped before entering the mux.
|
||||||
pub fn dropped(&self) -> u64 {
|
pub fn dropped(&self) -> u64 {
|
||||||
self.dropped.load(Ordering::Relaxed)
|
self.dropped.load(Ordering::Relaxed)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn now_unix_ns() -> u64 {
|
|
||||||
SystemTime::now()
|
|
||||||
.duration_since(UNIX_EPOCH)
|
|
||||||
.map(|duration| u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX))
|
|
||||||
.unwrap_or(0)
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -1,15 +1,15 @@
|
||||||
//! `DatastreamSink` — the cluster-side consumer of [`DatastreamFrame`] messages.
|
//! Legacy `DatastreamSink` for [`DatastreamFrame`] actor messages.
|
||||||
//!
|
//!
|
||||||
//! This is the counterpart to [`ClusterFrameSink`](super::emit::ClusterFrameSink):
|
//! New live transports should prefer catalog-aware [`crate::DatastreamEvent`]
|
||||||
//! a node ships its ordered telemetry as `DatastreamFrame` actor messages over the
|
//! streams. This actor remains as the counterpart to
|
||||||
//! regular swactor transport, and this actor — registered under a well-known name
|
//! [`ClusterFrameSink`](super::emit::ClusterFrameSink): a node ships positioned
|
||||||
//! on the collector (e.g. the orchestrator) — receives them, decodes each back
|
//! telemetry as legacy `DatastreamFrame` actor messages over regular Swactor
|
||||||
//! into a `(StreamId, Frame)` delivery, and hands it to a caller-supplied fold.
|
//! transport, and this actor receives them, decodes each back into a
|
||||||
|
//! `(StreamId, Frame)` delivery, and hands it to a caller-supplied fold.
|
||||||
//!
|
//!
|
||||||
//! It deliberately knows nothing about any view layer. The actor owns an opaque
|
//! It deliberately knows nothing about any view layer. The actor owns an opaque
|
||||||
//! callback so binaries can wire the decoded deliveries into whichever fold they
|
//! callback so binaries can wire decoded deliveries into whichever fold they
|
||||||
//! need. Malformed payloads are dropped silently — the same best-effort tolerance
|
//! need. Malformed payloads are dropped silently.
|
||||||
//! the UDP ingest had.
|
|
||||||
|
|
||||||
use swactor::actor::ActorInterface;
|
use swactor::actor::ActorInterface;
|
||||||
use swactor::runtime::Ctx;
|
use swactor::runtime::Ctx;
|
||||||
|
|
@ -17,9 +17,9 @@ use swactor::runtime::Ctx;
|
||||||
use super::frame::{Frame, StreamId};
|
use super::frame::{Frame, StreamId};
|
||||||
use super::wire::{DatastreamFrame, decode_delivery};
|
use super::wire::{DatastreamFrame, decode_delivery};
|
||||||
|
|
||||||
/// Receives [`DatastreamFrame`] cluster messages and folds each decoded delivery
|
/// Receives legacy [`DatastreamFrame`] cluster messages and folds each decoded
|
||||||
/// through `on_frame`. Spawn it, then publish its address under
|
/// delivery through `on_frame`. Spawn it, then publish its address under
|
||||||
/// [`DATASTREAM_SINK_NAME`] so emitters can resolve and ship to it.
|
/// [`DATASTREAM_SINK_NAME`] so legacy emitters can resolve and ship to it.
|
||||||
pub struct DatastreamSink {
|
pub struct DatastreamSink {
|
||||||
on_frame: Box<dyn FnMut(StreamId, Frame) + Send>,
|
on_frame: Box<dyn FnMut(StreamId, Frame) + Send>,
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,15 +1,15 @@
|
||||||
//! The stored stream — the consumer's source of truth (spec §8).
|
//! The stored stream — the consumer's frame source of truth (spec §7).
|
||||||
//!
|
//!
|
||||||
//! Storage holds each node's **complete** stream, whole and append-only.
|
//! Storage holds each node's complete stream, whole and append-only. Nothing is
|
||||||
//! Nothing is thinned, aggregated, decoded-and-discarded, or truncated at
|
//! thinned, aggregated, decoded-and-discarded, or truncated at ingest (spec
|
||||||
//! ingest (spec §8.2); everything a view ever shows is derived from here
|
//! §7.1, §7.2); everything a view shows is derived from here (spec §8.1).
|
||||||
//! (spec §9.1). Frames on channels the consumer cannot decode are kept as
|
//! Frames on channels the consumer cannot decode are kept as opaque bytes,
|
||||||
//! opaque bytes, in order, alongside the rest (spec §8.3) — the store never
|
//! alongside the rest (spec §7.2, §8.3) — the store never looks at a channel or
|
||||||
//! looks at a channel or a payload.
|
//! payload.
|
||||||
//!
|
//!
|
||||||
//! A [`StoredStream`] is keyed in the [`Store`] by [`StreamId`] — node plus
|
//! A [`StoredStream`] is keyed in the [`Store`] by [`StreamId`] — node plus
|
||||||
//! lifetime — so a re-incarnated node does not append to its prior life
|
//! lifetime — so a re-incarnated node does not append to its prior life (spec
|
||||||
//! (spec §8.4).
|
//! §2.2, §7.1).
|
||||||
|
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
|
|
@ -19,8 +19,8 @@ use super::frame::{Frame, Position, StreamId};
|
||||||
///
|
///
|
||||||
/// Backed by a position-keyed map so out-of-order arrivals land in order
|
/// Backed by a position-keyed map so out-of-order arrivals land in order
|
||||||
/// and a position seen twice collapses to one (the carrier may not
|
/// and a position seen twice collapses to one (the carrier may not
|
||||||
/// fabricate content, spec §9). Gaps are not stored — they are *derived*
|
/// fabricate content, spec §9.1). Gaps are not stored — they are *derived*
|
||||||
/// at read time from the positions that are present (spec §9.1).
|
/// at read time from the positions that are present (spec §7.3, §8.1).
|
||||||
#[derive(Debug, Clone, Default)]
|
#[derive(Debug, Clone, Default)]
|
||||||
pub struct StoredStream {
|
pub struct StoredStream {
|
||||||
frames: BTreeMap<u64, Frame>,
|
frames: BTreeMap<u64, Frame>,
|
||||||
|
|
@ -34,7 +34,7 @@ impl StoredStream {
|
||||||
|
|
||||||
/// Record a delivered frame. Idempotent by position: the first frame
|
/// Record a delivered frame. Idempotent by position: the first frame
|
||||||
/// seen for a position wins and is never mutated (append-only,
|
/// seen for a position wins and is never mutated (append-only,
|
||||||
/// spec §8.2). Returns `true` if this was the first time the position
|
/// spec §7.2). Returns `true` if this was the first time the position
|
||||||
/// was seen.
|
/// was seen.
|
||||||
pub fn record(&mut self, frame: Frame) -> bool {
|
pub fn record(&mut self, frame: Frame) -> bool {
|
||||||
match self.frames.entry(frame.position.0) {
|
match self.frames.entry(frame.position.0) {
|
||||||
|
|
@ -73,19 +73,19 @@ impl StoredStream {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The **interior** gaps — runs of positions assigned between the first
|
/// The **interior** gaps — runs of positions assigned between the first
|
||||||
/// and last delivered frame but never delivered (spec §7.5, §8), each as
|
/// and last delivered frame but never delivered (spec §7.3), each as one
|
||||||
/// one [`GapSpan`].
|
/// [`GapSpan`].
|
||||||
///
|
///
|
||||||
/// Cost is O(stored frames), never O(gap size): it walks adjacent stored
|
/// Cost is O(stored frames), never O(gap size): it walks adjacent stored
|
||||||
/// positions and reads each span's endpoints from them, rather than
|
/// positions and reads each span's endpoints from them, rather than
|
||||||
/// enumerating the (possibly enormous) range in between. A stream that
|
/// enumerating the (possibly enormous) range in between. A stream that
|
||||||
/// brackets a huge interior gap — what a long consumer outage produces
|
/// brackets a huge interior gap — what a long consumer outage produces
|
||||||
/// (spec §7.4), or a single wild position from a corrupt datagram — still
|
/// (spec §6.3, §7.3), or a single wild position from a corrupt datagram —
|
||||||
/// surfaces in work proportional to the frames held, not to `u64::MAX`.
|
/// still surfaces in work proportional to the frames held, not to `u64::MAX`.
|
||||||
///
|
///
|
||||||
/// Only interior gaps are knowable: a position lost *after* the last
|
/// Leading and trailing losses are not derivable from stored frames because
|
||||||
/// delivered frame leaves no bracketing frame to reveal it, so it shows
|
/// no bracketing position exists. A position lost after the last delivered
|
||||||
/// up as the stream simply ending (spec §7.4 node death), not a gap.
|
/// frame shows up as the stream simply ending (spec §5.7, §7.3), not a gap.
|
||||||
pub fn gap_spans(&self) -> Vec<GapSpan> {
|
pub fn gap_spans(&self) -> Vec<GapSpan> {
|
||||||
let mut spans = Vec::new();
|
let mut spans = Vec::new();
|
||||||
let mut prev: Option<u64> = None;
|
let mut prev: Option<u64> = None;
|
||||||
|
|
@ -105,7 +105,7 @@ impl StoredStream {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A contiguous run of missing positions surfaced in a stored stream
|
/// A contiguous run of missing positions surfaced in a stored stream
|
||||||
/// (spec §7.5). Inclusive on both ends.
|
/// (spec §7.3). Inclusive on both ends.
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
pub struct GapSpan {
|
pub struct GapSpan {
|
||||||
/// First missing position.
|
/// First missing position.
|
||||||
|
|
@ -121,7 +121,7 @@ impl GapSpan {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// All stored streams at the consumer, keyed by [`StreamId`] (spec §8.4).
|
/// All stored streams at the consumer, keyed by [`StreamId`] (spec §2.2, §7.1).
|
||||||
///
|
///
|
||||||
/// Two streams with the same node but different lifetime are distinct keys
|
/// Two streams with the same node but different lifetime are distinct keys
|
||||||
/// and never merge.
|
/// and never merge.
|
||||||
|
|
|
||||||
|
|
@ -1,32 +0,0 @@
|
||||||
//! Optional frame-construction timing sidecar records.
|
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
|
|
||||||
use crate::frame::ChannelId;
|
|
||||||
|
|
||||||
use crate::frame::Position;
|
|
||||||
use crate::record::Record;
|
|
||||||
|
|
||||||
/// Reserved channel carrying optional timing samples for frames in the same stream.
|
|
||||||
pub const FRAME_TIME_CHANNEL: &str = "datastream.frame_time";
|
|
||||||
pub const FRAME_TIME_CHANNEL_ID: ChannelId = ChannelId(0);
|
|
||||||
|
|
||||||
/// Sidecar timing sample keyed by the target frame's stream-local position.
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct FrameTimeSample {
|
|
||||||
pub target_position: u64,
|
|
||||||
pub created_at_unix_ns: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl FrameTimeSample {
|
|
||||||
pub fn new(target_position: Position, created_at_unix_ns: u64) -> Self {
|
|
||||||
Self {
|
|
||||||
target_position: target_position.0,
|
|
||||||
created_at_unix_ns,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Record for FrameTimeSample {
|
|
||||||
const CHANNEL: &'static str = FRAME_TIME_CHANNEL;
|
|
||||||
}
|
|
||||||
|
|
@ -1,28 +1,26 @@
|
||||||
//! Transport: best-effort carriage of a node's stream to the one consumer
|
//! Legacy transport/test seam for carrying positioned frames into ingest.
|
||||||
//! (spec §7), and a scripted in-process carrier for offline tests (testing
|
|
||||||
//! spec §2, §9).
|
|
||||||
//!
|
//!
|
||||||
//! A [`Delivery`] is the value on the transport→ingest seam: which stream a
|
//! The live endpoint path now fans out catalog-aware [`DatastreamEvent`] values;
|
||||||
//! frame belongs to, and the frame. A real carrier rides the connections
|
//! this module keeps the older [`Delivery`] shape used by ingest, storage tests,
|
||||||
//! the system already maintains (spec §7.1); a test replaces it with the
|
//! and scripted conformance checks.
|
||||||
//! [`ScriptedTransport`] here, whose faults are chosen by the scenario and
|
|
||||||
//! stay inside the **envelope** (testing spec §9): a carrier may *deliver*,
|
|
||||||
//! *drop*, *reorder*, or *delay*, and it MUST NOT corrupt a payload,
|
|
||||||
//! fabricate a frame, or alter a position.
|
|
||||||
//!
|
//!
|
||||||
//! Under position-ordering a *delay* is indistinguishable from a *reorder*
|
//! A [`Delivery`] pairs the producing stream id with one frame. A real carrier
|
||||||
//! (a delayed frame simply arrives later), so the envelope's delay is
|
//! rides connections the system already maintains; tests can replace it with
|
||||||
//! covered by [`Reorder`]. Everything the scripted carrier produces is a
|
//! [`ScriptedTransport`], whose faults stay inside the transport envelope: it
|
||||||
//! reordered subsequence of what was sent — never a superset, never a
|
//! may *deliver*, *drop*, *reorder*, or *delay*, and it MUST NOT corrupt a
|
||||||
//! mutation — which is exactly the property the real-transport conformance
|
//! payload, fabricate a frame, or alter a position.
|
||||||
//! check pins (testing spec §9).
|
//!
|
||||||
|
//! Under position-ordering a *delay* is indistinguishable from a *reorder*:
|
||||||
|
//! a delayed frame simply arrives later. Everything the scripted carrier
|
||||||
|
//! produces is a reordered subsequence of what was sent — never a superset and
|
||||||
|
//! never a mutation.
|
||||||
|
|
||||||
use std::collections::BTreeSet;
|
use std::collections::BTreeSet;
|
||||||
|
|
||||||
use super::frame::{Frame, StreamId};
|
use super::frame::{Frame, StreamId};
|
||||||
|
|
||||||
/// A frame as the consumer receives it from the transport (testing spec §2
|
/// A frame as the legacy transport/ingest seam receives it: tagged with the
|
||||||
/// seam): tagged with the stream it belongs to.
|
/// stream it belongs to.
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub struct Delivery {
|
pub struct Delivery {
|
||||||
/// Which node's life produced the frame (spec §8.1 ingest key).
|
/// Which node's life produced the frame (spec §8.1 ingest key).
|
||||||
|
|
@ -38,9 +36,9 @@ impl Delivery {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// How the surviving frames of a stream are reordered on arrival. This is
|
/// How the surviving frames of a stream are reordered on arrival. This is the
|
||||||
/// the envelope's *reorder* (and *delay*) axis (testing spec §9); each
|
/// envelope's *reorder* and *delay* axis (spec §6.3, §9.1); each variant is a
|
||||||
/// variant is a permutation of the survivors, never adding or dropping.
|
/// permutation of the survivors, never adding or dropping.
|
||||||
#[derive(Debug, Clone, Default)]
|
#[derive(Debug, Clone, Default)]
|
||||||
pub enum Reorder {
|
pub enum Reorder {
|
||||||
/// Delivered in the order sent.
|
/// Delivered in the order sent.
|
||||||
|
|
@ -57,7 +55,7 @@ pub enum Reorder {
|
||||||
Permutation(Vec<usize>),
|
Permutation(Vec<usize>),
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The faults a scripted carrier applies to one stream (testing spec §9
|
/// The faults a scripted carrier applies to one stream (spec §6.3, §9.1
|
||||||
/// envelope). Drops and reorders only — payloads and positions are never
|
/// envelope). Drops and reorders only — payloads and positions are never
|
||||||
/// touched.
|
/// touched.
|
||||||
#[derive(Debug, Clone, Default)]
|
#[derive(Debug, Clone, Default)]
|
||||||
|
|
@ -91,10 +89,9 @@ impl StreamScript {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A scripted, in-process transport (testing spec §2). It is a pure,
|
/// A scripted, in-process transport for conformance scenarios (spec §9.1). It
|
||||||
/// deterministic transform from what a node *sent* to what the consumer is
|
/// is a pure, deterministic transform from what a node sent to what the
|
||||||
/// *delivered* — the entanglement of real wires replaced by a script so a
|
/// consumer receives, replacing real wire behavior with a fast test script.
|
||||||
/// run completes in microseconds and returns the same result every time.
|
|
||||||
pub struct ScriptedTransport;
|
pub struct ScriptedTransport;
|
||||||
|
|
||||||
impl ScriptedTransport {
|
impl ScriptedTransport {
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
//! Views: read-time projections over a stored stream (spec §9).
|
//! Views: read-time projections over a stored stream (spec §8).
|
||||||
|
|
||||||
use std::fmt;
|
use std::fmt;
|
||||||
|
|
||||||
|
|
@ -70,6 +70,7 @@ where
|
||||||
C: ChannelClassifier + ?Sized,
|
C: ChannelClassifier + ?Sized,
|
||||||
R: Fn(ChannelId) -> Option<String>,
|
R: Fn(ChannelId) -> Option<String>,
|
||||||
{
|
{
|
||||||
|
// Capacity covers stored frames; surfaced gaps may add extra log entries.
|
||||||
let mut out = Vec::with_capacity(stream.len());
|
let mut out = Vec::with_capacity(stream.len());
|
||||||
let mut prev: Option<u64> = None;
|
let mut prev: Option<u64> = None;
|
||||||
for frame in stream.frames() {
|
for frame in stream.frames() {
|
||||||
|
|
@ -114,7 +115,8 @@ where
|
||||||
timeline_with_resolver(stream, classifier, resolve_name)
|
timeline_with_resolver(stream, classifier, resolve_name)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Transitional merged log using numeric channel ids as strings for classifier lookup.
|
/// Transitional helper for callers that still classify by rendered numeric ids;
|
||||||
|
/// named decoding should use [`merged_log_with_names`].
|
||||||
pub fn merged_log_with<C: ChannelClassifier + ?Sized>(
|
pub fn merged_log_with<C: ChannelClassifier + ?Sized>(
|
||||||
stream: &StoredStream,
|
stream: &StoredStream,
|
||||||
classifier: &C,
|
classifier: &C,
|
||||||
|
|
|
||||||
|
|
@ -59,7 +59,7 @@ pub fn decode_delivery(buf: &[u8]) -> Result<(StreamId, Frame), WireError> {
|
||||||
Ok((stream, frame))
|
Ok((stream, frame))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// One datastream event, addressed to a local/cluster datastream actor.
|
/// Legacy actor-message payload wrapper for datastream bytes.
|
||||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||||
pub struct DatastreamFrame {
|
pub struct DatastreamFrame {
|
||||||
pub payload: Vec<u8>,
|
pub payload: Vec<u8>,
|
||||||
|
|
|
||||||
|
|
@ -1,567 +0,0 @@
|
||||||
//! Shared support for the datastream tests (testing spec §2, §3).
|
|
||||||
//!
|
|
||||||
//! Two things live here, both deliberately separate from the system under
|
|
||||||
//! test so a test never asserts the code against itself:
|
|
||||||
//!
|
|
||||||
//! * the **payload library** — realistic record shapes and log lines, the
|
|
||||||
//! bytes the pipe will actually carry (testing spec §5: "never
|
|
||||||
//! placeholder text"); and
|
|
||||||
//! * the **reference model** — the spec's rules restated as small, total
|
|
||||||
//! functions over sequences (testing spec §3). It is the trusted oracle:
|
|
||||||
//! every test's `expected` is derived from it, never captured from a run.
|
|
||||||
//! It is written naively on purpose (collect, sort, scan) so a reader can
|
|
||||||
//! confirm it against the spec by eye, while the pipe computes the same
|
|
||||||
//! answers the long way.
|
|
||||||
//!
|
|
||||||
//! This module is `#[path]`-included into more than one test binary, so
|
|
||||||
//! some items are unused in some of them.
|
|
||||||
#![allow(dead_code)]
|
|
||||||
|
|
||||||
use datastream::mux::Mux;
|
|
||||||
use datastream::store::GapSpan;
|
|
||||||
pub use datastream::transport::Delivery;
|
|
||||||
use datastream::{ChannelId, Frame, Position, Record, StreamId};
|
|
||||||
|
|
||||||
pub mod schema {
|
|
||||||
use datastream::{ChannelId, Record};
|
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
|
|
||||||
pub const IDENTITY: &str = "identity";
|
|
||||||
pub const HOST_RESOURCE: &str = "host.resource";
|
|
||||||
pub const TRANSPORT_INTERNALS: &str = "transport.internals";
|
|
||||||
pub const MEMBERSHIP: &str = "membership";
|
|
||||||
pub const RUNTIME_STATS: &str = "runtime.stats";
|
|
||||||
pub const DIST_STATE: &str = "dist.state";
|
|
||||||
pub const RUNTIME_ACTORS: &str = "runtime.actors";
|
|
||||||
pub const RUNTIME_WORKERS: &str = "runtime.workers";
|
|
||||||
pub const DATASTREAM_HEALTH: &str = "datastream.health";
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
||||||
pub enum ProcStream {
|
|
||||||
Stdout,
|
|
||||||
Stderr,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ProcStream {
|
|
||||||
pub fn as_str(self) -> &'static str {
|
|
||||||
match self {
|
|
||||||
ProcStream::Stdout => "stdout",
|
|
||||||
ProcStream::Stderr => "stderr",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn process_output(label: &str, stream: ProcStream) -> ChannelId {
|
|
||||||
ChannelId::new(format!("proc.{label}.{}", stream.as_str()))
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct IdentityRecord {
|
|
||||||
pub node: String,
|
|
||||||
#[serde(default)]
|
|
||||||
pub life: u64,
|
|
||||||
#[serde(default)]
|
|
||||||
pub node_name: String,
|
|
||||||
#[serde(default)]
|
|
||||||
pub listen_addr: String,
|
|
||||||
#[serde(default)]
|
|
||||||
pub relay_url: String,
|
|
||||||
#[serde(default)]
|
|
||||||
pub version: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
|
||||||
pub struct ResourceSample {
|
|
||||||
#[serde(default)]
|
|
||||||
pub cpu_pct: f32,
|
|
||||||
#[serde(default)]
|
|
||||||
pub mem_used_mb: u32,
|
|
||||||
#[serde(default)]
|
|
||||||
pub mem_total_mb: u32,
|
|
||||||
#[serde(default)]
|
|
||||||
pub gpu_pct: f32,
|
|
||||||
#[serde(default)]
|
|
||||||
pub disk_used_gb: u32,
|
|
||||||
#[serde(default)]
|
|
||||||
pub net_rx_kbps: u32,
|
|
||||||
#[serde(default)]
|
|
||||||
pub net_tx_kbps: u32,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
|
||||||
pub struct TransportInternals {
|
|
||||||
#[serde(default)]
|
|
||||||
pub relay_connected: bool,
|
|
||||||
#[serde(default)]
|
|
||||||
pub direct_peers: u32,
|
|
||||||
#[serde(default)]
|
|
||||||
pub relay_peers: u32,
|
|
||||||
#[serde(default)]
|
|
||||||
pub rtt_ms_p50: u32,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct MembershipTransition {
|
|
||||||
pub peer: String,
|
|
||||||
pub from: String,
|
|
||||||
pub to: String,
|
|
||||||
#[serde(default)]
|
|
||||||
pub reason: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct RuntimeStats {
|
|
||||||
#[serde(default)]
|
|
||||||
pub actors_live: u32,
|
|
||||||
#[serde(default)]
|
|
||||||
pub mailbox_depth: u32,
|
|
||||||
#[serde(default)]
|
|
||||||
pub scheduled_tasks: u32,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct DistributionState {
|
|
||||||
#[serde(default)]
|
|
||||||
pub cache_size: u32,
|
|
||||||
#[serde(default)]
|
|
||||||
pub cache_entries: Vec<CacheEntryRec>,
|
|
||||||
#[serde(default)]
|
|
||||||
pub directory_route_count: u32,
|
|
||||||
#[serde(default)]
|
|
||||||
pub registry_size: u32,
|
|
||||||
#[serde(default)]
|
|
||||||
pub registry_tombstones: u32,
|
|
||||||
#[serde(default)]
|
|
||||||
pub registry_entries: Vec<RegistryEntryRec>,
|
|
||||||
#[serde(default)]
|
|
||||||
pub recent_probe_targets: Vec<String>,
|
|
||||||
#[serde(default)]
|
|
||||||
pub peer_auth_mode: String,
|
|
||||||
#[serde(default)]
|
|
||||||
pub authorized_peer_count: u32,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct CacheEntryRec {
|
|
||||||
#[serde(default)]
|
|
||||||
pub actor_addr: String,
|
|
||||||
#[serde(default)]
|
|
||||||
pub node_id: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct RegistryEntryRec {
|
|
||||||
#[serde(default)]
|
|
||||||
pub name: String,
|
|
||||||
#[serde(default)]
|
|
||||||
pub actor_addr: String,
|
|
||||||
#[serde(default)]
|
|
||||||
pub node_id: String,
|
|
||||||
#[serde(default)]
|
|
||||||
pub tombstone: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct WorkerCounters {
|
|
||||||
#[serde(default)]
|
|
||||||
pub num_workers: u32,
|
|
||||||
#[serde(default)]
|
|
||||||
pub scheduled_tasks: u32,
|
|
||||||
#[serde(default)]
|
|
||||||
pub local_sends: u64,
|
|
||||||
#[serde(default)]
|
|
||||||
pub cross_sends: u64,
|
|
||||||
#[serde(default)]
|
|
||||||
pub inbox_sends: u64,
|
|
||||||
#[serde(default)]
|
|
||||||
pub type_mismatches: u64,
|
|
||||||
#[serde(default)]
|
|
||||||
pub panics: u64,
|
|
||||||
#[serde(default)]
|
|
||||||
pub messages_dropped: u64,
|
|
||||||
#[serde(default)]
|
|
||||||
pub restarts: u64,
|
|
||||||
#[serde(default)]
|
|
||||||
pub stops: u64,
|
|
||||||
#[serde(default)]
|
|
||||||
pub messages_processed: u64,
|
|
||||||
#[serde(default)]
|
|
||||||
pub tick_p50_us: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct DatastreamHealth {
|
|
||||||
#[serde(default)]
|
|
||||||
pub assigned: u64,
|
|
||||||
#[serde(default)]
|
|
||||||
pub dropped: u64,
|
|
||||||
#[serde(default)]
|
|
||||||
pub loss_rate_ppm: u32,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct ActorRuntimeDetail {
|
|
||||||
#[serde(default)]
|
|
||||||
pub actors: Vec<ActorRec>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct ActorRec {
|
|
||||||
#[serde(default)]
|
|
||||||
pub address: String,
|
|
||||||
#[serde(default)]
|
|
||||||
pub name: String,
|
|
||||||
#[serde(default)]
|
|
||||||
pub mailbox_depth: u32,
|
|
||||||
#[serde(default)]
|
|
||||||
pub messages_processed: u64,
|
|
||||||
#[serde(default)]
|
|
||||||
pub last_msg_type: String,
|
|
||||||
#[serde(default)]
|
|
||||||
pub poisoned: bool,
|
|
||||||
#[serde(default)]
|
|
||||||
pub message_type_counts: Vec<(String, u64)>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Record for IdentityRecord {
|
|
||||||
const CHANNEL: &'static str = IDENTITY;
|
|
||||||
}
|
|
||||||
impl Record for ResourceSample {
|
|
||||||
const CHANNEL: &'static str = HOST_RESOURCE;
|
|
||||||
}
|
|
||||||
impl Record for TransportInternals {
|
|
||||||
const CHANNEL: &'static str = TRANSPORT_INTERNALS;
|
|
||||||
}
|
|
||||||
impl Record for MembershipTransition {
|
|
||||||
const CHANNEL: &'static str = MEMBERSHIP;
|
|
||||||
}
|
|
||||||
impl Record for RuntimeStats {
|
|
||||||
const CHANNEL: &'static str = RUNTIME_STATS;
|
|
||||||
}
|
|
||||||
impl Record for DistributionState {
|
|
||||||
const CHANNEL: &'static str = DIST_STATE;
|
|
||||||
}
|
|
||||||
impl Record for ActorRuntimeDetail {
|
|
||||||
const CHANNEL: &'static str = RUNTIME_ACTORS;
|
|
||||||
}
|
|
||||||
impl Record for WorkerCounters {
|
|
||||||
const CHANNEL: &'static str = RUNTIME_WORKERS;
|
|
||||||
}
|
|
||||||
impl Record for DatastreamHealth {
|
|
||||||
const CHANNEL: &'static str = DATASTREAM_HEALTH;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
use schema::*;
|
|
||||||
|
|
||||||
/// An in-process node (testing spec §2 — the faked machine boundary): a
|
|
||||||
/// *real* mux plus the producers that feed it. Producers push realistic
|
|
||||||
/// bytes in at the producer seam; [`Node::sent`] takes the mux's ordered
|
|
||||||
/// output stream — the value on the mux→transport seam. Only the machine
|
|
||||||
/// boundary is faked; the mux is the production code.
|
|
||||||
pub struct Node {
|
|
||||||
stream: StreamId,
|
|
||||||
mux: Mux,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Node {
|
|
||||||
/// Stand up a node for a given stream (node identity + lifetime).
|
|
||||||
pub fn new(stream: StreamId) -> Self {
|
|
||||||
Node {
|
|
||||||
mux: Mux::unbounded(stream.clone()),
|
|
||||||
stream,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The stream this node produces.
|
|
||||||
pub fn stream_id(&self) -> &StreamId {
|
|
||||||
&self.stream
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A producer emits a typed record on its own channel (spec §6.1).
|
|
||||||
pub fn emit<R: Record>(&self, record: &R) -> Position {
|
|
||||||
self.mux.submit(R::channel(), record.encode())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A producer emits a line of raw process output (spec §6.2).
|
|
||||||
pub fn emit_text(&self, label: &str, stream: ProcStream, line: &str) -> Position {
|
|
||||||
self.mux
|
|
||||||
.submit(process_output(label, stream), line.as_bytes().to_vec())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A producer emits bytes on a channel the consumer may not know
|
|
||||||
/// (spec §6.3) — opaque to everything until a view learns the channel.
|
|
||||||
pub fn emit_opaque(&self, channel: &str, bytes: &[u8]) -> Position {
|
|
||||||
self.mux.submit(ChannelId::new(channel), bytes.to_vec())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Take the node's ordered output stream (drains the mux).
|
|
||||||
pub fn sent(&self) -> Vec<Frame> {
|
|
||||||
self.mux.drain()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Realistic payloads, drawn on by both the verified vectors (testing spec
|
|
||||||
/// §5) and the deployment scenario (testing spec §8). Nothing here is
|
|
||||||
/// placeholder text.
|
|
||||||
pub mod payloads {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
/// A boot/identity record for a node, including the descriptive fields a
|
|
||||||
/// node fills once known (name, listen addr, embedded relay, build version).
|
|
||||||
pub fn identity(node: &str, life: u64) -> IdentityRecord {
|
|
||||||
IdentityRecord {
|
|
||||||
node: node.to_string(),
|
|
||||||
life,
|
|
||||||
node_name: format!("swift-{node}"),
|
|
||||||
listen_addr: format!("{node}.iroh:4242"),
|
|
||||||
relay_url: "https://relay.example:4443/".to_string(),
|
|
||||||
version: "ds-inference @ abc1234".to_string(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A consolidated distribution-subsystem state; `tick` nudges the values so
|
|
||||||
/// a series is not constant.
|
|
||||||
pub fn dist_state(tick: u64) -> DistributionState {
|
|
||||||
DistributionState {
|
|
||||||
cache_size: 3 + (tick % 4) as u32,
|
|
||||||
cache_entries: vec![CacheEntryRec {
|
|
||||||
actor_addr: format!("actor-{}", tick % 5),
|
|
||||||
node_id: format!("node-{}", tick % 3),
|
|
||||||
}],
|
|
||||||
directory_route_count: 5 + (tick % 7) as u32,
|
|
||||||
registry_size: 8 + (tick % 3) as u32,
|
|
||||||
registry_tombstones: (tick % 2) as u32,
|
|
||||||
registry_entries: vec![RegistryEntryRec {
|
|
||||||
name: format!("svc-{}", tick % 4),
|
|
||||||
actor_addr: format!("actor-{}", tick % 5),
|
|
||||||
node_id: format!("node-{}", tick % 3),
|
|
||||||
tombstone: tick.is_multiple_of(2),
|
|
||||||
}],
|
|
||||||
recent_probe_targets: vec![format!("peer-{}", tick % 6)],
|
|
||||||
peer_auth_mode: if tick.is_multiple_of(2) {
|
|
||||||
"open".into()
|
|
||||||
} else {
|
|
||||||
"allow-list".into()
|
|
||||||
},
|
|
||||||
authorized_peer_count: (tick % 5) as u32,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A per-actor runtime-detail record (the real actor table).
|
|
||||||
pub fn actor_detail(tick: u64) -> ActorRuntimeDetail {
|
|
||||||
ActorRuntimeDetail {
|
|
||||||
actors: vec![
|
|
||||||
ActorRec {
|
|
||||||
address: format!("{:064x}", tick),
|
|
||||||
name: "SwimActor".to_string(),
|
|
||||||
mailbox_depth: (tick % 5) as u32,
|
|
||||||
messages_processed: 100 + tick,
|
|
||||||
last_msg_type: "swactor_dist::Ping".to_string(),
|
|
||||||
poisoned: false,
|
|
||||||
message_type_counts: vec![("swactor_dist::Ping".to_string(), 40 + tick)],
|
|
||||||
},
|
|
||||||
ActorRec {
|
|
||||||
address: format!("{:064x}", tick + 1),
|
|
||||||
name: "RegistryActor".to_string(),
|
|
||||||
mailbox_depth: 0,
|
|
||||||
messages_processed: 10 + tick % 3,
|
|
||||||
last_msg_type: "Tick".to_string(),
|
|
||||||
poisoned: false,
|
|
||||||
message_type_counts: vec![("Tick".to_string(), 10 + tick % 3)],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A plausible resource sample; `tick` nudges the values so a series is
|
|
||||||
/// not constant.
|
|
||||||
pub fn resource(tick: u64) -> ResourceSample {
|
|
||||||
ResourceSample {
|
|
||||||
cpu_pct: 12.5 + (tick % 7) as f32 * 3.0,
|
|
||||||
mem_used_mb: 2048 + (tick % 5) as u32 * 128,
|
|
||||||
mem_total_mb: 16384,
|
|
||||||
gpu_pct: (tick % 4) as f32 * 25.0,
|
|
||||||
disk_used_gb: 40 + (tick % 3) as u32,
|
|
||||||
net_rx_kbps: 900 + (tick % 11) as u32 * 30,
|
|
||||||
net_tx_kbps: 300 + (tick % 13) as u32 * 20,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A transport-internals snapshot.
|
|
||||||
pub fn transport(tick: u64) -> TransportInternals {
|
|
||||||
TransportInternals {
|
|
||||||
relay_connected: !tick.is_multiple_of(9),
|
|
||||||
direct_peers: 2 + (tick % 3) as u32,
|
|
||||||
relay_peers: 1,
|
|
||||||
rtt_ms_p50: 18 + (tick % 5) as u32 * 4,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A membership transition between two peers' states.
|
|
||||||
pub fn membership(peer: &str, from: &str, to: &str) -> MembershipTransition {
|
|
||||||
MembershipTransition {
|
|
||||||
peer: peer.to_string(),
|
|
||||||
from: from.to_string(),
|
|
||||||
to: to.to_string(),
|
|
||||||
reason: "probe timeout".to_string(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A runtime-stats record.
|
|
||||||
pub fn runtime(tick: u64) -> RuntimeStats {
|
|
||||||
RuntimeStats {
|
|
||||||
actors_live: 30 + (tick % 6) as u32,
|
|
||||||
mailbox_depth: (tick % 17) as u32,
|
|
||||||
scheduled_tasks: 4 + (tick % 3) as u32,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Aggregated worker-runtime counters (the `runtime.workers` channel).
|
|
||||||
pub fn worker_counters(tick: u64) -> WorkerCounters {
|
|
||||||
WorkerCounters {
|
|
||||||
num_workers: 4,
|
|
||||||
scheduled_tasks: 4 + (tick % 3) as u32,
|
|
||||||
local_sends: 100 + tick,
|
|
||||||
cross_sends: 20 + tick,
|
|
||||||
inbox_sends: tick,
|
|
||||||
messages_processed: 1000 + tick * 7,
|
|
||||||
tick_p50_us: 50 + tick,
|
|
||||||
..Default::default()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Datastream self-health (the `datastream.health` channel). `assigned`
|
|
||||||
/// tracks the seed directly so a scenario can pick a frame out by its value.
|
|
||||||
pub fn datastream_health(tick: u64) -> DatastreamHealth {
|
|
||||||
DatastreamHealth {
|
|
||||||
assigned: tick,
|
|
||||||
dropped: tick % 4,
|
|
||||||
loss_rate_ppm: (tick % 4) as u32,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A realistic line of process output (without trailing newline).
|
|
||||||
pub fn log_line(label: &str, tick: u64) -> String {
|
|
||||||
format!(
|
|
||||||
"[{label}] step {tick} loss=0.{:03} lr=3e-4",
|
|
||||||
250 - (tick % 200)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The reference model: the spec's rules as plain total functions over
|
|
||||||
/// sequences. No transport, no storage, no concurrency, no time.
|
|
||||||
pub mod reference {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
/// The frames a consumer was delivered for one stream, in arrival
|
|
||||||
/// order — the raw material reconstruction works over.
|
|
||||||
pub fn delivered_frames(stream: &StreamId, deliveries: &[Delivery]) -> Vec<Frame> {
|
|
||||||
deliveries
|
|
||||||
.iter()
|
|
||||||
.filter(|d| &d.stream == stream)
|
|
||||||
.map(|d| d.frame.clone())
|
|
||||||
.collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Reconstruction (spec §8.1, testing spec §3): "keep the frames that
|
|
||||||
/// were delivered, in position order." Duplicates of a position
|
|
||||||
/// collapse to one (the carrier may not fabricate content, spec §9, so
|
|
||||||
/// a repeat carries identical bytes).
|
|
||||||
pub fn reconstruct(delivered: &[Frame]) -> Vec<Frame> {
|
|
||||||
let mut frames: Vec<Frame> = Vec::new();
|
|
||||||
for f in delivered {
|
|
||||||
if !frames.iter().any(|seen| seen.position == f.position) {
|
|
||||||
frames.push(f.clone());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
frames.sort_by_key(|f| f.position);
|
|
||||||
frames
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The surfaced gaps as spans: the **interior** runs of positions missing
|
|
||||||
/// between the first and last delivered position (spec §7.5, §8). A
|
|
||||||
/// consumer can only detect gaps it has bracketing frames for; positions
|
|
||||||
/// lost after the last delivered frame are invisible and manifest as the
|
|
||||||
/// stream ending (spec §7.4 node death = truncation, not a gap).
|
|
||||||
///
|
|
||||||
/// Walks the sorted delivered positions — O(frames), never the gap size —
|
|
||||||
/// so the oracle agrees with the store on the cheap path even across a
|
|
||||||
/// near-`u64::MAX` gap.
|
|
||||||
pub fn gap_spans(delivered: &[Frame]) -> Vec<GapSpan> {
|
|
||||||
let recon = reconstruct(delivered);
|
|
||||||
let mut spans = Vec::new();
|
|
||||||
let mut prev: Option<u64> = None;
|
|
||||||
for f in &recon {
|
|
||||||
let p = f.position.0;
|
|
||||||
if let Some(q) = prev
|
|
||||||
&& p > q + 1
|
|
||||||
{
|
|
||||||
spans.push(GapSpan {
|
|
||||||
start: q + 1,
|
|
||||||
end: p - 1,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
prev = Some(p);
|
|
||||||
}
|
|
||||||
spans
|
|
||||||
}
|
|
||||||
|
|
||||||
/// One structural item on the merged timeline (testing spec §3: "all
|
|
||||||
/// stored frames in position order, channels interleaved"). This is the
|
|
||||||
/// *structure* of the merged log — order and surfaced gaps — decoupled
|
|
||||||
/// from how each payload is rendered for display, which is a separate
|
|
||||||
/// §9.3 concern the tests assert on its own.
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
||||||
pub enum TimelineItem {
|
|
||||||
Frame { position: u64, channel: String },
|
|
||||||
Gap { start: u64, end: u64 },
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The merged log oracle (spec §9.2): every delivered frame in position
|
|
||||||
/// order, channels interleaved, with an interior gap surfaced wherever a
|
|
||||||
/// position is missing. Written naively so it is obviously the spec.
|
|
||||||
pub fn merged_log(delivered: &[Frame]) -> Vec<TimelineItem> {
|
|
||||||
let frames = reconstruct(delivered);
|
|
||||||
let mut out = Vec::new();
|
|
||||||
let mut prev: Option<u64> = None;
|
|
||||||
for f in &frames {
|
|
||||||
let pos = f.position.0;
|
|
||||||
if let Some(p) = prev
|
|
||||||
&& pos > p + 1
|
|
||||||
{
|
|
||||||
out.push(TimelineItem::Gap {
|
|
||||||
start: p + 1,
|
|
||||||
end: pos - 1,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
out.push(TimelineItem::Frame {
|
|
||||||
position: pos,
|
|
||||||
channel: f.channel.to_string(),
|
|
||||||
});
|
|
||||||
prev = Some(pos);
|
|
||||||
}
|
|
||||||
out
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Convenience: build a frame on a typed channel from a record.
|
|
||||||
pub fn typed_frame<R: Record>(record: &R, position: u64) -> Frame {
|
|
||||||
Frame::new(R::channel(), Position(position), record.encode())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Convenience: build a frame on a raw-text process-output channel.
|
|
||||||
pub fn text_frame(label: &str, stream: ProcStream, line: &str, position: u64) -> Frame {
|
|
||||||
Frame::new(
|
|
||||||
process_output(label, stream),
|
|
||||||
Position(position),
|
|
||||||
line.as_bytes().to_vec(),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Convenience: a frame on a channel id the consumer does not know — an
|
|
||||||
/// opaque channel (spec §6.3).
|
|
||||||
pub fn opaque_frame(id: &str, payload: &[u8], position: u64) -> Frame {
|
|
||||||
Frame::new(ChannelId::new(id), Position(position), payload.to_vec())
|
|
||||||
}
|
|
||||||
|
|
@ -7,8 +7,7 @@ use datastream::transport::{Delivery, Reorder, ScriptedTransport, StreamScript};
|
||||||
use datastream::views::{self, Body, LogEntry};
|
use datastream::views::{self, Body, LogEntry};
|
||||||
use datastream::wire::{decode_delivery, encode_delivery};
|
use datastream::wire::{decode_delivery, encode_delivery};
|
||||||
use datastream::{
|
use datastream::{
|
||||||
ChannelId, ChannelKind, ChannelRegistry, FRAME_TIME_CHANNEL_ID, Frame, FrameTimeSample,
|
ChannelId, ChannelKind, ChannelRegistry, Frame, Lifetime, NodeId, Position, Record, StreamId,
|
||||||
Lifetime, NodeId, Position, Record, StreamId,
|
|
||||||
};
|
};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
|
@ -58,67 +57,81 @@ fn record_codecs_round_trip_without_global_catalog() {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn mux_numbers_monotonic_and_gap_free() {
|
fn mux_assigns_positions_when_drained() {
|
||||||
let mux = Mux::unbounded(stream());
|
let mux = Mux::unbounded(stream());
|
||||||
mux.set_frame_timing_enabled(false);
|
|
||||||
|
|
||||||
for i in 0..64u64 {
|
for i in 0..64u64 {
|
||||||
let position = mux.submit(RESOURCE_CHANNEL, resource(i as u32).encode());
|
assert!(mux.submit(RESOURCE_CHANNEL, resource(i as u32).encode()));
|
||||||
assert_eq!(position, Position(i));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
assert_eq!(mux.assigned(), 64);
|
assert_eq!(mux.assigned(), 0);
|
||||||
assert_eq!(mux.dropped(), 0);
|
assert_eq!(mux.dropped(), 0);
|
||||||
let positions: Vec<u64> = mux.drain().iter().map(|frame| frame.position.0).collect();
|
let mut positions: Vec<u64> = mux.drain().iter().map(|frame| frame.position.0).collect();
|
||||||
|
positions.sort_unstable();
|
||||||
assert_eq!(positions, (0..64).collect::<Vec<_>>());
|
assert_eq!(positions, (0..64).collect::<Vec<_>>());
|
||||||
|
assert_eq!(mux.assigned(), 64);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn mux_queue_preserves_position_order_across_producers() {
|
fn mux_concurrent_producers_assign_unique_positions_on_drain() {
|
||||||
let mux = Arc::new(Mux::unbounded(stream()));
|
let mux = Arc::new(Mux::unbounded(stream()));
|
||||||
mux.set_frame_timing_enabled(false);
|
|
||||||
|
|
||||||
let mut threads = Vec::new();
|
let mut threads = Vec::new();
|
||||||
for producer in 0..4u8 {
|
for producer in 0..4u8 {
|
||||||
let mux = Arc::clone(&mux);
|
let mux = Arc::clone(&mux);
|
||||||
threads.push(thread::spawn(move || {
|
threads.push(thread::spawn(move || {
|
||||||
|
let mut accepted = 0;
|
||||||
for seq in 0..32u8 {
|
for seq in 0..32u8 {
|
||||||
mux.submit(LOG_CHANNEL, vec![producer, seq]);
|
if mux.submit(LOG_CHANNEL, vec![producer, seq]) {
|
||||||
|
accepted += 1;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
accepted
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
for thread in threads {
|
let accepted: usize = threads
|
||||||
thread.join().expect("producer thread completes");
|
.into_iter()
|
||||||
}
|
.map(|thread| thread.join().expect("producer thread completes"))
|
||||||
|
.sum();
|
||||||
|
|
||||||
|
assert_eq!(accepted, 128);
|
||||||
|
assert_eq!(mux.assigned(), 0);
|
||||||
let frames = mux.drain();
|
let frames = mux.drain();
|
||||||
assert_eq!(frames.len(), 128);
|
assert_eq!(frames.len(), 128);
|
||||||
for (expected, frame) in frames.iter().enumerate() {
|
let mut positions: Vec<u64> = frames.iter().map(|frame| frame.position.0).collect();
|
||||||
assert_eq!(frame.position, Position(expected as u64));
|
positions.sort_unstable();
|
||||||
}
|
assert_eq!(positions, (0..128).collect::<Vec<_>>());
|
||||||
|
assert_eq!(mux.assigned(), 128);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn mux_timing_sidecar_uses_reserved_numeric_channel_and_does_not_recurse() {
|
fn mux_full_queue_drops_without_consuming_position() {
|
||||||
let mux = Mux::unbounded(stream());
|
let mux = Mux::new(stream(), 1);
|
||||||
assert!(mux.frame_timing_enabled());
|
|
||||||
|
|
||||||
let data_position = mux.submit(RESOURCE_CHANNEL, resource(0).encode());
|
assert!(mux.submit(LOG_CHANNEL, b"first".to_vec()));
|
||||||
let timing_position = mux.submit(
|
assert!(!mux.submit(LOG_CHANNEL, b"second".to_vec()));
|
||||||
FRAME_TIME_CHANNEL_ID,
|
assert_eq!(mux.dropped(), 1);
|
||||||
FrameTimeSample::new(Position(42), 123).encode(),
|
assert_eq!(mux.assigned(), 0);
|
||||||
);
|
|
||||||
|
let frames = mux.drain();
|
||||||
|
assert_eq!(frames.len(), 1);
|
||||||
|
assert_eq!(frames[0].channel, LOG_CHANNEL);
|
||||||
|
assert_eq!(frames[0].position, Position(0));
|
||||||
|
assert_eq!(frames[0].payload, b"first");
|
||||||
|
assert_eq!(mux.assigned(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn mux_one_submit_one_drained_frame_without_timing_sidecar() {
|
||||||
|
let mux = Mux::unbounded(stream());
|
||||||
|
|
||||||
|
assert!(mux.submit(RESOURCE_CHANNEL, resource(0).encode()));
|
||||||
let frames = mux.drain();
|
let frames = mux.drain();
|
||||||
|
|
||||||
assert_eq!(data_position, Position(0));
|
assert_eq!(frames.len(), 1);
|
||||||
assert_eq!(timing_position, Position(2));
|
|
||||||
assert_eq!(mux.assigned(), 3);
|
|
||||||
assert_eq!(frames.len(), 3);
|
|
||||||
assert_eq!(frames[0].channel, RESOURCE_CHANNEL);
|
assert_eq!(frames[0].channel, RESOURCE_CHANNEL);
|
||||||
assert_eq!(frames[1].channel, FRAME_TIME_CHANNEL_ID);
|
assert_eq!(frames[0].position, Position(0));
|
||||||
assert_eq!(frames[2].channel, FRAME_TIME_CHANNEL_ID);
|
assert_eq!(frames[0].payload, resource(0).encode());
|
||||||
let sample = FrameTimeSample::decode(&frames[1].payload).expect("timing decodes");
|
|
||||||
assert_eq!(sample.target_position, data_position.0);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
|
||||||
|
|
@ -2,8 +2,8 @@ use std::time::Duration;
|
||||||
|
|
||||||
use datastream::{
|
use datastream::{
|
||||||
ChannelContent, ChannelContentKind, ChannelFilter, ChannelId, DatastreamEndpoint,
|
ChannelContent, ChannelContentKind, ChannelFilter, ChannelId, DatastreamEndpoint,
|
||||||
DatastreamEvent, FRAME_TIME_CHANNEL_ID, FrameDelivery, FrameTimeSample, Lifetime, NodeId,
|
DatastreamEvent, FrameDelivery, Lifetime, NodeId, Position, Record, SourceFilter, StreamId,
|
||||||
Position, Record, SourceFilter, StreamId, SubscriptionRequest,
|
SubscriptionRequest,
|
||||||
};
|
};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use swactor::actor::ActorAddress;
|
use swactor::actor::ActorAddress;
|
||||||
|
|
@ -37,7 +37,6 @@ fn frame_event(event: &DatastreamEvent) -> &FrameDelivery {
|
||||||
fn endpoint_without_subscribers_drains_to_bitbucket() {
|
fn endpoint_without_subscribers_drains_to_bitbucket() {
|
||||||
let endpoint = endpoint();
|
let endpoint = endpoint();
|
||||||
let producer = endpoint.producer();
|
let producer = endpoint.producer();
|
||||||
producer.set_frame_timing_enabled(false);
|
|
||||||
let log = producer.register_channel("runtime.log", ChannelContent::TextStream);
|
let log = producer.register_channel("runtime.log", ChannelContent::TextStream);
|
||||||
|
|
||||||
producer.submit_text(log, "before");
|
producer.submit_text(log, "before");
|
||||||
|
|
@ -63,12 +62,12 @@ fn channel_registration_allocates_numeric_ids() {
|
||||||
assert_eq!(stderr, ChannelId(2));
|
assert_eq!(stderr, ChannelId(2));
|
||||||
assert_eq!(runtime, ChannelId(3));
|
assert_eq!(runtime, ChannelId(3));
|
||||||
let catalog = endpoint.catalog_snapshot();
|
let catalog = endpoint.catalog_snapshot();
|
||||||
|
let removed_timing_name = ["datastream", "frame_time"].join(".");
|
||||||
assert!(
|
assert!(
|
||||||
catalog
|
!catalog
|
||||||
.channels
|
.channels
|
||||||
.values()
|
.values()
|
||||||
.any(|descriptor| descriptor.id == FRAME_TIME_CHANNEL_ID
|
.any(|descriptor| descriptor.name == removed_timing_name)
|
||||||
&& descriptor.name == "datastream.frame_time")
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -118,7 +117,6 @@ fn subscription_snapshot_contains_stream_and_channel_metadata() {
|
||||||
fn subscription_receives_only_future_matching_frames() {
|
fn subscription_receives_only_future_matching_frames() {
|
||||||
let endpoint = endpoint();
|
let endpoint = endpoint();
|
||||||
let producer = endpoint.producer();
|
let producer = endpoint.producer();
|
||||||
producer.set_frame_timing_enabled(false);
|
|
||||||
let log = producer.register_channel("runtime.log", ChannelContent::TextStream);
|
let log = producer.register_channel("runtime.log", ChannelContent::TextStream);
|
||||||
|
|
||||||
producer.submit_text(log, "pre-subscription");
|
producer.submit_text(log, "pre-subscription");
|
||||||
|
|
@ -138,10 +136,9 @@ fn subscription_receives_only_future_matching_frames() {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn subscription_filters_textstream_channels() {
|
fn subscription_snapshot_filters_but_future_fanout_broadcasts() {
|
||||||
let endpoint = endpoint();
|
let endpoint = endpoint();
|
||||||
let producer = endpoint.producer();
|
let producer = endpoint.producer();
|
||||||
producer.set_frame_timing_enabled(false);
|
|
||||||
let stdout = producer.register_channel("stdout", ChannelContent::TextStream);
|
let stdout = producer.register_channel("stdout", ChannelContent::TextStream);
|
||||||
let json = producer.register_record::<RuntimeRecord>();
|
let json = producer.register_record::<RuntimeRecord>();
|
||||||
let text_subscription = endpoint.subscribe(
|
let text_subscription = endpoint.subscribe(
|
||||||
|
|
@ -152,22 +149,26 @@ fn subscription_filters_textstream_channels() {
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
|
assert_eq!(text_subscription.snapshot().channels.len(), 1);
|
||||||
|
assert_eq!(text_subscription.snapshot().channels[0].id, stdout);
|
||||||
|
|
||||||
producer.submit_text(stdout, "line");
|
producer.submit_text(stdout, "line");
|
||||||
producer.submit_record(json, &RuntimeRecord { value: 5 });
|
producer.submit_record(json, &RuntimeRecord { value: 5 });
|
||||||
endpoint.tick();
|
endpoint.tick();
|
||||||
|
|
||||||
let events = text_subscription.drain_available();
|
let events = text_subscription.drain_available();
|
||||||
assert_eq!(events.len(), 1);
|
assert_eq!(events.len(), 2);
|
||||||
let delivery = frame_event(&events[0]);
|
let channels: Vec<ChannelId> = events
|
||||||
assert_eq!(delivery.channel.channel, stdout);
|
.iter()
|
||||||
assert_eq!(delivery.payload, b"line");
|
.map(|event| frame_event(event).channel.channel)
|
||||||
|
.collect();
|
||||||
|
assert_eq!(channels, vec![stdout, json]);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn endpoint_fans_out_ordered_frames_to_multiple_subscribers() {
|
fn endpoint_fans_out_ordered_frames_to_multiple_subscribers() {
|
||||||
let endpoint = endpoint();
|
let endpoint = endpoint();
|
||||||
let producer = endpoint.producer();
|
let producer = endpoint.producer();
|
||||||
producer.set_frame_timing_enabled(false);
|
|
||||||
let log = producer.register_channel("runtime.log", ChannelContent::TextStream);
|
let log = producer.register_channel("runtime.log", ChannelContent::TextStream);
|
||||||
let left = endpoint.subscribe_all("left");
|
let left = endpoint.subscribe_all("left");
|
||||||
let right = endpoint.subscribe_all("right");
|
let right = endpoint.subscribe_all("right");
|
||||||
|
|
@ -188,7 +189,6 @@ fn endpoint_fans_out_ordered_frames_to_multiple_subscribers() {
|
||||||
fn slow_subscriber_drops_without_blocking_fast_subscriber() {
|
fn slow_subscriber_drops_without_blocking_fast_subscriber() {
|
||||||
let endpoint = endpoint();
|
let endpoint = endpoint();
|
||||||
let producer = endpoint.producer();
|
let producer = endpoint.producer();
|
||||||
producer.set_frame_timing_enabled(false);
|
|
||||||
let log = producer.register_channel("runtime.log", ChannelContent::TextStream);
|
let log = producer.register_channel("runtime.log", ChannelContent::TextStream);
|
||||||
let slow = endpoint.subscribe_all_with_capacity("slow", 1);
|
let slow = endpoint.subscribe_all_with_capacity("slow", 1);
|
||||||
let fast = endpoint.subscribe_all_with_capacity("fast", 8);
|
let fast = endpoint.subscribe_all_with_capacity("fast", 8);
|
||||||
|
|
@ -212,35 +212,56 @@ fn slow_subscriber_drops_without_blocking_fast_subscriber() {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn endpoint_producer_timing_sidecars_are_fanned_out_when_enabled() {
|
fn channel_declared_is_broadcast_to_filtered_subscribers() {
|
||||||
|
let endpoint = endpoint();
|
||||||
|
let subscription = endpoint.subscribe(
|
||||||
|
"text-only",
|
||||||
|
SubscriptionRequest {
|
||||||
|
sources: SourceFilter::All,
|
||||||
|
channels: ChannelFilter::Content(ChannelContentKind::TextStream),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
let channel = endpoint.register_channel(
|
||||||
|
"runtime.json",
|
||||||
|
ChannelContent::JsonRecord {
|
||||||
|
schema: Some("runtime.json".to_owned()),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
let events = subscription.drain_available();
|
||||||
|
assert_eq!(events.len(), 1);
|
||||||
|
match &events[0] {
|
||||||
|
DatastreamEvent::ChannelDeclared(descriptor) => {
|
||||||
|
assert_eq!(descriptor.id, channel);
|
||||||
|
assert_eq!(descriptor.name, "runtime.json");
|
||||||
|
}
|
||||||
|
other => panic!("expected channel declaration, got {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn submit_text_owned_queues_owned_string() {
|
||||||
let endpoint = endpoint();
|
let endpoint = endpoint();
|
||||||
let producer = endpoint.producer();
|
let producer = endpoint.producer();
|
||||||
let log = producer.register_channel("runtime.log", ChannelContent::TextStream);
|
let log = producer.register_channel("runtime.log", ChannelContent::TextStream);
|
||||||
let sub = endpoint.subscribe_all("test");
|
let sub = endpoint.subscribe_all("test");
|
||||||
|
|
||||||
producer.set_frame_timing_enabled(true);
|
assert!(producer.submit_text_owned(log, String::from("hello")));
|
||||||
let data_position = producer.submit_text(log, "visible");
|
|
||||||
let tick = endpoint.tick();
|
let tick = endpoint.tick();
|
||||||
|
|
||||||
assert_eq!(data_position, Position(0));
|
assert_eq!(tick.drained, 1);
|
||||||
assert_eq!(tick.drained, 2);
|
assert_eq!(tick.delivered, 1);
|
||||||
let events = sub.drain_available();
|
let event = sub.recv_timeout(Duration::from_millis(50)).unwrap();
|
||||||
assert_eq!(events.len(), 2);
|
let delivery = frame_event(&event);
|
||||||
assert_eq!(frame_event(&events[0]).channel.channel, log);
|
assert_eq!(delivery.channel.channel, log);
|
||||||
assert_eq!(frame_event(&events[0]).payload, b"visible");
|
assert_eq!(delivery.payload, b"hello");
|
||||||
assert_eq!(
|
|
||||||
frame_event(&events[1]).channel.channel,
|
|
||||||
FRAME_TIME_CHANNEL_ID
|
|
||||||
);
|
|
||||||
let sample = FrameTimeSample::decode(&frame_event(&events[1]).payload).expect("timing decodes");
|
|
||||||
assert_eq!(sample.target_position, data_position.0);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn process_observer_adapter_submits_configured_channels() {
|
fn process_observer_adapter_submits_configured_channels() {
|
||||||
let endpoint = endpoint();
|
let endpoint = endpoint();
|
||||||
let producer = endpoint.producer();
|
let producer = endpoint.producer();
|
||||||
producer.set_frame_timing_enabled(false);
|
|
||||||
let stdout = producer.register_channel("proc.trainer.stdout", ChannelContent::TextStream);
|
let stdout = producer.register_channel("proc.trainer.stdout", ChannelContent::TextStream);
|
||||||
let stderr = producer.register_channel("proc.trainer.stderr", ChannelContent::TextStream);
|
let stderr = producer.register_channel("proc.trainer.stderr", ChannelContent::TextStream);
|
||||||
let observer = producer.process_observer_with(
|
let observer = producer.process_observer_with(
|
||||||
|
|
@ -300,8 +321,10 @@ fn stats_hook_adapter_submits_worker_snapshot_json() {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn positions(events: &[DatastreamEvent]) -> Vec<u64> {
|
fn positions(events: &[DatastreamEvent]) -> Vec<u64> {
|
||||||
events
|
let mut positions: Vec<u64> = events
|
||||||
.iter()
|
.iter()
|
||||||
.map(|event| frame_event(event).position.0)
|
.map(|event| frame_event(event).position.0)
|
||||||
.collect()
|
.collect();
|
||||||
|
positions.sort_unstable();
|
||||||
|
positions
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -28,7 +28,6 @@ impl Record for ResourceSample {
|
||||||
fn build_stream() -> (StreamId, Vec<Frame>) {
|
fn build_stream() -> (StreamId, Vec<Frame>) {
|
||||||
let id = StreamId::new(NodeId::new("node-real"), Lifetime(1));
|
let id = StreamId::new(NodeId::new("node-real"), Lifetime(1));
|
||||||
let mux = Mux::unbounded(id.clone());
|
let mux = Mux::unbounded(id.clone());
|
||||||
mux.set_frame_timing_enabled(false);
|
|
||||||
for tick in 0..20 {
|
for tick in 0..20 {
|
||||||
mux.submit(
|
mux.submit(
|
||||||
RESOURCE_CHANNEL,
|
RESOURCE_CHANNEL,
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,7 @@ mod datastream_records {
|
||||||
//! the mux.
|
//! the mux.
|
||||||
|
|
||||||
use datastream::frame::{Lifetime, NodeId, StreamId};
|
use datastream::frame::{Lifetime, NodeId, StreamId};
|
||||||
use datastream::{Mux, Record};
|
use datastream::{ChannelId, Mux, Position, Record};
|
||||||
use distribution::telemetry::{
|
use distribution::telemetry::{
|
||||||
CacheEntryRec, DIST_STATE, DistributionState, MembershipTransition, RegistryEntryRec,
|
CacheEntryRec, DIST_STATE, DistributionState, MembershipTransition, RegistryEntryRec,
|
||||||
};
|
};
|
||||||
|
|
@ -49,18 +49,18 @@ mod datastream_records {
|
||||||
fn distribution_emits_owned_channel_through_datastream_mux() {
|
fn distribution_emits_owned_channel_through_datastream_mux() {
|
||||||
let stream = StreamId::new(NodeId::new("dist-node"), Lifetime(1));
|
let stream = StreamId::new(NodeId::new("dist-node"), Lifetime(1));
|
||||||
let mux = Mux::unbounded(stream);
|
let mux = Mux::unbounded(stream);
|
||||||
mux.set_frame_timing_enabled(false);
|
|
||||||
let state = DistributionState {
|
let state = DistributionState {
|
||||||
registry_size: 9,
|
registry_size: 9,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
let pos = mux.submit(DistributionState::channel(), state.encode());
|
let channel = ChannelId(1);
|
||||||
|
assert!(mux.submit(channel, state.encode()));
|
||||||
let frames = mux.drain();
|
let frames = mux.drain();
|
||||||
|
|
||||||
assert_eq!(pos.0, 0);
|
|
||||||
assert_eq!(frames.len(), 1);
|
assert_eq!(frames.len(), 1);
|
||||||
assert_eq!(frames[0].channel.as_str(), DIST_STATE);
|
assert_eq!(frames[0].channel, channel);
|
||||||
|
assert_eq!(frames[0].position, Position(0));
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
DistributionState::decode(&frames[0].payload)
|
DistributionState::decode(&frames[0].payload)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
|
|
|
||||||
|
|
@ -7,11 +7,10 @@ autobins = false
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
default = []
|
default = []
|
||||||
local-e2e = ["dep:dashboard"]
|
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
datastream = { path = "../datastream" }
|
datastream = { path = "../datastream" }
|
||||||
dashboard = { path = "../dashboard", optional = true }
|
dashboard = { path = "../dashboard" }
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
swactor = { path = "../..", features = ["serde", "transport"] }
|
swactor = { path = "../..", features = ["serde", "transport"] }
|
||||||
|
|
@ -19,7 +18,7 @@ swactor-transport = { path = "../transport" }
|
||||||
distribution = { path = "../distribution" }
|
distribution = { path = "../distribution" }
|
||||||
iroh-driver = { path = "../iroh-driver" }
|
iroh-driver = { path = "../iroh-driver" }
|
||||||
iroh = "0.98"
|
iroh = "0.98"
|
||||||
tokio = { version = "1", features = ["rt-multi-thread", "macros", "process", "io-util", "sync", "time", "net"] }
|
tokio = { version = "1", features = ["rt-multi-thread", "macros", "process", "io-util", "sync", "time", "net", "signal"] }
|
||||||
swactor-vastai = { path = "../../tools/vastai" }
|
swactor-vastai = { path = "../../tools/vastai" }
|
||||||
parking_lot = "0.12"
|
parking_lot = "0.12"
|
||||||
blake3 = "1"
|
blake3 = "1"
|
||||||
|
|
@ -60,3 +59,8 @@ name = "local-e2e-cluster"
|
||||||
path = "tests/local_e2e_cluster.rs"
|
path = "tests/local_e2e_cluster.rs"
|
||||||
harness = false
|
harness = false
|
||||||
required-features = ["local-e2e"]
|
required-features = ["local-e2e"]
|
||||||
|
|
||||||
|
[[test]]
|
||||||
|
name = "mvp_chat_mock"
|
||||||
|
path = "tests/mvp_chat_mock.rs"
|
||||||
|
harness = false
|
||||||
|
|
|
||||||
|
|
@ -1430,23 +1430,28 @@ enum GpuWorkerCtlMsg {
|
||||||
ExecuteStep(ExecuteStep),
|
ExecuteStep(ExecuteStep),
|
||||||
ReleaseDeviceObject { device_handle: DeviceObjectHandle },
|
ReleaseDeviceObject { device_handle: DeviceObjectHandle },
|
||||||
ShutdownWorker(ShutdownWorker),
|
ShutdownWorker(ShutdownWorker),
|
||||||
Process(ProcessNotification),
|
Process(ProcessOutput),
|
||||||
|
WorkerAdapter(WorkerAdapterEvent),
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
`Process(ProcessNotification)` is delivered by a small `ProcessBridge` actor.
|
`Process(ProcessOutput)` is delivered by the configured upstream process owner;
|
||||||
`GpuWorkerCtl` sends process input through `ProcessCommand::WriteStdin`.
|
there is no process-local notification subscription bridge. If the
|
||||||
|
production worker still uses a subprocess stdin/stdout protocol, `GpuWorkerCtl`
|
||||||
|
talks to a separate worker I/O adapter. That adapter owns the child stdio handles
|
||||||
|
and is outside managed-process core.
|
||||||
|
|
||||||
The process adapter may use newline-delimited JSON for worker commands/events.
|
The optional worker I/O adapter may use newline-delimited JSON for worker
|
||||||
This is an adapter, not a second distributed protocol.
|
commands/events. This is an adapter-local protocol, not part of
|
||||||
|
`crates/process` and not a second distributed protocol.
|
||||||
|
|
||||||
Adapter rules:
|
Adapter rules:
|
||||||
|
|
||||||
- one command/event JSON object per line
|
- one command/event JSON object per line;
|
||||||
- stdout is reserved for worker events
|
- adapter-owned stdout may carry worker events;
|
||||||
- stderr is reserved for logs and diagnostics
|
- adapter-owned stderr may carry logs and diagnostics;
|
||||||
- payload bytes are forbidden in JSON
|
- payload bytes are forbidden in JSON;
|
||||||
- invalid JSON or unknown event shape is a worker/process fault
|
- invalid JSON or unknown event shape is a worker/adapter fault.
|
||||||
|
|
||||||
Worker environment:
|
Worker environment:
|
||||||
|
|
||||||
|
|
@ -1675,7 +1680,7 @@ Wake hints emitted by worker:
|
||||||
- egress rings: `RingReadable` after advancing `commit`
|
- egress rings: `RingReadable` after advancing `commit`
|
||||||
|
|
||||||
`GpuWorkerCtl` may synthesize `WorkerCrashed` and `RingFault` after process exit,
|
`GpuWorkerCtl` may synthesize `WorkerCrashed` and `RingFault` after process exit,
|
||||||
process error, or stdout control-stream failure.
|
process error, or worker I/O adapter control-stream failure.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
@ -1688,11 +1693,11 @@ NotStarted
|
||||||
on StartWorker -> Spawning
|
on StartWorker -> Spawning
|
||||||
|
|
||||||
Spawning
|
Spawning
|
||||||
spawn ProcessActor with ProcessSpec
|
spawn ProcessActor with ProcessSpec and upstream = GpuWorkerCtl/process owner
|
||||||
spawn ProcessBridge
|
wait for ProcessOutput::Started
|
||||||
subscribe bridge to ProcessActor
|
if subprocess worker protocol is enabled:
|
||||||
wait for ProcessNotification::Started
|
start/connect worker I/O adapter
|
||||||
send InitializeWorker through ProcessCommand::WriteStdin
|
send InitializeWorker through worker I/O adapter
|
||||||
-> Initializing
|
-> Initializing
|
||||||
|
|
||||||
Initializing
|
Initializing
|
||||||
|
|
|
||||||
|
|
@ -160,7 +160,7 @@ Accepted runtime fields:
|
||||||
- `[runtime].max_tokens`
|
- `[runtime].max_tokens`
|
||||||
|
|
||||||
`[runtime].max_tokens` sets the maximum number of tokens requested for each
|
`[runtime].max_tokens` sets the maximum number of tokens requested for each
|
||||||
prompt submission. It must be a positive integer.
|
prompt submission. A value of `0` implies no specified limit.
|
||||||
|
|
||||||
Accepted observability fields:
|
Accepted observability fields:
|
||||||
|
|
||||||
|
|
@ -352,16 +352,13 @@ Default values:
|
||||||
|
|
||||||
- provider: `process`;
|
- provider: `process`;
|
||||||
- pipeline stages: `1`;
|
- pipeline stages: `1`;
|
||||||
- max tokens: `64`;
|
- max tokens: `0`;
|
||||||
- dump logs: disabled;
|
- dump logs: disabled;
|
||||||
- cached model: disabled unless `--cached-model` is supplied;
|
- cached model: disabled unless `--cached-model` is supplied;
|
||||||
- rebuild: enabled unless `--skip-rebuild` is supplied.
|
- rebuild: enabled unless `--skip-rebuild` is supplied.
|
||||||
|
|
||||||
Invalid values must fail before runtime preparation begins.
|
Invalid values must fail before runtime preparation begins.
|
||||||
|
|
||||||
`[runtime].max_tokens` must be greater than zero. Zero and invalid values are
|
|
||||||
configuration errors detected before runtime preparation.
|
|
||||||
|
|
||||||
For provider `process`, no node image is required.
|
For provider `process`, no node image is required.
|
||||||
|
|
||||||
For provider `docker`, an image reference is required. It may be local or remote.
|
For provider `docker`, an image reference is required. It may be local or remote.
|
||||||
|
|
|
||||||
|
|
@ -5,8 +5,6 @@ pub mod actors;
|
||||||
pub mod arena_manager;
|
pub mod arena_manager;
|
||||||
pub mod bootstrap_datastream;
|
pub mod bootstrap_datastream;
|
||||||
pub mod config;
|
pub mod config;
|
||||||
#[cfg(feature = "local-e2e")]
|
|
||||||
pub mod dashboard_view;
|
|
||||||
pub mod device_bridge;
|
pub mod device_bridge;
|
||||||
pub mod distribution_stack;
|
pub mod distribution_stack;
|
||||||
pub mod docker_cluster_provisioning;
|
pub mod docker_cluster_provisioning;
|
||||||
|
|
@ -19,6 +17,7 @@ pub mod gpu_worker_egress_producer;
|
||||||
pub mod gpu_worker_ingress_parser;
|
pub mod gpu_worker_ingress_parser;
|
||||||
pub mod gpu_worker_process_adapter;
|
pub mod gpu_worker_process_adapter;
|
||||||
pub mod membership_pool_readiness;
|
pub mod membership_pool_readiness;
|
||||||
|
pub mod mvp_chat;
|
||||||
pub mod node_boot_lifecycle;
|
pub mod node_boot_lifecycle;
|
||||||
pub mod node_image;
|
pub mod node_image;
|
||||||
pub mod node_provisioning;
|
pub mod node_provisioning;
|
||||||
|
|
|
||||||
317
crates/mvp-system/src/mvp_chat.rs
Normal file
317
crates/mvp-system/src/mvp_chat.rs
Normal file
|
|
@ -0,0 +1,317 @@
|
||||||
|
use std::io::{self, BufRead, Write};
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use datastream::{DatastreamEndpoint, Lifetime, StreamDescriptor, StreamId, StreamOrigin};
|
||||||
|
use swactor::Error;
|
||||||
|
use tokio::sync::Notify;
|
||||||
|
type Result<T> = std::result::Result<T, swactor::Error>;
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
enum ChatSignals {}
|
||||||
|
|
||||||
|
const MVP_CHAT_DATASTREAM_NODE: &str = "mvp-chat";
|
||||||
|
const MVP_CHAT_DATASTREAM_LABEL: &str = "mvp chat";
|
||||||
|
|
||||||
|
struct MvpChatDatastream {
|
||||||
|
endpoint: Arc<DatastreamEndpoint>,
|
||||||
|
wake: Arc<Notify>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MvpChatDatastream {
|
||||||
|
fn new(run_id: u64) -> Self {
|
||||||
|
let stream = StreamId::new(MVP_CHAT_DATASTREAM_NODE, Lifetime(run_id));
|
||||||
|
let endpoint = DatastreamEndpoint::with_descriptor(
|
||||||
|
StreamDescriptor {
|
||||||
|
stream,
|
||||||
|
label: Some(MVP_CHAT_DATASTREAM_LABEL.to_owned()),
|
||||||
|
origin: StreamOrigin::Orchestrator,
|
||||||
|
},
|
||||||
|
4096,
|
||||||
|
1024,
|
||||||
|
);
|
||||||
|
|
||||||
|
Self {
|
||||||
|
endpoint: Arc::new(endpoint),
|
||||||
|
wake: Arc::new(Notify::new()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn spawn_handler(&self, handle: &tokio::runtime::Handle) -> tokio::task::JoinHandle<()> {
|
||||||
|
let endpoint = Arc::clone(&self.endpoint);
|
||||||
|
let wake = Arc::clone(&self.wake);
|
||||||
|
|
||||||
|
handle.spawn(async move {
|
||||||
|
loop {
|
||||||
|
wake.notified().await;
|
||||||
|
|
||||||
|
while endpoint.tick().drained != 0 {}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, PartialEq, Eq)]
|
||||||
|
enum PromptStep {
|
||||||
|
Response(String),
|
||||||
|
Ignore,
|
||||||
|
Exit,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_prompt_step<R>(input: &mut R) -> PromptStep
|
||||||
|
where
|
||||||
|
R: BufRead,
|
||||||
|
{
|
||||||
|
let mut line = String::new();
|
||||||
|
|
||||||
|
match input.read_line(&mut line) {
|
||||||
|
Ok(0) => return PromptStep::Exit,
|
||||||
|
Ok(_) => {}
|
||||||
|
Err(_) => return PromptStep::Exit,
|
||||||
|
}
|
||||||
|
|
||||||
|
let prompt = line.trim_end().to_owned();
|
||||||
|
|
||||||
|
if prompt.trim().is_empty() {
|
||||||
|
return PromptStep::Ignore;
|
||||||
|
}
|
||||||
|
|
||||||
|
PromptStep::Response(prompt_response(&prompt))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn prompt_response(prompt: &str) -> String {
|
||||||
|
format!("Hello, {prompt}!")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run_prompt_loop() -> Result<()> {
|
||||||
|
let stdin = io::stdin();
|
||||||
|
let mut input = stdin.lock();
|
||||||
|
|
||||||
|
let stdout = io::stdout();
|
||||||
|
let mut output = stdout.lock();
|
||||||
|
|
||||||
|
run_prompt_loop_with_io(&mut input, &mut output)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run_prompt_loop_with_io<R, W>(input: &mut R, output: &mut W) -> Result<()>
|
||||||
|
where
|
||||||
|
R: BufRead,
|
||||||
|
W: Write,
|
||||||
|
{
|
||||||
|
loop {
|
||||||
|
write!(output, "prompt:> ")
|
||||||
|
.map_err(|error| Error::from(format!("write prompt marker: {error}")))?;
|
||||||
|
output
|
||||||
|
.flush()
|
||||||
|
.map_err(|error| Error::from(format!("flush prompt marker: {error}")))?;
|
||||||
|
|
||||||
|
match read_prompt_step(input) {
|
||||||
|
PromptStep::Response(response) => {
|
||||||
|
writeln!(output, "{response}")
|
||||||
|
.map_err(|error| Error::from(format!("write prompt response: {error}")))?;
|
||||||
|
}
|
||||||
|
PromptStep::Ignore => continue,
|
||||||
|
PromptStep::Exit => return Ok(()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
enum ChatRuntimeEvent {
|
||||||
|
PromptExited(Result<()>),
|
||||||
|
PromptPanicked,
|
||||||
|
CtrlC(std::io::Result<()>),
|
||||||
|
}
|
||||||
|
|
||||||
|
struct PromptLoop {
|
||||||
|
join: Option<std::thread::JoinHandle<()>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PromptLoop {
|
||||||
|
fn spawn(events: tokio::sync::mpsc::UnboundedSender<ChatRuntimeEvent>) -> Result<Self> {
|
||||||
|
let join = std::thread::Builder::new()
|
||||||
|
.name("mvp-chat-prompt".to_owned())
|
||||||
|
.spawn(move || {
|
||||||
|
let event = match std::panic::catch_unwind(run_prompt_loop) {
|
||||||
|
Ok(result) => ChatRuntimeEvent::PromptExited(result),
|
||||||
|
Err(_) => ChatRuntimeEvent::PromptPanicked,
|
||||||
|
};
|
||||||
|
|
||||||
|
let _ = events.send(event);
|
||||||
|
})
|
||||||
|
.map_err(|error| Error::from(format!("spawn prompt loop: {error}")))?;
|
||||||
|
|
||||||
|
Ok(Self { join: Some(join) })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn join_finished(&mut self) -> Result<()> {
|
||||||
|
let Some(join) = self.join.take() else {
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
|
||||||
|
join.join()
|
||||||
|
.map_err(|_| Error::from("prompt loop thread panicked".to_owned()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn detach(mut self) {
|
||||||
|
let _ = self.join.take();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn spawn_ctrl_c_reporter(
|
||||||
|
handle: &tokio::runtime::Handle,
|
||||||
|
events: tokio::sync::mpsc::UnboundedSender<ChatRuntimeEvent>,
|
||||||
|
) {
|
||||||
|
handle.spawn(async move {
|
||||||
|
let result = tokio::signal::ctrl_c().await;
|
||||||
|
let _ = events.send(ChatRuntimeEvent::CtrlC(result));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
fn join_dashboard_http(result: std::result::Result<(), tokio::task::JoinError>) -> Result<()> {
|
||||||
|
result.map_err(|error| Error::from(format!("dashboard HTTP task failed: {error}")))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn join_datastream_handler(result: std::result::Result<(), tokio::task::JoinError>) -> Result<()> {
|
||||||
|
result.map_err(|error| Error::from(format!("datastream handler task failed: {error}")))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn supervise_chat_runtime(
|
||||||
|
dashboard: &dashboard::DashboardHandle,
|
||||||
|
mut dashboard_http: tokio::task::JoinHandle<()>,
|
||||||
|
mut datastream_handler: tokio::task::JoinHandle<()>,
|
||||||
|
prompt_loop: &mut PromptLoop,
|
||||||
|
events: &mut tokio::sync::mpsc::UnboundedReceiver<ChatRuntimeEvent>,
|
||||||
|
) -> Result<()> {
|
||||||
|
tokio::select! {
|
||||||
|
result = &mut dashboard_http => {
|
||||||
|
join_dashboard_http(result)?;
|
||||||
|
Err("dashboard HTTP server exited before shutdown"
|
||||||
|
.to_owned()
|
||||||
|
.into())
|
||||||
|
}
|
||||||
|
result = &mut datastream_handler => {
|
||||||
|
join_datastream_handler(result)?;
|
||||||
|
Err("datastream handler exited before shutdown"
|
||||||
|
.to_owned()
|
||||||
|
.into())
|
||||||
|
}
|
||||||
|
event = events.recv() => {
|
||||||
|
let event = event.ok_or_else(|| Error::from("runtime event channel closed".to_owned()))?;
|
||||||
|
let run_result = match event {
|
||||||
|
ChatRuntimeEvent::PromptExited(prompt_result) => {
|
||||||
|
match prompt_loop.join_finished() {
|
||||||
|
Ok(()) => prompt_result,
|
||||||
|
Err(error) => Err(error),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ChatRuntimeEvent::PromptPanicked => {
|
||||||
|
match prompt_loop.join_finished() {
|
||||||
|
Ok(()) => Err("prompt loop panicked".to_owned().into()),
|
||||||
|
Err(error) => Err(error),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ChatRuntimeEvent::CtrlC(result) => {
|
||||||
|
result
|
||||||
|
.map_err(|error| Error::from(format!("ctrl-c handler failed: {error}")))
|
||||||
|
.map(|_| ())
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
dashboard.shutdown();
|
||||||
|
join_dashboard_http(dashboard_http.await)?;
|
||||||
|
|
||||||
|
run_result
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn run_from_args<I>(args: I) -> Result<()>
|
||||||
|
where
|
||||||
|
I: IntoIterator<Item = String>,
|
||||||
|
{
|
||||||
|
let mut config_path: Option<PathBuf> = None;
|
||||||
|
let mut provider_selector: Option<&'static str> = None;
|
||||||
|
|
||||||
|
let mut args = args.into_iter();
|
||||||
|
while let Some(arg) = args.next() {
|
||||||
|
match arg.as_str() {
|
||||||
|
"--process" | "--docker" | "--vastai" => {
|
||||||
|
let selected = match arg.as_str() {
|
||||||
|
"--process" => "process",
|
||||||
|
"--docker" => "docker",
|
||||||
|
"--vastai" => "vastai",
|
||||||
|
_ => unreachable!(),
|
||||||
|
};
|
||||||
|
if provider_selector.replace(selected).is_some() {
|
||||||
|
return Err("conflicting provider selectors; use exactly one of --process, --docker, or --vastai".to_owned().into());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"--config" => {
|
||||||
|
let value = args
|
||||||
|
.next()
|
||||||
|
.ok_or_else(|| "--config requires a path".to_owned())?;
|
||||||
|
config_path = Some(PathBuf::from(value));
|
||||||
|
}
|
||||||
|
"--yes" | "-y" | "--dump-logs" | "--cached-model" | "--skip-rebuild" => {}
|
||||||
|
"--pipeline-stages" => {
|
||||||
|
let value = args
|
||||||
|
.next()
|
||||||
|
.ok_or_else(|| "--pipeline-stages requires a value".to_owned())?;
|
||||||
|
let stages = value
|
||||||
|
.parse::<u32>()
|
||||||
|
.map_err(|error| format!("parse --pipeline-stages: {error}"))?;
|
||||||
|
if stages == 0 {
|
||||||
|
return Err("--pipeline-stages must be greater than 0".into());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
value if value.starts_with("--dump-logs=") => {
|
||||||
|
if value["--dump-logs=".len()..].is_empty() {
|
||||||
|
return Err("--dump-logs= requires a path".into());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
value if value.starts_with("--cached-model=") => {
|
||||||
|
if value["--cached-model=".len()..].is_empty() {
|
||||||
|
return Err("--cached-model= requires a path".into());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
value => return Err(format!("unknown mvp-chat argument: {value}").into()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let loaded_config = crate::config::TomlConfigOverlay::load(config_path.as_deref())?;
|
||||||
|
let run_id = loaded_config.overlay.runtime.run_id.unwrap_or(0);
|
||||||
|
let rt = tokio::runtime::Builder::new_multi_thread()
|
||||||
|
.enable_all()
|
||||||
|
.build()
|
||||||
|
.map_err(|e| Error::from(e.to_string()))?;
|
||||||
|
|
||||||
|
let chat_datastream = MvpChatDatastream::new(run_id);
|
||||||
|
let datastream_handler = chat_datastream.spawn_handler(rt.handle());
|
||||||
|
|
||||||
|
let _swactor = swactor::runtime::Runtime::new(swactor::runtime::RuntimeConfig {
|
||||||
|
num_threads: 1,
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
let _chat_inbox = _swactor.new_inbox::<ChatSignals>()?;
|
||||||
|
|
||||||
|
let dashboard = dashboard::DashboardHandle::new(dashboard::DashboardConfig::default());
|
||||||
|
let dashboard_http = dashboard.spawn_http(rt.handle());
|
||||||
|
let (events_tx, mut events_rx) = tokio::sync::mpsc::unbounded_channel();
|
||||||
|
|
||||||
|
spawn_ctrl_c_reporter(rt.handle(), events_tx.clone());
|
||||||
|
let mut prompt_loop = PromptLoop::spawn(events_tx)?;
|
||||||
|
|
||||||
|
let result = rt.block_on(supervise_chat_runtime(
|
||||||
|
&dashboard,
|
||||||
|
dashboard_http,
|
||||||
|
datastream_handler,
|
||||||
|
&mut prompt_loop,
|
||||||
|
&mut events_rx,
|
||||||
|
));
|
||||||
|
|
||||||
|
if result.is_err() {
|
||||||
|
prompt_loop.detach();
|
||||||
|
}
|
||||||
|
|
||||||
|
result
|
||||||
|
}
|
||||||
3
crates/mvp-system/tests/mvp_chat_mock.rs
Normal file
3
crates/mvp-system/tests/mvp_chat_mock.rs
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
fn main() {
|
||||||
|
mvp_system::mvp_chat::run_from_args(std::env::args().skip(1)).expect("failed");
|
||||||
|
}
|
||||||
|
|
@ -5,12 +5,9 @@ edition = "2024"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
swactor = { path = "../..", default-features = false, features = ["no_random"] }
|
swactor = { path = "../..", default-features = false, features = ["no_random"] }
|
||||||
|
datastream = { path = "../datastream" }
|
||||||
crossbeam-queue = "0.3.12"
|
crossbeam-queue = "0.3.12"
|
||||||
libc = "0.2"
|
libc = "0.2"
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
serde_yaml = "0.9"
|
serde_yaml = "0.9"
|
||||||
|
|
||||||
[dev-dependencies]
|
|
||||||
proptest = "1"
|
|
||||||
proptest-state-machine = "0.3"
|
|
||||||
|
|
|
||||||
327
crates/process/SWACTOR_MANAGED_PROCESS_SPEC.md
Normal file
327
crates/process/SWACTOR_MANAGED_PROCESS_SPEC.md
Normal file
|
|
@ -0,0 +1,327 @@
|
||||||
|
# Swactor Managed Process Specification
|
||||||
|
|
||||||
|
**Status:** current contract for `swactor-process`.
|
||||||
|
|
||||||
|
`swactor-process` provides a Swactor actor interface for launching, supervising,
|
||||||
|
stopping, and observing one operating-system child process per process actor.
|
||||||
|
|
||||||
|
The crate owns process lifecycle/control only. Child stdin/stdout/stderr are not
|
||||||
|
managed or observed by this crate.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Public API
|
||||||
|
|
||||||
|
The crate exports:
|
||||||
|
|
||||||
|
```text
|
||||||
|
ProcessSpec
|
||||||
|
ProcessLifecycleObservability
|
||||||
|
ProcessOutputConfig
|
||||||
|
ProcessCommand
|
||||||
|
ProcessOutput
|
||||||
|
ExitStatus
|
||||||
|
spawn_local_process
|
||||||
|
send_process_command
|
||||||
|
```
|
||||||
|
|
||||||
|
Pipeline and YAML exports are separate crate features and are not part of the
|
||||||
|
managed-process protocol described here.
|
||||||
|
|
||||||
|
### 1.1 ProcessSpec
|
||||||
|
|
||||||
|
```text
|
||||||
|
ProcessSpec {
|
||||||
|
command: String,
|
||||||
|
args: Vec<String>,
|
||||||
|
env: HashMap<String, String>,
|
||||||
|
working_dir: Option<PathBuf>,
|
||||||
|
label: Option<String>,
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`command` is passed directly to `std::process::Command::new`.
|
||||||
|
|
||||||
|
`args` are passed as direct argv entries. The crate does not split, quote,
|
||||||
|
unquote, expand, or shell-parse argument strings.
|
||||||
|
|
||||||
|
`env` contains child environment overrides.
|
||||||
|
|
||||||
|
`working_dir`, when present, is passed as the child current working directory.
|
||||||
|
|
||||||
|
`label`, when present, is the lifecycle datastream label source. When `label` is
|
||||||
|
absent, the label source is the basename of `command`.
|
||||||
|
|
||||||
|
### 1.2 ProcessOutputConfig
|
||||||
|
|
||||||
|
```text
|
||||||
|
ProcessOutputConfig::disabled(upstream: ActorAddress) -> ProcessOutputConfig
|
||||||
|
ProcessOutputConfig::datastream_mirror(
|
||||||
|
upstream: ActorAddress,
|
||||||
|
producer: DatastreamProducer,
|
||||||
|
) -> ProcessOutputConfig
|
||||||
|
ProcessOutputConfig::upstream(&self) -> ActorAddress
|
||||||
|
ProcessOutputConfig::observability(&self) -> ProcessLifecycleObservability
|
||||||
|
```
|
||||||
|
|
||||||
|
`upstream` is the actor address that receives every public `ProcessOutput`.
|
||||||
|
|
||||||
|
`disabled` sends only upstream `ProcessOutput`.
|
||||||
|
|
||||||
|
`datastream_mirror` sends upstream `ProcessOutput` and also mirrors each
|
||||||
|
lifecycle/control output to one datastream channel.
|
||||||
|
|
||||||
|
### 1.3 ProcessLifecycleObservability
|
||||||
|
|
||||||
|
```text
|
||||||
|
ProcessLifecycleObservability::Disabled
|
||||||
|
ProcessLifecycleObservability::DatastreamMirror
|
||||||
|
```
|
||||||
|
|
||||||
|
This setting controls lifecycle/control mirroring only. It does not enable child
|
||||||
|
stdin/stdout/stderr handling.
|
||||||
|
|
||||||
|
### 1.4 ProcessCommand
|
||||||
|
|
||||||
|
```text
|
||||||
|
ProcessCommand::Stop {
|
||||||
|
kill_after: Option<Duration>,
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`Stop` asks the supervisor to terminate the child process. `kill_after`, when
|
||||||
|
present, is the grace duration before kill escalation.
|
||||||
|
|
||||||
|
### 1.5 ProcessOutput
|
||||||
|
|
||||||
|
```text
|
||||||
|
ProcessOutput::Started { pid: u32 }
|
||||||
|
ProcessOutput::SpawnFailed { error: String }
|
||||||
|
ProcessOutput::Exited { status: ExitStatus }
|
||||||
|
ProcessOutput::Error { error: String }
|
||||||
|
```
|
||||||
|
|
||||||
|
`Started` means the OS child spawned and `pid` is the child process id.
|
||||||
|
|
||||||
|
`SpawnFailed` means the process actor was created but the OS child did not spawn.
|
||||||
|
|
||||||
|
`Exited` means the OS child reached a terminal status.
|
||||||
|
|
||||||
|
`Error` means the process supervisor or process actor hit an operational failure
|
||||||
|
other than OS spawn failure.
|
||||||
|
|
||||||
|
### 1.6 ExitStatus
|
||||||
|
|
||||||
|
```text
|
||||||
|
ExitStatus::Code(i32)
|
||||||
|
ExitStatus::Signal(i32)
|
||||||
|
ExitStatus::Unknown
|
||||||
|
```
|
||||||
|
|
||||||
|
### 1.7 Spawn and command helpers
|
||||||
|
|
||||||
|
```text
|
||||||
|
spawn_local_process(
|
||||||
|
ctx: &Ctx,
|
||||||
|
sender: &ExternalSender,
|
||||||
|
spec: ProcessSpec,
|
||||||
|
output: ProcessOutputConfig,
|
||||||
|
) -> Result<ActorAddress, Error>
|
||||||
|
```
|
||||||
|
|
||||||
|
`spawn_local_process` validates lifecycle output configuration, creates one
|
||||||
|
process actor, wires its private supervisor wake path, and returns the process
|
||||||
|
actor address.
|
||||||
|
|
||||||
|
Success from `spawn_local_process` means the process actor was created. It does
|
||||||
|
not mean the OS child spawned successfully. OS spawn success or failure is
|
||||||
|
reported later as `ProcessOutput`.
|
||||||
|
|
||||||
|
```text
|
||||||
|
send_process_command(
|
||||||
|
sender: &ExternalSender,
|
||||||
|
process: ActorAddress,
|
||||||
|
command: ProcessCommand,
|
||||||
|
) -> Result<(), Error>
|
||||||
|
```
|
||||||
|
|
||||||
|
`send_process_command` is the public helper for sending process commands. It
|
||||||
|
wraps the public `ProcessCommand` in the actor's private mailbox type.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Runtime topology
|
||||||
|
|
||||||
|
One process actor owns one OS child process lifecycle.
|
||||||
|
|
||||||
|
The actor owns:
|
||||||
|
|
||||||
|
- lifecycle state;
|
||||||
|
- the configured upstream output address;
|
||||||
|
- optional lifecycle datastream mirror state;
|
||||||
|
- a private supervisor thread handle.
|
||||||
|
|
||||||
|
The private supervisor thread owns:
|
||||||
|
|
||||||
|
- the child process handle and pid;
|
||||||
|
- blocking-prone child exit polling;
|
||||||
|
- terminate/kill signal delivery;
|
||||||
|
- the stop kill deadline.
|
||||||
|
|
||||||
|
The child process owns its own execution.
|
||||||
|
|
||||||
|
The supervisor thread is not a public actor and not a public extension point.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Child spawn behavior
|
||||||
|
|
||||||
|
The supervisor starts the child with:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Command::new(&spec.command)
|
||||||
|
cmd.args(&spec.args)
|
||||||
|
cmd.env(key, value) for each spec.env entry
|
||||||
|
cmd.current_dir(dir) when spec.working_dir is Some(dir)
|
||||||
|
cmd.stdin(Stdio::null())
|
||||||
|
cmd.stdout(Stdio::null())
|
||||||
|
cmd.stderr(Stdio::null())
|
||||||
|
cmd.spawn()
|
||||||
|
```
|
||||||
|
|
||||||
|
The crate does not invoke a shell unless the caller explicitly sets `command` to
|
||||||
|
a shell executable and supplies shell arguments.
|
||||||
|
|
||||||
|
Child stdin/stdout/stderr are connected to null handles. The managed-process
|
||||||
|
protocol does not expose stdin writes, stdout/stderr output events, PTY resize,
|
||||||
|
or arbitrary signal commands.
|
||||||
|
|
||||||
|
If `cmd.spawn()` fails, the actor emits exactly one terminal
|
||||||
|
`ProcessOutput::SpawnFailed { error }` and does not emit `Started`, `Exited`, or
|
||||||
|
`Error` for that spawn failure.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Lifecycle output delivery
|
||||||
|
|
||||||
|
The actor sends each public `ProcessOutput` to the configured upstream actor.
|
||||||
|
|
||||||
|
When `ProcessOutputConfig::datastream_mirror` is used, the actor also mirrors
|
||||||
|
each output to the configured datastream producer. Datastream submit failure is
|
||||||
|
ignored and does not suppress upstream output or emit `ProcessOutput::Error`.
|
||||||
|
|
||||||
|
Public lifecycle/control output order follows observed lifecycle:
|
||||||
|
|
||||||
|
- successful spawn emits `Started` before any terminal `Exited`;
|
||||||
|
- spawn failure emits `SpawnFailed` without `Started` or `Exited`;
|
||||||
|
- supervisor failure emits `Error`;
|
||||||
|
- after a terminal output, later public stop attempts emit no additional
|
||||||
|
`ProcessOutput`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Lifecycle datastream labels and records
|
||||||
|
|
||||||
|
The label source is:
|
||||||
|
|
||||||
|
1. `ProcessSpec::label`, when present;
|
||||||
|
2. otherwise, the final non-empty path segment of `ProcessSpec::command`;
|
||||||
|
3. otherwise, the full `command` string.
|
||||||
|
|
||||||
|
The label sanitizer:
|
||||||
|
|
||||||
|
- trims source whitespace;
|
||||||
|
- lowercases ASCII alphanumeric characters;
|
||||||
|
- preserves `_` and `-`;
|
||||||
|
- replaces every other character with `_`;
|
||||||
|
- collapses repeated `_`;
|
||||||
|
- trims leading and trailing `_`;
|
||||||
|
- rejects an empty sanitized result with:
|
||||||
|
|
||||||
|
```text
|
||||||
|
invalid process lifecycle label: empty segment
|
||||||
|
```
|
||||||
|
|
||||||
|
The lifecycle channel name is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
proc.<label>.lifecycle
|
||||||
|
```
|
||||||
|
|
||||||
|
When lifecycle mirroring is enabled, the channel is registered as:
|
||||||
|
|
||||||
|
```text
|
||||||
|
ChannelContent::JsonRecord {
|
||||||
|
schema: Some("swactor_process.lifecycle.v1")
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Duplicate lifecycle labels on the same datastream stream are rejected before the
|
||||||
|
process actor is spawned with an error containing:
|
||||||
|
|
||||||
|
```text
|
||||||
|
duplicate process lifecycle datastream channel: proc.<label>.lifecycle on stream <stream>
|
||||||
|
```
|
||||||
|
|
||||||
|
Lifecycle JSON records are:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{"event":"started","pid":123}
|
||||||
|
{"event":"spawn_failed","error":"..."}
|
||||||
|
{"event":"exited","status":{"kind":"code","value":0}}
|
||||||
|
{"event":"exited","status":{"kind":"signal","value":9}}
|
||||||
|
{"event":"exited","status":{"kind":"unknown"}}
|
||||||
|
{"event":"error","error":"..."}
|
||||||
|
```
|
||||||
|
|
||||||
|
The managed-process core registers no `proc.<label>.stdout` or
|
||||||
|
`proc.<label>.stderr` channels.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Stop semantics
|
||||||
|
|
||||||
|
Use `send_process_command` with `ProcessCommand::Stop { kill_after }` to request
|
||||||
|
shutdown.
|
||||||
|
|
||||||
|
If stop is requested before the child reports successful spawn, the actor queues
|
||||||
|
the stop intent. When the child later starts, the actor still emits `Started`
|
||||||
|
first, asks the supervisor to terminate the child, and later emits terminal
|
||||||
|
`Exited` unless a supervisor failure occurs.
|
||||||
|
|
||||||
|
If stop is requested before an OS spawn failure is reported, the actor still
|
||||||
|
emits only `SpawnFailed` for that failed spawn.
|
||||||
|
|
||||||
|
If the child is running, stop sends terminate to the child process.
|
||||||
|
|
||||||
|
If `kill_after` is `Some(duration)`, the supervisor sends kill after that
|
||||||
|
deadline if the child has not exited.
|
||||||
|
|
||||||
|
If `kill_after` is `None`, the supervisor does not schedule kill escalation.
|
||||||
|
|
||||||
|
Duplicate stop while already stopping is a no-op. It does not tighten, extend,
|
||||||
|
or replace the original kill deadline.
|
||||||
|
|
||||||
|
Stop after terminal output is a no-op if the actor is still alive. If the actor
|
||||||
|
has already stopped, sending the command may fail at the runtime address layer;
|
||||||
|
that failure does not produce process output.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Actor and supervisor cleanup
|
||||||
|
|
||||||
|
The actor uses a private mailbox wrapper so supervisor wake messages and public
|
||||||
|
process commands share one actor input type without changing the Swactor runtime.
|
||||||
|
|
||||||
|
The actor drains supervisor events when it starts, when it receives a supervisor
|
||||||
|
wake, and before/after applying a public stop command.
|
||||||
|
|
||||||
|
The supervisor sends a private `ThreadFinished` event when its thread reaches the
|
||||||
|
end of process supervision. The actor stops itself only after a terminal state is
|
||||||
|
recorded and the supervisor handle has been cleared.
|
||||||
|
|
||||||
|
If the actor stops while the supervisor is still present, it sends best-effort
|
||||||
|
shutdown to the supervisor without blocking for a waiting join.
|
||||||
|
|
||||||
|
Dropping the supervisor handle sends best-effort shutdown and joins only when
|
||||||
|
the thread is already finished.
|
||||||
|
|
@ -1,53 +0,0 @@
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
use crate::types::{ExitStatus, ProcessError, ProcessSpec, PtySize, Signal};
|
|
||||||
use swactor::actor::ActorAddress;
|
|
||||||
|
|
||||||
/// Which output stream produced data.
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
||||||
pub enum OutputStream {
|
|
||||||
Stdout,
|
|
||||||
Stderr,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Actions emitted by ProcessSession for the driver or actor layer to execute.
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
||||||
pub enum ProcessAction {
|
|
||||||
// --- Driver commands ---
|
|
||||||
/// Spawn the process described by the spec.
|
|
||||||
SpawnProcess { spec: ProcessSpec },
|
|
||||||
/// Write bytes to the process's stdin.
|
|
||||||
WriteStdin { data: Vec<u8> },
|
|
||||||
/// Send a signal to the process.
|
|
||||||
SendSignal { signal: Signal },
|
|
||||||
/// Resize the process's PTY.
|
|
||||||
ResizePty { size: PtySize },
|
|
||||||
/// Close the process's stdin pipe.
|
|
||||||
CloseStdin,
|
|
||||||
/// Schedule a kill timeout that fires KillTimeout after the given duration.
|
|
||||||
ScheduleKillTimeout { duration: Duration },
|
|
||||||
|
|
||||||
// --- Subscriber notifications ---
|
|
||||||
/// Notify subscribers that the process started.
|
|
||||||
NotifyStarted { subscribers: Vec<ActorAddress> },
|
|
||||||
/// Notify subscribers of output.
|
|
||||||
NotifyOutput {
|
|
||||||
subscribers: Vec<ActorAddress>,
|
|
||||||
data: Vec<u8>,
|
|
||||||
stream: OutputStream,
|
|
||||||
},
|
|
||||||
/// Notify subscribers that the process exited.
|
|
||||||
NotifyExited {
|
|
||||||
subscribers: Vec<ActorAddress>,
|
|
||||||
status: ExitStatus,
|
|
||||||
},
|
|
||||||
/// Notify subscribers of an error.
|
|
||||||
NotifyError {
|
|
||||||
subscribers: Vec<ActorAddress>,
|
|
||||||
error: ProcessError,
|
|
||||||
},
|
|
||||||
|
|
||||||
// --- Lifecycle ---
|
|
||||||
/// The session is done; the owning actor should stop itself.
|
|
||||||
SelfTerminate,
|
|
||||||
}
|
|
||||||
|
|
@ -1,174 +1,283 @@
|
||||||
use std::sync::{Arc, OnceLock};
|
use std::sync::{Arc, OnceLock};
|
||||||
|
|
||||||
use swactor::actor::{ActorAddress, ActorInterface, Ctx};
|
use swactor::actor::{ActorAddress, ActorInterface, Ctx};
|
||||||
use swactor::process_observer::ProcessOutputObserver;
|
use swactor::runtime::ExternalSender;
|
||||||
|
|
||||||
use crate::action::{OutputStream, ProcessAction};
|
use crate::lifecycle::PreparedProcessOutput;
|
||||||
use crate::event::ProcessEvent;
|
use crate::message::{ProcessActorCommand, ProcessCommand, ProcessOutput};
|
||||||
use crate::message::{ProcessCommand, ProcessNotification};
|
use crate::supervisor::{
|
||||||
use crate::session::ProcessSession;
|
ProcessSupervisorThread, ProcessThreadHandle, ThreadEvent, ThreadEventReceiver,
|
||||||
use crate::types::{ProcessDriver, ProcessWaker};
|
thread_event_channel,
|
||||||
|
};
|
||||||
|
use crate::types::{ExitStatus, ProcessSpec};
|
||||||
|
|
||||||
/// Actor wrapper around a `ProcessSession` and its driver.
|
enum ProcessActorState {
|
||||||
///
|
Spawning { stop_requested: bool },
|
||||||
/// Generic over `D: ProcessDriver` so that tests can use `MockDriver` or
|
Running,
|
||||||
/// `TestDriver` while production uses `LocalDriver`.
|
Stopping,
|
||||||
pub struct ProcessActor<D: ProcessDriver> {
|
Done(ProcessDoneState),
|
||||||
session: ProcessSession,
|
|
||||||
driver: D,
|
|
||||||
self_addr: Option<ActorAddress>,
|
|
||||||
/// Actions from `ProcessSession::new()`, executed in `on_start`.
|
|
||||||
deferred_actions: Option<Vec<ProcessAction>>,
|
|
||||||
/// Shared slot for the waker — filled after the actor address is known.
|
|
||||||
pub waker_slot: Arc<OnceLock<ProcessWaker>>,
|
|
||||||
/// Per-node observer that taps this process's output (command-basename
|
|
||||||
/// `label`) onto the node's telemetry stream. `None` when the runtime has
|
|
||||||
/// no observer installed.
|
|
||||||
output_observer: Option<Arc<dyn ProcessOutputObserver>>,
|
|
||||||
/// Command basename used to label this process's output to the observer.
|
|
||||||
label: String,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<D: ProcessDriver> ProcessActor<D> {
|
enum ProcessDoneState {
|
||||||
pub fn new(
|
Exited(ExitStatus),
|
||||||
session: ProcessSession,
|
SpawnFailed(String),
|
||||||
driver: D,
|
SupervisorFailed(String),
|
||||||
initial_actions: Vec<ProcessAction>,
|
}
|
||||||
waker_slot: Arc<OnceLock<ProcessWaker>>,
|
|
||||||
output_observer: Option<Arc<dyn ProcessOutputObserver>>,
|
impl ProcessDoneState {
|
||||||
label: String,
|
fn observe(&self) {
|
||||||
|
match self {
|
||||||
|
Self::Exited(status) => {
|
||||||
|
let _ = *status;
|
||||||
|
}
|
||||||
|
Self::SpawnFailed(error) | Self::SupervisorFailed(error) => {
|
||||||
|
let _ = error.as_str();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) struct ProcessActor {
|
||||||
|
spec: Option<ProcessSpec>,
|
||||||
|
output: PreparedProcessOutput,
|
||||||
|
sender: ExternalSender,
|
||||||
|
addr_slot: Arc<OnceLock<ActorAddress>>,
|
||||||
|
state: ProcessActorState,
|
||||||
|
pid: Option<u32>,
|
||||||
|
supervisor: Option<ProcessThreadHandle>,
|
||||||
|
supervisor_events: Option<ThreadEventReceiver>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ProcessActor {
|
||||||
|
pub(crate) fn new(
|
||||||
|
spec: ProcessSpec,
|
||||||
|
output: PreparedProcessOutput,
|
||||||
|
sender: ExternalSender,
|
||||||
|
addr_slot: Arc<OnceLock<ActorAddress>>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
session,
|
spec: Some(spec),
|
||||||
driver,
|
output,
|
||||||
self_addr: None,
|
sender,
|
||||||
deferred_actions: Some(initial_actions),
|
addr_slot,
|
||||||
waker_slot,
|
state: ProcessActorState::Spawning {
|
||||||
output_observer,
|
stop_requested: false,
|
||||||
label,
|
},
|
||||||
|
pid: None,
|
||||||
|
supervisor: None,
|
||||||
|
supervisor_events: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Drain events from the driver, apply each to the session, and dispatch
|
fn emit_process_output(&self, ctx: &Ctx, output: ProcessOutput) {
|
||||||
/// all resulting actions.
|
let _ = ctx.send(self.output.upstream, output.clone());
|
||||||
fn drain_and_dispatch(&mut self, ctx: &Ctx) {
|
if let Some(mirror) = &self.output.mirror {
|
||||||
let events = self.driver.poll();
|
mirror.submit(&output);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn apply_stop(&mut self, ctx: &Ctx, kill_after: Option<std::time::Duration>) {
|
||||||
|
let mut next_state = None;
|
||||||
|
let mut terminal_error = None;
|
||||||
|
|
||||||
|
match &mut self.state {
|
||||||
|
ProcessActorState::Spawning { stop_requested } => {
|
||||||
|
if *stop_requested {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
match self
|
||||||
|
.supervisor
|
||||||
|
.as_ref()
|
||||||
|
.expect("supervisor started before commands")
|
||||||
|
.stop(kill_after)
|
||||||
|
{
|
||||||
|
Ok(()) => *stop_requested = true,
|
||||||
|
Err(error) => terminal_error = Some(error),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ProcessActorState::Running => {
|
||||||
|
match self
|
||||||
|
.supervisor
|
||||||
|
.as_ref()
|
||||||
|
.expect("supervisor started before commands")
|
||||||
|
.stop(kill_after)
|
||||||
|
{
|
||||||
|
Ok(()) => next_state = Some(ProcessActorState::Stopping),
|
||||||
|
Err(error) => terminal_error = Some(error),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ProcessActorState::Stopping | ProcessActorState::Done(_) => {}
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(state) = next_state {
|
||||||
|
self.state = state;
|
||||||
|
}
|
||||||
|
if let Some(error) = terminal_error {
|
||||||
|
self.emit_terminal_error(ctx, error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn drain_supervisor_events(&mut self, ctx: &Ctx) {
|
||||||
|
let events = self
|
||||||
|
.supervisor_events
|
||||||
|
.as_ref()
|
||||||
|
.map(ThreadEventReceiver::drain)
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
for event in events {
|
for event in events {
|
||||||
let actions = self.session.apply(event);
|
self.handle_thread_event(ctx, event);
|
||||||
self.dispatch_actions(ctx, actions);
|
self.finish_if_safe(ctx);
|
||||||
}
|
}
|
||||||
|
self.finish_if_safe(ctx);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Execute actions produced by the session state machine.
|
fn handle_thread_event(&mut self, ctx: &Ctx, event: ThreadEvent) {
|
||||||
fn dispatch_actions(&mut self, ctx: &Ctx, actions: Vec<ProcessAction>) {
|
match event {
|
||||||
let self_addr = self.self_addr.expect("self_addr not set");
|
ThreadEvent::Started { pid } => {
|
||||||
for action in actions {
|
let stop_requested = match &self.state {
|
||||||
match action {
|
ProcessActorState::Spawning { stop_requested } => *stop_requested,
|
||||||
// Driver commands — forward to the driver
|
ProcessActorState::Running
|
||||||
ProcessAction::SpawnProcess { .. }
|
| ProcessActorState::Stopping
|
||||||
| ProcessAction::WriteStdin { .. }
|
| ProcessActorState::Done(_) => return,
|
||||||
| ProcessAction::SendSignal { .. }
|
};
|
||||||
| ProcessAction::ResizePty { .. }
|
|
||||||
| ProcessAction::CloseStdin
|
|
||||||
| ProcessAction::ScheduleKillTimeout { .. } => {
|
|
||||||
self.driver.execute(action);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Subscriber notifications — send to each subscriber
|
self.pid = Some(pid);
|
||||||
ProcessAction::NotifyStarted { subscribers } => {
|
self.emit_process_output(ctx, ProcessOutput::Started { pid });
|
||||||
let notif = ProcessNotification::Started {
|
self.state = if stop_requested {
|
||||||
process: self_addr,
|
ProcessActorState::Stopping
|
||||||
pid: self.driver.pid(),
|
} else {
|
||||||
};
|
ProcessActorState::Running
|
||||||
for sub in subscribers {
|
};
|
||||||
let _ = ctx.send(sub, notif.clone());
|
}
|
||||||
|
ThreadEvent::SpawnFailed { error } => {
|
||||||
|
if !matches!(self.state, ProcessActorState::Done(_)) {
|
||||||
|
self.emit_process_output(
|
||||||
|
ctx,
|
||||||
|
ProcessOutput::SpawnFailed {
|
||||||
|
error: error.clone(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
self.state = ProcessActorState::Done(ProcessDoneState::SpawnFailed(error));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ThreadEvent::Exited { status } => {
|
||||||
|
if !matches!(self.state, ProcessActorState::Done(_)) {
|
||||||
|
self.emit_process_output(ctx, ProcessOutput::Exited { status });
|
||||||
|
self.state = ProcessActorState::Done(ProcessDoneState::Exited(status));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ThreadEvent::Error { error } => {
|
||||||
|
if !matches!(self.state, ProcessActorState::Done(_)) {
|
||||||
|
self.emit_process_output(
|
||||||
|
ctx,
|
||||||
|
ProcessOutput::Error {
|
||||||
|
error: error.clone(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
self.state = ProcessActorState::Done(ProcessDoneState::SupervisorFailed(error));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ThreadEvent::ThreadFinished => {
|
||||||
|
if let Some(supervisor) = self.supervisor.as_mut() {
|
||||||
|
match supervisor.join_if_finished() {
|
||||||
|
Ok(_) => {}
|
||||||
|
Err(_) => {
|
||||||
|
if !matches!(self.state, ProcessActorState::Done(_)) {
|
||||||
|
let error = "process supervisor thread failed".to_owned();
|
||||||
|
self.emit_process_output(
|
||||||
|
ctx,
|
||||||
|
ProcessOutput::Error {
|
||||||
|
error: error.clone(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
self.state = ProcessActorState::Done(
|
||||||
|
ProcessDoneState::SupervisorFailed(error),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
ProcessAction::NotifyOutput {
|
self.supervisor = None;
|
||||||
subscribers,
|
self.supervisor_events = None;
|
||||||
data,
|
|
||||||
stream,
|
|
||||||
} => {
|
|
||||||
// Tap the node's telemetry observer before the data is moved
|
|
||||||
// into the subscriber notification. Per-node, auto-attached.
|
|
||||||
if let Some(obs) = &self.output_observer {
|
|
||||||
obs.on_output(&self.label, matches!(stream, OutputStream::Stderr), &data);
|
|
||||||
}
|
|
||||||
let notif = ProcessNotification::Output {
|
|
||||||
process: self_addr,
|
|
||||||
data,
|
|
||||||
stream,
|
|
||||||
};
|
|
||||||
for sub in subscribers {
|
|
||||||
let _ = ctx.send(sub, notif.clone());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
ProcessAction::NotifyExited {
|
|
||||||
subscribers,
|
|
||||||
status,
|
|
||||||
} => {
|
|
||||||
let notif = ProcessNotification::Exited {
|
|
||||||
process: self_addr,
|
|
||||||
status,
|
|
||||||
};
|
|
||||||
for sub in subscribers {
|
|
||||||
let _ = ctx.send(sub, notif.clone());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
ProcessAction::NotifyError { subscribers, error } => {
|
|
||||||
let notif = ProcessNotification::Error {
|
|
||||||
process: self_addr,
|
|
||||||
error,
|
|
||||||
};
|
|
||||||
for sub in subscribers {
|
|
||||||
let _ = ctx.send(sub, notif.clone());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Lifecycle
|
|
||||||
ProcessAction::SelfTerminate => {
|
|
||||||
ctx.stop_self();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Map a `ProcessCommand` to the corresponding `ProcessEvent`.
|
fn finish_if_safe(&mut self, ctx: &Ctx) {
|
||||||
fn command_to_event(cmd: ProcessCommand) -> Option<ProcessEvent> {
|
if let ProcessActorState::Done(done) = &self.state {
|
||||||
match cmd {
|
done.observe();
|
||||||
ProcessCommand::WriteStdin { data } => Some(ProcessEvent::WriteStdin { data }),
|
let _ = self.pid;
|
||||||
ProcessCommand::SendSignal { signal } => Some(ProcessEvent::SendSignal { signal }),
|
if self.supervisor.is_none() {
|
||||||
ProcessCommand::ResizePty { size } => Some(ProcessEvent::ResizePty { size }),
|
ctx.stop_self();
|
||||||
ProcessCommand::CloseStdin => Some(ProcessEvent::CloseStdin),
|
}
|
||||||
ProcessCommand::Close => Some(ProcessEvent::CloseRequested),
|
|
||||||
ProcessCommand::Subscribe { address } => Some(ProcessEvent::Subscribe { address }),
|
|
||||||
ProcessCommand::Unsubscribe { address } => Some(ProcessEvent::Unsubscribe { address }),
|
|
||||||
ProcessCommand::PollTick => None, // handled by drain
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn emit_terminal_error(&mut self, ctx: &Ctx, error: String) {
|
||||||
|
if matches!(self.state, ProcessActorState::Done(_)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
self.emit_process_output(
|
||||||
|
ctx,
|
||||||
|
ProcessOutput::Error {
|
||||||
|
error: error.clone(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
self.state = ProcessActorState::Done(ProcessDoneState::SupervisorFailed(error));
|
||||||
|
ctx.stop_self();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<D: ProcessDriver + 'static> ActorInterface for ProcessActor<D> {
|
impl ActorInterface for ProcessActor {
|
||||||
type Incoming = ProcessCommand;
|
type Incoming = ProcessActorCommand;
|
||||||
type Response = ();
|
type Response = ();
|
||||||
|
|
||||||
fn on_start(&mut self, ctx: &Ctx) {
|
fn on_start(&mut self, ctx: &Ctx) {
|
||||||
self.self_addr = Some(ctx.self_addr());
|
let sender = self.sender.clone();
|
||||||
if let Some(actions) = self.deferred_actions.take() {
|
let addr_slot = self.addr_slot.clone();
|
||||||
self.dispatch_actions(ctx, actions);
|
let (event_sink, event_receiver) = thread_event_channel(move || {
|
||||||
|
if let Some(addr) = addr_slot.get() {
|
||||||
|
let _ = sender.send_to(*addr, ProcessActorCommand::SupervisorWake);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
let spec = self.spec.take().expect("process spec already taken");
|
||||||
|
match ProcessSupervisorThread::start(spec, event_sink) {
|
||||||
|
Ok(supervisor) => {
|
||||||
|
self.supervisor = Some(supervisor);
|
||||||
|
self.supervisor_events = Some(event_receiver);
|
||||||
|
self.drain_supervisor_events(ctx);
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
let error = format!("process supervisor thread failed: {err}");
|
||||||
|
self.emit_process_output(
|
||||||
|
ctx,
|
||||||
|
ProcessOutput::Error {
|
||||||
|
error: error.clone(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
self.state = ProcessActorState::Done(ProcessDoneState::SupervisorFailed(error));
|
||||||
|
ctx.stop_self();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn handle(&mut self, ctx: &Ctx, msg: ProcessCommand) {
|
fn handle(&mut self, ctx: &Ctx, msg: ProcessActorCommand) {
|
||||||
// Process the incoming command first — this ensures Subscribe
|
match msg {
|
||||||
// registers before drain dispatches notifications, and keeps
|
ProcessActorCommand::SupervisorWake => self.drain_supervisor_events(ctx),
|
||||||
// user commands (Close, WriteStdin) responsive.
|
ProcessActorCommand::Command(ProcessCommand::Stop { kill_after }) => {
|
||||||
if let Some(event) = Self::command_to_event(msg) {
|
self.drain_supervisor_events(ctx);
|
||||||
let actions = self.session.apply(event);
|
self.apply_stop(ctx, kill_after);
|
||||||
self.dispatch_actions(ctx, actions);
|
self.drain_supervisor_events(ctx);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Then drain pending I/O events from background threads.
|
fn on_stop(&mut self, _ctx: &Ctx) {
|
||||||
self.drain_and_dispatch(ctx);
|
if let Some(supervisor) = &self.supervisor {
|
||||||
|
let _ = supervisor.shutdown_now();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,71 +0,0 @@
|
||||||
use crate::types::{ExitStatus, PtySize, Signal};
|
|
||||||
use swactor::actor::ActorAddress;
|
|
||||||
|
|
||||||
/// Events that can be applied to a ProcessSession.
|
|
||||||
///
|
|
||||||
/// Some events come from the driver (Started, SpawnFailed, OutputReceived, etc.),
|
|
||||||
/// others come from the owning actor (WriteStdin, SendSignal, Subscribe, etc.).
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
||||||
pub enum ProcessEvent {
|
|
||||||
// --- Driver-sourced events ---
|
|
||||||
/// The process spawned successfully.
|
|
||||||
Started,
|
|
||||||
/// The kill timeout fired (process didn't exit after SIGTERM).
|
|
||||||
KillTimeout,
|
|
||||||
/// The process failed to spawn.
|
|
||||||
SpawnFailed { reason: String },
|
|
||||||
/// Output received on stdout or stderr.
|
|
||||||
OutputReceived { data: Vec<u8>, is_stderr: bool },
|
|
||||||
/// The process exited.
|
|
||||||
Exited { status: ExitStatus },
|
|
||||||
/// Connection to the process was lost unexpectedly.
|
|
||||||
ConnectionLost { reason: String },
|
|
||||||
|
|
||||||
// --- Driver acknowledgement events ---
|
|
||||||
/// Stdin bytes were successfully written.
|
|
||||||
StdinWritten { byte_count: usize },
|
|
||||||
/// A signal was delivered.
|
|
||||||
SignalSent,
|
|
||||||
/// The PTY was resized.
|
|
||||||
PtyResized,
|
|
||||||
|
|
||||||
// --- Actor-sourced events ---
|
|
||||||
/// Write data to the process's stdin.
|
|
||||||
WriteStdin { data: Vec<u8> },
|
|
||||||
/// Send a signal to the process.
|
|
||||||
SendSignal { signal: Signal },
|
|
||||||
/// Resize the process's PTY.
|
|
||||||
ResizePty { size: PtySize },
|
|
||||||
/// Close the process's stdin.
|
|
||||||
CloseStdin,
|
|
||||||
/// Request a graceful close of the process.
|
|
||||||
CloseRequested,
|
|
||||||
/// Subscribe an actor to process notifications.
|
|
||||||
Subscribe { address: ActorAddress },
|
|
||||||
/// Unsubscribe an actor from process notifications.
|
|
||||||
Unsubscribe { address: ActorAddress },
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ProcessEvent {
|
|
||||||
/// Human-readable name for error messages.
|
|
||||||
pub fn name(&self) -> &'static str {
|
|
||||||
match self {
|
|
||||||
Self::Started => "Started",
|
|
||||||
Self::KillTimeout => "KillTimeout",
|
|
||||||
Self::SpawnFailed { .. } => "SpawnFailed",
|
|
||||||
Self::OutputReceived { .. } => "OutputReceived",
|
|
||||||
Self::Exited { .. } => "Exited",
|
|
||||||
Self::ConnectionLost { .. } => "ConnectionLost",
|
|
||||||
Self::StdinWritten { .. } => "StdinWritten",
|
|
||||||
Self::SignalSent => "SignalSent",
|
|
||||||
Self::PtyResized => "PtyResized",
|
|
||||||
Self::WriteStdin { .. } => "WriteStdin",
|
|
||||||
Self::SendSignal { .. } => "SendSignal",
|
|
||||||
Self::ResizePty { .. } => "ResizePty",
|
|
||||||
Self::CloseStdin => "CloseStdin",
|
|
||||||
Self::CloseRequested => "CloseRequested",
|
|
||||||
Self::Subscribe { .. } => "Subscribe",
|
|
||||||
Self::Unsubscribe { .. } => "Unsubscribe",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,28 +1,18 @@
|
||||||
pub mod action;
|
mod actor;
|
||||||
pub mod actor;
|
mod lifecycle;
|
||||||
pub mod event;
|
mod message;
|
||||||
pub mod local;
|
mod spawn;
|
||||||
pub mod message;
|
mod supervisor;
|
||||||
pub mod mock;
|
mod types;
|
||||||
|
|
||||||
pub mod pipeline;
|
pub mod pipeline;
|
||||||
pub mod session;
|
|
||||||
pub mod spawn;
|
|
||||||
pub mod types;
|
|
||||||
pub mod yaml;
|
pub mod yaml;
|
||||||
|
|
||||||
pub use action::{OutputStream, ProcessAction};
|
pub use lifecycle::{ProcessLifecycleObservability, ProcessOutputConfig};
|
||||||
pub use actor::ProcessActor;
|
pub use message::{ProcessCommand, ProcessOutput};
|
||||||
pub use event::ProcessEvent;
|
|
||||||
pub use local::LocalDriver;
|
|
||||||
pub use message::{ProcessCommand, ProcessNotification};
|
|
||||||
pub use mock::MockDriver;
|
|
||||||
pub use pipeline::{
|
pub use pipeline::{
|
||||||
JobComplete, JobDefinition, JobFailure, JobId, JobProgress, JobStatus, JobSuccess,
|
JobComplete, JobDefinition, JobFailure, JobId, JobProgress, JobStatus, JobSuccess,
|
||||||
LocalPipelineConfig, LocalStartJob, PipelineId, PipelineStatus,
|
LocalPipelineConfig, LocalStartJob, PipelineId, PipelineStatus,
|
||||||
};
|
};
|
||||||
pub use session::{ProcessSession, ProcessState};
|
pub use spawn::{send_process_command, spawn_local_process};
|
||||||
pub use spawn::{spawn_local_process, spawn_process};
|
pub use types::{ExitStatus, ProcessSpec};
|
||||||
pub use types::{
|
|
||||||
EventQueue, ExitStatus, FlowControl, ProcessDriver, ProcessError, ProcessMode, ProcessSpec,
|
|
||||||
ProcessWaker, PtySize, Signal,
|
|
||||||
};
|
|
||||||
|
|
|
||||||
321
crates/process/src/lifecycle.rs
Normal file
321
crates/process/src/lifecycle.rs
Normal file
|
|
@ -0,0 +1,321 @@
|
||||||
|
use std::collections::HashSet;
|
||||||
|
use std::sync::{Mutex, OnceLock};
|
||||||
|
|
||||||
|
use datastream::{ChannelContent, DatastreamProducer, StreamId};
|
||||||
|
use serde_json::json;
|
||||||
|
use swactor::actor::ActorAddress;
|
||||||
|
|
||||||
|
use crate::message::ProcessOutput;
|
||||||
|
use crate::types::{ExitStatus, ProcessSpec};
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum ProcessLifecycleObservability {
|
||||||
|
Disabled,
|
||||||
|
DatastreamMirror,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct ProcessOutputConfig {
|
||||||
|
upstream: ActorAddress,
|
||||||
|
observability: ProcessLifecycleObservability,
|
||||||
|
datastream: Option<DatastreamProducer>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ProcessOutputConfig {
|
||||||
|
pub fn disabled(upstream: ActorAddress) -> Self {
|
||||||
|
Self {
|
||||||
|
upstream,
|
||||||
|
observability: ProcessLifecycleObservability::Disabled,
|
||||||
|
datastream: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn datastream_mirror(upstream: ActorAddress, producer: DatastreamProducer) -> Self {
|
||||||
|
Self {
|
||||||
|
upstream,
|
||||||
|
observability: ProcessLifecycleObservability::DatastreamMirror,
|
||||||
|
datastream: Some(producer),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn upstream(&self) -> ActorAddress {
|
||||||
|
self.upstream
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn observability(&self) -> ProcessLifecycleObservability {
|
||||||
|
self.observability
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) struct PreparedProcessOutput {
|
||||||
|
pub(crate) upstream: ActorAddress,
|
||||||
|
pub(crate) mirror: Option<LifecycleDatastreamMirror>,
|
||||||
|
pub(crate) _label_reservation: Option<LifecycleLabelReservation>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) struct LifecycleDatastreamMirror {
|
||||||
|
channel: datastream::ChannelId,
|
||||||
|
producer: DatastreamProducer,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LifecycleDatastreamMirror {
|
||||||
|
pub(crate) fn submit(&self, output: &ProcessOutput) {
|
||||||
|
let payload = match output {
|
||||||
|
ProcessOutput::Started { pid } => json!({"event": "started", "pid": pid}),
|
||||||
|
ProcessOutput::SpawnFailed { error } => {
|
||||||
|
json!({"event": "spawn_failed", "error": error})
|
||||||
|
}
|
||||||
|
ProcessOutput::Exited { status } => match status {
|
||||||
|
ExitStatus::Code(value) => {
|
||||||
|
json!({"event": "exited", "status": {"kind": "code", "value": value}})
|
||||||
|
}
|
||||||
|
ExitStatus::Signal(value) => {
|
||||||
|
json!({"event": "exited", "status": {"kind": "signal", "value": value}})
|
||||||
|
}
|
||||||
|
ExitStatus::Unknown => json!({"event": "exited", "status": {"kind": "unknown"}}),
|
||||||
|
},
|
||||||
|
ProcessOutput::Error { error } => json!({"event": "error", "error": error}),
|
||||||
|
};
|
||||||
|
let bytes = serde_json::to_vec(&payload).expect("process lifecycle record serializes");
|
||||||
|
let _ = self.producer.submit_bytes(self.channel, bytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn prepare_process_output(
|
||||||
|
spec: &ProcessSpec,
|
||||||
|
config: ProcessOutputConfig,
|
||||||
|
) -> Result<PreparedProcessOutput, swactor::Error> {
|
||||||
|
let label = derive_lifecycle_label(spec)?;
|
||||||
|
match config.observability {
|
||||||
|
ProcessLifecycleObservability::Disabled => Ok(PreparedProcessOutput {
|
||||||
|
upstream: config.upstream,
|
||||||
|
mirror: None,
|
||||||
|
_label_reservation: None,
|
||||||
|
}),
|
||||||
|
ProcessLifecycleObservability::DatastreamMirror => {
|
||||||
|
let producer = config
|
||||||
|
.datastream
|
||||||
|
.expect("datastream mirror config stores producer");
|
||||||
|
let reservation =
|
||||||
|
LifecycleLabelReservation::reserve(producer.stream_id().clone(), &label)?;
|
||||||
|
let channel_name = label.channel_name();
|
||||||
|
let channel = producer
|
||||||
|
.try_register_channel(
|
||||||
|
channel_name,
|
||||||
|
ChannelContent::JsonRecord {
|
||||||
|
schema: Some("swactor_process.lifecycle.v1".to_owned()),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.map_err(|err| match err {
|
||||||
|
datastream::ChannelRegistrationError::ConflictingName { name } => {
|
||||||
|
swactor::Error::from(format!(
|
||||||
|
"conflicting datastream channel registration for {name}"
|
||||||
|
))
|
||||||
|
}
|
||||||
|
})?;
|
||||||
|
|
||||||
|
Ok(PreparedProcessOutput {
|
||||||
|
upstream: config.upstream,
|
||||||
|
mirror: Some(LifecycleDatastreamMirror { channel, producer }),
|
||||||
|
_label_reservation: Some(reservation),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||||
|
pub(crate) struct LifecycleLabel(String);
|
||||||
|
|
||||||
|
impl LifecycleLabel {
|
||||||
|
pub(crate) fn as_str(&self) -> &str {
|
||||||
|
&self.0
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn channel_name(&self) -> String {
|
||||||
|
format!("proc.{}.lifecycle", self.0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn derive_lifecycle_label(spec: &ProcessSpec) -> Result<LifecycleLabel, swactor::Error> {
|
||||||
|
let source = match spec.label.as_deref() {
|
||||||
|
Some(label) => label,
|
||||||
|
None => spec
|
||||||
|
.command
|
||||||
|
.rsplit(['/', '\\'])
|
||||||
|
.find(|segment| !segment.is_empty())
|
||||||
|
.unwrap_or(&spec.command),
|
||||||
|
};
|
||||||
|
sanitize_lifecycle_label(source)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sanitize_lifecycle_label(source: &str) -> Result<LifecycleLabel, swactor::Error> {
|
||||||
|
let mut sanitized = String::new();
|
||||||
|
let mut last_was_underscore = false;
|
||||||
|
|
||||||
|
for ch in source.trim().chars() {
|
||||||
|
let next = if ch.is_ascii_alphanumeric() {
|
||||||
|
ch.to_ascii_lowercase()
|
||||||
|
} else if ch == '_' || ch == '-' {
|
||||||
|
ch
|
||||||
|
} else {
|
||||||
|
'_'
|
||||||
|
};
|
||||||
|
|
||||||
|
if next == '_' {
|
||||||
|
if !last_was_underscore {
|
||||||
|
sanitized.push(next);
|
||||||
|
}
|
||||||
|
last_was_underscore = true;
|
||||||
|
} else {
|
||||||
|
sanitized.push(next);
|
||||||
|
last_was_underscore = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let sanitized = sanitized.trim_matches('_').to_owned();
|
||||||
|
if sanitized.is_empty() {
|
||||||
|
return Err(swactor::Error::from(
|
||||||
|
"invalid process lifecycle label: empty segment",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(LifecycleLabel(sanitized))
|
||||||
|
}
|
||||||
|
|
||||||
|
static LIFECYCLE_LABELS: OnceLock<Mutex<HashSet<(StreamId, String)>>> = OnceLock::new();
|
||||||
|
|
||||||
|
pub(crate) struct LifecycleLabelReservation {
|
||||||
|
key: Option<(StreamId, String)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LifecycleLabelReservation {
|
||||||
|
fn reserve(stream: StreamId, label: &LifecycleLabel) -> Result<Self, swactor::Error> {
|
||||||
|
let key = (stream, label.as_str().to_owned());
|
||||||
|
let mut labels = LIFECYCLE_LABELS
|
||||||
|
.get_or_init(|| Mutex::new(HashSet::new()))
|
||||||
|
.lock()
|
||||||
|
.expect("process lifecycle label registry poisoned");
|
||||||
|
if !labels.insert(key.clone()) {
|
||||||
|
return Err(swactor::Error::from(format!(
|
||||||
|
"duplicate process lifecycle datastream channel: {} on stream {}",
|
||||||
|
label.channel_name(),
|
||||||
|
key.0
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(Self { key: Some(key) })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for LifecycleLabelReservation {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
if let Some(key) = self.key.take() {
|
||||||
|
LIFECYCLE_LABELS
|
||||||
|
.get_or_init(|| Mutex::new(HashSet::new()))
|
||||||
|
.lock()
|
||||||
|
.expect("process lifecycle label registry poisoned")
|
||||||
|
.remove(&key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use std::collections::HashMap;
|
||||||
|
|
||||||
|
use datastream::{DatastreamEndpoint, Lifetime, NodeId, StreamId};
|
||||||
|
use swactor::actor::ActorAddress;
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn spec(command: &str, label: Option<&str>) -> ProcessSpec {
|
||||||
|
ProcessSpec {
|
||||||
|
command: command.to_owned(),
|
||||||
|
args: Vec::new(),
|
||||||
|
env: HashMap::new(),
|
||||||
|
working_dir: None,
|
||||||
|
label: label.map(str::to_owned),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn explicit_labels_are_sanitized() {
|
||||||
|
assert_eq!(
|
||||||
|
derive_lifecycle_label(&spec("ignored", Some("trainer.0/foo")))
|
||||||
|
.unwrap()
|
||||||
|
.as_str(),
|
||||||
|
"trainer_0_foo"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
derive_lifecycle_label(&spec("ignored", Some(" GPU Worker ")))
|
||||||
|
.unwrap()
|
||||||
|
.as_str(),
|
||||||
|
"gpu_worker"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn command_basename_labels_are_sanitized() {
|
||||||
|
assert_eq!(
|
||||||
|
derive_lifecycle_label(&spec("/usr/bin/python3", None))
|
||||||
|
.unwrap()
|
||||||
|
.as_str(),
|
||||||
|
"python3"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
derive_lifecycle_label(&spec("./bin/train.v2", None))
|
||||||
|
.unwrap()
|
||||||
|
.as_str(),
|
||||||
|
"train_v2"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn empty_lifecycle_labels_are_rejected() {
|
||||||
|
assert_eq!(
|
||||||
|
derive_lifecycle_label(&spec("ignored", Some("../")))
|
||||||
|
.unwrap_err()
|
||||||
|
.to_string(),
|
||||||
|
"invalid process lifecycle label: empty segment"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
derive_lifecycle_label(&spec("", None))
|
||||||
|
.unwrap_err()
|
||||||
|
.to_string(),
|
||||||
|
"invalid process lifecycle label: empty segment"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn duplicate_datastream_label_reservations_are_released_on_drop() {
|
||||||
|
let endpoint = DatastreamEndpoint::new(StreamId::new(NodeId::new("stage2"), Lifetime(1)));
|
||||||
|
let upstream = ActorAddress::new_random();
|
||||||
|
let spec = spec("sh", Some("trainer.0/foo"));
|
||||||
|
|
||||||
|
let first = prepare_process_output(
|
||||||
|
&spec,
|
||||||
|
ProcessOutputConfig::datastream_mirror(upstream, endpoint.producer()),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let err = match prepare_process_output(
|
||||||
|
&spec,
|
||||||
|
ProcessOutputConfig::datastream_mirror(upstream, endpoint.producer()),
|
||||||
|
) {
|
||||||
|
Ok(_) => panic!("duplicate lifecycle label should be rejected"),
|
||||||
|
Err(err) => err,
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
err.to_string(),
|
||||||
|
"duplicate process lifecycle datastream channel: proc.trainer_0_foo.lifecycle on stream stage2#1"
|
||||||
|
);
|
||||||
|
|
||||||
|
drop(first);
|
||||||
|
|
||||||
|
let second = prepare_process_output(
|
||||||
|
&spec,
|
||||||
|
ProcessOutputConfig::datastream_mirror(upstream, endpoint.producer()),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
drop(second);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,291 +0,0 @@
|
||||||
use std::io::{Read, Write};
|
|
||||||
use std::process::{Child, ChildStdin, Command, Stdio};
|
|
||||||
use std::sync::{Arc, OnceLock};
|
|
||||||
use std::thread::{self, JoinHandle};
|
|
||||||
|
|
||||||
use crate::action::ProcessAction;
|
|
||||||
use crate::event::ProcessEvent;
|
|
||||||
use crate::types::EventQueue;
|
|
||||||
use crate::types::ProcessDriver;
|
|
||||||
use crate::types::ProcessWaker;
|
|
||||||
use crate::types::{ExitStatus, ProcessSpec, Signal};
|
|
||||||
|
|
||||||
// ─── Signal ────────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
/// Map a `Signal` enum variant to the corresponding libc signal constant.
|
|
||||||
fn signal_to_libc(signal: Signal) -> libc::c_int {
|
|
||||||
match signal {
|
|
||||||
Signal::Terminate => libc::SIGTERM,
|
|
||||||
Signal::Kill => libc::SIGKILL,
|
|
||||||
Signal::Hangup => libc::SIGHUP,
|
|
||||||
Signal::Interrupt => libc::SIGINT,
|
|
||||||
Signal::Other(n) => n,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Send a signal to a process by PID. Returns `Ok(())` on success.
|
|
||||||
fn send_signal(pid: u32, signal: Signal) -> Result<(), String> {
|
|
||||||
let sig = signal_to_libc(signal);
|
|
||||||
// Safety: kill() is safe to call with any pid/signal combo;
|
|
||||||
// it returns -1 on error which we check.
|
|
||||||
let ret = unsafe { libc::kill(pid as libc::pid_t, sig) };
|
|
||||||
if ret == 0 {
|
|
||||||
Ok(())
|
|
||||||
} else {
|
|
||||||
Err(format!(
|
|
||||||
"kill({}, {}) failed: {}",
|
|
||||||
pid,
|
|
||||||
sig,
|
|
||||||
std::io::Error::last_os_error()
|
|
||||||
))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Pipes ─────────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
/// Read from a pipe in a loop, pushing events to the queue and waking the actor.
|
|
||||||
///
|
|
||||||
/// Runs in a background thread. Exits when the pipe reaches EOF or errors.
|
|
||||||
fn read_pipe(
|
|
||||||
mut pipe: impl Read + Send + 'static,
|
|
||||||
is_stderr: bool,
|
|
||||||
queue: EventQueue,
|
|
||||||
waker: Arc<OnceLock<ProcessWaker>>,
|
|
||||||
) {
|
|
||||||
let mut buf = [0u8; 8192];
|
|
||||||
loop {
|
|
||||||
match pipe.read(&mut buf) {
|
|
||||||
Ok(0) => break, // EOF
|
|
||||||
Ok(n) => {
|
|
||||||
queue.push(ProcessEvent::OutputReceived {
|
|
||||||
data: buf[..n].to_vec(),
|
|
||||||
is_stderr,
|
|
||||||
});
|
|
||||||
if let Some(w) = waker.get() {
|
|
||||||
w.wake();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
|
|
||||||
Err(_) => break,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Wait ──────────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
/// Wait for a child process to exit, then push the appropriate event.
|
|
||||||
///
|
|
||||||
/// Runs in a background thread. Uses `libc::waitpid` for accurate exit status.
|
|
||||||
/// After waitpid returns, joins the pipe reader threads so all buffered
|
|
||||||
/// stdout/stderr is drained before the `Exited` event is enqueued.
|
|
||||||
fn wait_for_exit(
|
|
||||||
pid: u32,
|
|
||||||
reader_threads: Vec<JoinHandle<()>>,
|
|
||||||
queue: EventQueue,
|
|
||||||
waker: Arc<OnceLock<ProcessWaker>>,
|
|
||||||
) {
|
|
||||||
let mut status: libc::c_int = 0;
|
|
||||||
let ret = unsafe { libc::waitpid(pid as libc::pid_t, &mut status, 0) };
|
|
||||||
|
|
||||||
let exit_status = if ret < 0 {
|
|
||||||
ExitStatus::Unknown
|
|
||||||
} else {
|
|
||||||
decode_wait_status(status)
|
|
||||||
};
|
|
||||||
|
|
||||||
// Wait for pipe readers to finish draining all output before signaling exit.
|
|
||||||
// Once the process exits, its pipe ends close, so readers will hit EOF shortly.
|
|
||||||
for handle in reader_threads {
|
|
||||||
let _ = handle.join();
|
|
||||||
}
|
|
||||||
|
|
||||||
queue.push(ProcessEvent::Exited {
|
|
||||||
status: exit_status,
|
|
||||||
});
|
|
||||||
if let Some(w) = waker.get() {
|
|
||||||
w.wake();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn decode_wait_status(status: libc::c_int) -> ExitStatus {
|
|
||||||
if libc::WIFEXITED(status) {
|
|
||||||
ExitStatus::Code(libc::WEXITSTATUS(status))
|
|
||||||
} else if libc::WIFSIGNALED(status) {
|
|
||||||
ExitStatus::Signal(libc::WTERMSIG(status))
|
|
||||||
} else {
|
|
||||||
ExitStatus::Unknown
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── LocalDriver ───────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
/// A `ProcessDriver` that spawns real OS subprocesses via `std::process::Command`.
|
|
||||||
///
|
|
||||||
/// Background threads read stdout/stderr and wait for process exit,
|
|
||||||
/// pushing events into a shared `EventQueue`. The actor polls via `poll()`.
|
|
||||||
pub struct LocalDriver {
|
|
||||||
queue: EventQueue,
|
|
||||||
waker_slot: Arc<OnceLock<ProcessWaker>>,
|
|
||||||
child: Option<Child>,
|
|
||||||
stdin: Option<ChildStdin>,
|
|
||||||
_wait_thread: Option<JoinHandle<()>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl LocalDriver {
|
|
||||||
pub fn new(queue: EventQueue, waker_slot: Arc<OnceLock<ProcessWaker>>) -> Self {
|
|
||||||
Self {
|
|
||||||
queue,
|
|
||||||
waker_slot,
|
|
||||||
child: None,
|
|
||||||
stdin: None,
|
|
||||||
_wait_thread: None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn spawn_process(&mut self, spec: &ProcessSpec) {
|
|
||||||
let mut cmd = Command::new(&spec.command);
|
|
||||||
cmd.args(&spec.args);
|
|
||||||
for (k, v) in &spec.env {
|
|
||||||
cmd.env(k, v);
|
|
||||||
}
|
|
||||||
if let Some(ref dir) = spec.working_dir {
|
|
||||||
cmd.current_dir(dir);
|
|
||||||
}
|
|
||||||
cmd.stdin(Stdio::piped())
|
|
||||||
.stdout(Stdio::piped())
|
|
||||||
.stderr(Stdio::piped());
|
|
||||||
|
|
||||||
match cmd.spawn() {
|
|
||||||
Ok(mut child) => {
|
|
||||||
let pid = child.id();
|
|
||||||
|
|
||||||
// Take the stdin handle
|
|
||||||
self.stdin = child.stdin.take();
|
|
||||||
|
|
||||||
// Spawn stdout reader thread
|
|
||||||
let mut reader_threads = Vec::new();
|
|
||||||
if let Some(stdout) = child.stdout.take() {
|
|
||||||
let queue = self.queue.clone();
|
|
||||||
let waker = self.waker_slot.clone();
|
|
||||||
reader_threads.push(
|
|
||||||
thread::Builder::new()
|
|
||||||
.name(format!("proc-{}-stdout", pid))
|
|
||||||
.spawn(move || read_pipe(stdout, false, queue, waker))
|
|
||||||
.expect("failed to spawn stdout reader"),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Spawn stderr reader thread
|
|
||||||
if let Some(stderr) = child.stderr.take() {
|
|
||||||
let queue = self.queue.clone();
|
|
||||||
let waker = self.waker_slot.clone();
|
|
||||||
reader_threads.push(
|
|
||||||
thread::Builder::new()
|
|
||||||
.name(format!("proc-{}-stderr", pid))
|
|
||||||
.spawn(move || read_pipe(stderr, true, queue, waker))
|
|
||||||
.expect("failed to spawn stderr reader"),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Spawn wait thread — it joins the reader threads before pushing Exited,
|
|
||||||
// ensuring all output is drained before the exit event.
|
|
||||||
let queue = self.queue.clone();
|
|
||||||
let waker = self.waker_slot.clone();
|
|
||||||
self._wait_thread = Some(
|
|
||||||
thread::Builder::new()
|
|
||||||
.name(format!("proc-{}-wait", pid))
|
|
||||||
.spawn(move || wait_for_exit(pid, reader_threads, queue, waker))
|
|
||||||
.expect("failed to spawn wait thread"),
|
|
||||||
);
|
|
||||||
|
|
||||||
self.child = Some(child);
|
|
||||||
self.queue.push(ProcessEvent::Started);
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
self.queue.push(ProcessEvent::SpawnFailed {
|
|
||||||
reason: e.to_string(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ProcessDriver for LocalDriver {
|
|
||||||
fn execute(&mut self, action: ProcessAction) {
|
|
||||||
match action {
|
|
||||||
ProcessAction::SpawnProcess { spec } => {
|
|
||||||
self.spawn_process(&spec);
|
|
||||||
}
|
|
||||||
ProcessAction::WriteStdin { data } => {
|
|
||||||
if let Some(ref mut stdin) = self.stdin {
|
|
||||||
match stdin.write_all(&data) {
|
|
||||||
Ok(()) => {
|
|
||||||
self.queue.push(ProcessEvent::StdinWritten {
|
|
||||||
byte_count: data.len(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
self.queue.push(ProcessEvent::ConnectionLost {
|
|
||||||
reason: format!("stdin write failed: {}", e),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
ProcessAction::SendSignal { signal } => {
|
|
||||||
if let Some(ref child) = self.child {
|
|
||||||
let pid = child.id();
|
|
||||||
match send_signal(pid, signal) {
|
|
||||||
Ok(()) => {
|
|
||||||
self.queue.push(ProcessEvent::SignalSent);
|
|
||||||
}
|
|
||||||
Err(reason) => {
|
|
||||||
self.queue.push(ProcessEvent::ConnectionLost { reason });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
ProcessAction::ResizePty { .. } => {
|
|
||||||
// No-op for Phase 1 (pipes only, no PTY support)
|
|
||||||
self.queue.push(ProcessEvent::PtyResized);
|
|
||||||
}
|
|
||||||
ProcessAction::CloseStdin => {
|
|
||||||
// Drop the stdin handle to close the pipe
|
|
||||||
self.stdin.take();
|
|
||||||
}
|
|
||||||
ProcessAction::ScheduleKillTimeout { duration } => {
|
|
||||||
let queue = self.queue.clone();
|
|
||||||
let waker = self.waker_slot.clone();
|
|
||||||
thread::spawn(move || {
|
|
||||||
thread::sleep(duration);
|
|
||||||
queue.push(ProcessEvent::KillTimeout);
|
|
||||||
if let Some(w) = waker.get() {
|
|
||||||
w.wake();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
// Notification actions are not driver commands
|
|
||||||
_ => {}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn poll(&mut self) -> Vec<ProcessEvent> {
|
|
||||||
self.queue.drain()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn pid(&self) -> Option<u32> {
|
|
||||||
self.child.as_ref().map(|c| c.id())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Drop for LocalDriver {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
// Close stdin to let the process know we're done
|
|
||||||
self.stdin.take();
|
|
||||||
// Kill the process if still alive
|
|
||||||
if let Some(ref mut child) = self.child {
|
|
||||||
let _ = child.kill();
|
|
||||||
let _ = child.wait();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,58 +1,24 @@
|
||||||
use swactor::actor::ActorAddress;
|
use crate::types::ExitStatus;
|
||||||
|
|
||||||
use crate::action::OutputStream;
|
/// Public managed-process commands.
|
||||||
use crate::types::{ExitStatus, ProcessError, PtySize, Signal};
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
|
||||||
/// Commands sent to a process actor.
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
pub enum ProcessCommand {
|
pub enum ProcessCommand {
|
||||||
/// Write data to the process's stdin.
|
Stop {
|
||||||
WriteStdin { data: Vec<u8> },
|
kill_after: Option<std::time::Duration>,
|
||||||
/// Send a signal to the process.
|
},
|
||||||
SendSignal { signal: Signal },
|
|
||||||
/// Resize the process's PTY.
|
|
||||||
ResizePty { size: PtySize },
|
|
||||||
/// Close the process's stdin pipe.
|
|
||||||
CloseStdin,
|
|
||||||
/// Request a graceful close of the process.
|
|
||||||
Close,
|
|
||||||
/// Subscribe to process notifications.
|
|
||||||
Subscribe { address: ActorAddress },
|
|
||||||
/// Unsubscribe from process notifications.
|
|
||||||
Unsubscribe { address: ActorAddress },
|
|
||||||
/// Internal: sent by the waker to trigger event draining.
|
|
||||||
#[doc(hidden)]
|
|
||||||
PollTick,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Notifications sent from a process actor to subscribers.
|
/// Public managed-process outputs.
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub enum ProcessNotification {
|
pub enum ProcessOutput {
|
||||||
/// The process started successfully.
|
Started { pid: u32 },
|
||||||
///
|
SpawnFailed { error: String },
|
||||||
/// `pid` is `Some(u32)` when the underlying driver knows the OS
|
Exited { status: ExitStatus },
|
||||||
/// pid (real `LocalDriver`) and `None` when it doesn't
|
Error { error: String },
|
||||||
/// (mock drivers, future SSH-tunnel-style drivers). Telemetry hooks
|
}
|
||||||
/// can read this to attribute the subprocess in a per-node datastream.
|
|
||||||
Started {
|
#[derive(Debug, Clone)]
|
||||||
process: ActorAddress,
|
pub(crate) enum ProcessActorCommand {
|
||||||
#[doc(hidden)]
|
Command(ProcessCommand),
|
||||||
pid: Option<u32>,
|
SupervisorWake,
|
||||||
},
|
|
||||||
/// Output was received from the process.
|
|
||||||
Output {
|
|
||||||
process: ActorAddress,
|
|
||||||
data: Vec<u8>,
|
|
||||||
stream: OutputStream,
|
|
||||||
},
|
|
||||||
/// The process exited.
|
|
||||||
Exited {
|
|
||||||
process: ActorAddress,
|
|
||||||
status: ExitStatus,
|
|
||||||
},
|
|
||||||
/// An error occurred.
|
|
||||||
Error {
|
|
||||||
process: ActorAddress,
|
|
||||||
error: ProcessError,
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,61 +0,0 @@
|
||||||
use std::collections::VecDeque;
|
|
||||||
|
|
||||||
use crate::action::ProcessAction;
|
|
||||||
use crate::event::ProcessEvent;
|
|
||||||
use crate::types::ProcessDriver;
|
|
||||||
|
|
||||||
/// A test-oriented driver that records executed actions and lets you inject events.
|
|
||||||
pub struct MockDriver {
|
|
||||||
pending_events: VecDeque<ProcessEvent>,
|
|
||||||
executed_actions: Vec<ProcessAction>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl MockDriver {
|
|
||||||
pub fn new() -> Self {
|
|
||||||
Self {
|
|
||||||
pending_events: VecDeque::new(),
|
|
||||||
executed_actions: Vec::new(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Queue a single event to be returned by the next `poll()`.
|
|
||||||
pub fn inject(&mut self, event: ProcessEvent) {
|
|
||||||
self.pending_events.push_back(event);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Queue multiple events to be returned by subsequent `poll()` calls.
|
|
||||||
pub fn inject_many(&mut self, events: impl IntoIterator<Item = ProcessEvent>) {
|
|
||||||
self.pending_events.extend(events);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// View all actions that have been executed so far.
|
|
||||||
pub fn executed_actions(&self) -> &[ProcessAction] {
|
|
||||||
&self.executed_actions
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Take all executed actions, clearing the internal log.
|
|
||||||
pub fn take_executed_actions(&mut self) -> Vec<ProcessAction> {
|
|
||||||
std::mem::take(&mut self.executed_actions)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Number of events waiting to be polled.
|
|
||||||
pub fn pending_event_count(&self) -> usize {
|
|
||||||
self.pending_events.len()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for MockDriver {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self::new()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ProcessDriver for MockDriver {
|
|
||||||
fn execute(&mut self, action: ProcessAction) {
|
|
||||||
self.executed_actions.push(action);
|
|
||||||
}
|
|
||||||
|
|
||||||
fn poll(&mut self) -> Vec<ProcessEvent> {
|
|
||||||
self.pending_events.drain(..).collect()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,398 +0,0 @@
|
||||||
use std::collections::VecDeque;
|
|
||||||
|
|
||||||
use swactor::actor::ActorAddress;
|
|
||||||
|
|
||||||
use crate::action::{OutputStream, ProcessAction};
|
|
||||||
use crate::event::ProcessEvent;
|
|
||||||
use crate::types::{ExitStatus, FlowControl, ProcessError, ProcessMode, ProcessSpec, Signal};
|
|
||||||
|
|
||||||
// ─── SubscriberSet ─────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
/// A deduplicated collection of subscriber addresses.
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
pub(crate) struct SubscriberSet {
|
|
||||||
inner: Vec<ActorAddress>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl SubscriberSet {
|
|
||||||
pub(crate) fn new() -> Self {
|
|
||||||
Self { inner: Vec::new() }
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Add an address. No-op if already present.
|
|
||||||
pub(crate) fn add(&mut self, address: ActorAddress) {
|
|
||||||
if !self.inner.contains(&address) {
|
|
||||||
self.inner.push(address);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Remove an address. No-op if not present.
|
|
||||||
pub(crate) fn remove(&mut self, address: &ActorAddress) {
|
|
||||||
self.inner.retain(|a| a != address);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Snapshot of current subscribers.
|
|
||||||
pub(crate) fn snapshot(&self) -> Vec<ActorAddress> {
|
|
||||||
self.inner.clone()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Number of subscribers.
|
|
||||||
pub(crate) fn count(&self) -> usize {
|
|
||||||
self.inner.len()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The lifecycle states of a process session.
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
||||||
pub enum ProcessState {
|
|
||||||
Starting,
|
|
||||||
Running,
|
|
||||||
Stopping,
|
|
||||||
Exited,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ProcessState {
|
|
||||||
pub fn name(&self) -> &'static str {
|
|
||||||
match self {
|
|
||||||
Self::Starting => "Starting",
|
|
||||||
Self::Running => "Running",
|
|
||||||
Self::Stopping => "Stopping",
|
|
||||||
Self::Exited => "Exited",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Pure-logic state machine for managing a process lifecycle.
|
|
||||||
///
|
|
||||||
/// Created via `new()` which returns the session plus initial actions (SpawnProcess).
|
|
||||||
/// Drive it forward by calling `apply(event)` which returns actions to execute.
|
|
||||||
pub struct ProcessSession {
|
|
||||||
spec: ProcessSpec,
|
|
||||||
state: ProcessState,
|
|
||||||
subscribers: SubscriberSet,
|
|
||||||
flow: FlowControl,
|
|
||||||
exit_status: Option<ExitStatus>,
|
|
||||||
stdin_closed: bool,
|
|
||||||
close_requested_before_start: bool,
|
|
||||||
stdin_buffer: VecDeque<Vec<u8>>,
|
|
||||||
stdin_buffer_bytes: usize,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ProcessSession {
|
|
||||||
/// Create a new session. Returns the session and the initial actions to execute
|
|
||||||
/// (always a single `SpawnProcess` action).
|
|
||||||
pub fn new(spec: ProcessSpec) -> (Self, Vec<ProcessAction>) {
|
|
||||||
let actions = vec![ProcessAction::SpawnProcess { spec: spec.clone() }];
|
|
||||||
let session = Self {
|
|
||||||
spec,
|
|
||||||
state: ProcessState::Starting,
|
|
||||||
subscribers: SubscriberSet::new(),
|
|
||||||
flow: FlowControl::default(),
|
|
||||||
exit_status: None,
|
|
||||||
stdin_closed: false,
|
|
||||||
close_requested_before_start: false,
|
|
||||||
stdin_buffer: VecDeque::new(),
|
|
||||||
stdin_buffer_bytes: 0,
|
|
||||||
};
|
|
||||||
(session, actions)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Apply an event and return the resulting actions.
|
|
||||||
pub fn apply(&mut self, event: ProcessEvent) -> Vec<ProcessAction> {
|
|
||||||
// Subscribe/Unsubscribe handled in all states
|
|
||||||
match &event {
|
|
||||||
ProcessEvent::Subscribe { address } => {
|
|
||||||
self.subscribers.add(*address);
|
|
||||||
return vec![];
|
|
||||||
}
|
|
||||||
ProcessEvent::Unsubscribe { address } => {
|
|
||||||
self.subscribers.remove(address);
|
|
||||||
return vec![];
|
|
||||||
}
|
|
||||||
_ => {}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Driver acks — silently consumed in all states
|
|
||||||
match &event {
|
|
||||||
ProcessEvent::StdinWritten { byte_count } => {
|
|
||||||
self.flow.pending_stdin_bytes =
|
|
||||||
self.flow.pending_stdin_bytes.saturating_sub(*byte_count);
|
|
||||||
return self.drain_stdin_buffer();
|
|
||||||
}
|
|
||||||
ProcessEvent::SignalSent | ProcessEvent::PtyResized => {
|
|
||||||
return vec![];
|
|
||||||
}
|
|
||||||
_ => {}
|
|
||||||
}
|
|
||||||
|
|
||||||
// KillTimeout — handled in all states before per-state dispatch
|
|
||||||
if matches!(event, ProcessEvent::KillTimeout) {
|
|
||||||
return if self.state == ProcessState::Stopping {
|
|
||||||
vec![ProcessAction::SendSignal {
|
|
||||||
signal: Signal::Kill,
|
|
||||||
}]
|
|
||||||
} else {
|
|
||||||
vec![]
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
// Dispatch to per-state handler
|
|
||||||
match self.state {
|
|
||||||
ProcessState::Starting => self.handle_starting(event),
|
|
||||||
ProcessState::Running => self.handle_running(event),
|
|
||||||
ProcessState::Stopping => self.handle_stopping(event),
|
|
||||||
ProcessState::Exited => self.handle_exited(event),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- Per-state handlers ---
|
|
||||||
|
|
||||||
fn handle_starting(&mut self, event: ProcessEvent) -> Vec<ProcessAction> {
|
|
||||||
match event {
|
|
||||||
ProcessEvent::Started => {
|
|
||||||
self.state = ProcessState::Running;
|
|
||||||
let mut actions = vec![ProcessAction::NotifyStarted {
|
|
||||||
subscribers: self.subscribers.snapshot(),
|
|
||||||
}];
|
|
||||||
// If close was requested before the process started, transition to Stopping
|
|
||||||
if self.close_requested_before_start {
|
|
||||||
self.state = ProcessState::Stopping;
|
|
||||||
actions.push(ProcessAction::SendSignal {
|
|
||||||
signal: Signal::Terminate,
|
|
||||||
});
|
|
||||||
if let Some(duration) = self.spec.kill_timeout {
|
|
||||||
actions.push(ProcessAction::ScheduleKillTimeout { duration });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
actions
|
|
||||||
}
|
|
||||||
ProcessEvent::SpawnFailed { reason } => {
|
|
||||||
self.state = ProcessState::Exited;
|
|
||||||
vec![
|
|
||||||
ProcessAction::NotifyError {
|
|
||||||
subscribers: self.subscribers.snapshot(),
|
|
||||||
error: ProcessError::SpawnFailed { reason },
|
|
||||||
},
|
|
||||||
ProcessAction::SelfTerminate,
|
|
||||||
]
|
|
||||||
}
|
|
||||||
ProcessEvent::CloseRequested => {
|
|
||||||
self.close_requested_before_start = true;
|
|
||||||
vec![]
|
|
||||||
}
|
|
||||||
_ => self.invalid_state_error(&event),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn handle_running(&mut self, event: ProcessEvent) -> Vec<ProcessAction> {
|
|
||||||
match event {
|
|
||||||
ProcessEvent::OutputReceived { data, is_stderr } => {
|
|
||||||
let stream = if is_stderr {
|
|
||||||
OutputStream::Stderr
|
|
||||||
} else {
|
|
||||||
OutputStream::Stdout
|
|
||||||
};
|
|
||||||
vec![ProcessAction::NotifyOutput {
|
|
||||||
subscribers: self.subscribers.snapshot(),
|
|
||||||
data,
|
|
||||||
stream,
|
|
||||||
}]
|
|
||||||
}
|
|
||||||
ProcessEvent::Exited { status } => self.enter_exited(status),
|
|
||||||
ProcessEvent::ConnectionLost { reason } => {
|
|
||||||
self.state = ProcessState::Exited;
|
|
||||||
self.exit_status = Some(ExitStatus::Unknown);
|
|
||||||
self.clear_stdin_buffer();
|
|
||||||
vec![
|
|
||||||
ProcessAction::NotifyError {
|
|
||||||
subscribers: self.subscribers.snapshot(),
|
|
||||||
error: ProcessError::ConnectionLost { reason },
|
|
||||||
},
|
|
||||||
ProcessAction::SelfTerminate,
|
|
||||||
]
|
|
||||||
}
|
|
||||||
ProcessEvent::WriteStdin { data } => {
|
|
||||||
if self.stdin_closed {
|
|
||||||
return self.notify_error(ProcessError::InvalidState {
|
|
||||||
attempted: "WriteStdin",
|
|
||||||
current_state: "Running (stdin closed)",
|
|
||||||
});
|
|
||||||
}
|
|
||||||
// Backpressure: buffer if over limit
|
|
||||||
if let Some(limit) = self.spec.stdin_buffer_limit
|
|
||||||
&& self.flow.pending_stdin_bytes >= limit
|
|
||||||
{
|
|
||||||
self.stdin_buffer_bytes += data.len();
|
|
||||||
self.stdin_buffer.push_back(data);
|
|
||||||
return vec![];
|
|
||||||
}
|
|
||||||
self.flow.pending_stdin_bytes += data.len();
|
|
||||||
vec![ProcessAction::WriteStdin { data }]
|
|
||||||
}
|
|
||||||
ProcessEvent::SendSignal { signal } => {
|
|
||||||
vec![ProcessAction::SendSignal { signal }]
|
|
||||||
}
|
|
||||||
ProcessEvent::ResizePty { size } => {
|
|
||||||
vec![ProcessAction::ResizePty { size }]
|
|
||||||
}
|
|
||||||
ProcessEvent::CloseStdin => {
|
|
||||||
if self.stdin_closed {
|
|
||||||
return vec![];
|
|
||||||
}
|
|
||||||
self.stdin_closed = true;
|
|
||||||
self.clear_stdin_buffer();
|
|
||||||
vec![ProcessAction::CloseStdin]
|
|
||||||
}
|
|
||||||
ProcessEvent::CloseRequested => {
|
|
||||||
self.state = ProcessState::Stopping;
|
|
||||||
self.clear_stdin_buffer();
|
|
||||||
let mut actions = vec![ProcessAction::SendSignal {
|
|
||||||
signal: Signal::Terminate,
|
|
||||||
}];
|
|
||||||
if let Some(duration) = self.spec.kill_timeout {
|
|
||||||
actions.push(ProcessAction::ScheduleKillTimeout { duration });
|
|
||||||
}
|
|
||||||
actions
|
|
||||||
}
|
|
||||||
_ => self.invalid_state_error(&event),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn handle_stopping(&mut self, event: ProcessEvent) -> Vec<ProcessAction> {
|
|
||||||
match event {
|
|
||||||
ProcessEvent::OutputReceived { data, is_stderr } => {
|
|
||||||
let stream = if is_stderr {
|
|
||||||
OutputStream::Stderr
|
|
||||||
} else {
|
|
||||||
OutputStream::Stdout
|
|
||||||
};
|
|
||||||
vec![ProcessAction::NotifyOutput {
|
|
||||||
subscribers: self.subscribers.snapshot(),
|
|
||||||
data,
|
|
||||||
stream,
|
|
||||||
}]
|
|
||||||
}
|
|
||||||
ProcessEvent::Exited { status } => self.enter_exited(status),
|
|
||||||
ProcessEvent::ConnectionLost { reason } => {
|
|
||||||
self.state = ProcessState::Exited;
|
|
||||||
self.exit_status = Some(ExitStatus::Unknown);
|
|
||||||
vec![
|
|
||||||
ProcessAction::NotifyError {
|
|
||||||
subscribers: self.subscribers.snapshot(),
|
|
||||||
error: ProcessError::ConnectionLost { reason },
|
|
||||||
},
|
|
||||||
ProcessAction::SelfTerminate,
|
|
||||||
]
|
|
||||||
}
|
|
||||||
ProcessEvent::SendSignal { signal } => {
|
|
||||||
// Escalation (e.g., Kill after Terminate) is allowed in Stopping
|
|
||||||
vec![ProcessAction::SendSignal { signal }]
|
|
||||||
}
|
|
||||||
ProcessEvent::CloseStdin => {
|
|
||||||
if self.stdin_closed {
|
|
||||||
return vec![];
|
|
||||||
}
|
|
||||||
self.stdin_closed = true;
|
|
||||||
vec![ProcessAction::CloseStdin]
|
|
||||||
}
|
|
||||||
ProcessEvent::CloseRequested => {
|
|
||||||
// Already stopping, no-op
|
|
||||||
vec![]
|
|
||||||
}
|
|
||||||
_ => self.invalid_state_error(&event),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn handle_exited(&mut self, event: ProcessEvent) -> Vec<ProcessAction> {
|
|
||||||
// Everything in Exited is invalid — produce an error.
|
|
||||||
// (Acks and Subscribe/Unsubscribe are already handled before dispatch.)
|
|
||||||
self.invalid_state_error(&event)
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- Helpers ---
|
|
||||||
|
|
||||||
fn enter_exited(&mut self, status: ExitStatus) -> Vec<ProcessAction> {
|
|
||||||
self.state = ProcessState::Exited;
|
|
||||||
self.exit_status = Some(status);
|
|
||||||
self.clear_stdin_buffer();
|
|
||||||
vec![
|
|
||||||
ProcessAction::NotifyExited {
|
|
||||||
subscribers: self.subscribers.snapshot(),
|
|
||||||
status,
|
|
||||||
},
|
|
||||||
ProcessAction::SelfTerminate,
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
fn clear_stdin_buffer(&mut self) {
|
|
||||||
self.stdin_buffer.clear();
|
|
||||||
self.stdin_buffer_bytes = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
fn drain_stdin_buffer(&mut self) -> Vec<ProcessAction> {
|
|
||||||
let limit = match self.spec.stdin_buffer_limit {
|
|
||||||
Some(limit) => limit,
|
|
||||||
None => return vec![],
|
|
||||||
};
|
|
||||||
let mut actions = Vec::new();
|
|
||||||
while self.flow.pending_stdin_bytes < limit {
|
|
||||||
match self.stdin_buffer.pop_front() {
|
|
||||||
Some(data) => {
|
|
||||||
self.stdin_buffer_bytes -= data.len();
|
|
||||||
self.flow.pending_stdin_bytes += data.len();
|
|
||||||
actions.push(ProcessAction::WriteStdin { data });
|
|
||||||
}
|
|
||||||
None => break,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
actions
|
|
||||||
}
|
|
||||||
|
|
||||||
fn invalid_state_error(&self, event: &ProcessEvent) -> Vec<ProcessAction> {
|
|
||||||
self.notify_error(ProcessError::InvalidState {
|
|
||||||
attempted: event.name(),
|
|
||||||
current_state: self.state.name(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn notify_error(&self, error: ProcessError) -> Vec<ProcessAction> {
|
|
||||||
vec![ProcessAction::NotifyError {
|
|
||||||
subscribers: self.subscribers.snapshot(),
|
|
||||||
error,
|
|
||||||
}]
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- Query methods ---
|
|
||||||
|
|
||||||
pub fn state(&self) -> ProcessState {
|
|
||||||
self.state
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn spec(&self) -> &ProcessSpec {
|
|
||||||
&self.spec
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn mode(&self) -> ProcessMode {
|
|
||||||
self.spec.mode
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn exit_status(&self) -> Option<ExitStatus> {
|
|
||||||
self.exit_status
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn flow_control(&self) -> &FlowControl {
|
|
||||||
&self.flow
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn subscriber_count(&self) -> usize {
|
|
||||||
self.subscribers.count()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn stdin_closed(&self) -> bool {
|
|
||||||
self.stdin_closed
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn stdin_buffer_bytes(&self) -> usize {
|
|
||||||
self.stdin_buffer_bytes
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -5,89 +5,32 @@ use swactor::actor::{ActorAddress, Ctx};
|
||||||
use swactor::runtime::ExternalSender;
|
use swactor::runtime::ExternalSender;
|
||||||
|
|
||||||
use crate::actor::ProcessActor;
|
use crate::actor::ProcessActor;
|
||||||
use crate::local::LocalDriver;
|
use crate::lifecycle::{ProcessOutputConfig, prepare_process_output};
|
||||||
use crate::message::ProcessCommand;
|
use crate::message::{ProcessActorCommand, ProcessCommand};
|
||||||
use crate::session::ProcessSession;
|
use crate::types::ProcessSpec;
|
||||||
use crate::types::{EventQueue, ProcessDriver, ProcessSpec, ProcessWaker};
|
|
||||||
|
|
||||||
/// Spawn a process actor using the real `LocalDriver` (OS subprocess).
|
/// Spawn a process actor using the OS subprocess supervisor.
|
||||||
///
|
|
||||||
/// Creates a `ProcessActor<LocalDriver>`, spawns it in the runtime, and
|
|
||||||
/// wires up the waker so that I/O thread events automatically wake the actor.
|
|
||||||
///
|
|
||||||
/// Returns the actor's address. Send `ProcessCommand` messages to control it.
|
|
||||||
pub fn spawn_local_process(
|
pub fn spawn_local_process(
|
||||||
ctx: &Ctx,
|
ctx: &Ctx,
|
||||||
sender: &ExternalSender,
|
sender: &ExternalSender,
|
||||||
spec: ProcessSpec,
|
spec: ProcessSpec,
|
||||||
|
output: ProcessOutputConfig,
|
||||||
) -> Result<ActorAddress, Error> {
|
) -> Result<ActorAddress, Error> {
|
||||||
let waker_slot = Arc::new(OnceLock::new());
|
let prepared_output = prepare_process_output(&spec, output)?;
|
||||||
let queue = EventQueue::new();
|
let addr_slot = Arc::new(OnceLock::new());
|
||||||
let driver = LocalDriver::new(queue, waker_slot.clone());
|
let actor = ProcessActor::new(spec, prepared_output, sender.clone(), addr_slot.clone());
|
||||||
spawn_process_inner(ctx, sender, spec, driver, waker_slot, None)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Spawn a process actor with a custom driver.
|
|
||||||
///
|
|
||||||
/// Useful for testing with `MockDriver` or other custom drivers while
|
|
||||||
/// still getting the full actor integration (waker, lifecycle, etc.).
|
|
||||||
pub fn spawn_process<D: ProcessDriver + 'static>(
|
|
||||||
ctx: &Ctx,
|
|
||||||
sender: &ExternalSender,
|
|
||||||
spec: ProcessSpec,
|
|
||||||
driver: D,
|
|
||||||
waker_slot: Arc<OnceLock<ProcessWaker>>,
|
|
||||||
) -> Result<ActorAddress, Error> {
|
|
||||||
spawn_process_inner(ctx, sender, spec, driver, waker_slot, None)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The basename of a command path, used as the process's telemetry label.
|
|
||||||
/// `/usr/bin/python3` → `python3`, `python` → `python`. Falls back to the whole
|
|
||||||
/// string when there is no path separator or trailing component.
|
|
||||||
fn command_basename(command: &str) -> String {
|
|
||||||
command
|
|
||||||
.rsplit(['/', '\\'])
|
|
||||||
.find(|s| !s.is_empty())
|
|
||||||
.unwrap_or(command)
|
|
||||||
.to_string()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn spawn_process_inner<D: ProcessDriver + 'static>(
|
|
||||||
ctx: &Ctx,
|
|
||||||
sender: &ExternalSender,
|
|
||||||
spec: ProcessSpec,
|
|
||||||
driver: D,
|
|
||||||
waker_slot: Arc<OnceLock<ProcessWaker>>,
|
|
||||||
label_override: Option<String>,
|
|
||||||
) -> Result<ActorAddress, Error> {
|
|
||||||
// Label this process's output so the node's per-runtime observer (if any)
|
|
||||||
// taps it onto `proc.<label>.*` automatically. Default to the command
|
|
||||||
// basename; a caller can override (e.g. to keep several remote shells
|
|
||||||
// running the same command distinguishable).
|
|
||||||
let label = label_override.unwrap_or_else(|| command_basename(&spec.command));
|
|
||||||
let observer = ctx.process_output_observer();
|
|
||||||
let (session, initial_actions) = ProcessSession::new(spec);
|
|
||||||
let actor = ProcessActor::new(
|
|
||||||
session,
|
|
||||||
driver,
|
|
||||||
initial_actions,
|
|
||||||
waker_slot.clone(),
|
|
||||||
observer,
|
|
||||||
label,
|
|
||||||
);
|
|
||||||
let addr = ctx.spawn(actor)?;
|
let addr = ctx.spawn(actor)?;
|
||||||
|
addr_slot
|
||||||
// Now that we have the address, fill the waker
|
.set(addr)
|
||||||
let sender = sender.clone();
|
.expect("process actor address already set");
|
||||||
let waker = ProcessWaker::new(move || {
|
let _ = sender.send_to(addr, ProcessActorCommand::SupervisorWake);
|
||||||
let _ = sender.send_to(addr, ProcessCommand::PollTick);
|
|
||||||
});
|
|
||||||
waker_slot
|
|
||||||
.set(waker.clone())
|
|
||||||
.expect("waker slot already set");
|
|
||||||
|
|
||||||
// Flush any events from the startup race window
|
|
||||||
waker.wake();
|
|
||||||
|
|
||||||
Ok(addr)
|
Ok(addr)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn send_process_command(
|
||||||
|
sender: &ExternalSender,
|
||||||
|
process: ActorAddress,
|
||||||
|
command: ProcessCommand,
|
||||||
|
) -> Result<(), Error> {
|
||||||
|
sender.send_to(process, ProcessActorCommand::Command(command))
|
||||||
|
}
|
||||||
|
|
|
||||||
955
crates/process/src/supervisor.rs
Normal file
955
crates/process/src/supervisor.rs
Normal file
|
|
@ -0,0 +1,955 @@
|
||||||
|
use std::io;
|
||||||
|
use std::mem;
|
||||||
|
use std::os::fd::RawFd;
|
||||||
|
use std::process::{Child, Command, Stdio};
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::thread::{self, JoinHandle};
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use crossbeam_queue::SegQueue;
|
||||||
|
|
||||||
|
use crate::types::{ExitStatus, ProcessSpec, Signal};
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub(crate) enum ThreadCommand {
|
||||||
|
Stop { kill_after: Option<Duration> },
|
||||||
|
ShutdownNow,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub(crate) enum ThreadEvent {
|
||||||
|
Started { pid: u32 },
|
||||||
|
SpawnFailed { error: String },
|
||||||
|
Exited { status: ExitStatus },
|
||||||
|
Error { error: String },
|
||||||
|
ThreadFinished,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub(crate) struct ThreadEventSink {
|
||||||
|
queue: Arc<SegQueue<ThreadEvent>>,
|
||||||
|
wake: Arc<dyn Fn() + Send + Sync>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) struct ThreadEventReceiver {
|
||||||
|
queue: Arc<SegQueue<ThreadEvent>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn thread_event_channel(
|
||||||
|
wake: impl Fn() + Send + Sync + 'static,
|
||||||
|
) -> (ThreadEventSink, ThreadEventReceiver) {
|
||||||
|
let queue = Arc::new(SegQueue::new());
|
||||||
|
(
|
||||||
|
ThreadEventSink {
|
||||||
|
queue: queue.clone(),
|
||||||
|
wake: Arc::new(wake),
|
||||||
|
},
|
||||||
|
ThreadEventReceiver { queue },
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ThreadEventSink {
|
||||||
|
pub(crate) fn push(&self, event: ThreadEvent) {
|
||||||
|
self.queue.push(event);
|
||||||
|
(self.wake)();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ThreadEventReceiver {
|
||||||
|
pub(crate) fn drain(&self) -> Vec<ThreadEvent> {
|
||||||
|
let mut events = Vec::new();
|
||||||
|
while let Some(event) = self.queue.pop() {
|
||||||
|
events.push(event);
|
||||||
|
}
|
||||||
|
events
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct WakeFd(Arc<WakeFdInner>);
|
||||||
|
|
||||||
|
struct WakeFdInner {
|
||||||
|
fd: RawFd,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WakeFd {
|
||||||
|
fn new() -> io::Result<Self> {
|
||||||
|
let fd = unsafe { libc::eventfd(0, libc::EFD_NONBLOCK | libc::EFD_CLOEXEC) };
|
||||||
|
if fd < 0 {
|
||||||
|
return Err(io::Error::last_os_error());
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(Self(Arc::new(WakeFdInner { fd })))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn fd(&self) -> RawFd {
|
||||||
|
self.0.fd
|
||||||
|
}
|
||||||
|
|
||||||
|
fn wake(&self) -> io::Result<()> {
|
||||||
|
let value: u64 = 1;
|
||||||
|
let ptr = (&value as *const u64).cast::<libc::c_void>();
|
||||||
|
let len = mem::size_of::<u64>();
|
||||||
|
|
||||||
|
loop {
|
||||||
|
let written = unsafe { libc::write(self.fd(), ptr, len) };
|
||||||
|
if written == len as libc::ssize_t {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
if written < 0 {
|
||||||
|
let err = io::Error::last_os_error();
|
||||||
|
if err.kind() == io::ErrorKind::Interrupted {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if is_would_block(&err) {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
return Err(err);
|
||||||
|
}
|
||||||
|
|
||||||
|
return Err(io::Error::new(
|
||||||
|
io::ErrorKind::WriteZero,
|
||||||
|
"short write to process supervisor wake fd",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn drain(&self) -> io::Result<()> {
|
||||||
|
let mut value: u64 = 0;
|
||||||
|
let ptr = (&mut value as *mut u64).cast::<libc::c_void>();
|
||||||
|
let len = mem::size_of::<u64>();
|
||||||
|
|
||||||
|
loop {
|
||||||
|
let read = unsafe { libc::read(self.fd(), ptr, len) };
|
||||||
|
if read == len as libc::ssize_t {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if read < 0 {
|
||||||
|
let err = io::Error::last_os_error();
|
||||||
|
if err.kind() == io::ErrorKind::Interrupted {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if is_would_block(&err) {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
return Err(err);
|
||||||
|
}
|
||||||
|
|
||||||
|
return Err(io::Error::new(
|
||||||
|
io::ErrorKind::UnexpectedEof,
|
||||||
|
"short read from process supervisor wake fd",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for WakeFdInner {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let _ = unsafe { libc::close(self.fd) };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn poll_command_wake(wake: &WakeFd, timeout: Duration) -> io::Result<bool> {
|
||||||
|
let timeout_ms = poll_timeout_ms(timeout);
|
||||||
|
let mut pollfd = libc::pollfd {
|
||||||
|
fd: wake.fd(),
|
||||||
|
events: libc::POLLIN,
|
||||||
|
revents: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
loop {
|
||||||
|
let result = unsafe { libc::poll(&mut pollfd, 1, timeout_ms) };
|
||||||
|
if result == 0 {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
if result > 0 {
|
||||||
|
let revents = pollfd.revents;
|
||||||
|
if revents & (libc::POLLERR | libc::POLLNVAL | libc::POLLHUP) != 0 {
|
||||||
|
return Err(io::Error::other(format!(
|
||||||
|
"process supervisor wake fd poll failed: revents={revents}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
return Ok(revents & libc::POLLIN != 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
let err = io::Error::last_os_error();
|
||||||
|
if err.kind() == io::ErrorKind::Interrupted {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
return Err(err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn poll_timeout_ms(timeout: Duration) -> libc::c_int {
|
||||||
|
if timeout.is_zero() {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
let millis = timeout.as_millis();
|
||||||
|
if millis == 0 {
|
||||||
|
1
|
||||||
|
} else {
|
||||||
|
millis.min(libc::c_int::MAX as u128) as libc::c_int
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_would_block(err: &io::Error) -> bool {
|
||||||
|
matches!(
|
||||||
|
err.raw_os_error(),
|
||||||
|
Some(code) if code == libc::EAGAIN || code == libc::EWOULDBLOCK
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) struct ProcessThreadHandle {
|
||||||
|
commands: Arc<SegQueue<ThreadCommand>>,
|
||||||
|
wake: WakeFd,
|
||||||
|
join: Option<JoinHandle<()>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) struct ProcessSupervisorThread;
|
||||||
|
|
||||||
|
impl ProcessSupervisorThread {
|
||||||
|
pub(crate) fn start(
|
||||||
|
spec: ProcessSpec,
|
||||||
|
events: ThreadEventSink,
|
||||||
|
) -> Result<ProcessThreadHandle, swactor::Error> {
|
||||||
|
let commands = Arc::new(SegQueue::new());
|
||||||
|
let wake = WakeFd::new().map_err(|err| {
|
||||||
|
swactor::Error::from(format!(
|
||||||
|
"failed to create process supervisor wake fd: {err}"
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let thread_commands = commands.clone();
|
||||||
|
let thread_wake = wake.clone();
|
||||||
|
let join = thread::Builder::new()
|
||||||
|
.name("swactor-process-supervisor".to_owned())
|
||||||
|
.spawn(move || supervisor_thread_main(spec, events, thread_commands, thread_wake))
|
||||||
|
.map_err(|err| {
|
||||||
|
swactor::Error::from(format!("failed to start process supervisor thread: {err}"))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
Ok(ProcessThreadHandle {
|
||||||
|
commands,
|
||||||
|
wake,
|
||||||
|
join: Some(join),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ProcessThreadHandle {
|
||||||
|
pub(crate) fn send(&self, command: ThreadCommand) -> Result<(), String> {
|
||||||
|
self.commands.push(command);
|
||||||
|
self.wake
|
||||||
|
.wake()
|
||||||
|
.map_err(|err| format!("failed to wake process supervisor: {err}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn stop(&self, kill_after: Option<Duration>) -> Result<(), String> {
|
||||||
|
self.send(ThreadCommand::Stop { kill_after })
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn shutdown_now(&self) -> Result<(), String> {
|
||||||
|
self.send(ThreadCommand::ShutdownNow)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn is_finished(&self) -> bool {
|
||||||
|
match &self.join {
|
||||||
|
Some(join) => join.is_finished(),
|
||||||
|
None => true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn join_if_finished(&mut self) -> Result<bool, String> {
|
||||||
|
if !self.is_finished() {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(join) = self.join.take() {
|
||||||
|
join.join()
|
||||||
|
.map_err(|_| "process supervisor thread panicked".to_owned())?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for ProcessThreadHandle {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let _ = self.shutdown_now();
|
||||||
|
let _ = self.join_if_finished();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const WAITPID_POLL_INTERVAL: Duration = Duration::from_millis(10);
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
enum SupervisorState {
|
||||||
|
Spawning,
|
||||||
|
Running,
|
||||||
|
Stopping,
|
||||||
|
Done,
|
||||||
|
}
|
||||||
|
|
||||||
|
enum CommandOutcome {
|
||||||
|
Continue,
|
||||||
|
Exited(ExitStatus),
|
||||||
|
Error(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
fn supervisor_thread_main(
|
||||||
|
spec: ProcessSpec,
|
||||||
|
events: ThreadEventSink,
|
||||||
|
commands: Arc<SegQueue<ThreadCommand>>,
|
||||||
|
wake: WakeFd,
|
||||||
|
) {
|
||||||
|
let mut state = SupervisorState::Spawning;
|
||||||
|
let mut child: Option<Child>;
|
||||||
|
let pid: Option<u32>;
|
||||||
|
let mut kill_deadline: Option<Instant> = None;
|
||||||
|
let mut kill_sent = false;
|
||||||
|
debug_assert!(matches!(state, SupervisorState::Spawning));
|
||||||
|
|
||||||
|
let mut cmd = Command::new(&spec.command);
|
||||||
|
cmd.args(&spec.args);
|
||||||
|
for (key, value) in &spec.env {
|
||||||
|
cmd.env(key, value);
|
||||||
|
}
|
||||||
|
if let Some(dir) = &spec.working_dir {
|
||||||
|
cmd.current_dir(dir);
|
||||||
|
}
|
||||||
|
cmd.stdin(Stdio::null())
|
||||||
|
.stdout(Stdio::null())
|
||||||
|
.stderr(Stdio::null());
|
||||||
|
|
||||||
|
match cmd.spawn() {
|
||||||
|
Ok(spawned_child) => {
|
||||||
|
let child_pid = spawned_child.id();
|
||||||
|
pid = Some(child_pid);
|
||||||
|
child = Some(spawned_child);
|
||||||
|
events.push(ThreadEvent::Started { pid: child_pid });
|
||||||
|
state = SupervisorState::Running;
|
||||||
|
|
||||||
|
match drain_queued_commands(
|
||||||
|
child_pid,
|
||||||
|
&commands,
|
||||||
|
&mut state,
|
||||||
|
&mut kill_deadline,
|
||||||
|
&mut kill_sent,
|
||||||
|
) {
|
||||||
|
CommandOutcome::Continue => {}
|
||||||
|
CommandOutcome::Exited(status) => {
|
||||||
|
finish_with_exit(&events, &mut state, &mut child, status);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
CommandOutcome::Error(error) => {
|
||||||
|
finish_with_error(&events, &mut state, &mut child, error);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
events.push(ThreadEvent::SpawnFailed {
|
||||||
|
error: err.to_string(),
|
||||||
|
});
|
||||||
|
events.push(ThreadEvent::ThreadFinished);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let child_pid = pid.expect("supervisor pid stored after successful spawn");
|
||||||
|
loop {
|
||||||
|
let timeout = command_poll_timeout(kill_deadline);
|
||||||
|
match poll_command_wake(&wake, timeout) {
|
||||||
|
Ok(true) => {
|
||||||
|
if let Err(err) = wake.drain() {
|
||||||
|
finish_with_error(
|
||||||
|
&events,
|
||||||
|
&mut state,
|
||||||
|
&mut child,
|
||||||
|
format!("process supervisor command wake failed: {err}"),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
match drain_queued_commands(
|
||||||
|
child_pid,
|
||||||
|
&commands,
|
||||||
|
&mut state,
|
||||||
|
&mut kill_deadline,
|
||||||
|
&mut kill_sent,
|
||||||
|
) {
|
||||||
|
CommandOutcome::Continue => {}
|
||||||
|
CommandOutcome::Exited(status) => {
|
||||||
|
finish_with_exit(&events, &mut state, &mut child, status);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
CommandOutcome::Error(error) => {
|
||||||
|
finish_with_error(&events, &mut state, &mut child, error);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(false) => {}
|
||||||
|
Err(err) => {
|
||||||
|
finish_with_error(
|
||||||
|
&events,
|
||||||
|
&mut state,
|
||||||
|
&mut child,
|
||||||
|
format!("process supervisor command wake failed: {err}"),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
match try_wait_pid(child_pid) {
|
||||||
|
Ok(Some(status)) => {
|
||||||
|
finish_with_exit(&events, &mut state, &mut child, status);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Ok(None) => {}
|
||||||
|
Err(error) => {
|
||||||
|
finish_with_error(&events, &mut state, &mut child, error);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if kill_deadline.is_some_and(|deadline| deadline <= Instant::now()) && !kill_sent {
|
||||||
|
match send_kill_or_observe_exit(child_pid) {
|
||||||
|
CommandOutcome::Continue => {
|
||||||
|
kill_sent = true;
|
||||||
|
kill_deadline = None;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
CommandOutcome::Exited(status) => {
|
||||||
|
finish_with_exit(&events, &mut state, &mut child, status);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
CommandOutcome::Error(error) => {
|
||||||
|
finish_with_error(&events, &mut state, &mut child, error);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn command_poll_timeout(kill_deadline: Option<Instant>) -> Duration {
|
||||||
|
let Some(deadline) = kill_deadline else {
|
||||||
|
return WAITPID_POLL_INTERVAL;
|
||||||
|
};
|
||||||
|
|
||||||
|
let now = Instant::now();
|
||||||
|
if deadline <= now {
|
||||||
|
Duration::ZERO
|
||||||
|
} else {
|
||||||
|
(deadline - now).min(WAITPID_POLL_INTERVAL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn drain_queued_commands(
|
||||||
|
pid: u32,
|
||||||
|
commands: &SegQueue<ThreadCommand>,
|
||||||
|
state: &mut SupervisorState,
|
||||||
|
kill_deadline: &mut Option<Instant>,
|
||||||
|
kill_sent: &mut bool,
|
||||||
|
) -> CommandOutcome {
|
||||||
|
while let Some(command) = commands.pop() {
|
||||||
|
match apply_thread_command(pid, command, state, kill_deadline, kill_sent) {
|
||||||
|
CommandOutcome::Continue => {}
|
||||||
|
outcome => return outcome,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
CommandOutcome::Continue
|
||||||
|
}
|
||||||
|
|
||||||
|
fn apply_thread_command(
|
||||||
|
pid: u32,
|
||||||
|
command: ThreadCommand,
|
||||||
|
state: &mut SupervisorState,
|
||||||
|
kill_deadline: &mut Option<Instant>,
|
||||||
|
kill_sent: &mut bool,
|
||||||
|
) -> CommandOutcome {
|
||||||
|
match command {
|
||||||
|
ThreadCommand::Stop { kill_after } => match state {
|
||||||
|
SupervisorState::Running => match send_terminate_or_observe_exit(pid) {
|
||||||
|
CommandOutcome::Continue => {
|
||||||
|
if let Some(duration) = kill_after {
|
||||||
|
*kill_deadline = Some(Instant::now() + duration);
|
||||||
|
} else {
|
||||||
|
*kill_deadline = None;
|
||||||
|
}
|
||||||
|
*state = SupervisorState::Stopping;
|
||||||
|
CommandOutcome::Continue
|
||||||
|
}
|
||||||
|
outcome => outcome,
|
||||||
|
},
|
||||||
|
SupervisorState::Spawning | SupervisorState::Stopping | SupervisorState::Done => {
|
||||||
|
CommandOutcome::Continue
|
||||||
|
}
|
||||||
|
},
|
||||||
|
ThreadCommand::ShutdownNow => match state {
|
||||||
|
SupervisorState::Done => CommandOutcome::Continue,
|
||||||
|
SupervisorState::Spawning | SupervisorState::Running | SupervisorState::Stopping => {
|
||||||
|
*state = SupervisorState::Stopping;
|
||||||
|
*kill_deadline = None;
|
||||||
|
match send_terminate_or_observe_exit(pid) {
|
||||||
|
CommandOutcome::Continue => match try_wait_pid(pid) {
|
||||||
|
Ok(Some(status)) => CommandOutcome::Exited(status),
|
||||||
|
Ok(None) => match send_kill_or_observe_exit(pid) {
|
||||||
|
CommandOutcome::Continue => {
|
||||||
|
*kill_sent = true;
|
||||||
|
CommandOutcome::Continue
|
||||||
|
}
|
||||||
|
outcome => outcome,
|
||||||
|
},
|
||||||
|
Err(error) => CommandOutcome::Error(error),
|
||||||
|
},
|
||||||
|
outcome => outcome,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn send_terminate_or_observe_exit(pid: u32) -> CommandOutcome {
|
||||||
|
match send_signal(pid, Signal::Terminate) {
|
||||||
|
Ok(()) => CommandOutcome::Continue,
|
||||||
|
Err(kill_error) => match try_wait_pid(pid) {
|
||||||
|
Ok(Some(status)) => CommandOutcome::Exited(status),
|
||||||
|
Ok(None) => {
|
||||||
|
CommandOutcome::Error(format!("failed to terminate process {pid}: {kill_error}"))
|
||||||
|
}
|
||||||
|
Err(error) => CommandOutcome::Error(error),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn send_kill_or_observe_exit(pid: u32) -> CommandOutcome {
|
||||||
|
match send_signal(pid, Signal::Kill) {
|
||||||
|
Ok(()) => CommandOutcome::Continue,
|
||||||
|
Err(kill_error) => match try_wait_pid(pid) {
|
||||||
|
Ok(Some(status)) => CommandOutcome::Exited(status),
|
||||||
|
Ok(None) => {
|
||||||
|
CommandOutcome::Error(format!("failed to kill process {pid}: {kill_error}"))
|
||||||
|
}
|
||||||
|
Err(error) => CommandOutcome::Error(error),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn finish_with_exit(
|
||||||
|
events: &ThreadEventSink,
|
||||||
|
state: &mut SupervisorState,
|
||||||
|
child: &mut Option<Child>,
|
||||||
|
status: ExitStatus,
|
||||||
|
) {
|
||||||
|
*state = SupervisorState::Done;
|
||||||
|
debug_assert!(matches!(*state, SupervisorState::Done));
|
||||||
|
let _ = child.take();
|
||||||
|
events.push(ThreadEvent::Exited { status });
|
||||||
|
events.push(ThreadEvent::ThreadFinished);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn finish_with_error(
|
||||||
|
events: &ThreadEventSink,
|
||||||
|
state: &mut SupervisorState,
|
||||||
|
child: &mut Option<Child>,
|
||||||
|
error: String,
|
||||||
|
) {
|
||||||
|
*state = SupervisorState::Done;
|
||||||
|
debug_assert!(matches!(*state, SupervisorState::Done));
|
||||||
|
let _ = child.take();
|
||||||
|
events.push(ThreadEvent::Error { error });
|
||||||
|
events.push(ThreadEvent::ThreadFinished);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn signal_to_libc(signal: Signal) -> libc::c_int {
|
||||||
|
match signal {
|
||||||
|
Signal::Terminate => libc::SIGTERM,
|
||||||
|
Signal::Kill => libc::SIGKILL,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn send_signal(pid: u32, signal: Signal) -> Result<(), String> {
|
||||||
|
let sig = signal_to_libc(signal);
|
||||||
|
let ret = unsafe { libc::kill(pid as libc::pid_t, sig) };
|
||||||
|
if ret == 0 {
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(format!(
|
||||||
|
"kill({}, {}) failed: {}",
|
||||||
|
pid,
|
||||||
|
sig,
|
||||||
|
std::io::Error::last_os_error()
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode_wait_status(status: libc::c_int) -> ExitStatus {
|
||||||
|
if libc::WIFEXITED(status) {
|
||||||
|
ExitStatus::Code(libc::WEXITSTATUS(status))
|
||||||
|
} else if libc::WIFSIGNALED(status) {
|
||||||
|
ExitStatus::Signal(libc::WTERMSIG(status))
|
||||||
|
} else {
|
||||||
|
ExitStatus::Unknown
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn try_wait_pid(pid: u32) -> Result<Option<ExitStatus>, String> {
|
||||||
|
loop {
|
||||||
|
let mut status: libc::c_int = 0;
|
||||||
|
let result = unsafe { libc::waitpid(pid as libc::pid_t, &mut status, libc::WNOHANG) };
|
||||||
|
|
||||||
|
if result == 0 {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
if result == pid as libc::pid_t {
|
||||||
|
return Ok(Some(decode_wait_status(status)));
|
||||||
|
}
|
||||||
|
if result > 0 {
|
||||||
|
return Ok(Some(ExitStatus::Unknown));
|
||||||
|
}
|
||||||
|
|
||||||
|
let err = std::io::Error::last_os_error();
|
||||||
|
if err.kind() == io::ErrorKind::Interrupted {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
return Err(format!("waitpid({pid}) failed: {err}"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
|
||||||
|
fn spec(command: &str, args: Vec<&str>) -> ProcessSpec {
|
||||||
|
ProcessSpec {
|
||||||
|
command: command.to_owned(),
|
||||||
|
args: args.into_iter().map(str::to_owned).collect(),
|
||||||
|
env: HashMap::new(),
|
||||||
|
working_dir: None,
|
||||||
|
label: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn shell_spec(script: &str) -> ProcessSpec {
|
||||||
|
spec("sh", vec!["-c", script])
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect_until_finished(
|
||||||
|
receiver: &ThreadEventReceiver,
|
||||||
|
timeout: Duration,
|
||||||
|
) -> Vec<ThreadEvent> {
|
||||||
|
let deadline = Instant::now() + timeout;
|
||||||
|
let mut events = Vec::new();
|
||||||
|
|
||||||
|
while Instant::now() < deadline {
|
||||||
|
events.extend(receiver.drain());
|
||||||
|
if matches!(events.last(), Some(ThreadEvent::ThreadFinished)) {
|
||||||
|
return events;
|
||||||
|
}
|
||||||
|
thread::sleep(Duration::from_millis(5));
|
||||||
|
}
|
||||||
|
|
||||||
|
events.extend(receiver.drain());
|
||||||
|
events
|
||||||
|
}
|
||||||
|
|
||||||
|
fn join_finished(handle: &mut ProcessThreadHandle) {
|
||||||
|
let deadline = Instant::now() + Duration::from_secs(2);
|
||||||
|
loop {
|
||||||
|
if handle
|
||||||
|
.join_if_finished()
|
||||||
|
.expect("supervisor thread should join")
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
Instant::now() < deadline,
|
||||||
|
"supervisor thread did not finish before join timeout"
|
||||||
|
);
|
||||||
|
thread::sleep(Duration::from_millis(5));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn thread_command_and_event_shapes_match_target_contract() {
|
||||||
|
let stop_with_deadline = ThreadCommand::Stop {
|
||||||
|
kill_after: Some(Duration::from_millis(10)),
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
stop_with_deadline,
|
||||||
|
ThreadCommand::Stop {
|
||||||
|
kill_after: Some(Duration::from_millis(10))
|
||||||
|
}
|
||||||
|
);
|
||||||
|
match stop_with_deadline {
|
||||||
|
ThreadCommand::Stop {
|
||||||
|
kill_after: Some(duration),
|
||||||
|
} => assert_eq!(duration, Duration::from_millis(10)),
|
||||||
|
_ => panic!("expected stop command with deadline"),
|
||||||
|
}
|
||||||
|
|
||||||
|
let stop_without_deadline = ThreadCommand::Stop { kill_after: None };
|
||||||
|
assert_eq!(
|
||||||
|
stop_without_deadline,
|
||||||
|
ThreadCommand::Stop { kill_after: None }
|
||||||
|
);
|
||||||
|
match stop_without_deadline {
|
||||||
|
ThreadCommand::Stop { kill_after: None } => {}
|
||||||
|
_ => panic!("expected stop command without deadline"),
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_eq!(ThreadCommand::ShutdownNow, ThreadCommand::ShutdownNow);
|
||||||
|
match ThreadCommand::ShutdownNow {
|
||||||
|
ThreadCommand::ShutdownNow => {}
|
||||||
|
_ => panic!("expected shutdown command"),
|
||||||
|
}
|
||||||
|
|
||||||
|
let started = ThreadEvent::Started { pid: 42 };
|
||||||
|
assert_eq!(started, ThreadEvent::Started { pid: 42 });
|
||||||
|
match started {
|
||||||
|
ThreadEvent::Started { pid } => assert_eq!(pid, 42),
|
||||||
|
_ => panic!("expected started event"),
|
||||||
|
}
|
||||||
|
|
||||||
|
let spawn_failed = ThreadEvent::SpawnFailed {
|
||||||
|
error: "spawn failed".to_owned(),
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
spawn_failed,
|
||||||
|
ThreadEvent::SpawnFailed {
|
||||||
|
error: "spawn failed".to_owned()
|
||||||
|
}
|
||||||
|
);
|
||||||
|
match spawn_failed {
|
||||||
|
ThreadEvent::SpawnFailed { error } => assert_eq!(error, "spawn failed"),
|
||||||
|
_ => panic!("expected spawn failed event"),
|
||||||
|
}
|
||||||
|
|
||||||
|
let exited = ThreadEvent::Exited {
|
||||||
|
status: ExitStatus::Code(7),
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
exited,
|
||||||
|
ThreadEvent::Exited {
|
||||||
|
status: ExitStatus::Code(7)
|
||||||
|
}
|
||||||
|
);
|
||||||
|
match exited {
|
||||||
|
ThreadEvent::Exited { status } => assert_eq!(status, ExitStatus::Code(7)),
|
||||||
|
_ => panic!("expected exited event"),
|
||||||
|
}
|
||||||
|
|
||||||
|
let error = ThreadEvent::Error {
|
||||||
|
error: "lost".to_owned(),
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
error,
|
||||||
|
ThreadEvent::Error {
|
||||||
|
error: "lost".to_owned()
|
||||||
|
}
|
||||||
|
);
|
||||||
|
match error {
|
||||||
|
ThreadEvent::Error { error } => assert_eq!(error, "lost"),
|
||||||
|
_ => panic!("expected error event"),
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_eq!(ThreadEvent::ThreadFinished, ThreadEvent::ThreadFinished);
|
||||||
|
match ThreadEvent::ThreadFinished {
|
||||||
|
ThreadEvent::ThreadFinished => {}
|
||||||
|
_ => panic!("expected thread finished event"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn thread_event_sink_enqueues_then_wakes() {
|
||||||
|
let wake_count = Arc::new(AtomicUsize::new(0));
|
||||||
|
let wake_count_for_callback = wake_count.clone();
|
||||||
|
let (sink, receiver) = thread_event_channel(move || {
|
||||||
|
wake_count_for_callback.fetch_add(1, Ordering::SeqCst);
|
||||||
|
});
|
||||||
|
|
||||||
|
sink.push(ThreadEvent::Started { pid: 1 });
|
||||||
|
sink.push(ThreadEvent::Exited {
|
||||||
|
status: ExitStatus::Code(0),
|
||||||
|
});
|
||||||
|
|
||||||
|
assert_eq!(wake_count.load(Ordering::SeqCst), 2);
|
||||||
|
assert_eq!(
|
||||||
|
receiver.drain(),
|
||||||
|
vec![
|
||||||
|
ThreadEvent::Started { pid: 1 },
|
||||||
|
ThreadEvent::Exited {
|
||||||
|
status: ExitStatus::Code(0)
|
||||||
|
}
|
||||||
|
]
|
||||||
|
);
|
||||||
|
assert!(receiver.drain().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn process_thread_handle_queues_commands_and_wakes_supervisor() {
|
||||||
|
let commands = Arc::new(SegQueue::new());
|
||||||
|
let wake = WakeFd::new().expect("wake fd should be created");
|
||||||
|
let handle = ProcessThreadHandle {
|
||||||
|
commands: commands.clone(),
|
||||||
|
wake: wake.clone(),
|
||||||
|
join: None,
|
||||||
|
};
|
||||||
|
|
||||||
|
handle
|
||||||
|
.send(ThreadCommand::Stop {
|
||||||
|
kill_after: Some(Duration::from_millis(25)),
|
||||||
|
})
|
||||||
|
.expect("stop command should queue");
|
||||||
|
handle
|
||||||
|
.send(ThreadCommand::ShutdownNow)
|
||||||
|
.expect("shutdown command should queue");
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
poll_command_wake(&wake, Duration::ZERO).expect("wake poll should succeed"),
|
||||||
|
"wake fd should be readable after queued commands"
|
||||||
|
);
|
||||||
|
wake.drain().expect("wake fd should drain");
|
||||||
|
assert_eq!(
|
||||||
|
commands.pop(),
|
||||||
|
Some(ThreadCommand::Stop {
|
||||||
|
kill_after: Some(Duration::from_millis(25))
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(commands.pop(), Some(ThreadCommand::ShutdownNow));
|
||||||
|
assert_eq!(commands.pop(), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn supervisor_reports_started_exited_and_finished() {
|
||||||
|
let (sink, receiver) = thread_event_channel(|| {});
|
||||||
|
let mut handle = ProcessSupervisorThread::start(shell_spec("exit 7"), sink)
|
||||||
|
.expect("supervisor should start");
|
||||||
|
|
||||||
|
let events = collect_until_finished(&receiver, Duration::from_secs(2));
|
||||||
|
join_finished(&mut handle);
|
||||||
|
|
||||||
|
assert_eq!(events.len(), 3);
|
||||||
|
assert!(matches!(
|
||||||
|
events.first(),
|
||||||
|
Some(ThreadEvent::Started { pid }) if *pid > 0
|
||||||
|
));
|
||||||
|
assert_eq!(
|
||||||
|
events.get(1),
|
||||||
|
Some(&ThreadEvent::Exited {
|
||||||
|
status: ExitStatus::Code(7)
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(events.last(), Some(&ThreadEvent::ThreadFinished));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn supervisor_reports_spawn_failed_and_finished() {
|
||||||
|
let (sink, receiver) = thread_event_channel(|| {});
|
||||||
|
let mut handle =
|
||||||
|
ProcessSupervisorThread::start(spec("/definitely/not/a/real/binary", vec![]), sink)
|
||||||
|
.expect("supervisor should start");
|
||||||
|
|
||||||
|
let events = collect_until_finished(&receiver, Duration::from_secs(2));
|
||||||
|
join_finished(&mut handle);
|
||||||
|
|
||||||
|
assert_eq!(events.len(), 2);
|
||||||
|
assert!(matches!(
|
||||||
|
events.first(),
|
||||||
|
Some(ThreadEvent::SpawnFailed { error }) if !error.is_empty()
|
||||||
|
));
|
||||||
|
assert_eq!(events.last(), Some(&ThreadEvent::ThreadFinished));
|
||||||
|
assert!(!events.iter().any(|event| {
|
||||||
|
matches!(
|
||||||
|
event,
|
||||||
|
ThreadEvent::Started { .. }
|
||||||
|
| ThreadEvent::Exited { .. }
|
||||||
|
| ThreadEvent::Error { .. }
|
||||||
|
)
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn supervisor_uses_null_stdio_and_reports_only_lifecycle() {
|
||||||
|
let (sink, receiver) = thread_event_channel(|| {});
|
||||||
|
let mut handle = ProcessSupervisorThread::start(
|
||||||
|
shell_spec("echo stdout; echo stderr >&2; exit 0"),
|
||||||
|
sink,
|
||||||
|
)
|
||||||
|
.expect("supervisor should start");
|
||||||
|
|
||||||
|
let events = collect_until_finished(&receiver, Duration::from_secs(2));
|
||||||
|
join_finished(&mut handle);
|
||||||
|
|
||||||
|
assert_eq!(events.len(), 3);
|
||||||
|
assert!(matches!(
|
||||||
|
events.first(),
|
||||||
|
Some(ThreadEvent::Started { pid }) if *pid > 0
|
||||||
|
));
|
||||||
|
assert_eq!(
|
||||||
|
events.get(1),
|
||||||
|
Some(&ThreadEvent::Exited {
|
||||||
|
status: ExitStatus::Code(0)
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(events.last(), Some(&ThreadEvent::ThreadFinished));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn supervisor_stop_escalates_to_kill_after_deadline() {
|
||||||
|
let (sink, receiver) = thread_event_channel(|| {});
|
||||||
|
let mut handle = ProcessSupervisorThread::start(
|
||||||
|
shell_spec("trap '' TERM; while true; do sleep 1; done"),
|
||||||
|
sink,
|
||||||
|
)
|
||||||
|
.expect("supervisor should start");
|
||||||
|
let deadline = Instant::now() + Duration::from_secs(3);
|
||||||
|
let mut events = Vec::new();
|
||||||
|
let mut stop_sent = false;
|
||||||
|
|
||||||
|
while Instant::now() < deadline {
|
||||||
|
events.extend(receiver.drain());
|
||||||
|
if !stop_sent
|
||||||
|
&& events
|
||||||
|
.iter()
|
||||||
|
.any(|event| matches!(event, ThreadEvent::Started { pid } if *pid > 0))
|
||||||
|
{
|
||||||
|
handle
|
||||||
|
.stop(Some(Duration::from_millis(20)))
|
||||||
|
.expect("stop command should queue");
|
||||||
|
stop_sent = true;
|
||||||
|
}
|
||||||
|
if matches!(events.last(), Some(ThreadEvent::ThreadFinished)) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
thread::sleep(Duration::from_millis(5));
|
||||||
|
}
|
||||||
|
|
||||||
|
if !matches!(events.last(), Some(ThreadEvent::ThreadFinished)) {
|
||||||
|
let _ = handle.shutdown_now();
|
||||||
|
events.extend(collect_until_finished(&receiver, Duration::from_secs(2)));
|
||||||
|
}
|
||||||
|
join_finished(&mut handle);
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
stop_sent,
|
||||||
|
"supervisor did not report Started before timeout"
|
||||||
|
);
|
||||||
|
assert_eq!(events.last(), Some(&ThreadEvent::ThreadFinished));
|
||||||
|
assert!(events.iter().any(|event| {
|
||||||
|
matches!(
|
||||||
|
event,
|
||||||
|
ThreadEvent::Exited {
|
||||||
|
status: ExitStatus::Signal(9)
|
||||||
|
}
|
||||||
|
)
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,41 +1,12 @@
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::sync::Arc;
|
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
use crossbeam_queue::SegQueue;
|
|
||||||
|
|
||||||
use crate::action::ProcessAction;
|
|
||||||
use crate::event::ProcessEvent;
|
|
||||||
|
|
||||||
/// Describes how to spawn a process.
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub struct ProcessSpec {
|
pub struct ProcessSpec {
|
||||||
pub command: String,
|
pub command: String,
|
||||||
pub args: Vec<String>,
|
pub args: Vec<String>,
|
||||||
pub env: HashMap<String, String>,
|
pub env: HashMap<String, String>,
|
||||||
pub working_dir: Option<String>,
|
pub working_dir: Option<std::path::PathBuf>,
|
||||||
pub mode: ProcessMode,
|
pub label: Option<String>,
|
||||||
pub initial_pty_size: Option<PtySize>,
|
|
||||||
/// If set, escalate to SIGKILL after this duration if the process hasn't exited
|
|
||||||
/// after SIGTERM. None = no escalation.
|
|
||||||
pub kill_timeout: Option<Duration>,
|
|
||||||
/// If set, buffer stdin writes when pending bytes exceed this limit.
|
|
||||||
/// None = unlimited (current behavior).
|
|
||||||
pub stdin_buffer_limit: Option<usize>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Whether the process is interactive (PTY) or automated (pipes).
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
||||||
pub enum ProcessMode {
|
|
||||||
Interactive,
|
|
||||||
Automated,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Dimensions of a pseudo-terminal.
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
||||||
pub struct PtySize {
|
|
||||||
pub cols: u16,
|
|
||||||
pub rows: u16,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// How a process exited.
|
/// How a process exited.
|
||||||
|
|
@ -46,128 +17,8 @@ pub enum ExitStatus {
|
||||||
Unknown,
|
Unknown,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Signals that can be sent to a process.
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
pub enum Signal {
|
pub(crate) enum Signal {
|
||||||
Terminate,
|
Terminate,
|
||||||
Kill,
|
Kill,
|
||||||
Hangup,
|
|
||||||
Interrupt,
|
|
||||||
Other(i32),
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Errors produced by the process session.
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
||||||
pub enum ProcessError {
|
|
||||||
SpawnFailed {
|
|
||||||
reason: String,
|
|
||||||
},
|
|
||||||
ConnectionLost {
|
|
||||||
reason: String,
|
|
||||||
},
|
|
||||||
InvalidState {
|
|
||||||
attempted: &'static str,
|
|
||||||
current_state: &'static str,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Passive tracking of stdin backpressure.
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Default)]
|
|
||||||
pub struct FlowControl {
|
|
||||||
pub pending_stdin_bytes: usize,
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── ProcessDriver ─────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
/// Abstraction over the mechanism that actually runs a process.
|
|
||||||
///
|
|
||||||
/// Implementations translate `ProcessAction` commands into real I/O (or mock I/O)
|
|
||||||
/// and produce `ProcessEvent`s by polling for state changes.
|
|
||||||
pub trait ProcessDriver: Send {
|
|
||||||
/// Execute an action (spawn, write stdin, send signal, etc.).
|
|
||||||
fn execute(&mut self, action: ProcessAction);
|
|
||||||
|
|
||||||
/// Poll for new events from the underlying process.
|
|
||||||
fn poll(&mut self) -> Vec<ProcessEvent>;
|
|
||||||
|
|
||||||
/// PID of the underlying OS process when the driver knows one.
|
|
||||||
///
|
|
||||||
/// Returns `None` before the child has spawned, after it has been
|
|
||||||
/// reaped, or for drivers that do not run an OS process (mocks,
|
|
||||||
/// SSH-tunnel drivers that wrap a remote shell). The default
|
|
||||||
/// impl returns `None` so existing drivers compile unchanged.
|
|
||||||
///
|
|
||||||
/// Read by [`crate::actor::ProcessActor`] when it builds the
|
|
||||||
/// outbound `ProcessNotification::Started { pid }` — this is the
|
|
||||||
/// channel observability hooks use to learn the subprocess's PID
|
|
||||||
/// without coupling to a particular driver implementation
|
|
||||||
/// (`N3_OBSERVABILITY_UPGRADE_SPEC.md` §4 wiring contract).
|
|
||||||
fn pid(&self) -> Option<u32> {
|
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── ProcessWaker ──────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
/// A handle that I/O threads use to wake the owning actor.
|
|
||||||
///
|
|
||||||
/// Constructed with a closure that sends a `ProcessCommand::PollTick`
|
|
||||||
/// to the actor via `ExternalSender`. Thread-safe and cloneable.
|
|
||||||
#[derive(Clone)]
|
|
||||||
pub struct ProcessWaker(Arc<dyn Fn() + Send + Sync>);
|
|
||||||
|
|
||||||
impl std::fmt::Debug for ProcessWaker {
|
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
||||||
f.debug_struct("ProcessWaker").finish_non_exhaustive()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ProcessWaker {
|
|
||||||
pub fn new(f: impl Fn() + Send + Sync + 'static) -> Self {
|
|
||||||
Self(Arc::new(f))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Wake the owning actor so it drains pending events.
|
|
||||||
pub fn wake(&self) {
|
|
||||||
(self.0)();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── EventQueue ────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
/// Thread-safe queue for buffering process events from I/O threads.
|
|
||||||
///
|
|
||||||
/// Cloneable via inner `Arc` — I/O threads push events, the driver's
|
|
||||||
/// `poll()` drains them.
|
|
||||||
#[derive(Clone)]
|
|
||||||
pub struct EventQueue {
|
|
||||||
inner: Arc<SegQueue<ProcessEvent>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl EventQueue {
|
|
||||||
pub fn new() -> Self {
|
|
||||||
Self {
|
|
||||||
inner: Arc::new(SegQueue::new()),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Push an event (called from I/O threads).
|
|
||||||
pub fn push(&self, event: ProcessEvent) {
|
|
||||||
self.inner.push(event);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Drain all pending events (called from driver's `poll()`).
|
|
||||||
pub fn drain(&self) -> Vec<ProcessEvent> {
|
|
||||||
let mut events = Vec::new();
|
|
||||||
while let Some(event) = self.inner.pop() {
|
|
||||||
events.push(event);
|
|
||||||
}
|
|
||||||
events
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for EventQueue {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self::new()
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,570 +0,0 @@
|
||||||
//! Layer 3 — Actor integration tests.
|
|
||||||
//!
|
|
||||||
//! Uses a TestDriver backed by a shared EventQueue so tests can inject
|
|
||||||
//! events and observe actions without real OS processes.
|
|
||||||
|
|
||||||
use std::collections::HashMap;
|
|
||||||
use std::sync::{Arc, Mutex, OnceLock};
|
|
||||||
|
|
||||||
use swactor::actor::{ActorAddress, ActorInterface, Ctx};
|
|
||||||
use swactor::runtime::{Inbox, Runtime, RuntimeConfig};
|
|
||||||
|
|
||||||
use swactor_process::*;
|
|
||||||
|
|
||||||
// ── TestDriver ──────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
/// Shared harness for injecting events and inspecting driver actions.
|
|
||||||
#[derive(Clone)]
|
|
||||||
struct TestHarness {
|
|
||||||
queue: EventQueue,
|
|
||||||
actions: Arc<Mutex<Vec<ProcessAction>>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl TestHarness {
|
|
||||||
fn new() -> Self {
|
|
||||||
Self {
|
|
||||||
queue: EventQueue::new(),
|
|
||||||
actions: Arc::new(Mutex::new(Vec::new())),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn inject(&self, event: ProcessEvent) {
|
|
||||||
self.queue.push(event);
|
|
||||||
}
|
|
||||||
|
|
||||||
fn take_actions(&self) -> Vec<ProcessAction> {
|
|
||||||
std::mem::take(&mut self.actions.lock().unwrap())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A ProcessDriver that records actions and drains from a shared queue.
|
|
||||||
struct TestDriver {
|
|
||||||
queue: EventQueue,
|
|
||||||
actions: Arc<Mutex<Vec<ProcessAction>>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl TestDriver {
|
|
||||||
fn from_harness(harness: &TestHarness) -> Self {
|
|
||||||
Self {
|
|
||||||
queue: harness.queue.clone(),
|
|
||||||
actions: harness.actions.clone(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ProcessDriver for TestDriver {
|
|
||||||
fn execute(&mut self, action: ProcessAction) {
|
|
||||||
self.actions.lock().unwrap().push(action);
|
|
||||||
}
|
|
||||||
|
|
||||||
fn poll(&mut self) -> Vec<ProcessEvent> {
|
|
||||||
self.queue.drain()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── Helpers ─────────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
fn automated_spec() -> ProcessSpec {
|
|
||||||
ProcessSpec {
|
|
||||||
command: "echo".into(),
|
|
||||||
args: vec!["hello".into()],
|
|
||||||
env: HashMap::new(),
|
|
||||||
working_dir: None,
|
|
||||||
mode: ProcessMode::Automated,
|
|
||||||
initial_pty_size: None,
|
|
||||||
kill_timeout: None,
|
|
||||||
stdin_buffer_limit: None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn setup() -> (Runtime, ExternalSender) {
|
|
||||||
let rt = Runtime::new(RuntimeConfig::default());
|
|
||||||
let sender = rt.create_sender();
|
|
||||||
(rt, sender)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Helper: tick until we receive N messages, returning them.
|
|
||||||
fn tick_collect<M: swactor::actor::Message>(
|
|
||||||
rt: &Runtime,
|
|
||||||
inbox: &Inbox<M>,
|
|
||||||
n: usize,
|
|
||||||
max_ticks: usize,
|
|
||||||
) -> Vec<M> {
|
|
||||||
let mut msgs = Vec::new();
|
|
||||||
for _ in 0..max_ticks {
|
|
||||||
rt.tick();
|
|
||||||
while let Some(m) = inbox.try_recv() {
|
|
||||||
msgs.push(m);
|
|
||||||
if msgs.len() >= n {
|
|
||||||
return msgs;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
msgs
|
|
||||||
}
|
|
||||||
|
|
||||||
use swactor::runtime::ExternalSender;
|
|
||||||
|
|
||||||
// ── Spawner actor ───────────────────────────────────────────────────────────
|
|
||||||
// We can't call ctx.spawn from outside a handle(), so we use a small "spawner"
|
|
||||||
// actor that spawns the process actor and reports its address.
|
|
||||||
|
|
||||||
#[derive(Clone)]
|
|
||||||
struct SpawnRequest {
|
|
||||||
spec: ProcessSpec,
|
|
||||||
harness: TestHarness,
|
|
||||||
reply_to: ActorAddress,
|
|
||||||
sender: ExternalSender,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
|
||||||
struct SpawnedAddr(ActorAddress);
|
|
||||||
|
|
||||||
struct SpawnerActor;
|
|
||||||
|
|
||||||
impl ActorInterface for SpawnerActor {
|
|
||||||
type Incoming = SpawnRequest;
|
|
||||||
type Response = SpawnedAddr;
|
|
||||||
|
|
||||||
fn handle(&mut self, ctx: &Ctx, msg: SpawnRequest) {
|
|
||||||
let waker_slot = Arc::new(OnceLock::new());
|
|
||||||
let driver = TestDriver::from_harness(&msg.harness);
|
|
||||||
let addr = spawn_process(ctx, &msg.sender, msg.spec, driver, waker_slot)
|
|
||||||
.expect("spawn_process failed");
|
|
||||||
let _ = ctx.send(msg.reply_to, SpawnedAddr(addr));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── Tests ───────────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn happy_path_spawn_output_exit_notifies_subscriber() {
|
|
||||||
let (rt, sender) = setup();
|
|
||||||
let harness = TestHarness::new();
|
|
||||||
let notif_inbox = rt.new_inbox::<ProcessNotification>().unwrap();
|
|
||||||
|
|
||||||
// Spawn the spawner actor
|
|
||||||
let spawner_addr = rt.spawn(SpawnerActor).unwrap();
|
|
||||||
let reply_inbox = rt.new_inbox::<SpawnedAddr>().unwrap();
|
|
||||||
rt.tick();
|
|
||||||
|
|
||||||
// Ask spawner to create a process actor
|
|
||||||
rt.send_to(
|
|
||||||
spawner_addr,
|
|
||||||
SpawnRequest {
|
|
||||||
spec: automated_spec(),
|
|
||||||
harness: harness.clone(),
|
|
||||||
reply_to: *reply_inbox.addr(),
|
|
||||||
sender: sender.clone(),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
// Tick to process spawn request
|
|
||||||
for _ in 0..5 {
|
|
||||||
rt.tick();
|
|
||||||
}
|
|
||||||
|
|
||||||
let spawned = reply_inbox.try_recv().expect("should get spawned addr");
|
|
||||||
let proc_addr = spawned.0;
|
|
||||||
|
|
||||||
// Verify SpawnProcess action was sent to driver
|
|
||||||
let actions = harness.take_actions();
|
|
||||||
assert!(
|
|
||||||
actions
|
|
||||||
.iter()
|
|
||||||
.any(|a| matches!(a, ProcessAction::SpawnProcess { .. })),
|
|
||||||
"driver should receive SpawnProcess, got: {:?}",
|
|
||||||
actions
|
|
||||||
);
|
|
||||||
|
|
||||||
// Subscribe to notifications
|
|
||||||
rt.send_to(
|
|
||||||
proc_addr,
|
|
||||||
ProcessCommand::Subscribe {
|
|
||||||
address: *notif_inbox.addr(),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
rt.tick();
|
|
||||||
|
|
||||||
// Inject Started event from "driver"
|
|
||||||
harness.inject(ProcessEvent::Started);
|
|
||||||
// Send PollTick to trigger drain
|
|
||||||
rt.send_to(proc_addr, ProcessCommand::PollTick).unwrap();
|
|
||||||
for _ in 0..3 {
|
|
||||||
rt.tick();
|
|
||||||
}
|
|
||||||
|
|
||||||
let msgs = tick_collect(&rt, ¬if_inbox, 1, 10);
|
|
||||||
assert!(
|
|
||||||
msgs.iter()
|
|
||||||
.any(|m| matches!(m, ProcessNotification::Started { .. })),
|
|
||||||
"subscriber should get Started notification, got: {:?}",
|
|
||||||
msgs
|
|
||||||
);
|
|
||||||
|
|
||||||
// Inject output
|
|
||||||
harness.inject(ProcessEvent::OutputReceived {
|
|
||||||
data: b"hello\n".to_vec(),
|
|
||||||
is_stderr: false,
|
|
||||||
});
|
|
||||||
rt.send_to(proc_addr, ProcessCommand::PollTick).unwrap();
|
|
||||||
let msgs = tick_collect(&rt, ¬if_inbox, 1, 10);
|
|
||||||
assert!(
|
|
||||||
msgs.iter()
|
|
||||||
.any(|m| matches!(m, ProcessNotification::Output { .. })),
|
|
||||||
"subscriber should get Output notification"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Inject exit
|
|
||||||
harness.inject(ProcessEvent::Exited {
|
|
||||||
status: ExitStatus::Code(0),
|
|
||||||
});
|
|
||||||
rt.send_to(proc_addr, ProcessCommand::PollTick).unwrap();
|
|
||||||
let msgs = tick_collect(&rt, ¬if_inbox, 1, 10);
|
|
||||||
assert!(
|
|
||||||
msgs.iter().any(|m| matches!(
|
|
||||||
m,
|
|
||||||
ProcessNotification::Exited {
|
|
||||||
status: ExitStatus::Code(0),
|
|
||||||
..
|
|
||||||
}
|
|
||||||
)),
|
|
||||||
"subscriber should get Exited(0) notification"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn polltick_drains_queued_events() {
|
|
||||||
let (rt, sender) = setup();
|
|
||||||
let harness = TestHarness::new();
|
|
||||||
let notif_inbox = rt.new_inbox::<ProcessNotification>().unwrap();
|
|
||||||
|
|
||||||
let spawner_addr = rt.spawn(SpawnerActor).unwrap();
|
|
||||||
let reply_inbox = rt.new_inbox::<SpawnedAddr>().unwrap();
|
|
||||||
rt.tick();
|
|
||||||
|
|
||||||
rt.send_to(
|
|
||||||
spawner_addr,
|
|
||||||
SpawnRequest {
|
|
||||||
spec: automated_spec(),
|
|
||||||
harness: harness.clone(),
|
|
||||||
reply_to: *reply_inbox.addr(),
|
|
||||||
sender: sender.clone(),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
for _ in 0..5 {
|
|
||||||
rt.tick();
|
|
||||||
}
|
|
||||||
|
|
||||||
let proc_addr = reply_inbox.try_recv().unwrap().0;
|
|
||||||
|
|
||||||
// Subscribe
|
|
||||||
rt.send_to(
|
|
||||||
proc_addr,
|
|
||||||
ProcessCommand::Subscribe {
|
|
||||||
address: *notif_inbox.addr(),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
rt.tick();
|
|
||||||
|
|
||||||
// Queue multiple events before sending PollTick
|
|
||||||
harness.inject(ProcessEvent::Started);
|
|
||||||
harness.inject(ProcessEvent::OutputReceived {
|
|
||||||
data: b"line1\n".to_vec(),
|
|
||||||
is_stderr: false,
|
|
||||||
});
|
|
||||||
harness.inject(ProcessEvent::OutputReceived {
|
|
||||||
data: b"line2\n".to_vec(),
|
|
||||||
is_stderr: false,
|
|
||||||
});
|
|
||||||
|
|
||||||
// Single PollTick should drain all
|
|
||||||
rt.send_to(proc_addr, ProcessCommand::PollTick).unwrap();
|
|
||||||
let msgs = tick_collect(&rt, ¬if_inbox, 3, 20);
|
|
||||||
|
|
||||||
assert_eq!(
|
|
||||||
msgs.len(),
|
|
||||||
3,
|
|
||||||
"all three events should produce notifications"
|
|
||||||
);
|
|
||||||
assert!(matches!(msgs[0], ProcessNotification::Started { .. }));
|
|
||||||
assert!(matches!(msgs[1], ProcessNotification::Output { .. }));
|
|
||||||
assert!(matches!(msgs[2], ProcessNotification::Output { .. }));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn close_command_triggers_graceful_shutdown() {
|
|
||||||
let (rt, sender) = setup();
|
|
||||||
let harness = TestHarness::new();
|
|
||||||
let notif_inbox = rt.new_inbox::<ProcessNotification>().unwrap();
|
|
||||||
|
|
||||||
let spawner_addr = rt.spawn(SpawnerActor).unwrap();
|
|
||||||
let reply_inbox = rt.new_inbox::<SpawnedAddr>().unwrap();
|
|
||||||
rt.tick();
|
|
||||||
|
|
||||||
rt.send_to(
|
|
||||||
spawner_addr,
|
|
||||||
SpawnRequest {
|
|
||||||
spec: automated_spec(),
|
|
||||||
harness: harness.clone(),
|
|
||||||
reply_to: *reply_inbox.addr(),
|
|
||||||
sender: sender.clone(),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
for _ in 0..5 {
|
|
||||||
rt.tick();
|
|
||||||
}
|
|
||||||
let proc_addr = reply_inbox.try_recv().unwrap().0;
|
|
||||||
|
|
||||||
// Subscribe and get to Running state
|
|
||||||
rt.send_to(
|
|
||||||
proc_addr,
|
|
||||||
ProcessCommand::Subscribe {
|
|
||||||
address: *notif_inbox.addr(),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
rt.tick();
|
|
||||||
harness.inject(ProcessEvent::Started);
|
|
||||||
rt.send_to(proc_addr, ProcessCommand::PollTick).unwrap();
|
|
||||||
let _ = tick_collect::<ProcessNotification>(&rt, ¬if_inbox, 1, 10);
|
|
||||||
|
|
||||||
// Send Close
|
|
||||||
harness.take_actions(); // clear previous actions
|
|
||||||
rt.send_to(proc_addr, ProcessCommand::Close).unwrap();
|
|
||||||
for _ in 0..5 {
|
|
||||||
rt.tick();
|
|
||||||
}
|
|
||||||
|
|
||||||
let actions = harness.take_actions();
|
|
||||||
assert!(
|
|
||||||
actions.iter().any(|a| matches!(
|
|
||||||
a,
|
|
||||||
ProcessAction::SendSignal {
|
|
||||||
signal: Signal::Terminate
|
|
||||||
}
|
|
||||||
)),
|
|
||||||
"Close should trigger SIGTERM, got: {:?}",
|
|
||||||
actions
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn write_stdin_and_signal_forwarded_to_driver() {
|
|
||||||
let (rt, sender) = setup();
|
|
||||||
let harness = TestHarness::new();
|
|
||||||
|
|
||||||
let spawner_addr = rt.spawn(SpawnerActor).unwrap();
|
|
||||||
let reply_inbox = rt.new_inbox::<SpawnedAddr>().unwrap();
|
|
||||||
rt.tick();
|
|
||||||
|
|
||||||
rt.send_to(
|
|
||||||
spawner_addr,
|
|
||||||
SpawnRequest {
|
|
||||||
spec: automated_spec(),
|
|
||||||
harness: harness.clone(),
|
|
||||||
reply_to: *reply_inbox.addr(),
|
|
||||||
sender: sender.clone(),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
for _ in 0..5 {
|
|
||||||
rt.tick();
|
|
||||||
}
|
|
||||||
let proc_addr = reply_inbox.try_recv().unwrap().0;
|
|
||||||
|
|
||||||
// Get to Running
|
|
||||||
harness.inject(ProcessEvent::Started);
|
|
||||||
rt.send_to(proc_addr, ProcessCommand::PollTick).unwrap();
|
|
||||||
for _ in 0..5 {
|
|
||||||
rt.tick();
|
|
||||||
}
|
|
||||||
|
|
||||||
harness.take_actions(); // clear SpawnProcess action
|
|
||||||
|
|
||||||
// Write stdin
|
|
||||||
rt.send_to(
|
|
||||||
proc_addr,
|
|
||||||
ProcessCommand::WriteStdin {
|
|
||||||
data: b"input\n".to_vec(),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
for _ in 0..3 {
|
|
||||||
rt.tick();
|
|
||||||
}
|
|
||||||
|
|
||||||
let actions = harness.take_actions();
|
|
||||||
assert!(
|
|
||||||
actions
|
|
||||||
.iter()
|
|
||||||
.any(|a| matches!(a, ProcessAction::WriteStdin { .. })),
|
|
||||||
"WriteStdin should be forwarded to driver, got: {:?}",
|
|
||||||
actions
|
|
||||||
);
|
|
||||||
|
|
||||||
// Send signal
|
|
||||||
rt.send_to(
|
|
||||||
proc_addr,
|
|
||||||
ProcessCommand::SendSignal {
|
|
||||||
signal: Signal::Interrupt,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
for _ in 0..3 {
|
|
||||||
rt.tick();
|
|
||||||
}
|
|
||||||
|
|
||||||
let actions = harness.take_actions();
|
|
||||||
assert!(
|
|
||||||
actions.iter().any(|a| matches!(
|
|
||||||
a,
|
|
||||||
ProcessAction::SendSignal {
|
|
||||||
signal: Signal::Interrupt
|
|
||||||
}
|
|
||||||
)),
|
|
||||||
"SendSignal should be forwarded to driver, got: {:?}",
|
|
||||||
actions
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn spawn_failure_notifies_error_and_stops_actor() {
|
|
||||||
let (rt, sender) = setup();
|
|
||||||
let harness = TestHarness::new();
|
|
||||||
let notif_inbox = rt.new_inbox::<ProcessNotification>().unwrap();
|
|
||||||
|
|
||||||
let spawner_addr = rt.spawn(SpawnerActor).unwrap();
|
|
||||||
let reply_inbox = rt.new_inbox::<SpawnedAddr>().unwrap();
|
|
||||||
rt.tick();
|
|
||||||
|
|
||||||
rt.send_to(
|
|
||||||
spawner_addr,
|
|
||||||
SpawnRequest {
|
|
||||||
spec: automated_spec(),
|
|
||||||
harness: harness.clone(),
|
|
||||||
reply_to: *reply_inbox.addr(),
|
|
||||||
sender: sender.clone(),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
for _ in 0..5 {
|
|
||||||
rt.tick();
|
|
||||||
}
|
|
||||||
let proc_addr = reply_inbox.try_recv().unwrap().0;
|
|
||||||
|
|
||||||
// Subscribe
|
|
||||||
rt.send_to(
|
|
||||||
proc_addr,
|
|
||||||
ProcessCommand::Subscribe {
|
|
||||||
address: *notif_inbox.addr(),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
rt.tick();
|
|
||||||
|
|
||||||
// Inject spawn failure
|
|
||||||
harness.inject(ProcessEvent::SpawnFailed {
|
|
||||||
reason: "command not found".into(),
|
|
||||||
});
|
|
||||||
rt.send_to(proc_addr, ProcessCommand::PollTick).unwrap();
|
|
||||||
|
|
||||||
let msgs = tick_collect(&rt, ¬if_inbox, 1, 20);
|
|
||||||
assert!(
|
|
||||||
msgs.iter()
|
|
||||||
.any(|m| matches!(m, ProcessNotification::Error { .. })),
|
|
||||||
"subscriber should get Error notification on spawn failure"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Actor should have stopped — sending further messages should fail or be ignored
|
|
||||||
// (the address may still be in the map briefly, but the actor won't process)
|
|
||||||
for _ in 0..10 {
|
|
||||||
rt.tick();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn subscribe_and_unsubscribe_routing() {
|
|
||||||
let (rt, sender) = setup();
|
|
||||||
let harness = TestHarness::new();
|
|
||||||
let inbox_a = rt.new_inbox::<ProcessNotification>().unwrap();
|
|
||||||
let inbox_b = rt.new_inbox::<ProcessNotification>().unwrap();
|
|
||||||
|
|
||||||
let spawner_addr = rt.spawn(SpawnerActor).unwrap();
|
|
||||||
let reply_inbox = rt.new_inbox::<SpawnedAddr>().unwrap();
|
|
||||||
rt.tick();
|
|
||||||
|
|
||||||
rt.send_to(
|
|
||||||
spawner_addr,
|
|
||||||
SpawnRequest {
|
|
||||||
spec: automated_spec(),
|
|
||||||
harness: harness.clone(),
|
|
||||||
reply_to: *reply_inbox.addr(),
|
|
||||||
sender: sender.clone(),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
for _ in 0..5 {
|
|
||||||
rt.tick();
|
|
||||||
}
|
|
||||||
let proc_addr = reply_inbox.try_recv().unwrap().0;
|
|
||||||
|
|
||||||
// Subscribe both
|
|
||||||
rt.send_to(
|
|
||||||
proc_addr,
|
|
||||||
ProcessCommand::Subscribe {
|
|
||||||
address: *inbox_a.addr(),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
rt.send_to(
|
|
||||||
proc_addr,
|
|
||||||
ProcessCommand::Subscribe {
|
|
||||||
address: *inbox_b.addr(),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
rt.tick();
|
|
||||||
|
|
||||||
// Get to Running
|
|
||||||
harness.inject(ProcessEvent::Started);
|
|
||||||
rt.send_to(proc_addr, ProcessCommand::PollTick).unwrap();
|
|
||||||
for _ in 0..5 {
|
|
||||||
rt.tick();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Both should have received Started
|
|
||||||
assert!(inbox_a.try_recv().is_some(), "inbox_a should get Started");
|
|
||||||
assert!(inbox_b.try_recv().is_some(), "inbox_b should get Started");
|
|
||||||
|
|
||||||
// Unsubscribe inbox_b
|
|
||||||
rt.send_to(
|
|
||||||
proc_addr,
|
|
||||||
ProcessCommand::Unsubscribe {
|
|
||||||
address: *inbox_b.addr(),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
rt.tick();
|
|
||||||
|
|
||||||
// Inject output — only inbox_a should receive it
|
|
||||||
harness.inject(ProcessEvent::OutputReceived {
|
|
||||||
data: b"data".to_vec(),
|
|
||||||
is_stderr: false,
|
|
||||||
});
|
|
||||||
rt.send_to(proc_addr, ProcessCommand::PollTick).unwrap();
|
|
||||||
for _ in 0..5 {
|
|
||||||
rt.tick();
|
|
||||||
}
|
|
||||||
|
|
||||||
assert!(inbox_a.try_recv().is_some(), "inbox_a should get Output");
|
|
||||||
assert!(
|
|
||||||
inbox_b.try_recv().is_none(),
|
|
||||||
"inbox_b should NOT get Output after unsubscribe"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
@ -1,269 +0,0 @@
|
||||||
//! End-to-end tests: full Runtime + ExternalSender + ProcessActor<LocalDriver>.
|
|
||||||
//!
|
|
||||||
//! Spawns real OS processes through the actor system and verifies the
|
|
||||||
//! complete notification flow.
|
|
||||||
|
|
||||||
use std::collections::HashMap;
|
|
||||||
|
|
||||||
use swactor::actor::{ActorAddress, ActorInterface, Ctx};
|
|
||||||
use swactor::runtime::{ExternalSender, Inbox, Runtime, RuntimeConfig};
|
|
||||||
|
|
||||||
use swactor_process::*;
|
|
||||||
|
|
||||||
fn automated_spec(cmd: &str, args: &[&str]) -> ProcessSpec {
|
|
||||||
ProcessSpec {
|
|
||||||
command: cmd.into(),
|
|
||||||
args: args.iter().map(|s| s.to_string()).collect(),
|
|
||||||
env: HashMap::new(),
|
|
||||||
working_dir: None,
|
|
||||||
mode: ProcessMode::Automated,
|
|
||||||
initial_pty_size: None,
|
|
||||||
kill_timeout: None,
|
|
||||||
stdin_buffer_limit: None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Tick and collect up to `n` notifications, with a max tick budget.
|
|
||||||
fn tick_collect(
|
|
||||||
rt: &Runtime,
|
|
||||||
inbox: &Inbox<ProcessNotification>,
|
|
||||||
n: usize,
|
|
||||||
max_ticks: usize,
|
|
||||||
) -> Vec<ProcessNotification> {
|
|
||||||
let mut msgs = Vec::new();
|
|
||||||
for _ in 0..max_ticks {
|
|
||||||
rt.tick();
|
|
||||||
// Small sleep to let I/O threads produce events
|
|
||||||
std::thread::sleep(std::time::Duration::from_millis(5));
|
|
||||||
while let Some(m) = inbox.try_recv() {
|
|
||||||
msgs.push(m);
|
|
||||||
if msgs.len() >= n {
|
|
||||||
return msgs;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
msgs
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── Spawner actor (needed because spawn_local_process requires &Ctx) ────────
|
|
||||||
|
|
||||||
#[derive(Clone)]
|
|
||||||
struct E2eSpawnRequest {
|
|
||||||
spec: ProcessSpec,
|
|
||||||
subscriber: ActorAddress,
|
|
||||||
reply_to: ActorAddress,
|
|
||||||
sender: ExternalSender,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
|
||||||
struct E2eSpawned(ActorAddress);
|
|
||||||
|
|
||||||
struct E2eSpawnerActor;
|
|
||||||
|
|
||||||
impl ActorInterface for E2eSpawnerActor {
|
|
||||||
type Incoming = E2eSpawnRequest;
|
|
||||||
type Response = E2eSpawned;
|
|
||||||
|
|
||||||
fn handle(&mut self, ctx: &Ctx, msg: E2eSpawnRequest) {
|
|
||||||
let addr =
|
|
||||||
spawn_local_process(ctx, &msg.sender, msg.spec).expect("spawn_local_process failed");
|
|
||||||
// Subscribe the notification inbox
|
|
||||||
let _ = ctx.send(
|
|
||||||
addr,
|
|
||||||
ProcessCommand::Subscribe {
|
|
||||||
address: msg.subscriber,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
let _ = ctx.send(msg.reply_to, E2eSpawned(addr));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── Tests ───────────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn echo_hello_full_lifecycle() {
|
|
||||||
let rt = Runtime::new(RuntimeConfig::default());
|
|
||||||
let sender = rt.create_sender();
|
|
||||||
let notif_inbox = rt.new_inbox::<ProcessNotification>().unwrap();
|
|
||||||
|
|
||||||
let spawner_addr = rt.spawn(E2eSpawnerActor).unwrap();
|
|
||||||
let reply_inbox = rt.new_inbox::<E2eSpawned>().unwrap();
|
|
||||||
rt.tick();
|
|
||||||
|
|
||||||
rt.send_to(
|
|
||||||
spawner_addr,
|
|
||||||
E2eSpawnRequest {
|
|
||||||
spec: automated_spec("echo", &["hello"]),
|
|
||||||
subscriber: *notif_inbox.addr(),
|
|
||||||
reply_to: *reply_inbox.addr(),
|
|
||||||
sender: sender.clone(),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
// Tick enough for the spawner to process + the process actor to start
|
|
||||||
for _ in 0..10 {
|
|
||||||
rt.tick();
|
|
||||||
std::thread::sleep(std::time::Duration::from_millis(5));
|
|
||||||
}
|
|
||||||
|
|
||||||
let spawned = reply_inbox.try_recv().expect("should get spawned address");
|
|
||||||
let _proc_addr = spawned.0;
|
|
||||||
|
|
||||||
// Collect notifications: Started, Output("hello\n"), Exited(0)
|
|
||||||
let msgs = tick_collect(&rt, ¬if_inbox, 3, 200);
|
|
||||||
|
|
||||||
let has_started = msgs
|
|
||||||
.iter()
|
|
||||||
.any(|m| matches!(m, ProcessNotification::Started { .. }));
|
|
||||||
let has_output = msgs.iter().any(|m| {
|
|
||||||
if let ProcessNotification::Output { data, .. } = m {
|
|
||||||
String::from_utf8_lossy(data).contains("hello")
|
|
||||||
} else {
|
|
||||||
false
|
|
||||||
}
|
|
||||||
});
|
|
||||||
let has_exited = msgs.iter().any(|m| {
|
|
||||||
matches!(
|
|
||||||
m,
|
|
||||||
ProcessNotification::Exited {
|
|
||||||
status: ExitStatus::Code(0),
|
|
||||||
..
|
|
||||||
}
|
|
||||||
)
|
|
||||||
});
|
|
||||||
|
|
||||||
assert!(
|
|
||||||
has_started,
|
|
||||||
"should receive Started notification, got: {:?}",
|
|
||||||
msgs
|
|
||||||
);
|
|
||||||
assert!(
|
|
||||||
has_output,
|
|
||||||
"should receive Output with 'hello', got: {:?}",
|
|
||||||
msgs
|
|
||||||
);
|
|
||||||
assert!(
|
|
||||||
has_exited,
|
|
||||||
"should receive Exited(0) notification, got: {:?}",
|
|
||||||
msgs
|
|
||||||
);
|
|
||||||
|
|
||||||
// Verify ordering: Started before Output before Exited
|
|
||||||
let started_idx = msgs
|
|
||||||
.iter()
|
|
||||||
.position(|m| matches!(m, ProcessNotification::Started { .. }))
|
|
||||||
.unwrap();
|
|
||||||
let output_idx = msgs
|
|
||||||
.iter()
|
|
||||||
.position(|m| matches!(m, ProcessNotification::Output { .. }))
|
|
||||||
.unwrap();
|
|
||||||
let exited_idx = msgs
|
|
||||||
.iter()
|
|
||||||
.position(|m| matches!(m, ProcessNotification::Exited { .. }))
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
assert!(
|
|
||||||
started_idx < output_idx,
|
|
||||||
"Started should come before Output"
|
|
||||||
);
|
|
||||||
assert!(output_idx < exited_idx, "Output should come before Exited");
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The per-node process-output observer is the mechanism a telemetry node uses
|
|
||||||
/// to tap *every* managed process with no per-spawn wiring: it installs one
|
|
||||||
/// observer on its runtime, and any process spawned through the facility hands
|
|
||||||
/// its output there, labeled by command basename. This is the contract a node's
|
|
||||||
/// `proc.<label>.*` capture relies on, so it must hold for a real spawn driven
|
|
||||||
/// only through the public Runtime + `spawn_local_process` API.
|
|
||||||
#[test]
|
|
||||||
fn runtime_observer_taps_managed_process_output_by_basename() {
|
|
||||||
use std::sync::{Arc, Mutex};
|
|
||||||
use swactor::process_observer::ProcessOutputObserver;
|
|
||||||
|
|
||||||
#[derive(Default)]
|
|
||||||
struct Recorder {
|
|
||||||
// (label, is_stderr, text) for each chunk observed.
|
|
||||||
seen: Mutex<Vec<(String, bool, String)>>,
|
|
||||||
}
|
|
||||||
impl ProcessOutputObserver for Recorder {
|
|
||||||
fn on_output(&self, label: &str, is_stderr: bool, data: &[u8]) {
|
|
||||||
self.seen.lock().unwrap().push((
|
|
||||||
label.to_string(),
|
|
||||||
is_stderr,
|
|
||||||
String::from_utf8_lossy(data).into_owned(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let rt = Runtime::new(RuntimeConfig::default());
|
|
||||||
let recorder = Arc::new(Recorder::default());
|
|
||||||
// Install the observer before the first managed process is spawned.
|
|
||||||
rt.set_process_output_observer(recorder.clone());
|
|
||||||
|
|
||||||
let sender = rt.create_sender();
|
|
||||||
let notif_inbox = rt.new_inbox::<ProcessNotification>().unwrap();
|
|
||||||
let spawner_addr = rt.spawn(E2eSpawnerActor).unwrap();
|
|
||||||
let reply_inbox = rt.new_inbox::<E2eSpawned>().unwrap();
|
|
||||||
rt.tick();
|
|
||||||
|
|
||||||
// A path command so the basename (`echo`) is what labels the output, not the
|
|
||||||
// full path — the node keys `proc.<label>.*` on the basename.
|
|
||||||
rt.send_to(
|
|
||||||
spawner_addr,
|
|
||||||
E2eSpawnRequest {
|
|
||||||
spec: automated_spec("/bin/echo", &["telemetry-line"]),
|
|
||||||
subscriber: *notif_inbox.addr(),
|
|
||||||
reply_to: *reply_inbox.addr(),
|
|
||||||
sender: sender.clone(),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
// Drive until the process has produced output (also drains the inbox).
|
|
||||||
let _ = tick_collect(&rt, ¬if_inbox, 3, 200);
|
|
||||||
|
|
||||||
let seen = recorder.seen.lock().unwrap().clone();
|
|
||||||
let captured = seen.iter().any(|(label, is_stderr, text)| {
|
|
||||||
label == "echo" && !*is_stderr && text.contains("telemetry-line")
|
|
||||||
});
|
|
||||||
assert!(
|
|
||||||
captured,
|
|
||||||
"observer should capture stdout of the managed process under its command basename, got: {seen:?}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn bad_command_reports_error_e2e() {
|
|
||||||
let rt = Runtime::new(RuntimeConfig::default());
|
|
||||||
let sender = rt.create_sender();
|
|
||||||
let notif_inbox = rt.new_inbox::<ProcessNotification>().unwrap();
|
|
||||||
|
|
||||||
let spawner_addr = rt.spawn(E2eSpawnerActor).unwrap();
|
|
||||||
let reply_inbox = rt.new_inbox::<E2eSpawned>().unwrap();
|
|
||||||
rt.tick();
|
|
||||||
|
|
||||||
rt.send_to(
|
|
||||||
spawner_addr,
|
|
||||||
E2eSpawnRequest {
|
|
||||||
spec: automated_spec("/nonexistent/binary/xyz", &[]),
|
|
||||||
subscriber: *notif_inbox.addr(),
|
|
||||||
reply_to: *reply_inbox.addr(),
|
|
||||||
sender: sender.clone(),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
for _ in 0..10 {
|
|
||||||
rt.tick();
|
|
||||||
std::thread::sleep(std::time::Duration::from_millis(5));
|
|
||||||
}
|
|
||||||
|
|
||||||
let msgs = tick_collect(&rt, ¬if_inbox, 1, 200);
|
|
||||||
assert!(
|
|
||||||
msgs.iter()
|
|
||||||
.any(|m| matches!(m, ProcessNotification::Error { .. })),
|
|
||||||
"should receive Error notification for bad command, got: {:?}",
|
|
||||||
msgs
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
@ -1,342 +0,0 @@
|
||||||
//! Layer 4 — LocalDriver integration tests.
|
|
||||||
//!
|
|
||||||
//! Real OS processes, no actor layer. Tests LocalDriver in isolation.
|
|
||||||
|
|
||||||
use std::collections::HashMap;
|
|
||||||
use std::sync::{Arc, OnceLock};
|
|
||||||
use std::thread;
|
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
use swactor_process::*;
|
|
||||||
|
|
||||||
fn automated_spec(cmd: &str, args: &[&str]) -> ProcessSpec {
|
|
||||||
ProcessSpec {
|
|
||||||
command: cmd.into(),
|
|
||||||
args: args.iter().map(|s| s.to_string()).collect(),
|
|
||||||
env: HashMap::new(),
|
|
||||||
working_dir: None,
|
|
||||||
mode: ProcessMode::Automated,
|
|
||||||
initial_pty_size: None,
|
|
||||||
kill_timeout: None,
|
|
||||||
stdin_buffer_limit: None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Poll the driver until `pred` matches at least one collected event, or timeout.
|
|
||||||
fn poll_until_match(
|
|
||||||
driver: &mut LocalDriver,
|
|
||||||
timeout: Duration,
|
|
||||||
pred: impl Fn(&ProcessEvent) -> bool,
|
|
||||||
) -> Vec<ProcessEvent> {
|
|
||||||
let start = std::time::Instant::now();
|
|
||||||
let mut all_events = Vec::new();
|
|
||||||
loop {
|
|
||||||
let events = driver.poll();
|
|
||||||
if events.is_empty() {
|
|
||||||
if start.elapsed() >= timeout {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
thread::sleep(Duration::from_millis(10));
|
|
||||||
}
|
|
||||||
all_events.extend(events);
|
|
||||||
if all_events.iter().any(&pred) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
all_events
|
|
||||||
}
|
|
||||||
|
|
||||||
fn has_event(events: &[ProcessEvent], pred: impl Fn(&ProcessEvent) -> bool) -> bool {
|
|
||||||
events.iter().any(pred)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn echo_produces_started_output_and_exit_zero() {
|
|
||||||
let queue = EventQueue::new();
|
|
||||||
let waker_slot = Arc::new(OnceLock::new());
|
|
||||||
let mut driver = LocalDriver::new(queue, waker_slot);
|
|
||||||
|
|
||||||
let spec = automated_spec("echo", &["hello"]);
|
|
||||||
driver.execute(ProcessAction::SpawnProcess { spec });
|
|
||||||
|
|
||||||
// Wait for Exited (which means Started + output + exit are all in)
|
|
||||||
let events = poll_until_match(&mut driver, Duration::from_secs(5), |e| {
|
|
||||||
matches!(e, ProcessEvent::Exited { .. })
|
|
||||||
});
|
|
||||||
|
|
||||||
assert!(
|
|
||||||
has_event(&events, |e| matches!(e, ProcessEvent::Started)),
|
|
||||||
"should have Started event, got: {:?}",
|
|
||||||
events
|
|
||||||
);
|
|
||||||
assert!(
|
|
||||||
has_event(&events, |e| matches!(
|
|
||||||
e,
|
|
||||||
ProcessEvent::OutputReceived {
|
|
||||||
is_stderr: false,
|
|
||||||
..
|
|
||||||
}
|
|
||||||
)),
|
|
||||||
"should have stdout OutputReceived"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Check the output contains "hello"
|
|
||||||
let output: Vec<u8> = events
|
|
||||||
.iter()
|
|
||||||
.filter_map(|e| match e {
|
|
||||||
ProcessEvent::OutputReceived {
|
|
||||||
data,
|
|
||||||
is_stderr: false,
|
|
||||||
} => Some(data.clone()),
|
|
||||||
_ => None,
|
|
||||||
})
|
|
||||||
.flatten()
|
|
||||||
.collect();
|
|
||||||
let output_str = String::from_utf8_lossy(&output);
|
|
||||||
assert!(
|
|
||||||
output_str.contains("hello"),
|
|
||||||
"output should contain 'hello', got: {:?}",
|
|
||||||
output_str
|
|
||||||
);
|
|
||||||
|
|
||||||
assert!(
|
|
||||||
has_event(&events, |e| matches!(
|
|
||||||
e,
|
|
||||||
ProcessEvent::Exited {
|
|
||||||
status: ExitStatus::Code(0)
|
|
||||||
}
|
|
||||||
)),
|
|
||||||
"should have Exited(0)"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn cat_stdin_echo_and_close() {
|
|
||||||
let queue = EventQueue::new();
|
|
||||||
let waker_slot = Arc::new(OnceLock::new());
|
|
||||||
let mut driver = LocalDriver::new(queue, waker_slot);
|
|
||||||
|
|
||||||
let spec = automated_spec("cat", &[]);
|
|
||||||
driver.execute(ProcessAction::SpawnProcess { spec });
|
|
||||||
|
|
||||||
// Wait for Started
|
|
||||||
let events = poll_until_match(&mut driver, Duration::from_secs(5), |e| {
|
|
||||||
matches!(e, ProcessEvent::Started)
|
|
||||||
});
|
|
||||||
assert!(has_event(&events, |e| matches!(e, ProcessEvent::Started)));
|
|
||||||
|
|
||||||
// Write to stdin
|
|
||||||
driver.execute(ProcessAction::WriteStdin {
|
|
||||||
data: b"ping\n".to_vec(),
|
|
||||||
});
|
|
||||||
|
|
||||||
// Wait until we see actual output (not just the StdinWritten ack)
|
|
||||||
let events = poll_until_match(&mut driver, Duration::from_secs(5), |e| {
|
|
||||||
matches!(e, ProcessEvent::OutputReceived { .. })
|
|
||||||
});
|
|
||||||
let output: Vec<u8> = events
|
|
||||||
.iter()
|
|
||||||
.filter_map(|e| match e {
|
|
||||||
ProcessEvent::OutputReceived { data, .. } => Some(data.clone()),
|
|
||||||
_ => None,
|
|
||||||
})
|
|
||||||
.flatten()
|
|
||||||
.collect();
|
|
||||||
let output_str = String::from_utf8_lossy(&output);
|
|
||||||
assert!(
|
|
||||||
output_str.contains("ping"),
|
|
||||||
"cat should echo back 'ping', got: {:?}",
|
|
||||||
output_str
|
|
||||||
);
|
|
||||||
|
|
||||||
// Close stdin — cat should exit
|
|
||||||
driver.execute(ProcessAction::CloseStdin);
|
|
||||||
let events = poll_until_match(&mut driver, Duration::from_secs(5), |e| {
|
|
||||||
matches!(e, ProcessEvent::Exited { .. })
|
|
||||||
});
|
|
||||||
assert!(
|
|
||||||
has_event(&events, |e| matches!(
|
|
||||||
e,
|
|
||||||
ProcessEvent::Exited {
|
|
||||||
status: ExitStatus::Code(0)
|
|
||||||
}
|
|
||||||
)),
|
|
||||||
"cat should exit cleanly after stdin close, got: {:?}",
|
|
||||||
events
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn signal_terminates_long_running_process() {
|
|
||||||
let queue = EventQueue::new();
|
|
||||||
let waker_slot = Arc::new(OnceLock::new());
|
|
||||||
let mut driver = LocalDriver::new(queue, waker_slot);
|
|
||||||
|
|
||||||
let spec = automated_spec("sleep", &["60"]);
|
|
||||||
driver.execute(ProcessAction::SpawnProcess { spec });
|
|
||||||
|
|
||||||
// Wait for Started
|
|
||||||
let events = poll_until_match(&mut driver, Duration::from_secs(5), |e| {
|
|
||||||
matches!(e, ProcessEvent::Started)
|
|
||||||
});
|
|
||||||
assert!(has_event(&events, |e| matches!(e, ProcessEvent::Started)));
|
|
||||||
|
|
||||||
// Send SIGTERM
|
|
||||||
driver.execute(ProcessAction::SendSignal {
|
|
||||||
signal: Signal::Terminate,
|
|
||||||
});
|
|
||||||
|
|
||||||
// Wait for Exited (may also see SignalSent ack first)
|
|
||||||
let events = poll_until_match(&mut driver, Duration::from_secs(5), |e| {
|
|
||||||
matches!(e, ProcessEvent::Exited { .. })
|
|
||||||
});
|
|
||||||
assert!(
|
|
||||||
has_event(&events, |e| matches!(
|
|
||||||
e,
|
|
||||||
ProcessEvent::Exited {
|
|
||||||
status: ExitStatus::Signal(_)
|
|
||||||
}
|
|
||||||
)),
|
|
||||||
"sleep should exit with signal status after SIGTERM, got: {:?}",
|
|
||||||
events
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn bad_command_produces_spawn_failed() {
|
|
||||||
let queue = EventQueue::new();
|
|
||||||
let waker_slot = Arc::new(OnceLock::new());
|
|
||||||
let mut driver = LocalDriver::new(queue, waker_slot);
|
|
||||||
|
|
||||||
let spec = automated_spec("/nonexistent/binary/that/does/not/exist", &[]);
|
|
||||||
driver.execute(ProcessAction::SpawnProcess { spec });
|
|
||||||
|
|
||||||
let events = poll_until_match(&mut driver, Duration::from_secs(5), |e| {
|
|
||||||
matches!(e, ProcessEvent::SpawnFailed { .. })
|
|
||||||
});
|
|
||||||
assert!(
|
|
||||||
has_event(&events, |e| matches!(e, ProcessEvent::SpawnFailed { .. })),
|
|
||||||
"nonexistent binary should produce SpawnFailed, got: {:?}",
|
|
||||||
events
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn large_output_no_data_loss() {
|
|
||||||
let queue = EventQueue::new();
|
|
||||||
let waker_slot = Arc::new(OnceLock::new());
|
|
||||||
let mut driver = LocalDriver::new(queue, waker_slot);
|
|
||||||
|
|
||||||
// Generate a large amount of output: seq 1 10000
|
|
||||||
let spec = automated_spec("seq", &["1", "10000"]);
|
|
||||||
driver.execute(ProcessAction::SpawnProcess { spec });
|
|
||||||
|
|
||||||
// Collect all events until exit
|
|
||||||
let events = poll_until_match(&mut driver, Duration::from_secs(10), |e| {
|
|
||||||
matches!(e, ProcessEvent::Exited { .. })
|
|
||||||
});
|
|
||||||
|
|
||||||
// Gather all output
|
|
||||||
let output: Vec<u8> = events
|
|
||||||
.iter()
|
|
||||||
.filter_map(|e| match e {
|
|
||||||
ProcessEvent::OutputReceived { data, .. } => Some(data.clone()),
|
|
||||||
_ => None,
|
|
||||||
})
|
|
||||||
.flatten()
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
let output_str = String::from_utf8_lossy(&output);
|
|
||||||
// seq 1 10000 should end with "10000\n"
|
|
||||||
assert!(
|
|
||||||
output_str.contains("10000"),
|
|
||||||
"large output should contain '10000'"
|
|
||||||
);
|
|
||||||
// Check that it starts with "1\n"
|
|
||||||
assert!(
|
|
||||||
output_str.starts_with("1\n"),
|
|
||||||
"large output should start with '1\\n'"
|
|
||||||
);
|
|
||||||
|
|
||||||
assert!(
|
|
||||||
has_event(&events, |e| matches!(
|
|
||||||
e,
|
|
||||||
ProcessEvent::Exited {
|
|
||||||
status: ExitStatus::Code(0)
|
|
||||||
}
|
|
||||||
)),
|
|
||||||
"seq should exit cleanly"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn kill_timeout_escalates_to_sigkill() {
|
|
||||||
let queue = EventQueue::new();
|
|
||||||
let waker_slot = Arc::new(OnceLock::new());
|
|
||||||
let mut driver = LocalDriver::new(queue, waker_slot);
|
|
||||||
|
|
||||||
// Spawn a process that traps SIGTERM. Use exec to replace the shell so
|
|
||||||
// SIGTERM goes directly to the perl process (avoids shell vs child races).
|
|
||||||
let spec = automated_spec("perl", &["-e", "$SIG{TERM} = 'IGNORE'; sleep 300"]);
|
|
||||||
driver.execute(ProcessAction::SpawnProcess { spec });
|
|
||||||
|
|
||||||
// Wait for Started
|
|
||||||
let events = poll_until_match(&mut driver, Duration::from_secs(5), |e| {
|
|
||||||
matches!(e, ProcessEvent::Started)
|
|
||||||
});
|
|
||||||
assert!(has_event(&events, |e| matches!(e, ProcessEvent::Started)));
|
|
||||||
|
|
||||||
// Give the process a moment to set up the trap
|
|
||||||
thread::sleep(Duration::from_millis(100));
|
|
||||||
|
|
||||||
// Send SIGTERM (the process ignores it)
|
|
||||||
driver.execute(ProcessAction::SendSignal {
|
|
||||||
signal: Signal::Terminate,
|
|
||||||
});
|
|
||||||
poll_until_match(&mut driver, Duration::from_secs(1), |e| {
|
|
||||||
matches!(e, ProcessEvent::SignalSent)
|
|
||||||
});
|
|
||||||
|
|
||||||
// Verify the process is still alive after a short wait (SIGTERM was ignored)
|
|
||||||
thread::sleep(Duration::from_millis(200));
|
|
||||||
let events = driver.poll();
|
|
||||||
assert!(
|
|
||||||
!has_event(&events, |e| matches!(e, ProcessEvent::Exited { .. })),
|
|
||||||
"process should still be alive after SIGTERM (trap should ignore it)"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Schedule a short kill timeout
|
|
||||||
driver.execute(ProcessAction::ScheduleKillTimeout {
|
|
||||||
duration: Duration::from_millis(200),
|
|
||||||
});
|
|
||||||
|
|
||||||
// Wait for KillTimeout event
|
|
||||||
let events = poll_until_match(&mut driver, Duration::from_secs(3), |e| {
|
|
||||||
matches!(e, ProcessEvent::KillTimeout)
|
|
||||||
});
|
|
||||||
assert!(
|
|
||||||
has_event(&events, |e| matches!(e, ProcessEvent::KillTimeout)),
|
|
||||||
"should receive KillTimeout, got: {:?}",
|
|
||||||
events
|
|
||||||
);
|
|
||||||
|
|
||||||
// Now send SIGKILL
|
|
||||||
driver.execute(ProcessAction::SendSignal {
|
|
||||||
signal: Signal::Kill,
|
|
||||||
});
|
|
||||||
|
|
||||||
// Wait for exit
|
|
||||||
let events = poll_until_match(&mut driver, Duration::from_secs(5), |e| {
|
|
||||||
matches!(e, ProcessEvent::Exited { .. })
|
|
||||||
});
|
|
||||||
assert!(
|
|
||||||
has_event(&events, |e| matches!(
|
|
||||||
e,
|
|
||||||
ProcessEvent::Exited {
|
|
||||||
status: ExitStatus::Signal(_)
|
|
||||||
}
|
|
||||||
)),
|
|
||||||
"process should exit with signal after SIGKILL, got: {:?}",
|
|
||||||
events
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
@ -1,203 +0,0 @@
|
||||||
use std::collections::HashMap;
|
|
||||||
|
|
||||||
use proptest::prelude::*;
|
|
||||||
use swactor::actor::ActorAddress;
|
|
||||||
use swactor_process::*;
|
|
||||||
|
|
||||||
fn automated_spec() -> ProcessSpec {
|
|
||||||
ProcessSpec {
|
|
||||||
command: "test".into(),
|
|
||||||
args: vec![],
|
|
||||||
env: HashMap::new(),
|
|
||||||
working_dir: None,
|
|
||||||
mode: ProcessMode::Automated,
|
|
||||||
initial_pty_size: None,
|
|
||||||
kill_timeout: None,
|
|
||||||
stdin_buffer_limit: None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn addr(n: u8) -> ActorAddress {
|
|
||||||
let mut bytes = [0u8; 32];
|
|
||||||
bytes[0] = n;
|
|
||||||
ActorAddress(bytes)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn arb_signal() -> impl Strategy<Value = Signal> {
|
|
||||||
prop_oneof![
|
|
||||||
Just(Signal::Terminate),
|
|
||||||
Just(Signal::Kill),
|
|
||||||
Just(Signal::Hangup),
|
|
||||||
Just(Signal::Interrupt),
|
|
||||||
(0..32i32).prop_map(Signal::Other),
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
fn arb_event() -> impl Strategy<Value = ProcessEvent> {
|
|
||||||
prop_oneof![
|
|
||||||
Just(ProcessEvent::Started),
|
|
||||||
Just(ProcessEvent::KillTimeout),
|
|
||||||
".*".prop_map(|reason| ProcessEvent::SpawnFailed { reason }),
|
|
||||||
proptest::collection::vec(any::<u8>(), 0..64).prop_map(|data| {
|
|
||||||
ProcessEvent::OutputReceived {
|
|
||||||
data,
|
|
||||||
is_stderr: false,
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
proptest::collection::vec(any::<u8>(), 0..64).prop_map(|data| {
|
|
||||||
ProcessEvent::OutputReceived {
|
|
||||||
data,
|
|
||||||
is_stderr: true,
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
prop_oneof![
|
|
||||||
any::<i32>().prop_map(ExitStatus::Code),
|
|
||||||
any::<i32>().prop_map(ExitStatus::Signal),
|
|
||||||
Just(ExitStatus::Unknown),
|
|
||||||
]
|
|
||||||
.prop_map(|status| ProcessEvent::Exited { status }),
|
|
||||||
".*".prop_map(|reason| ProcessEvent::ConnectionLost { reason }),
|
|
||||||
(0..5usize).prop_map(|n| ProcessEvent::StdinWritten { byte_count: n * 10 }),
|
|
||||||
Just(ProcessEvent::SignalSent),
|
|
||||||
Just(ProcessEvent::PtyResized),
|
|
||||||
proptest::collection::vec(any::<u8>(), 0..64)
|
|
||||||
.prop_map(|data| ProcessEvent::WriteStdin { data }),
|
|
||||||
arb_signal().prop_map(|signal| ProcessEvent::SendSignal { signal }),
|
|
||||||
Just(ProcessEvent::ResizePty {
|
|
||||||
size: PtySize { cols: 80, rows: 24 },
|
|
||||||
}),
|
|
||||||
Just(ProcessEvent::CloseStdin),
|
|
||||||
Just(ProcessEvent::CloseRequested),
|
|
||||||
(0..4u8).prop_map(|n| ProcessEvent::Subscribe { address: addr(n) }),
|
|
||||||
(0..4u8).prop_map(|n| ProcessEvent::Unsubscribe { address: addr(n) }),
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
// 1. No panics for arbitrary event sequences
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
|
|
||||||
proptest! {
|
|
||||||
#[test]
|
|
||||||
fn no_panics_on_arbitrary_events(events in proptest::collection::vec(arb_event(), 0..50)) {
|
|
||||||
let (mut session, _) = ProcessSession::new(automated_spec());
|
|
||||||
for event in events {
|
|
||||||
let _ = session.apply(event);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
// 2. Exited is terminal
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
|
|
||||||
proptest! {
|
|
||||||
#[test]
|
|
||||||
fn exited_is_terminal(events in proptest::collection::vec(arb_event(), 0..50)) {
|
|
||||||
let (mut session, _) = ProcessSession::new(automated_spec());
|
|
||||||
let mut reached_exited = false;
|
|
||||||
|
|
||||||
for event in events {
|
|
||||||
let _ = session.apply(event);
|
|
||||||
if session.state() == ProcessState::Exited {
|
|
||||||
reached_exited = true;
|
|
||||||
}
|
|
||||||
if reached_exited {
|
|
||||||
prop_assert_eq!(session.state(), ProcessState::Exited);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
// 3. SelfTerminate always last action when entering Exited
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
|
|
||||||
proptest! {
|
|
||||||
#[test]
|
|
||||||
fn self_terminate_is_last_when_entering_exited(events in proptest::collection::vec(arb_event(), 0..50)) {
|
|
||||||
let (mut session, _) = ProcessSession::new(automated_spec());
|
|
||||||
let mut was_exited = false;
|
|
||||||
|
|
||||||
for event in events {
|
|
||||||
let prev_state = session.state();
|
|
||||||
let actions = session.apply(event);
|
|
||||||
|
|
||||||
// If we just transitioned into Exited
|
|
||||||
if session.state() == ProcessState::Exited && !was_exited && prev_state != ProcessState::Exited {
|
|
||||||
prop_assert!(
|
|
||||||
matches!(actions.last(), Some(ProcessAction::SelfTerminate)),
|
|
||||||
"SelfTerminate must be last action when entering Exited, got: {:?}", actions
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
if session.state() == ProcessState::Exited {
|
|
||||||
was_exited = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
// 4. Subscriber count matches add/remove operations
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
|
|
||||||
proptest! {
|
|
||||||
#[test]
|
|
||||||
fn subscriber_count_is_consistent(
|
|
||||||
ops in proptest::collection::vec(
|
|
||||||
prop_oneof![
|
|
||||||
(0..8u8).prop_map(|n| (true, n)),
|
|
||||||
(0..8u8).prop_map(|n| (false, n)),
|
|
||||||
],
|
|
||||||
0..30
|
|
||||||
)
|
|
||||||
) {
|
|
||||||
let (mut session, _) = ProcessSession::new(automated_spec());
|
|
||||||
let mut expected: Vec<u8> = Vec::new();
|
|
||||||
|
|
||||||
for (is_add, n) in ops {
|
|
||||||
if is_add {
|
|
||||||
session.apply(ProcessEvent::Subscribe { address: addr(n) });
|
|
||||||
if !expected.contains(&n) {
|
|
||||||
expected.push(n);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
session.apply(ProcessEvent::Unsubscribe { address: addr(n) });
|
|
||||||
expected.retain(|&x| x != n);
|
|
||||||
}
|
|
||||||
prop_assert_eq!(session.subscriber_count(), expected.len());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
// 5. State monotonicity (never goes backward)
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
|
|
||||||
fn state_ordinal(s: ProcessState) -> u8 {
|
|
||||||
match s {
|
|
||||||
ProcessState::Starting => 0,
|
|
||||||
ProcessState::Running => 1,
|
|
||||||
ProcessState::Stopping => 2,
|
|
||||||
ProcessState::Exited => 3,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
proptest! {
|
|
||||||
#[test]
|
|
||||||
fn state_never_goes_backward(events in proptest::collection::vec(arb_event(), 0..50)) {
|
|
||||||
let (mut session, _) = ProcessSession::new(automated_spec());
|
|
||||||
let mut max_ordinal = state_ordinal(session.state());
|
|
||||||
|
|
||||||
for event in events {
|
|
||||||
let _ = session.apply(event);
|
|
||||||
let current = state_ordinal(session.state());
|
|
||||||
prop_assert!(
|
|
||||||
current >= max_ordinal,
|
|
||||||
"State went backward: ordinal {} -> {}", max_ordinal, current
|
|
||||||
);
|
|
||||||
max_ordinal = current;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
54
crates/process/tests/public_api_stage1.rs
Normal file
54
crates/process/tests/public_api_stage1.rs
Normal file
|
|
@ -0,0 +1,54 @@
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use swactor_process::{
|
||||||
|
ExitStatus, ProcessCommand, ProcessOutput, ProcessSpec, send_process_command,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn stage1_public_api_exposes_only_target_process_shapes() {
|
||||||
|
let spec = ProcessSpec {
|
||||||
|
command: "echo".to_string(),
|
||||||
|
args: vec!["ok".to_string()],
|
||||||
|
env: HashMap::new(),
|
||||||
|
working_dir: None,
|
||||||
|
label: Some("echo_ok".to_string()),
|
||||||
|
};
|
||||||
|
assert_eq!(spec.label.as_deref(), Some("echo_ok"));
|
||||||
|
|
||||||
|
let stop = ProcessCommand::Stop {
|
||||||
|
kill_after: Some(Duration::from_millis(10)),
|
||||||
|
};
|
||||||
|
assert!(matches!(
|
||||||
|
stop,
|
||||||
|
ProcessCommand::Stop {
|
||||||
|
kill_after: Some(_)
|
||||||
|
}
|
||||||
|
));
|
||||||
|
|
||||||
|
let outputs = [
|
||||||
|
ProcessOutput::Started { pid: 1 },
|
||||||
|
ProcessOutput::SpawnFailed {
|
||||||
|
error: "spawn failed".to_string(),
|
||||||
|
},
|
||||||
|
ProcessOutput::Exited {
|
||||||
|
status: ExitStatus::Code(0),
|
||||||
|
},
|
||||||
|
ProcessOutput::Error {
|
||||||
|
error: "supervisor failed".to_string(),
|
||||||
|
},
|
||||||
|
];
|
||||||
|
assert!(matches!(outputs[0], ProcessOutput::Started { pid: 1 }));
|
||||||
|
assert!(matches!(
|
||||||
|
outputs[2],
|
||||||
|
ProcessOutput::Exited {
|
||||||
|
status: ExitStatus::Code(0)
|
||||||
|
}
|
||||||
|
));
|
||||||
|
|
||||||
|
let _send: fn(
|
||||||
|
&swactor::runtime::ExternalSender,
|
||||||
|
swactor::actor::ActorAddress,
|
||||||
|
ProcessCommand,
|
||||||
|
) -> Result<(), swactor::Error> = send_process_command;
|
||||||
|
}
|
||||||
974
crates/process/tests/public_api_stage2.rs
Normal file
974
crates/process/tests/public_api_stage2.rs
Normal file
|
|
@ -0,0 +1,974 @@
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use datastream::{DatastreamEndpoint, DatastreamEvent, Lifetime, NodeId, StreamId};
|
||||||
|
use serde_json::{Value, json};
|
||||||
|
use swactor::actor::{ActorAddress, ActorInterface, Ctx};
|
||||||
|
use swactor::runtime::{ExternalSender, Inbox, Runtime, RuntimeConfig};
|
||||||
|
use swactor_process::{
|
||||||
|
ExitStatus, ProcessCommand, ProcessOutput, ProcessOutputConfig, ProcessSpec,
|
||||||
|
send_process_command, spawn_local_process,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct SpawnRequest {
|
||||||
|
spec: ProcessSpec,
|
||||||
|
output: ProcessOutputConfig,
|
||||||
|
sender: ExternalSender,
|
||||||
|
reply_to: ActorAddress,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
enum SpawnReply {
|
||||||
|
Spawned(ActorAddress),
|
||||||
|
Failed(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
struct SpawnerActor;
|
||||||
|
|
||||||
|
impl ActorInterface for SpawnerActor {
|
||||||
|
type Incoming = SpawnRequest;
|
||||||
|
type Response = ();
|
||||||
|
|
||||||
|
fn handle(&mut self, ctx: &Ctx, msg: SpawnRequest) {
|
||||||
|
let reply = match spawn_local_process(ctx, &msg.sender, msg.spec, msg.output) {
|
||||||
|
Ok(addr) => SpawnReply::Spawned(addr),
|
||||||
|
Err(err) => SpawnReply::Failed(err.to_string()),
|
||||||
|
};
|
||||||
|
let _ = ctx.send(msg.reply_to, reply);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static NEXT_STREAM: AtomicU64 = AtomicU64::new(1);
|
||||||
|
|
||||||
|
fn stage2_stream() -> StreamId {
|
||||||
|
let id = NEXT_STREAM.fetch_add(1, Ordering::Relaxed);
|
||||||
|
StreamId::new(NodeId::new(format!("process-stage2-{id}")), Lifetime(2))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn shell_spec(command: &str, args: Vec<&str>, label: Option<&str>) -> ProcessSpec {
|
||||||
|
ProcessSpec {
|
||||||
|
command: command.to_owned(),
|
||||||
|
args: args.into_iter().map(str::to_owned).collect(),
|
||||||
|
env: HashMap::new(),
|
||||||
|
working_dir: None,
|
||||||
|
label: label.map(str::to_owned),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn drain_outputs(inbox: &Inbox<ProcessOutput>, outputs: &mut Vec<ProcessOutput>) {
|
||||||
|
while let Some(output) = inbox.try_recv() {
|
||||||
|
outputs.push(output);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn drive_once(
|
||||||
|
rt: &Runtime,
|
||||||
|
endpoint: Option<&DatastreamEndpoint>,
|
||||||
|
upstream: &Inbox<ProcessOutput>,
|
||||||
|
outputs: &mut Vec<ProcessOutput>,
|
||||||
|
) {
|
||||||
|
rt.tick();
|
||||||
|
std::thread::sleep(Duration::from_millis(5));
|
||||||
|
if let Some(endpoint) = endpoint {
|
||||||
|
endpoint.tick();
|
||||||
|
}
|
||||||
|
drain_outputs(upstream, outputs);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn send_spawn(
|
||||||
|
rt: &Runtime,
|
||||||
|
spawner: ActorAddress,
|
||||||
|
sender: &ExternalSender,
|
||||||
|
spec: ProcessSpec,
|
||||||
|
output: ProcessOutputConfig,
|
||||||
|
reply: &Inbox<SpawnReply>,
|
||||||
|
) {
|
||||||
|
rt.send_to(
|
||||||
|
spawner,
|
||||||
|
SpawnRequest {
|
||||||
|
spec,
|
||||||
|
output,
|
||||||
|
sender: sender.clone(),
|
||||||
|
reply_to: *reply.addr(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn drive_until_spawn_reply(
|
||||||
|
rt: &Runtime,
|
||||||
|
endpoint: Option<&DatastreamEndpoint>,
|
||||||
|
upstream: &Inbox<ProcessOutput>,
|
||||||
|
outputs: &mut Vec<ProcessOutput>,
|
||||||
|
reply: &Inbox<SpawnReply>,
|
||||||
|
) -> SpawnReply {
|
||||||
|
for _ in 0..400 {
|
||||||
|
drive_once(rt, endpoint, upstream, outputs);
|
||||||
|
if let Some(reply) = reply.try_recv() {
|
||||||
|
return reply;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
panic!("spawner did not reply; outputs={outputs:?}");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn drive_until(
|
||||||
|
rt: &Runtime,
|
||||||
|
endpoint: Option<&DatastreamEndpoint>,
|
||||||
|
upstream: &Inbox<ProcessOutput>,
|
||||||
|
outputs: &mut Vec<ProcessOutput>,
|
||||||
|
mut done: impl FnMut(&[ProcessOutput]) -> bool,
|
||||||
|
) {
|
||||||
|
for _ in 0..800 {
|
||||||
|
drive_once(rt, endpoint, upstream, outputs);
|
||||||
|
if done(outputs) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
panic!("runtime condition was not reached; outputs={outputs:?}");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn expect_spawned(reply: SpawnReply) -> ActorAddress {
|
||||||
|
match reply {
|
||||||
|
SpawnReply::Spawned(addr) => {
|
||||||
|
assert_ne!(addr, ActorAddress::default());
|
||||||
|
addr
|
||||||
|
}
|
||||||
|
SpawnReply::Failed(error) => panic!("spawn helper failed: {error}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn expect_failed(reply: SpawnReply) -> String {
|
||||||
|
match reply {
|
||||||
|
SpawnReply::Spawned(addr) => panic!("spawn helper unexpectedly spawned {addr:?}"),
|
||||||
|
SpawnReply::Failed(error) => error,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_terminal(output: &ProcessOutput) -> bool {
|
||||||
|
matches!(
|
||||||
|
output,
|
||||||
|
ProcessOutput::SpawnFailed { .. }
|
||||||
|
| ProcessOutput::Exited { .. }
|
||||||
|
| ProcessOutput::Error { .. }
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn terminal_count(outputs: &[ProcessOutput]) -> usize {
|
||||||
|
outputs.iter().filter(|output| is_terminal(output)).count()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn has_started(outputs: &[ProcessOutput]) -> bool {
|
||||||
|
outputs
|
||||||
|
.iter()
|
||||||
|
.any(|output| matches!(output, ProcessOutput::Started { pid } if *pid > 0))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn has_exited(outputs: &[ProcessOutput], status: ExitStatus) -> bool {
|
||||||
|
outputs.iter().any(|output| {
|
||||||
|
matches!(
|
||||||
|
output,
|
||||||
|
ProcessOutput::Exited { status: observed } if *observed == status
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn has_spawn_failed(outputs: &[ProcessOutput]) -> bool {
|
||||||
|
outputs
|
||||||
|
.iter()
|
||||||
|
.any(|output| matches!(output, ProcessOutput::SpawnFailed { .. }))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_no_errors(outputs: &[ProcessOutput]) {
|
||||||
|
assert!(
|
||||||
|
!outputs
|
||||||
|
.iter()
|
||||||
|
.any(|output| matches!(output, ProcessOutput::Error { .. })),
|
||||||
|
"unexpected process error output: {outputs:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_no_spawn_failed(outputs: &[ProcessOutput]) {
|
||||||
|
assert!(
|
||||||
|
!has_spawn_failed(outputs),
|
||||||
|
"unexpected spawn failure output: {outputs:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn started_index(outputs: &[ProcessOutput]) -> usize {
|
||||||
|
outputs
|
||||||
|
.iter()
|
||||||
|
.position(|output| matches!(output, ProcessOutput::Started { pid } if *pid > 0))
|
||||||
|
.unwrap_or_else(|| panic!("missing Started output: {outputs:?}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn exited_index(outputs: &[ProcessOutput]) -> usize {
|
||||||
|
outputs
|
||||||
|
.iter()
|
||||||
|
.position(|output| matches!(output, ProcessOutput::Exited { .. }))
|
||||||
|
.unwrap_or_else(|| panic!("missing Exited output: {outputs:?}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn channel_names(endpoint: &DatastreamEndpoint) -> Vec<String> {
|
||||||
|
endpoint
|
||||||
|
.catalog_snapshot()
|
||||||
|
.channels
|
||||||
|
.values()
|
||||||
|
.map(|descriptor| descriptor.name.clone())
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn lifecycle_outputs_are_sent_upstream_and_mirrored_to_datastream() {
|
||||||
|
let rt = Runtime::new(RuntimeConfig::default());
|
||||||
|
let sender = rt.create_sender();
|
||||||
|
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
|
||||||
|
let reply = rt.new_inbox::<SpawnReply>().unwrap();
|
||||||
|
let spawner = rt.spawn(SpawnerActor).unwrap();
|
||||||
|
let endpoint = DatastreamEndpoint::new(stage2_stream());
|
||||||
|
let subscription = endpoint.subscribe_all("process-stage2-lifecycle");
|
||||||
|
|
||||||
|
send_spawn(
|
||||||
|
&rt,
|
||||||
|
spawner,
|
||||||
|
&sender,
|
||||||
|
shell_spec(
|
||||||
|
"sh",
|
||||||
|
vec!["-c", "echo stdout; echo stderr >&2; exit 0"],
|
||||||
|
Some("trainer.0/foo"),
|
||||||
|
),
|
||||||
|
ProcessOutputConfig::datastream_mirror(*upstream.addr(), endpoint.producer()),
|
||||||
|
&reply,
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut outputs = Vec::new();
|
||||||
|
let process_addr = expect_spawned(drive_until_spawn_reply(
|
||||||
|
&rt,
|
||||||
|
Some(&endpoint),
|
||||||
|
&upstream,
|
||||||
|
&mut outputs,
|
||||||
|
&reply,
|
||||||
|
));
|
||||||
|
assert_ne!(process_addr, ActorAddress::default());
|
||||||
|
|
||||||
|
let mut datastream_events = subscription.drain_available();
|
||||||
|
drive_until(&rt, Some(&endpoint), &upstream, &mut outputs, |outputs| {
|
||||||
|
has_exited(outputs, ExitStatus::Code(0))
|
||||||
|
});
|
||||||
|
for _ in 0..5 {
|
||||||
|
drive_once(&rt, Some(&endpoint), &upstream, &mut outputs);
|
||||||
|
datastream_events.extend(subscription.drain_available());
|
||||||
|
}
|
||||||
|
|
||||||
|
let started = started_index(&outputs);
|
||||||
|
let exited = exited_index(&outputs);
|
||||||
|
assert!(
|
||||||
|
started < exited,
|
||||||
|
"Started should be delivered before Exited: {outputs:?}"
|
||||||
|
);
|
||||||
|
assert_no_spawn_failed(&outputs);
|
||||||
|
assert_no_errors(&outputs);
|
||||||
|
|
||||||
|
let names = channel_names(&endpoint);
|
||||||
|
assert!(
|
||||||
|
names
|
||||||
|
.iter()
|
||||||
|
.any(|name| name == "proc.trainer_0_foo.lifecycle"),
|
||||||
|
"catalog should contain lifecycle channel, got {names:?}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
names
|
||||||
|
.iter()
|
||||||
|
.all(|name| !name.contains("stdout") && !name.contains("stderr")),
|
||||||
|
"process core should not register stdout/stderr channels: {names:?}"
|
||||||
|
);
|
||||||
|
|
||||||
|
let payloads: Vec<Value> = datastream_events
|
||||||
|
.iter()
|
||||||
|
.filter_map(|event| match event {
|
||||||
|
DatastreamEvent::Frame(delivery) => serde_json::from_slice(&delivery.payload).ok(),
|
||||||
|
_ => None,
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
assert!(
|
||||||
|
payloads.iter().any(|payload| {
|
||||||
|
payload.get("event") == Some(&Value::String("started".to_owned()))
|
||||||
|
&& payload
|
||||||
|
.get("pid")
|
||||||
|
.and_then(Value::as_u64)
|
||||||
|
.is_some_and(|pid| pid > 0)
|
||||||
|
}),
|
||||||
|
"lifecycle mirror should include started JSON, got {payloads:?}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
payloads.iter().any(|payload| {
|
||||||
|
payload.get("event") == Some(&Value::String("exited".to_owned()))
|
||||||
|
&& payload.get("status") == Some(&json!({"kind": "code", "value": 0}))
|
||||||
|
}),
|
||||||
|
"lifecycle mirror should include exited JSON, got {payloads:?}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
payloads.iter().all(|payload| {
|
||||||
|
payload.get("event") != Some(&Value::String("stdout".to_owned()))
|
||||||
|
&& payload.get("event") != Some(&Value::String("stderr".to_owned()))
|
||||||
|
}),
|
||||||
|
"lifecycle mirror should not include child output events: {payloads:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn spawn_failure_maps_to_public_spawn_failed_output() {
|
||||||
|
let rt = Runtime::new(RuntimeConfig::default());
|
||||||
|
let sender = rt.create_sender();
|
||||||
|
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
|
||||||
|
let reply = rt.new_inbox::<SpawnReply>().unwrap();
|
||||||
|
let spawner = rt.spawn(SpawnerActor).unwrap();
|
||||||
|
|
||||||
|
send_spawn(
|
||||||
|
&rt,
|
||||||
|
spawner,
|
||||||
|
&sender,
|
||||||
|
shell_spec(
|
||||||
|
"/definitely/not/a/real/binary",
|
||||||
|
Vec::new(),
|
||||||
|
Some("missing-binary"),
|
||||||
|
),
|
||||||
|
ProcessOutputConfig::disabled(*upstream.addr()),
|
||||||
|
&reply,
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut outputs = Vec::new();
|
||||||
|
let _process_addr = expect_spawned(drive_until_spawn_reply(
|
||||||
|
&rt,
|
||||||
|
None,
|
||||||
|
&upstream,
|
||||||
|
&mut outputs,
|
||||||
|
&reply,
|
||||||
|
));
|
||||||
|
drive_until(&rt, None, &upstream, &mut outputs, has_spawn_failed);
|
||||||
|
|
||||||
|
let spawn_failures: Vec<&String> = outputs
|
||||||
|
.iter()
|
||||||
|
.filter_map(|output| match output {
|
||||||
|
ProcessOutput::SpawnFailed { error } => Some(error),
|
||||||
|
_ => None,
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
assert_eq!(
|
||||||
|
spawn_failures.len(),
|
||||||
|
1,
|
||||||
|
"expected exactly one spawn failure output, got {outputs:?}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!spawn_failures[0].is_empty(),
|
||||||
|
"spawn failure error should not be empty"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!outputs
|
||||||
|
.iter()
|
||||||
|
.any(|output| matches!(output, ProcessOutput::Started { .. })),
|
||||||
|
"spawn failure should not emit Started: {outputs:?}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!outputs
|
||||||
|
.iter()
|
||||||
|
.any(|output| matches!(output, ProcessOutput::Exited { .. })),
|
||||||
|
"spawn failure should not emit Exited: {outputs:?}"
|
||||||
|
);
|
||||||
|
assert_no_errors(&outputs);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn command_basename_is_default_lifecycle_label_source() {
|
||||||
|
let rt = Runtime::new(RuntimeConfig::default());
|
||||||
|
let sender = rt.create_sender();
|
||||||
|
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
|
||||||
|
let reply = rt.new_inbox::<SpawnReply>().unwrap();
|
||||||
|
let spawner = rt.spawn(SpawnerActor).unwrap();
|
||||||
|
let endpoint = DatastreamEndpoint::new(stage2_stream());
|
||||||
|
|
||||||
|
send_spawn(
|
||||||
|
&rt,
|
||||||
|
spawner,
|
||||||
|
&sender,
|
||||||
|
shell_spec("/bin/sh", vec!["-c", "exit 0"], None),
|
||||||
|
ProcessOutputConfig::datastream_mirror(*upstream.addr(), endpoint.producer()),
|
||||||
|
&reply,
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut outputs = Vec::new();
|
||||||
|
let _process_addr = expect_spawned(drive_until_spawn_reply(
|
||||||
|
&rt,
|
||||||
|
Some(&endpoint),
|
||||||
|
&upstream,
|
||||||
|
&mut outputs,
|
||||||
|
&reply,
|
||||||
|
));
|
||||||
|
drive_until(&rt, Some(&endpoint), &upstream, &mut outputs, |outputs| {
|
||||||
|
has_exited(outputs, ExitStatus::Code(0))
|
||||||
|
});
|
||||||
|
|
||||||
|
let names = channel_names(&endpoint);
|
||||||
|
assert!(
|
||||||
|
names.iter().any(|name| name == "proc.sh.lifecycle"),
|
||||||
|
"catalog should contain basename lifecycle channel, got {names:?}"
|
||||||
|
);
|
||||||
|
assert!(started_index(&outputs) < exited_index(&outputs));
|
||||||
|
assert_no_spawn_failed(&outputs);
|
||||||
|
assert_no_errors(&outputs);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn explicit_label_overrides_basename_and_duplicate_labels_are_rejected() {
|
||||||
|
let rt = Runtime::new(RuntimeConfig::default());
|
||||||
|
let sender = rt.create_sender();
|
||||||
|
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
|
||||||
|
let reply = rt.new_inbox::<SpawnReply>().unwrap();
|
||||||
|
let spawner = rt.spawn(SpawnerActor).unwrap();
|
||||||
|
let endpoint = DatastreamEndpoint::new(stage2_stream());
|
||||||
|
|
||||||
|
send_spawn(
|
||||||
|
&rt,
|
||||||
|
spawner,
|
||||||
|
&sender,
|
||||||
|
shell_spec("/bin/sh", vec!["-c", "sleep 2"], Some("trainer.0/foo")),
|
||||||
|
ProcessOutputConfig::datastream_mirror(*upstream.addr(), endpoint.producer()),
|
||||||
|
&reply,
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut outputs = Vec::new();
|
||||||
|
let first_addr = expect_spawned(drive_until_spawn_reply(
|
||||||
|
&rt,
|
||||||
|
Some(&endpoint),
|
||||||
|
&upstream,
|
||||||
|
&mut outputs,
|
||||||
|
&reply,
|
||||||
|
));
|
||||||
|
|
||||||
|
send_spawn(
|
||||||
|
&rt,
|
||||||
|
spawner,
|
||||||
|
&sender,
|
||||||
|
shell_spec("/bin/echo", vec!["unused"], Some("trainer.0/foo")),
|
||||||
|
ProcessOutputConfig::datastream_mirror(*upstream.addr(), endpoint.producer()),
|
||||||
|
&reply,
|
||||||
|
);
|
||||||
|
let error = expect_failed(drive_until_spawn_reply(
|
||||||
|
&rt,
|
||||||
|
Some(&endpoint),
|
||||||
|
&upstream,
|
||||||
|
&mut outputs,
|
||||||
|
&reply,
|
||||||
|
));
|
||||||
|
assert!(
|
||||||
|
error.contains(
|
||||||
|
"duplicate process lifecycle datastream channel: proc.trainer_0_foo.lifecycle"
|
||||||
|
),
|
||||||
|
"duplicate label error should name lifecycle channel, got {error}"
|
||||||
|
);
|
||||||
|
|
||||||
|
send_process_command(
|
||||||
|
&sender,
|
||||||
|
first_addr,
|
||||||
|
ProcessCommand::Stop {
|
||||||
|
kill_after: Some(Duration::from_millis(20)),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
drive_until(&rt, Some(&endpoint), &upstream, &mut outputs, |outputs| {
|
||||||
|
outputs
|
||||||
|
.iter()
|
||||||
|
.any(|output| matches!(output, ProcessOutput::Exited { .. }))
|
||||||
|
});
|
||||||
|
assert_no_errors(&outputs);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn stop_before_spawn_success_reports_started_then_exited() {
|
||||||
|
let rt = Runtime::new(RuntimeConfig::default());
|
||||||
|
let sender = rt.create_sender();
|
||||||
|
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
|
||||||
|
let reply = rt.new_inbox::<SpawnReply>().unwrap();
|
||||||
|
let spawner = rt.spawn(SpawnerActor).unwrap();
|
||||||
|
|
||||||
|
send_spawn(
|
||||||
|
&rt,
|
||||||
|
spawner,
|
||||||
|
&sender,
|
||||||
|
shell_spec("sh", vec!["-c", "sleep 60"], Some("stop-before-start")),
|
||||||
|
ProcessOutputConfig::disabled(*upstream.addr()),
|
||||||
|
&reply,
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut outputs = Vec::new();
|
||||||
|
let process_addr = expect_spawned(drive_until_spawn_reply(
|
||||||
|
&rt,
|
||||||
|
None,
|
||||||
|
&upstream,
|
||||||
|
&mut outputs,
|
||||||
|
&reply,
|
||||||
|
));
|
||||||
|
send_process_command(
|
||||||
|
&sender,
|
||||||
|
process_addr,
|
||||||
|
ProcessCommand::Stop {
|
||||||
|
kill_after: Some(Duration::from_millis(20)),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
drive_until(&rt, None, &upstream, &mut outputs, |outputs| {
|
||||||
|
outputs
|
||||||
|
.iter()
|
||||||
|
.any(|output| matches!(output, ProcessOutput::Exited { .. }))
|
||||||
|
});
|
||||||
|
|
||||||
|
assert!(started_index(&outputs) < exited_index(&outputs));
|
||||||
|
assert_eq!(
|
||||||
|
outputs
|
||||||
|
.iter()
|
||||||
|
.filter(|output| matches!(output, ProcessOutput::Exited { .. }))
|
||||||
|
.count(),
|
||||||
|
1,
|
||||||
|
"expected exactly one Exited output, got {outputs:?}"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
terminal_count(&outputs),
|
||||||
|
1,
|
||||||
|
"unexpected terminal outputs: {outputs:?}"
|
||||||
|
);
|
||||||
|
assert_no_spawn_failed(&outputs);
|
||||||
|
assert_no_errors(&outputs);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn stop_before_spawn_failure_reports_only_spawn_failed() {
|
||||||
|
let rt = Runtime::new(RuntimeConfig::default());
|
||||||
|
let sender = rt.create_sender();
|
||||||
|
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
|
||||||
|
let reply = rt.new_inbox::<SpawnReply>().unwrap();
|
||||||
|
let spawner = rt.spawn(SpawnerActor).unwrap();
|
||||||
|
|
||||||
|
send_spawn(
|
||||||
|
&rt,
|
||||||
|
spawner,
|
||||||
|
&sender,
|
||||||
|
shell_spec(
|
||||||
|
"/definitely/not/a/real/binary",
|
||||||
|
Vec::new(),
|
||||||
|
Some("stop-before-spawn-failure"),
|
||||||
|
),
|
||||||
|
ProcessOutputConfig::disabled(*upstream.addr()),
|
||||||
|
&reply,
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut outputs = Vec::new();
|
||||||
|
let process_addr = expect_spawned(drive_until_spawn_reply(
|
||||||
|
&rt,
|
||||||
|
None,
|
||||||
|
&upstream,
|
||||||
|
&mut outputs,
|
||||||
|
&reply,
|
||||||
|
));
|
||||||
|
send_process_command(
|
||||||
|
&sender,
|
||||||
|
process_addr,
|
||||||
|
ProcessCommand::Stop {
|
||||||
|
kill_after: Some(Duration::from_millis(20)),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
drive_until(&rt, None, &upstream, &mut outputs, has_spawn_failed);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
terminal_count(&outputs),
|
||||||
|
1,
|
||||||
|
"unexpected terminal outputs: {outputs:?}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!has_started(&outputs),
|
||||||
|
"spawn failure should not emit Started: {outputs:?}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!outputs
|
||||||
|
.iter()
|
||||||
|
.any(|output| matches!(output, ProcessOutput::Exited { .. })),
|
||||||
|
"spawn failure should not emit Exited: {outputs:?}"
|
||||||
|
);
|
||||||
|
assert_no_errors(&outputs);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn stop_running_with_kill_after_escalates_to_kill() {
|
||||||
|
let rt = Runtime::new(RuntimeConfig::default());
|
||||||
|
let sender = rt.create_sender();
|
||||||
|
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
|
||||||
|
let reply = rt.new_inbox::<SpawnReply>().unwrap();
|
||||||
|
let spawner = rt.spawn(SpawnerActor).unwrap();
|
||||||
|
|
||||||
|
send_spawn(
|
||||||
|
&rt,
|
||||||
|
spawner,
|
||||||
|
&sender,
|
||||||
|
shell_spec(
|
||||||
|
"sh",
|
||||||
|
vec!["-c", "trap '' TERM; while true; do sleep 1; done"],
|
||||||
|
Some("kill-escalates"),
|
||||||
|
),
|
||||||
|
ProcessOutputConfig::disabled(*upstream.addr()),
|
||||||
|
&reply,
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut outputs = Vec::new();
|
||||||
|
let process_addr = expect_spawned(drive_until_spawn_reply(
|
||||||
|
&rt,
|
||||||
|
None,
|
||||||
|
&upstream,
|
||||||
|
&mut outputs,
|
||||||
|
&reply,
|
||||||
|
));
|
||||||
|
drive_until(&rt, None, &upstream, &mut outputs, has_started);
|
||||||
|
send_process_command(
|
||||||
|
&sender,
|
||||||
|
process_addr,
|
||||||
|
ProcessCommand::Stop {
|
||||||
|
kill_after: Some(Duration::from_millis(20)),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
drive_until(&rt, None, &upstream, &mut outputs, |outputs| {
|
||||||
|
has_exited(outputs, ExitStatus::Signal(9))
|
||||||
|
});
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
has_exited(&outputs, ExitStatus::Signal(9)),
|
||||||
|
"expected SIGKILL exit: {outputs:?}"
|
||||||
|
);
|
||||||
|
assert_no_errors(&outputs);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn stop_running_without_kill_after_terminates_without_kill_escalation() {
|
||||||
|
let rt = Runtime::new(RuntimeConfig::default());
|
||||||
|
let sender = rt.create_sender();
|
||||||
|
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
|
||||||
|
let reply = rt.new_inbox::<SpawnReply>().unwrap();
|
||||||
|
let spawner = rt.spawn(SpawnerActor).unwrap();
|
||||||
|
|
||||||
|
send_spawn(
|
||||||
|
&rt,
|
||||||
|
spawner,
|
||||||
|
&sender,
|
||||||
|
shell_spec(
|
||||||
|
"sh",
|
||||||
|
vec!["-c", "trap 'exit 0' TERM; while true; do sleep 1; done"],
|
||||||
|
Some("terminate-without-kill"),
|
||||||
|
),
|
||||||
|
ProcessOutputConfig::disabled(*upstream.addr()),
|
||||||
|
&reply,
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut outputs = Vec::new();
|
||||||
|
let process_addr = expect_spawned(drive_until_spawn_reply(
|
||||||
|
&rt,
|
||||||
|
None,
|
||||||
|
&upstream,
|
||||||
|
&mut outputs,
|
||||||
|
&reply,
|
||||||
|
));
|
||||||
|
drive_until(&rt, None, &upstream, &mut outputs, has_started);
|
||||||
|
send_process_command(
|
||||||
|
&sender,
|
||||||
|
process_addr,
|
||||||
|
ProcessCommand::Stop { kill_after: None },
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
drive_until(&rt, None, &upstream, &mut outputs, |outputs| {
|
||||||
|
has_exited(outputs, ExitStatus::Code(0))
|
||||||
|
});
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
has_exited(&outputs, ExitStatus::Code(0)),
|
||||||
|
"expected graceful exit: {outputs:?}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!has_exited(&outputs, ExitStatus::Signal(9)),
|
||||||
|
"stop without deadline should not escalate to SIGKILL: {outputs:?}"
|
||||||
|
);
|
||||||
|
assert_no_errors(&outputs);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn child_exit_before_kill_deadline_suppresses_kill_escalation() {
|
||||||
|
let rt = Runtime::new(RuntimeConfig::default());
|
||||||
|
let sender = rt.create_sender();
|
||||||
|
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
|
||||||
|
let reply = rt.new_inbox::<SpawnReply>().unwrap();
|
||||||
|
let spawner = rt.spawn(SpawnerActor).unwrap();
|
||||||
|
|
||||||
|
send_spawn(
|
||||||
|
&rt,
|
||||||
|
spawner,
|
||||||
|
&sender,
|
||||||
|
shell_spec(
|
||||||
|
"sh",
|
||||||
|
vec!["-c", "trap 'exit 0' TERM; while true; do sleep 1; done"],
|
||||||
|
Some("deadline-suppresses-kill"),
|
||||||
|
),
|
||||||
|
ProcessOutputConfig::disabled(*upstream.addr()),
|
||||||
|
&reply,
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut outputs = Vec::new();
|
||||||
|
let process_addr = expect_spawned(drive_until_spawn_reply(
|
||||||
|
&rt,
|
||||||
|
None,
|
||||||
|
&upstream,
|
||||||
|
&mut outputs,
|
||||||
|
&reply,
|
||||||
|
));
|
||||||
|
drive_until(&rt, None, &upstream, &mut outputs, has_started);
|
||||||
|
send_process_command(
|
||||||
|
&sender,
|
||||||
|
process_addr,
|
||||||
|
ProcessCommand::Stop {
|
||||||
|
kill_after: Some(Duration::from_secs(1)),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
drive_until(&rt, None, &upstream, &mut outputs, |outputs| {
|
||||||
|
has_exited(outputs, ExitStatus::Code(0))
|
||||||
|
});
|
||||||
|
for _ in 0..5 {
|
||||||
|
drive_once(&rt, None, &upstream, &mut outputs);
|
||||||
|
}
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
has_exited(&outputs, ExitStatus::Code(0)),
|
||||||
|
"expected graceful exit: {outputs:?}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!has_exited(&outputs, ExitStatus::Signal(9)),
|
||||||
|
"child exit before deadline should suppress SIGKILL: {outputs:?}"
|
||||||
|
);
|
||||||
|
assert_no_errors(&outputs);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn duplicate_stop_while_stopping_is_noop_and_keeps_original_deadline() {
|
||||||
|
let rt = Runtime::new(RuntimeConfig::default());
|
||||||
|
let sender = rt.create_sender();
|
||||||
|
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
|
||||||
|
let reply = rt.new_inbox::<SpawnReply>().unwrap();
|
||||||
|
let spawner = rt.spawn(SpawnerActor).unwrap();
|
||||||
|
|
||||||
|
send_spawn(
|
||||||
|
&rt,
|
||||||
|
spawner,
|
||||||
|
&sender,
|
||||||
|
shell_spec(
|
||||||
|
"sh",
|
||||||
|
vec!["-c", "trap '' TERM; while true; do sleep 1; done"],
|
||||||
|
Some("duplicate-stop"),
|
||||||
|
),
|
||||||
|
ProcessOutputConfig::disabled(*upstream.addr()),
|
||||||
|
&reply,
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut outputs = Vec::new();
|
||||||
|
let process_addr = expect_spawned(drive_until_spawn_reply(
|
||||||
|
&rt,
|
||||||
|
None,
|
||||||
|
&upstream,
|
||||||
|
&mut outputs,
|
||||||
|
&reply,
|
||||||
|
));
|
||||||
|
drive_until(&rt, None, &upstream, &mut outputs, has_started);
|
||||||
|
|
||||||
|
let first_stop = Instant::now();
|
||||||
|
send_process_command(
|
||||||
|
&sender,
|
||||||
|
process_addr,
|
||||||
|
ProcessCommand::Stop {
|
||||||
|
kill_after: Some(Duration::from_millis(250)),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
send_process_command(
|
||||||
|
&sender,
|
||||||
|
process_addr,
|
||||||
|
ProcessCommand::Stop {
|
||||||
|
kill_after: Some(Duration::from_secs(5)),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
while first_stop.elapsed() < Duration::from_secs(2) {
|
||||||
|
drive_once(&rt, None, &upstream, &mut outputs);
|
||||||
|
if has_exited(&outputs, ExitStatus::Signal(9)) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
has_exited(&outputs, ExitStatus::Signal(9)),
|
||||||
|
"duplicate stop should keep the original kill deadline: {outputs:?}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
first_stop.elapsed() < Duration::from_secs(2),
|
||||||
|
"SIGKILL arrived too late after duplicate stop: {:?}",
|
||||||
|
first_stop.elapsed()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
terminal_count(&outputs),
|
||||||
|
1,
|
||||||
|
"unexpected terminal outputs: {outputs:?}"
|
||||||
|
);
|
||||||
|
assert_no_errors(&outputs);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn stop_after_terminal_output_emits_no_additional_output() {
|
||||||
|
let rt = Runtime::new(RuntimeConfig::default());
|
||||||
|
let sender = rt.create_sender();
|
||||||
|
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
|
||||||
|
let reply = rt.new_inbox::<SpawnReply>().unwrap();
|
||||||
|
let spawner = rt.spawn(SpawnerActor).unwrap();
|
||||||
|
|
||||||
|
send_spawn(
|
||||||
|
&rt,
|
||||||
|
spawner,
|
||||||
|
&sender,
|
||||||
|
shell_spec("sh", vec!["-c", "exit 0"], Some("post-terminal-stop")),
|
||||||
|
ProcessOutputConfig::disabled(*upstream.addr()),
|
||||||
|
&reply,
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut outputs = Vec::new();
|
||||||
|
let process_addr = expect_spawned(drive_until_spawn_reply(
|
||||||
|
&rt,
|
||||||
|
None,
|
||||||
|
&upstream,
|
||||||
|
&mut outputs,
|
||||||
|
&reply,
|
||||||
|
));
|
||||||
|
drive_until(&rt, None, &upstream, &mut outputs, |outputs| {
|
||||||
|
has_exited(outputs, ExitStatus::Code(0))
|
||||||
|
});
|
||||||
|
let output_len = outputs.len();
|
||||||
|
|
||||||
|
let _ = send_process_command(
|
||||||
|
&sender,
|
||||||
|
process_addr,
|
||||||
|
ProcessCommand::Stop {
|
||||||
|
kill_after: Some(Duration::from_millis(10)),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
for _ in 0..10 {
|
||||||
|
drive_once(&rt, None, &upstream, &mut outputs);
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
outputs.len(),
|
||||||
|
output_len,
|
||||||
|
"post-terminal stop should not emit more output: {outputs:?}"
|
||||||
|
);
|
||||||
|
assert_no_errors(&outputs);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn lifecycle_mirror_submit_failure_does_not_suppress_upstream_or_emit_error() {
|
||||||
|
let rt = Runtime::new(RuntimeConfig::default());
|
||||||
|
let sender = rt.create_sender();
|
||||||
|
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
|
||||||
|
let reply = rt.new_inbox::<SpawnReply>().unwrap();
|
||||||
|
let spawner = rt.spawn(SpawnerActor).unwrap();
|
||||||
|
let endpoint = DatastreamEndpoint::with_capacity(stage2_stream(), 1, 16);
|
||||||
|
|
||||||
|
send_spawn(
|
||||||
|
&rt,
|
||||||
|
spawner,
|
||||||
|
&sender,
|
||||||
|
shell_spec("sh", vec!["-c", "exit 0"], Some("mirror-drop")),
|
||||||
|
ProcessOutputConfig::datastream_mirror(*upstream.addr(), endpoint.producer()),
|
||||||
|
&reply,
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut outputs = Vec::new();
|
||||||
|
let _process_addr = expect_spawned(drive_until_spawn_reply(
|
||||||
|
&rt,
|
||||||
|
None,
|
||||||
|
&upstream,
|
||||||
|
&mut outputs,
|
||||||
|
&reply,
|
||||||
|
));
|
||||||
|
drive_until(&rt, None, &upstream, &mut outputs, |outputs| {
|
||||||
|
has_exited(outputs, ExitStatus::Code(0))
|
||||||
|
});
|
||||||
|
let dropped = endpoint.mux().dropped();
|
||||||
|
endpoint.tick();
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
has_started(&outputs),
|
||||||
|
"upstream should receive Started: {outputs:?}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
has_exited(&outputs, ExitStatus::Code(0)),
|
||||||
|
"upstream should receive Exited: {outputs:?}"
|
||||||
|
);
|
||||||
|
assert_no_errors(&outputs);
|
||||||
|
assert!(
|
||||||
|
dropped > 0,
|
||||||
|
"full lifecycle mirror mux should drop at least one frame"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn stdout_and_stderr_writes_do_not_affect_lifecycle() {
|
||||||
|
let rt = Runtime::new(RuntimeConfig::default());
|
||||||
|
let sender = rt.create_sender();
|
||||||
|
let upstream = rt.new_inbox::<ProcessOutput>().unwrap();
|
||||||
|
let reply = rt.new_inbox::<SpawnReply>().unwrap();
|
||||||
|
let spawner = rt.spawn(SpawnerActor).unwrap();
|
||||||
|
|
||||||
|
send_spawn(
|
||||||
|
&rt,
|
||||||
|
spawner,
|
||||||
|
&sender,
|
||||||
|
shell_spec(
|
||||||
|
"sh",
|
||||||
|
vec![
|
||||||
|
"-c",
|
||||||
|
"for i in $(seq 1 1000); do echo out; echo err >&2; done; exit 0",
|
||||||
|
],
|
||||||
|
Some("child-output-is-null"),
|
||||||
|
),
|
||||||
|
ProcessOutputConfig::disabled(*upstream.addr()),
|
||||||
|
&reply,
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut outputs = Vec::new();
|
||||||
|
let _process_addr = expect_spawned(drive_until_spawn_reply(
|
||||||
|
&rt,
|
||||||
|
None,
|
||||||
|
&upstream,
|
||||||
|
&mut outputs,
|
||||||
|
&reply,
|
||||||
|
));
|
||||||
|
drive_until(&rt, None, &upstream, &mut outputs, |outputs| {
|
||||||
|
has_exited(outputs, ExitStatus::Code(0))
|
||||||
|
});
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
outputs.len(),
|
||||||
|
2,
|
||||||
|
"process core should emit only lifecycle outputs: {outputs:?}"
|
||||||
|
);
|
||||||
|
assert!(started_index(&outputs) < exited_index(&outputs));
|
||||||
|
assert!(
|
||||||
|
has_exited(&outputs, ExitStatus::Code(0)),
|
||||||
|
"expected clean exit: {outputs:?}"
|
||||||
|
);
|
||||||
|
assert_no_spawn_failed(&outputs);
|
||||||
|
assert_no_errors(&outputs);
|
||||||
|
}
|
||||||
|
|
@ -1,787 +0,0 @@
|
||||||
use std::collections::HashMap;
|
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
use swactor::actor::ActorAddress;
|
|
||||||
use swactor_process::*;
|
|
||||||
|
|
||||||
fn automated_spec() -> ProcessSpec {
|
|
||||||
ProcessSpec {
|
|
||||||
command: "echo".into(),
|
|
||||||
args: vec!["hello".into()],
|
|
||||||
env: HashMap::new(),
|
|
||||||
working_dir: None,
|
|
||||||
mode: ProcessMode::Automated,
|
|
||||||
initial_pty_size: None,
|
|
||||||
kill_timeout: None,
|
|
||||||
stdin_buffer_limit: None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn spec_with_kill_timeout(timeout: Duration) -> ProcessSpec {
|
|
||||||
ProcessSpec {
|
|
||||||
kill_timeout: Some(timeout),
|
|
||||||
..automated_spec()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn spec_with_stdin_limit(limit: usize) -> ProcessSpec {
|
|
||||||
ProcessSpec {
|
|
||||||
stdin_buffer_limit: Some(limit),
|
|
||||||
..automated_spec()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn interactive_spec() -> ProcessSpec {
|
|
||||||
ProcessSpec {
|
|
||||||
command: "/bin/bash".into(),
|
|
||||||
args: vec![],
|
|
||||||
env: HashMap::new(),
|
|
||||||
working_dir: None,
|
|
||||||
mode: ProcessMode::Interactive,
|
|
||||||
initial_pty_size: Some(PtySize { cols: 80, rows: 24 }),
|
|
||||||
kill_timeout: None,
|
|
||||||
stdin_buffer_limit: None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn addr(n: u8) -> ActorAddress {
|
|
||||||
let mut bytes = [0u8; 32];
|
|
||||||
bytes[0] = n;
|
|
||||||
ActorAddress(bytes)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Verify that a SelfTerminate is present and is the last action.
|
|
||||||
fn assert_self_terminate_is_last(actions: &[ProcessAction]) {
|
|
||||||
assert!(
|
|
||||||
matches!(actions.last(), Some(ProcessAction::SelfTerminate)),
|
|
||||||
"SelfTerminate must be the last action, got: {actions:?}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
// 1. Happy path — automated process
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn automated_process_runs_produces_output_and_exits_cleanly() {
|
|
||||||
let (mut session, init) = ProcessSession::new(automated_spec());
|
|
||||||
assert_eq!(session.state(), ProcessState::Starting);
|
|
||||||
assert!(matches!(&init[0], ProcessAction::SpawnProcess { .. }));
|
|
||||||
|
|
||||||
// Process starts
|
|
||||||
let actions = session.apply(ProcessEvent::Started);
|
|
||||||
assert_eq!(session.state(), ProcessState::Running);
|
|
||||||
assert!(matches!(&actions[0], ProcessAction::NotifyStarted { .. }));
|
|
||||||
|
|
||||||
// Some output arrives
|
|
||||||
let actions = session.apply(ProcessEvent::OutputReceived {
|
|
||||||
data: b"hello\n".to_vec(),
|
|
||||||
is_stderr: false,
|
|
||||||
});
|
|
||||||
assert!(matches!(
|
|
||||||
&actions[0],
|
|
||||||
ProcessAction::NotifyOutput {
|
|
||||||
stream: OutputStream::Stdout,
|
|
||||||
..
|
|
||||||
}
|
|
||||||
));
|
|
||||||
|
|
||||||
// More output on stderr
|
|
||||||
let actions = session.apply(ProcessEvent::OutputReceived {
|
|
||||||
data: b"warn\n".to_vec(),
|
|
||||||
is_stderr: true,
|
|
||||||
});
|
|
||||||
assert!(matches!(
|
|
||||||
&actions[0],
|
|
||||||
ProcessAction::NotifyOutput {
|
|
||||||
stream: OutputStream::Stderr,
|
|
||||||
..
|
|
||||||
}
|
|
||||||
));
|
|
||||||
|
|
||||||
// Process exits
|
|
||||||
let actions = session.apply(ProcessEvent::Exited {
|
|
||||||
status: ExitStatus::Code(0),
|
|
||||||
});
|
|
||||||
assert_eq!(session.state(), ProcessState::Exited);
|
|
||||||
assert_eq!(session.exit_status(), Some(ExitStatus::Code(0)));
|
|
||||||
assert_self_terminate_is_last(&actions);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
// 2. Interactive process with subscriber lifecycle
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn interactive_session_manages_subscribers_correctly() {
|
|
||||||
let (mut session, _) = ProcessSession::new(interactive_spec());
|
|
||||||
|
|
||||||
// Add two subscribers before start
|
|
||||||
session.apply(ProcessEvent::Subscribe { address: addr(1) });
|
|
||||||
session.apply(ProcessEvent::Subscribe { address: addr(2) });
|
|
||||||
assert_eq!(session.subscriber_count(), 2);
|
|
||||||
|
|
||||||
// Duplicate add is a no-op
|
|
||||||
session.apply(ProcessEvent::Subscribe { address: addr(1) });
|
|
||||||
assert_eq!(session.subscriber_count(), 2);
|
|
||||||
|
|
||||||
// Start — both subscribers notified
|
|
||||||
let actions = session.apply(ProcessEvent::Started);
|
|
||||||
match &actions[0] {
|
|
||||||
ProcessAction::NotifyStarted { subscribers } => {
|
|
||||||
assert_eq!(subscribers.len(), 2);
|
|
||||||
}
|
|
||||||
other => panic!("expected NotifyStarted, got {other:?}"),
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove one subscriber
|
|
||||||
session.apply(ProcessEvent::Unsubscribe { address: addr(1) });
|
|
||||||
assert_eq!(session.subscriber_count(), 1);
|
|
||||||
|
|
||||||
// Output only goes to remaining subscriber
|
|
||||||
let actions = session.apply(ProcessEvent::OutputReceived {
|
|
||||||
data: b"data".to_vec(),
|
|
||||||
is_stderr: false,
|
|
||||||
});
|
|
||||||
match &actions[0] {
|
|
||||||
ProcessAction::NotifyOutput { subscribers, .. } => {
|
|
||||||
assert_eq!(subscribers, &vec![addr(2)]);
|
|
||||||
}
|
|
||||||
other => panic!("expected NotifyOutput, got {other:?}"),
|
|
||||||
}
|
|
||||||
|
|
||||||
// Exit
|
|
||||||
let actions = session.apply(ProcessEvent::Exited {
|
|
||||||
status: ExitStatus::Code(0),
|
|
||||||
});
|
|
||||||
match &actions[0] {
|
|
||||||
ProcessAction::NotifyExited { subscribers, .. } => {
|
|
||||||
assert_eq!(subscribers, &vec![addr(2)]);
|
|
||||||
}
|
|
||||||
other => panic!("expected NotifyExited, got {other:?}"),
|
|
||||||
}
|
|
||||||
assert_self_terminate_is_last(&actions);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
// 3. Spawn failure
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn spawn_failure_notifies_and_self_terminates() {
|
|
||||||
let (mut session, _) = ProcessSession::new(automated_spec());
|
|
||||||
session.apply(ProcessEvent::Subscribe { address: addr(1) });
|
|
||||||
|
|
||||||
let actions = session.apply(ProcessEvent::SpawnFailed {
|
|
||||||
reason: "command not found".into(),
|
|
||||||
});
|
|
||||||
assert_eq!(session.state(), ProcessState::Exited);
|
|
||||||
assert!(matches!(
|
|
||||||
&actions[0],
|
|
||||||
ProcessAction::NotifyError {
|
|
||||||
error: ProcessError::SpawnFailed { .. },
|
|
||||||
..
|
|
||||||
}
|
|
||||||
));
|
|
||||||
assert_self_terminate_is_last(&actions);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
// 4. Connection loss mid-run
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn connection_loss_during_running_transitions_to_exited() {
|
|
||||||
let (mut session, _) = ProcessSession::new(automated_spec());
|
|
||||||
session.apply(ProcessEvent::Started);
|
|
||||||
|
|
||||||
let actions = session.apply(ProcessEvent::ConnectionLost {
|
|
||||||
reason: "pipe broken".into(),
|
|
||||||
});
|
|
||||||
assert_eq!(session.state(), ProcessState::Exited);
|
|
||||||
assert_eq!(session.exit_status(), Some(ExitStatus::Unknown));
|
|
||||||
assert!(matches!(
|
|
||||||
&actions[0],
|
|
||||||
ProcessAction::NotifyError {
|
|
||||||
error: ProcessError::ConnectionLost { .. },
|
|
||||||
..
|
|
||||||
}
|
|
||||||
));
|
|
||||||
assert_self_terminate_is_last(&actions);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
// 5. Close requested before start
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn close_before_start_sends_signal_on_belated_start() {
|
|
||||||
let (mut session, _) = ProcessSession::new(automated_spec());
|
|
||||||
|
|
||||||
// Close requested while still Starting
|
|
||||||
let actions = session.apply(ProcessEvent::CloseRequested);
|
|
||||||
assert!(actions.is_empty());
|
|
||||||
assert_eq!(session.state(), ProcessState::Starting);
|
|
||||||
|
|
||||||
// Process starts belatedly — should immediately get SIGTERM
|
|
||||||
let actions = session.apply(ProcessEvent::Started);
|
|
||||||
assert_eq!(session.state(), ProcessState::Stopping);
|
|
||||||
assert!(matches!(&actions[0], ProcessAction::NotifyStarted { .. }));
|
|
||||||
assert!(matches!(
|
|
||||||
&actions[1],
|
|
||||||
ProcessAction::SendSignal {
|
|
||||||
signal: Signal::Terminate
|
|
||||||
}
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
// 6. Invalid operations produce errors, not panics
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn invalid_event_in_starting_produces_error() {
|
|
||||||
let (mut session, _) = ProcessSession::new(automated_spec());
|
|
||||||
|
|
||||||
let actions = session.apply(ProcessEvent::WriteStdin {
|
|
||||||
data: b"hi".to_vec(),
|
|
||||||
});
|
|
||||||
assert!(matches!(
|
|
||||||
&actions[0],
|
|
||||||
ProcessAction::NotifyError {
|
|
||||||
error: ProcessError::InvalidState {
|
|
||||||
attempted: "WriteStdin",
|
|
||||||
current_state: "Starting"
|
|
||||||
},
|
|
||||||
..
|
|
||||||
}
|
|
||||||
));
|
|
||||||
// State unchanged
|
|
||||||
assert_eq!(session.state(), ProcessState::Starting);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn invalid_event_in_exited_produces_error() {
|
|
||||||
let (mut session, _) = ProcessSession::new(automated_spec());
|
|
||||||
session.apply(ProcessEvent::SpawnFailed {
|
|
||||||
reason: "no".into(),
|
|
||||||
});
|
|
||||||
assert_eq!(session.state(), ProcessState::Exited);
|
|
||||||
|
|
||||||
let actions = session.apply(ProcessEvent::WriteStdin {
|
|
||||||
data: b"hi".to_vec(),
|
|
||||||
});
|
|
||||||
assert!(matches!(
|
|
||||||
&actions[0],
|
|
||||||
ProcessAction::NotifyError {
|
|
||||||
error: ProcessError::InvalidState {
|
|
||||||
attempted: "WriteStdin",
|
|
||||||
current_state: "Exited"
|
|
||||||
},
|
|
||||||
..
|
|
||||||
}
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
// 7. Stdin closed then write → error
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn write_after_stdin_closed_produces_error() {
|
|
||||||
let (mut session, _) = ProcessSession::new(automated_spec());
|
|
||||||
session.apply(ProcessEvent::Started);
|
|
||||||
|
|
||||||
let actions = session.apply(ProcessEvent::CloseStdin);
|
|
||||||
assert!(matches!(&actions[0], ProcessAction::CloseStdin));
|
|
||||||
assert!(session.stdin_closed());
|
|
||||||
|
|
||||||
// Duplicate close is a no-op
|
|
||||||
let actions = session.apply(ProcessEvent::CloseStdin);
|
|
||||||
assert!(actions.is_empty());
|
|
||||||
|
|
||||||
// Write after close → error
|
|
||||||
let actions = session.apply(ProcessEvent::WriteStdin {
|
|
||||||
data: b"too late".to_vec(),
|
|
||||||
});
|
|
||||||
assert!(matches!(
|
|
||||||
&actions[0],
|
|
||||||
ProcessAction::NotifyError {
|
|
||||||
error: ProcessError::InvalidState { .. },
|
|
||||||
..
|
|
||||||
}
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
// 8. MockDriver round-trip (driver + session tick loop)
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn mock_driver_round_trip() {
|
|
||||||
let (mut session, init_actions) = ProcessSession::new(automated_spec());
|
|
||||||
let mut driver = MockDriver::new();
|
|
||||||
|
|
||||||
// Execute initial actions (SpawnProcess)
|
|
||||||
for action in init_actions {
|
|
||||||
driver.execute(action);
|
|
||||||
}
|
|
||||||
assert!(matches!(
|
|
||||||
&driver.executed_actions()[0],
|
|
||||||
ProcessAction::SpawnProcess { .. }
|
|
||||||
));
|
|
||||||
|
|
||||||
// Simulate: driver produces Started
|
|
||||||
driver.inject(ProcessEvent::Started);
|
|
||||||
|
|
||||||
// Tick loop: poll → apply → execute
|
|
||||||
let events = driver.poll();
|
|
||||||
for event in events {
|
|
||||||
let actions = session.apply(event);
|
|
||||||
for action in actions {
|
|
||||||
driver.execute(action);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
assert_eq!(session.state(), ProcessState::Running);
|
|
||||||
|
|
||||||
// Simulate output and exit
|
|
||||||
driver.inject(ProcessEvent::OutputReceived {
|
|
||||||
data: b"done".to_vec(),
|
|
||||||
is_stderr: false,
|
|
||||||
});
|
|
||||||
driver.inject(ProcessEvent::Exited {
|
|
||||||
status: ExitStatus::Code(0),
|
|
||||||
});
|
|
||||||
|
|
||||||
let events = driver.poll();
|
|
||||||
for event in events {
|
|
||||||
let actions = session.apply(event);
|
|
||||||
for action in actions {
|
|
||||||
driver.execute(action);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
assert_eq!(session.state(), ProcessState::Exited);
|
|
||||||
|
|
||||||
// Verify the driver saw the expected sequence
|
|
||||||
let all_actions = driver.take_executed_actions();
|
|
||||||
assert!(matches!(
|
|
||||||
&all_actions[0],
|
|
||||||
ProcessAction::SpawnProcess { .. }
|
|
||||||
));
|
|
||||||
assert!(matches!(
|
|
||||||
&all_actions[1],
|
|
||||||
ProcessAction::NotifyStarted { .. }
|
|
||||||
));
|
|
||||||
assert!(matches!(
|
|
||||||
&all_actions[2],
|
|
||||||
ProcessAction::NotifyOutput { .. }
|
|
||||||
));
|
|
||||||
assert!(matches!(
|
|
||||||
&all_actions[3],
|
|
||||||
ProcessAction::NotifyExited { .. }
|
|
||||||
));
|
|
||||||
assert!(matches!(&all_actions[4], ProcessAction::SelfTerminate));
|
|
||||||
}
|
|
||||||
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
// 9. Signal escalation in Stopping
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn signal_escalation_allowed_in_stopping() {
|
|
||||||
let (mut session, _) = ProcessSession::new(automated_spec());
|
|
||||||
session.apply(ProcessEvent::Started);
|
|
||||||
session.apply(ProcessEvent::CloseRequested);
|
|
||||||
assert_eq!(session.state(), ProcessState::Stopping);
|
|
||||||
|
|
||||||
// Escalate to Kill
|
|
||||||
let actions = session.apply(ProcessEvent::SendSignal {
|
|
||||||
signal: Signal::Kill,
|
|
||||||
});
|
|
||||||
assert!(matches!(
|
|
||||||
&actions[0],
|
|
||||||
ProcessAction::SendSignal {
|
|
||||||
signal: Signal::Kill
|
|
||||||
}
|
|
||||||
));
|
|
||||||
|
|
||||||
// Can still receive output while stopping
|
|
||||||
let actions = session.apply(ProcessEvent::OutputReceived {
|
|
||||||
data: b"final".to_vec(),
|
|
||||||
is_stderr: false,
|
|
||||||
});
|
|
||||||
assert!(matches!(&actions[0], ProcessAction::NotifyOutput { .. }));
|
|
||||||
|
|
||||||
// Finally exits
|
|
||||||
let actions = session.apply(ProcessEvent::Exited {
|
|
||||||
status: ExitStatus::Signal(9),
|
|
||||||
});
|
|
||||||
assert_eq!(session.exit_status(), Some(ExitStatus::Signal(9)));
|
|
||||||
assert_self_terminate_is_last(&actions);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
// 10. Late acks in Exited silently consumed
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn late_acks_in_exited_are_silently_consumed() {
|
|
||||||
let (mut session, _) = ProcessSession::new(automated_spec());
|
|
||||||
session.apply(ProcessEvent::Started);
|
|
||||||
session.apply(ProcessEvent::Exited {
|
|
||||||
status: ExitStatus::Code(0),
|
|
||||||
});
|
|
||||||
assert_eq!(session.state(), ProcessState::Exited);
|
|
||||||
|
|
||||||
// Acks should produce no actions, no errors
|
|
||||||
assert!(
|
|
||||||
session
|
|
||||||
.apply(ProcessEvent::StdinWritten { byte_count: 10 })
|
|
||||||
.is_empty()
|
|
||||||
);
|
|
||||||
assert!(session.apply(ProcessEvent::SignalSent).is_empty());
|
|
||||||
assert!(session.apply(ProcessEvent::PtyResized).is_empty());
|
|
||||||
|
|
||||||
// Subscribe/Unsubscribe also still works in Exited
|
|
||||||
assert!(
|
|
||||||
session
|
|
||||||
.apply(ProcessEvent::Subscribe { address: addr(1) })
|
|
||||||
.is_empty()
|
|
||||||
);
|
|
||||||
assert_eq!(session.subscriber_count(), 1);
|
|
||||||
assert!(
|
|
||||||
session
|
|
||||||
.apply(ProcessEvent::Unsubscribe { address: addr(1) })
|
|
||||||
.is_empty()
|
|
||||||
);
|
|
||||||
assert_eq!(session.subscriber_count(), 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
// Flow control tracking
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn flow_control_tracks_pending_stdin_bytes() {
|
|
||||||
let (mut session, _) = ProcessSession::new(automated_spec());
|
|
||||||
session.apply(ProcessEvent::Started);
|
|
||||||
|
|
||||||
session.apply(ProcessEvent::WriteStdin {
|
|
||||||
data: vec![0u8; 100],
|
|
||||||
});
|
|
||||||
assert_eq!(session.flow_control().pending_stdin_bytes, 100);
|
|
||||||
|
|
||||||
session.apply(ProcessEvent::WriteStdin {
|
|
||||||
data: vec![0u8; 50],
|
|
||||||
});
|
|
||||||
assert_eq!(session.flow_control().pending_stdin_bytes, 150);
|
|
||||||
|
|
||||||
session.apply(ProcessEvent::StdinWritten { byte_count: 80 });
|
|
||||||
assert_eq!(session.flow_control().pending_stdin_bytes, 70);
|
|
||||||
|
|
||||||
// Ack more than pending → saturates at 0
|
|
||||||
session.apply(ProcessEvent::StdinWritten { byte_count: 200 });
|
|
||||||
assert_eq!(session.flow_control().pending_stdin_bytes, 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
// CloseStdin in Stopping
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn close_stdin_allowed_in_stopping() {
|
|
||||||
let (mut session, _) = ProcessSession::new(automated_spec());
|
|
||||||
session.apply(ProcessEvent::Started);
|
|
||||||
session.apply(ProcessEvent::CloseRequested);
|
|
||||||
assert_eq!(session.state(), ProcessState::Stopping);
|
|
||||||
|
|
||||||
let actions = session.apply(ProcessEvent::CloseStdin);
|
|
||||||
assert!(matches!(&actions[0], ProcessAction::CloseStdin));
|
|
||||||
assert!(session.stdin_closed());
|
|
||||||
}
|
|
||||||
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
// Connection loss in Stopping
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn connection_loss_in_stopping_transitions_to_exited() {
|
|
||||||
let (mut session, _) = ProcessSession::new(automated_spec());
|
|
||||||
session.apply(ProcessEvent::Started);
|
|
||||||
session.apply(ProcessEvent::CloseRequested);
|
|
||||||
assert_eq!(session.state(), ProcessState::Stopping);
|
|
||||||
|
|
||||||
let actions = session.apply(ProcessEvent::ConnectionLost {
|
|
||||||
reason: "gone".into(),
|
|
||||||
});
|
|
||||||
assert_eq!(session.state(), ProcessState::Exited);
|
|
||||||
assert_self_terminate_is_last(&actions);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
// Redundant CloseRequested in Stopping is no-op
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn duplicate_close_requested_in_stopping_is_noop() {
|
|
||||||
let (mut session, _) = ProcessSession::new(automated_spec());
|
|
||||||
session.apply(ProcessEvent::Started);
|
|
||||||
session.apply(ProcessEvent::CloseRequested);
|
|
||||||
assert_eq!(session.state(), ProcessState::Stopping);
|
|
||||||
|
|
||||||
let actions = session.apply(ProcessEvent::CloseRequested);
|
|
||||||
assert!(actions.is_empty());
|
|
||||||
assert_eq!(session.state(), ProcessState::Stopping);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
// Kill timeout — A1–A6
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn close_requested_with_kill_timeout_schedules_timer() {
|
|
||||||
let (mut session, _) = ProcessSession::new(spec_with_kill_timeout(Duration::from_secs(5)));
|
|
||||||
session.apply(ProcessEvent::Started);
|
|
||||||
|
|
||||||
let actions = session.apply(ProcessEvent::CloseRequested);
|
|
||||||
assert_eq!(session.state(), ProcessState::Stopping);
|
|
||||||
assert!(matches!(
|
|
||||||
&actions[0],
|
|
||||||
ProcessAction::SendSignal {
|
|
||||||
signal: Signal::Terminate
|
|
||||||
}
|
|
||||||
));
|
|
||||||
assert!(matches!(
|
|
||||||
&actions[1],
|
|
||||||
ProcessAction::ScheduleKillTimeout { duration } if *duration == Duration::from_secs(5)
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn close_before_start_with_kill_timeout_schedules_timer_on_belated_start() {
|
|
||||||
let (mut session, _) = ProcessSession::new(spec_with_kill_timeout(Duration::from_secs(3)));
|
|
||||||
session.apply(ProcessEvent::CloseRequested);
|
|
||||||
|
|
||||||
let actions = session.apply(ProcessEvent::Started);
|
|
||||||
assert_eq!(session.state(), ProcessState::Stopping);
|
|
||||||
assert!(matches!(&actions[0], ProcessAction::NotifyStarted { .. }));
|
|
||||||
assert!(matches!(
|
|
||||||
&actions[1],
|
|
||||||
ProcessAction::SendSignal {
|
|
||||||
signal: Signal::Terminate
|
|
||||||
}
|
|
||||||
));
|
|
||||||
assert!(matches!(
|
|
||||||
&actions[2],
|
|
||||||
ProcessAction::ScheduleKillTimeout { duration } if *duration == Duration::from_secs(3)
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn kill_timeout_in_stopping_sends_sigkill() {
|
|
||||||
let (mut session, _) = ProcessSession::new(spec_with_kill_timeout(Duration::from_secs(5)));
|
|
||||||
session.apply(ProcessEvent::Started);
|
|
||||||
session.apply(ProcessEvent::CloseRequested);
|
|
||||||
assert_eq!(session.state(), ProcessState::Stopping);
|
|
||||||
|
|
||||||
let actions = session.apply(ProcessEvent::KillTimeout);
|
|
||||||
assert!(matches!(
|
|
||||||
&actions[0],
|
|
||||||
ProcessAction::SendSignal {
|
|
||||||
signal: Signal::Kill
|
|
||||||
}
|
|
||||||
));
|
|
||||||
assert_eq!(session.state(), ProcessState::Stopping);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn kill_timeout_silently_consumed_outside_stopping() {
|
|
||||||
// Starting
|
|
||||||
let (mut session, _) = ProcessSession::new(automated_spec());
|
|
||||||
assert!(session.apply(ProcessEvent::KillTimeout).is_empty());
|
|
||||||
assert_eq!(session.state(), ProcessState::Starting);
|
|
||||||
|
|
||||||
// Running
|
|
||||||
session.apply(ProcessEvent::Started);
|
|
||||||
assert!(session.apply(ProcessEvent::KillTimeout).is_empty());
|
|
||||||
assert_eq!(session.state(), ProcessState::Running);
|
|
||||||
|
|
||||||
// Exited
|
|
||||||
session.apply(ProcessEvent::Exited {
|
|
||||||
status: ExitStatus::Code(0),
|
|
||||||
});
|
|
||||||
assert!(session.apply(ProcessEvent::KillTimeout).is_empty());
|
|
||||||
assert_eq!(session.state(), ProcessState::Exited);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn close_requested_without_kill_timeout_no_schedule_action() {
|
|
||||||
let (mut session, _) = ProcessSession::new(automated_spec());
|
|
||||||
session.apply(ProcessEvent::Started);
|
|
||||||
|
|
||||||
let actions = session.apply(ProcessEvent::CloseRequested);
|
|
||||||
assert_eq!(actions.len(), 1);
|
|
||||||
assert!(matches!(
|
|
||||||
&actions[0],
|
|
||||||
ProcessAction::SendSignal {
|
|
||||||
signal: Signal::Terminate
|
|
||||||
}
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn kill_timeout_full_escalation_to_sigkill_then_exit() {
|
|
||||||
let (mut session, _) = ProcessSession::new(spec_with_kill_timeout(Duration::from_secs(1)));
|
|
||||||
session.apply(ProcessEvent::Started);
|
|
||||||
|
|
||||||
// CloseRequested → SIGTERM + schedule
|
|
||||||
let actions = session.apply(ProcessEvent::CloseRequested);
|
|
||||||
assert_eq!(session.state(), ProcessState::Stopping);
|
|
||||||
assert!(matches!(
|
|
||||||
&actions[0],
|
|
||||||
ProcessAction::SendSignal {
|
|
||||||
signal: Signal::Terminate
|
|
||||||
}
|
|
||||||
));
|
|
||||||
assert!(matches!(
|
|
||||||
&actions[1],
|
|
||||||
ProcessAction::ScheduleKillTimeout { .. }
|
|
||||||
));
|
|
||||||
|
|
||||||
// KillTimeout fires → SIGKILL
|
|
||||||
let actions = session.apply(ProcessEvent::KillTimeout);
|
|
||||||
assert!(matches!(
|
|
||||||
&actions[0],
|
|
||||||
ProcessAction::SendSignal {
|
|
||||||
signal: Signal::Kill
|
|
||||||
}
|
|
||||||
));
|
|
||||||
|
|
||||||
// Process finally exits via signal 9
|
|
||||||
let actions = session.apply(ProcessEvent::Exited {
|
|
||||||
status: ExitStatus::Signal(9),
|
|
||||||
});
|
|
||||||
assert_eq!(session.state(), ProcessState::Exited);
|
|
||||||
assert_eq!(session.exit_status(), Some(ExitStatus::Signal(9)));
|
|
||||||
assert_self_terminate_is_last(&actions);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
// Backpressure — B1–B5
|
|
||||||
// ──────────────────────────────────────────────
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn backpressure_buffers_when_over_limit() {
|
|
||||||
let (mut session, _) = ProcessSession::new(spec_with_stdin_limit(100));
|
|
||||||
session.apply(ProcessEvent::Started);
|
|
||||||
|
|
||||||
// First write (50 bytes) — under limit, passes through
|
|
||||||
let actions = session.apply(ProcessEvent::WriteStdin {
|
|
||||||
data: vec![1u8; 50],
|
|
||||||
});
|
|
||||||
assert_eq!(actions.len(), 1);
|
|
||||||
assert!(matches!(&actions[0], ProcessAction::WriteStdin { .. }));
|
|
||||||
assert_eq!(session.flow_control().pending_stdin_bytes, 50);
|
|
||||||
|
|
||||||
// Second write (60 bytes) — still under limit (50 < 100), passes through
|
|
||||||
let actions = session.apply(ProcessEvent::WriteStdin {
|
|
||||||
data: vec![2u8; 60],
|
|
||||||
});
|
|
||||||
assert_eq!(actions.len(), 1);
|
|
||||||
assert_eq!(session.flow_control().pending_stdin_bytes, 110);
|
|
||||||
|
|
||||||
// Third write (30 bytes) — now at 110 >= 100, buffered
|
|
||||||
let actions = session.apply(ProcessEvent::WriteStdin {
|
|
||||||
data: vec![3u8; 30],
|
|
||||||
});
|
|
||||||
assert!(actions.is_empty());
|
|
||||||
assert_eq!(session.stdin_buffer_bytes(), 30);
|
|
||||||
// pending_stdin_bytes unchanged (buffered data not counted as pending)
|
|
||||||
assert_eq!(session.flow_control().pending_stdin_bytes, 110);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn stdin_written_ack_drains_buffer() {
|
|
||||||
let (mut session, _) = ProcessSession::new(spec_with_stdin_limit(100));
|
|
||||||
session.apply(ProcessEvent::Started);
|
|
||||||
|
|
||||||
// Fill up: 100 bytes pending
|
|
||||||
session.apply(ProcessEvent::WriteStdin {
|
|
||||||
data: vec![1u8; 100],
|
|
||||||
});
|
|
||||||
assert_eq!(session.flow_control().pending_stdin_bytes, 100);
|
|
||||||
|
|
||||||
// Buffer two chunks
|
|
||||||
session.apply(ProcessEvent::WriteStdin {
|
|
||||||
data: vec![2u8; 40],
|
|
||||||
});
|
|
||||||
session.apply(ProcessEvent::WriteStdin {
|
|
||||||
data: vec![3u8; 30],
|
|
||||||
});
|
|
||||||
assert_eq!(session.stdin_buffer_bytes(), 70);
|
|
||||||
|
|
||||||
// Ack 80 bytes → pending drops to 20, buffer should drain in FIFO order
|
|
||||||
let actions = session.apply(ProcessEvent::StdinWritten { byte_count: 80 });
|
|
||||||
// pending was 100, now 20. Drain first chunk (40 bytes) → pending = 60.
|
|
||||||
// 60 < 100, drain second chunk (30 bytes) → pending = 90.
|
|
||||||
// 90 < 100, buffer empty.
|
|
||||||
assert_eq!(actions.len(), 2);
|
|
||||||
assert!(matches!(&actions[0], ProcessAction::WriteStdin { data } if data.len() == 40));
|
|
||||||
assert!(matches!(&actions[1], ProcessAction::WriteStdin { data } if data.len() == 30));
|
|
||||||
assert_eq!(session.flow_control().pending_stdin_bytes, 90);
|
|
||||||
assert_eq!(session.stdin_buffer_bytes(), 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn close_requested_clears_stdin_buffer() {
|
|
||||||
let (mut session, _) = ProcessSession::new(spec_with_stdin_limit(50));
|
|
||||||
session.apply(ProcessEvent::Started);
|
|
||||||
|
|
||||||
session.apply(ProcessEvent::WriteStdin {
|
|
||||||
data: vec![1u8; 60],
|
|
||||||
});
|
|
||||||
session.apply(ProcessEvent::WriteStdin {
|
|
||||||
data: vec![2u8; 30],
|
|
||||||
});
|
|
||||||
assert_eq!(session.stdin_buffer_bytes(), 30);
|
|
||||||
|
|
||||||
session.apply(ProcessEvent::CloseRequested);
|
|
||||||
assert_eq!(session.stdin_buffer_bytes(), 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn no_backpressure_when_limit_is_none() {
|
|
||||||
let (mut session, _) = ProcessSession::new(automated_spec());
|
|
||||||
session.apply(ProcessEvent::Started);
|
|
||||||
|
|
||||||
// All writes pass through regardless of pending bytes
|
|
||||||
for _ in 0..10 {
|
|
||||||
let actions = session.apply(ProcessEvent::WriteStdin {
|
|
||||||
data: vec![0u8; 1000],
|
|
||||||
});
|
|
||||||
assert_eq!(actions.len(), 1);
|
|
||||||
assert!(matches!(&actions[0], ProcessAction::WriteStdin { .. }));
|
|
||||||
}
|
|
||||||
assert_eq!(session.flow_control().pending_stdin_bytes, 10_000);
|
|
||||||
assert_eq!(session.stdin_buffer_bytes(), 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn exit_clears_stdin_buffer() {
|
|
||||||
let (mut session, _) = ProcessSession::new(spec_with_stdin_limit(50));
|
|
||||||
session.apply(ProcessEvent::Started);
|
|
||||||
|
|
||||||
session.apply(ProcessEvent::WriteStdin {
|
|
||||||
data: vec![1u8; 60],
|
|
||||||
});
|
|
||||||
session.apply(ProcessEvent::WriteStdin {
|
|
||||||
data: vec![2u8; 30],
|
|
||||||
});
|
|
||||||
assert_eq!(session.stdin_buffer_bytes(), 30);
|
|
||||||
|
|
||||||
session.apply(ProcessEvent::Exited {
|
|
||||||
status: ExitStatus::Code(0),
|
|
||||||
});
|
|
||||||
assert_eq!(session.stdin_buffer_bytes(), 0);
|
|
||||||
}
|
|
||||||
|
|
@ -681,14 +681,6 @@ impl<'a> Ctx<'a> {
|
||||||
self.inner.extension()
|
self.inner.extension()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Access the per-runtime process-output observer (if installed). The
|
|
||||||
/// process facility reads this when spawning so every managed process's
|
|
||||||
/// output is taped automatically.
|
|
||||||
pub fn process_output_observer(
|
|
||||||
&self,
|
|
||||||
) -> Option<std::sync::Arc<dyn crate::process_observer::ProcessOutputObserver>> {
|
|
||||||
self.inner.process_output_observer()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Return system-level information (worker count, actor count, uptime).
|
/// Return system-level information (worker count, actor count, uptime).
|
||||||
pub fn system_info(&self) -> SystemInfo {
|
pub fn system_info(&self) -> SystemInfo {
|
||||||
|
|
|
||||||
|
|
@ -1,20 +1,16 @@
|
||||||
//! Per-runtime hook for observing output from processes spawned through the
|
//! Legacy/custom hook for adapters that choose to observe stdout/stderr bytes.
|
||||||
//! process facility.
|
|
||||||
//!
|
//!
|
||||||
//! This trait lives in swactor rather than `swactor-process` so the runtime can
|
//! Current `swactor-process` managed-process core does not call this hook.
|
||||||
//! store an observer without a dependency cycle: `swactor-process` depends on
|
//! Managed-process lifecycle/control output is delivered as `ProcessOutput` to
|
||||||
//! swactor, and a telemetry emitter (which lives even higher up) implements this
|
//! the configured upstream owner and may be mirrored to datastream through
|
||||||
//! trait. The runtime is merely the carrier — it never interprets the bytes.
|
//! `ProcessOutputConfig::datastream_mirror`.
|
||||||
//!
|
//!
|
||||||
//! The observer is per-node (per-runtime), installed via
|
//! The runtime is only the storage location for callers that still wire this
|
||||||
//! [`Runtime::set_process_output_observer`](crate::runtime::Runtime::set_process_output_observer).
|
//! hook themselves; it does not interpret the bytes.
|
||||||
//! The process facility hands every managed process's output to it automatically,
|
|
||||||
//! labeled by the process's command basename, so a node taps all of its managed
|
|
||||||
//! processes with no per-spawn wiring.
|
|
||||||
|
|
||||||
/// Observes every chunk of stdout/stderr from processes the runtime spawns.
|
/// Observes chunks supplied by legacy/custom process-output adapters.
|
||||||
pub trait ProcessOutputObserver: Send + Sync {
|
pub trait ProcessOutputObserver: Send + Sync {
|
||||||
/// `label` identifies the process (its command basename); `is_stderr`
|
/// `label` identifies the adapter-defined source; `is_stderr` selects the
|
||||||
/// selects the stream; `data` is one raw output chunk.
|
/// stream; `data` is one raw output chunk.
|
||||||
fn on_output(&self, label: &str, is_stderr: bool, data: &[u8]);
|
fn on_output(&self, label: &str, is_stderr: bool, data: &[u8]);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -598,17 +598,6 @@ impl Runtime {
|
||||||
self.stats_hook = Some(hook);
|
self.stats_hook = Some(hook);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Install the per-node process-output observer. Every process spawned
|
|
||||||
/// through the process facility on this runtime hands its stdout/stderr to
|
|
||||||
/// `obs`, labeled by command basename. Takes `&self` (the slot is a
|
|
||||||
/// `OnceLock`) so it can be installed on an already-shared `Arc<Runtime>`,
|
|
||||||
/// before the first managed process is spawned. Subsequent calls are no-ops.
|
|
||||||
pub fn set_process_output_observer(
|
|
||||||
&self,
|
|
||||||
obs: Arc<dyn crate::process_observer::ProcessOutputObserver>,
|
|
||||||
) {
|
|
||||||
let _ = self.process_output_observer.set(obs);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Set the sink for non-local (remote) message delivery.
|
/// Set the sink for non-local (remote) message delivery.
|
||||||
///
|
///
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue