diff --git a/Cargo.lock b/Cargo.lock index 47912b1..d1e1c1e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -937,6 +937,7 @@ dependencies = [ name = "datastream" version = "0.1.0" dependencies = [ + "crossbeam-channel", "iroh", "libc", "serde", @@ -2104,6 +2105,7 @@ dependencies = [ name = "iroh-driver" version = "0.1.0" dependencies = [ + "crossbeam-channel", "datastream", "distribution", "iroh", @@ -4342,6 +4344,7 @@ name = "swactor-process" version = "0.1.0" dependencies = [ "crossbeam-queue", + "datastream", "libc", "proptest", "proptest-state-machine", diff --git a/crates/dashboard/src/lib.rs b/crates/dashboard/src/lib.rs index 6d9ecd5..bd5a37f 100644 --- a/crates/dashboard/src/lib.rs +++ b/crates/dashboard/src/lib.rs @@ -8,7 +8,6 @@ pub mod view; use std::sync::Arc; use datastream::frame::{ChannelId, Frame, Lifetime, NodeId, Position, StreamId}; -use parking_lot::Mutex; use serde::Serialize; use tokio::sync::broadcast; @@ -86,10 +85,32 @@ pub struct DashboardHandle { store: Arc, views: Arc, shutdown_notify: Arc, - standalone_rt: Mutex>, } impl DashboardHandle { + /// Create the datastream dashboard state. + /// + /// The HTTP server is not started until `start_http` or `start_http_standalone` + /// is called. + pub fn new(config: DashboardConfig) -> Self { + let views = Arc::new(ViewRegistry::new()); + views.register(Arc::new(live_explorer::LiveDatastreamExplorer::default())); + views.register(Arc::new(hardware_view::HardwareDashboardView::default())); + views.register(swactor::worker_view()); + let store = Arc::new(DashboardStore::new( + config.raw_frame_history, + Arc::clone(&views), + )); + let (frames, _) = broadcast::channel(config.frame_buffer.max(1)); + Self { + port: config.port, + frames, + store, + views, + shutdown_notify: Arc::new(tokio::sync::Notify::new()), + } + } + /// Register a read-only view. External crates can keep their interpretation /// code beside their component and plug it into this registry. pub fn register_view(&self, view: Arc) { @@ -113,8 +134,8 @@ impl DashboardHandle { self.shutdown_notify.notify_waiters(); } - /// Start the HTTP server on an existing Tokio runtime. - pub fn start_http(&self, handle: tokio::runtime::Handle) { + /// Build the HTTP server future for an embedding runtime to poll directly. + pub fn http_server(&self) -> impl Future + Send + 'static { let state = server::AppState { frames: self.frames.clone(), store: Arc::clone(&self.store), @@ -122,44 +143,13 @@ impl DashboardHandle { shutdown_notify: Arc::clone(&self.shutdown_notify), }; let port = self.port; - handle.spawn(async move { + async move { server::run_server(state, port).await; - }); + } } - /// Start the HTTP server on a standalone Tokio runtime. - pub fn start_http_standalone(&self) { - let rt = tokio::runtime::Builder::new_multi_thread() - .worker_threads(1) - .enable_all() - .build() - .expect("failed to create tokio runtime for dashboard HTTP"); - let handle = rt.handle().clone(); - *self.standalone_rt.lock() = Some(rt); - self.start_http(handle); - } -} - -/// Create the datastream dashboard state. -/// -/// The HTTP server is not started until `start_http` or `start_http_standalone` -/// is called. -pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle { - let views = Arc::new(ViewRegistry::new()); - views.register(Arc::new(live_explorer::LiveDatastreamExplorer::default())); - views.register(Arc::new(hardware_view::HardwareDashboardView::default())); - views.register(swactor::worker_view()); - let store = Arc::new(DashboardStore::new( - config.raw_frame_history, - Arc::clone(&views), - )); - let (frames, _) = broadcast::channel(config.frame_buffer.max(1)); - DashboardHandle { - port: config.port, - frames, - store, - views, - shutdown_notify: Arc::new(tokio::sync::Notify::new()), - standalone_rt: Mutex::new(None), + /// Spawn the HTTP server on an existing Tokio runtime and return its task handle. + pub fn spawn_http(&self, handle: &tokio::runtime::Handle) -> tokio::task::JoinHandle<()> { + handle.spawn(self.http_server()) } } diff --git a/crates/datastream/Cargo.toml b/crates/datastream/Cargo.toml index 06198e8..c6510c1 100644 --- a/crates/datastream/Cargo.toml +++ b/crates/datastream/Cargo.toml @@ -9,6 +9,7 @@ swactor-transport = { path = "../transport" } serde = { version = "1", features = ["derive"] } serde_json = "1" iroh = "0.98" +crossbeam-channel = "0.5" [target.'cfg(target_os = "linux")'.dependencies] libc = "0.2" diff --git a/crates/datastream/DATASTREAM_SPEC.md b/crates/datastream/DATASTREAM_SPEC.md index 3e1b9c2..35266b8 100644 --- a/crates/datastream/DATASTREAM_SPEC.md +++ b/crates/datastream/DATASTREAM_SPEC.md @@ -1,462 +1,738 @@ # The Datastream — Specification ->>**STALE** Stale and not the final advisor on current or future state of the datastream. - -> **Status.** This is the canonical specification for the `datastream` crate. -> It supersedes the implicit spec the old `crates/distribution/src/datastream` -> doc-comments referred to (`DATASTREAM_SPEC.md §4.1`, etc.). Those section -> numbers are not preserved; updating the stale `(spec §…)` references is part -> of the migration this document drives. +> **Status.** Current-state specification for the `datastream` crate. This keeps +> the original outline, but updates the model to the implementation that now +> exists: stream-local numeric channel ids, a channel catalog, broadcast endpoint +> subscriptions, and catalog-aware transport events. --- ## 1. What the datastream is -The datastream is a **per-node, append-only telemetry pipe**. Every node -produces exactly one stream of its own observations; one consumer reconstructs -those streams and reads them. +The datastream is a **per-node, append-only telemetry pipe**. Every stream is +produced by one node incarnation, identified by `(node, life)`. A node can feed +zero, one, or many local/remote consumers: the mux is drained once, then the +endpoint fans out the resulting events to subscribers. -Its entire value comes from one rule: +Its core rule is unchanged: -> **Nothing between a producer and a view ever interprets a payload.** +> **Nothing between a producer and a view ever interprets a producer payload.** -Producers tag bytes with a channel name and hand them off. A single per-node -**mux** stamps each with a position and interleaves them into one ordered -stream. A best-effort **transport** carries the stream. **Ingest** reconstructs -each node's stream by position into a **store**. **Views** decode bytes back -into meaning — and only here, at read time, does anything look inside a payload. +The current implementation is catalog-aware. Producers register a channel name +and content class with their stream owner, receive a stream-local numeric +`ChannelId`, and submit opaque payload bytes on that id. A single per-stream +**mux** queues accepted payloads and stamps each drained frame with a position, +interleaving every channel into one ordered stream. A **catalog** maps numeric +channel ids back to human-readable names and decode/display metadata. +A best-effort **transport** +carries catalog declarations and frame events. **Ingest** reconstructs streams +by `StreamId` and position into a **store**. **Views** resolve channel names +from metadata and decode bytes back into meaning — and only here, at read time, +does anything look inside a producer payload. ```text - producers tag bytes with a channel + producers register names, submit opaque bytes by ChannelId │ ▼ - per-node MUX stamp position, interleave into one ordered stream + endpoint / catalog allocate ids, declare stream/channel metadata │ ▼ - transport best-effort: may drop / reorder / delay, never corrupt + per-stream MUX queue payloads, assign positions on drain │ ▼ - ingest reconstruct each node's stream by position + endpoint fanout drain once, broadcast catalog-aware events │ ▼ - store (the truth) whole, append-only, position-keyed + transport best-effort: may drop / reorder / delay / duplicate │ ▼ - views read-time projections — the only place bytes are decoded + ingest reconstruct each stream by position + │ + ▼ + store (truth for frames) whole, append-only, position-keyed + │ + ▼ + views read-time projections; payload decoding lives here ``` -This collapses the usual telemetry zoo — counters, gauges, histograms, logs, -events, traces — into **one** thing: positioned bytes on a named channel. -Collection, transport, and storage are uniform because none of them know which -of those a frame "is." That distinction does not exist in the pipe. It exists, -if at all, in a view: "the latest frame on this channel" is a gauge, "every -frame on this channel" is an event log, and they are the same bytes read two -ways. +This still collapses counters, gauges, histograms, logs, events, traces, and +binary blobs into one mechanism: positioned bytes on named channels. The pipe +stores and moves frames uniformly because it does not know what a payload +"means." That distinction exists in a view: "the last frame on this channel" is +a gauge, "every frame on this channel" is an event log, and both are projections +over the same stored frames. -### 1.1 Consequences that the rest of this spec just spells out +### 1.1 Consequences that the rest of this spec spells out -- There is **one way data enters** (§4). No per-channel "kind of producer." -- A channel is **a name, not a resource** (§5). Nothing to allocate or free. -- All **semantics live in views** (§8). The pipe is mechanism only. +- There is **one data-entry shape** (§4): register or reuse a channel id, then + submit bytes on that id. +- A channel name is not a queue, counter, or buffer, but it **is cataloged** + (§5): the stream owner allocates a stream-local numeric id and declares its + name/content metadata. +- The **store is frame truth**, not catalog truth (§7). Durable name recovery + requires catalog descriptors alongside numeric frames. +- All **producer-payload semantics live in views** (§8). Catalog content classes + help route and display; they do not let mux, transport, ingest, or store + inspect producer payloads. --- ## 2. The data model -### 2.1 The frame is the only unit +### 2.1 The frame remains the unit of ordered data ```rust pub struct Frame { - pub channel: ChannelId, // the named lane these bytes belong to - pub position: Position, // the mux-assigned order within the node's stream + pub channel: ChannelId, // stream-local numeric lane + pub position: Position, // mux-assigned order within the stream pub payload: Vec, // opaque bytes — never interpreted by the pipe } ``` -A typed record, a log line, and a binary blob are the same kind of thing here: -bytes on a channel. The `payload` is opaque to the mux, the transport, and the -store. +A typed record, a log line, and a binary blob are all frames: bytes on a +stream-local channel id at a stream-local position. `payload` is opaque to the +mux, transport, ingest, and store. -There is **no per-frame wall-clock timestamp.** Frames are ordered and -correlated by position alone. A producer that wants wall-clock time puts it -*inside* the payload, as a field of its record — it is data, not a property of -the pipe. +There is still **no timestamp field on `Frame`**. Time, wall-clock correlation, +latency, or tracing data is producer payload, not a datastream-owned sidecar or +property on the frame envelope. -### 2.2 The coordinate - -Every frame in the system has exactly one address: - -```text -Frame @ (node, life, channel, position) - └── stream ──┘ source order -``` - -| Part | Type | Answers | -|------------|--------------|------------------| -| `node` | `NodeId` | *who* produced it | -| `life` | `Lifetime` | *which incarnation* of that node | -| `channel` | `ChannelId` | *which source* within that node | -| `position` | `Position` | *where in order* within that stream | - -`(node, life)` together are the **stream**; `channel` is the **source** within -it; `position` is the **order**. This four-tuple locates any datum the system -has ever produced. Addressing (§3) is built entirely on it. - -### 2.3 Position: per-stream, monotonic, gap-free - -```rust -pub struct Position(pub u64); -``` - -- **One sequence per stream**, not per channel. A node's mux holds a single - counter shared across every channel, so a given channel's frames carry - *non-contiguous* positions — a sparse projection of the node's one global - sequence, interleaved with every other channel. -- **Monotonic and gap-free in assignment.** The mux never reuses a position and - never skips one when numbering. A position that is assigned but never - delivered surfaces downstream as a *missing* position — a detectable gap - (§4.4, §6.3). -- **Not comparable across streams.** Positions order frames within one - `(node, life)` only. There is no global clock. - -**Why one counter and not one per channel.** It keeps the mux a single position -authority and makes gap detection a whole-stream property: a hole means -*something* was lost. The cost is that a drop **cannot be attributed to a -specific channel** — you know a frame is missing, not which channel it carried. -That is the right trade for telemetry. - -> **Decision of record — drop attribution.** If a particular source ever needs -> guaranteed contiguous accounting, it -> carries its *own* sequence number as a field in its record. The pipe's -> position stays global and dumb; domain counting is domain data. - -### 2.4 Stream identity — incarnations never merge +### 2.2 Stream identity — incarnations never merge ```rust pub struct StreamId { pub node: NodeId, pub life: Lifetime } ``` `StreamId` is the ingest key. Two streams with the same `node` but different -`life` are **different streams and must never merge.** A node that dies and is -restarted (re-rented, re-scheduled) begins a new `life`, so its fresh stream -does not append to — or collide with — its prior one. Restart is visible, not -silently glued over. +`life` are **different streams and must never merge**. A node that dies and is +restarted (re-rented, re-scheduled, or bootstrapped for a new run) begins a new +`life`, so the fresh stream does not append to or collide with the prior one. +Restart is visible, not silently glued over. + +The stream descriptor declares where a stream came from: + +```rust +pub enum StreamOrigin { Orchestrator, Bootstrap, RemoteNode } + +pub struct StreamDescriptor { + pub stream: StreamId, + pub label: Option, + pub origin: StreamOrigin, +} +``` + +`StreamOrigin` is subscription metadata. It is not a clock and does not order +streams. + +### 2.3 Position: per-stream, monotonic when assigned + +```rust +pub struct Position(pub u64); +``` + +- **One sequence per stream**, not per channel. A stream's mux holds a single + counter shared across every channel. A single channel therefore has sparse, + non-contiguous positions interleaved with every other channel. +- **Assigned while draining.** `submit` only attempts to enqueue payload bytes. + `drain` assigns positions to accepted payloads. A queue-full rejection happens + before position assignment and therefore does not create a position gap. +- **Not comparable across streams.** Positions order frames within one + `StreamId` only. There is no global stream order. + +**Why one counter and not one per channel.** It keeps the mux a single position +authority once frames leave the queue and makes assigned-frame loss detection a +whole-stream property: a hole means *something already assigned* was lost. A +source needing contiguous domain accounting carries its own sequence number +inside its payload. + +### 2.4 Channels: numeric ids plus catalog descriptors + +A raw frame contains a numeric channel id: + +```rust +pub struct ChannelId(pub u32); +``` + +`ChannelId` values are **stream-local**. `ChannelId(7)` in one stream is not the +same channel as `ChannelId(7)` in another stream unless their descriptors say +so. The current endpoint allocates user channels starting at `ChannelId(1)`; +`ChannelId(0)` is not allocated by the public registration path. + +The human-readable channel name and display/routing metadata live in a channel +descriptor: + +```rust +pub enum ChannelContentKind { Bytes, TextStream, JsonRecord } + +pub enum ChannelContent { + Bytes, + TextStream, + JsonRecord { schema: Option }, +} + +pub struct ChannelDescriptor { + pub stream: StreamId, + pub id: ChannelId, + pub name: String, + pub label: Option, + pub content: ChannelContent, +} + +pub struct ChannelRef { + pub stream: StreamId, + pub channel: ChannelId, +} +``` + +The globally resolved raw channel identity is `ChannelRef`, not a bare +`ChannelId`. A view usually needs the `ChannelDescriptor` for that `ChannelRef` +to recover the channel name and choose a decode/display path. + +### 2.5 Datastream events + +The live endpoint/subscription path carries catalog-aware events: + +```rust +pub struct FrameDelivery { + pub channel: ChannelRef, + pub position: Position, + pub payload: Vec, +} + +pub enum DatastreamEvent { + StreamDeclared(StreamDescriptor), + ChannelDeclared(ChannelDescriptor), + Frame(FrameDelivery), + StreamEnded(StreamId), +} +``` + +`FrameDelivery` is the live-event form of a frame: stream and channel are +resolved into a `ChannelRef`, then position and payload follow. The older +`Delivery { stream, frame }` shape still exists for ingest, store tests, and +legacy adapters (§6.1, §7.1). + +`StreamDeclared` is part of the event vocabulary, but current QUIC transport +puts the stream descriptor in the stream header and does not emit a separate +`StreamDeclared` event. `StreamEnded` is also part of the event vocabulary, but +current `DatastreamEndpoint` exposes no public `end_stream` method; terminal +source records remain a producer convention until a stream-ending API is added +(§5.7). + --- ## 3. Addressing -### 3.1 The address is the coordinate +### 3.1 Raw and resolved coordinates -A frame is addressed by `(node, life, channel, position)`. A *source* is -addressed by the prefix `(node, life, channel)` — drop `position` and you are -naming a lane rather than a single datum. A *node's whole stream* is -`(node, life)`. Nothing else is needed; there is no separate registry of -producer identities. +A stored raw frame is addressed by: -### 3.2 Channel paths: opaque on the wire, structured at the edges - -```rust -pub struct ChannelId(Arc); // an opaque token to the pipe +```text +raw-frame @ (stream, numeric-channel, position) + └ node/life ┘ ChannelId order ``` -To the mux, transport, ingest, and store a `ChannelId` is an uninterpreted -string. They never split it, match it, or validate it. This is what keeps the -pipe dumb and lets a brand-new channel flow end to end with zero pipe changes. +A named channel source is addressed by resolving that raw channel through the +catalog: -Its **structure is a read-side convention** — known only to producers (to mint -paths they own) and to consumers/classifiers (to match and decode them). It -is therefore *both*: a flat string on the wire, a structured path at the edges. -There is no conflict, because the two views never meet inside the pipe. +```text +source @ (stream, channel-name) + └ node/life ┘ descriptor.name +``` -### 3.3 The path grammar +A payload datum is therefore either: -A channel path is a dotted sequence of segments: +- raw: `(StreamId, ChannelId, Position)`, sufficient for storage and ordering; +- resolved: `(StreamId, ChannelDescriptor.name, Position)`, required for + human-facing selection and decoding. + +No bare `ChannelId` is globally meaningful. A consumer that receives a frame +before its descriptor can store it raw, but cannot render a stable name until the +catalog descriptor arrives or is recovered from durable catalog state. + +### 3.2 Channel names: structured catalog paths, not wire ids + +Channel names remain dotted paths, but they are catalog descriptor fields rather +than the frame's wire identity. The mux, ingest, and store do not split names; +subscription matching and views may match names through descriptors. + +A channel name is a dotted sequence of segments: ```text channel := namespace ( "." qualifier )* -namespace := the owning subsystem e.g. host, transport, swim, runtime, - dist, proc, identity -qualifier := instance-key | leaf instance-key identifies *which* of a - dynamic source; leaf names the signal +namespace := owning subsystem e.g. datastream, host, runtime, proc, + mvp, transport, dist, identity +qualifier := instance-key | leaf instance-key identifies a dynamic source; + leaf names the signal ``` -Examples, current and proposed: +Current names and families seen in code include: -| Path | Namespace | Instance | Leaf | -|-------------------------------|-------------|---------------|------------| -| `identity` | identity | — | — | -| `host.resource` | host | — | resource | -| `transport.internals` | transport | — | internals | -| `proc.trainer.stdout` | proc | `trainer` | stdout | -| `transport.peer..conn` *(proposed)* | transport | `peer/` | conn | +| Path or family | Owner / meaning | +|-----------------------------------------------------|-----------------| +| `datastream.health` | datastream self-health record | +| `host.cpu`, `host.gpu`, `host.net` | host hardware samples | +| `runtime.actors` | swactor runtime actor stats | +| `proc.