Zachery Aaron Shores-Chmielewski
5bbdfb041e
Integrate namespace and source-route lifecycle changes, contextual process cleanup, Python binding updates, and Myelin worker/orchestrator recovery. Keep the shared control contracts, deployment identity fencing, SSH bootstrap adapters, paid admission accounting, and VastAI cleanup implementation together with their consumers. Migrate Iroh dependencies and telemetry transport/collection with dashboard and demo callsites, workspace build configuration, and actor-control-flow policy updates. This is an intermediate development checkpoint, not paid-provider qualification. Review verification: contextual_process_guarantees (4 tests), telemetry_transport (4 tests), and shared control contracts (5 tests) passed. Historical five-node redeployment and campaign execution passed individually; complete ordered qualification remains pending.
1232 lines
42 KiB
Rust
1232 lines
42 KiB
Rust
//! Arena-resident byte-movement rings.
|
|
//!
|
|
//! Shared-memory protocol beneath the Python data-plane surface: one
|
|
//! producer and one consumer move a byte stream through a ring leased from
|
|
//! the arena. This module owns the wire layout, the cursor protocol, and
|
|
//! the record framing; nothing here knows about paths or identities.
|
|
//!
|
|
//! Wire layout (all integers little-endian):
|
|
//!
|
|
//! | offset | field | owner |
|
|
//! |--------|--------------------------------|------------------------|
|
|
//! | 0 | magic u32 (`SWRG`) | fixed at install |
|
|
//! | 4 | version u16 | fixed at install |
|
|
//! | 6 | reserved u16 (zero) | fixed at install |
|
|
//! | 8 | capacity u64 | fixed at install |
|
|
//! | 16 | generation u64 | fixed at install |
|
|
//! | 24 | commit u64 (atomic) | producer writes only |
|
|
//! | 32 | consume u64 (atomic) | consumer writes only |
|
|
//! | 40 | peer terminal reason u64 (atomic) | host writes only |
|
|
//! | 128 | data[capacity] | protocol |
|
|
//!
|
|
//! Memory model (property P4): the producer copies bytes and then
|
|
//! release-stores `commit`; the consumer acquire-loads `commit`, copies
|
|
//! bytes out, and release-stores `consume`; the producer acquire-loads
|
|
//! `consume` before reusing freed space. That is the entire ordering story.
|
|
//!
|
|
//! Properties a correct implementation must uphold (each defended in
|
|
//! `tests/byte_ring_guarantees.rs`):
|
|
//!
|
|
//! - P1 exactly-once, in-order byte delivery across interleavings,
|
|
//! wake timings, and wraparound;
|
|
//! - P2 unread bytes are never overwritten by the producer;
|
|
//! - P3 single-writer cursor fields (SPSC), enforced by endpoint roles;
|
|
//! - P4 data publication is release-ordered before `commit` advance,
|
|
//! and `consume` advance after reads (acquire);
|
|
//! - P5 untrusted-header containment: arbitrary header bytes yield typed
|
|
//! errors or bounded access, never out-of-region access;
|
|
//! - P6 (binding slice) Python performs ring operations via helpers only;
|
|
//! - P7 backpressure: reserving beyond free space fails cleanly with the
|
|
//! exact free amount, and succeeds again once the peer consumes;
|
|
//! - P8 (binding slice) no lost wakeups: signals follow publication;
|
|
//! - P9 completion is explicit: `Data`/`Eof`/`Fault` records, `Eof` only
|
|
//! after all bytes, torn (uncommitted) records stay invisible;
|
|
//! - P10 generation fence: stale reservations and stale generations are
|
|
//! rejected, never applied;
|
|
//! - P11 (binding slice) fast path performs no syscalls;
|
|
//! - P12 one copy per side per byte.
|
|
|
|
use serde::{Deserialize, Serialize};
|
|
use std::ptr::NonNull;
|
|
use std::sync::atomic::{AtomicU64, Ordering};
|
|
|
|
use crate::arena::{
|
|
ArenaEvent, ArenaManager, ArenaRequest, LeaseRequestId, LeaseRing, RingLease,
|
|
RingLeaseRejection, RingSpec,
|
|
};
|
|
|
|
/// `"SWRG"` read little-endian.
|
|
pub const RING_MAGIC: u32 = u32::from_le_bytes(*b"SWRG");
|
|
pub const RING_VERSION: u16 = 1;
|
|
|
|
/// Byte offset of the data region inside the ring lease.
|
|
pub const DATA_OFFSET: u64 = 128;
|
|
|
|
pub const OFF_MAGIC: u64 = 0;
|
|
pub const OFF_VERSION: u64 = 4;
|
|
pub const OFF_RESERVED: u64 = 6;
|
|
pub const OFF_CAPACITY: u64 = 8;
|
|
pub const OFF_GENERATION: u64 = 16;
|
|
pub const OFF_COMMIT: u64 = 24;
|
|
pub const OFF_CONSUME: u64 = 32;
|
|
pub const OFF_TERMINAL: u64 = 40;
|
|
|
|
const ZERO_CHUNK: usize = 4 * 1024;
|
|
|
|
/// Record prefix: `kind u8 | len u32 LE`.
|
|
const RECORD_HEADER_LEN: u64 = 5;
|
|
pub struct ByteRingSpec {
|
|
/// Data-region capacity in bytes (the header adds `DATA_OFFSET`).
|
|
pub capacity: u64,
|
|
/// First generation of the installed ring (nonzero).
|
|
pub generation: u64,
|
|
/// Placement alignment requested from the arena.
|
|
pub alignment: u64,
|
|
/// Lease request identity; distinct rings must use distinct ids.
|
|
pub request_id: u64,
|
|
}
|
|
|
|
/// A located, installed ring: what `attach` needs to find it again.
|
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
pub struct RingHandle {
|
|
/// Lease start (the header lives here).
|
|
pub offset: u64,
|
|
/// Data-region capacity.
|
|
pub capacity: u64,
|
|
/// Ring generation.
|
|
pub generation: u64,
|
|
/// Arena lease identity used for deterministic release.
|
|
pub lease_id: u64,
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
pub enum Role {
|
|
Producer,
|
|
Consumer,
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
|
pub enum PeerTermination {
|
|
Closed,
|
|
PathReplaced,
|
|
}
|
|
|
|
/// A reserved, not-yet-committed span of the data region.
|
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
|
pub struct Reservation {
|
|
/// Logical stream position (monotonic, wraps modulo capacity on
|
|
/// physical access).
|
|
pub start: u64,
|
|
pub len: u64,
|
|
pub generation: u64,
|
|
}
|
|
|
|
/// Record framing layered on the byte stream (property P9).
|
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
pub enum RecordKind {
|
|
Data,
|
|
Eof,
|
|
Fault,
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
|
pub struct RecordMeta {
|
|
pub kind: RecordKind,
|
|
pub len: u64,
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
|
pub struct RecordCursor {
|
|
kind: RecordKind,
|
|
payload_start: u64,
|
|
payload_len: u64,
|
|
record_start: u64,
|
|
record_len: u64,
|
|
generation: u64,
|
|
}
|
|
|
|
impl RecordCursor {
|
|
pub const fn kind(self) -> RecordKind {
|
|
self.kind
|
|
}
|
|
|
|
pub fn len(self) -> usize {
|
|
self.payload_len as usize
|
|
}
|
|
|
|
pub const fn is_empty(self) -> bool {
|
|
self.payload_len == 0
|
|
}
|
|
}
|
|
|
|
/// Borrowed committed record. Dropping the view releases the complete framed
|
|
/// record back to the producer.
|
|
pub struct PinnedRecord<'a> {
|
|
endpoint: &'a mut Endpoint,
|
|
kind: RecordKind,
|
|
payload_start: u64,
|
|
payload_len: u64,
|
|
record_start: u64,
|
|
record_len: u64,
|
|
generation: u64,
|
|
released: bool,
|
|
}
|
|
|
|
impl std::fmt::Debug for PinnedRecord<'_> {
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
f.debug_struct("PinnedRecord")
|
|
.field("kind", &self.kind)
|
|
.field("payload_len", &self.payload_len)
|
|
.field("record_start", &self.record_start)
|
|
.finish_non_exhaustive()
|
|
}
|
|
}
|
|
|
|
impl PinnedRecord<'_> {
|
|
pub fn kind(&self) -> RecordKind {
|
|
self.kind
|
|
}
|
|
|
|
pub fn len(&self) -> usize {
|
|
self.payload_len as usize
|
|
}
|
|
|
|
pub fn is_empty(&self) -> bool {
|
|
self.payload_len == 0
|
|
}
|
|
|
|
pub fn spans(&self) -> (&[u8], &[u8]) {
|
|
self.endpoint
|
|
.read_spans(self.payload_start, self.payload_len)
|
|
}
|
|
|
|
pub fn release(mut self) -> Result<(), FlowError> {
|
|
self.release_inner()?;
|
|
self.released = true;
|
|
Ok(())
|
|
}
|
|
|
|
fn release_inner(&mut self) -> Result<(), FlowError> {
|
|
self.endpoint.check("release_record", Role::Consumer)?;
|
|
let generation = self.endpoint.field_u64(OFF_GENERATION);
|
|
if generation != self.generation {
|
|
return Err(FlowError::StaleReservation {
|
|
reservation: self.generation,
|
|
ring: generation,
|
|
});
|
|
}
|
|
let consume = self.endpoint.consume_cursor();
|
|
if consume != self.record_start {
|
|
return Err(FlowError::PinnedRecordMoved {
|
|
expected: self.record_start,
|
|
found: consume,
|
|
});
|
|
}
|
|
self.endpoint
|
|
.atomic(OFF_CONSUME)
|
|
.store(self.record_start + self.record_len, Ordering::Release);
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
impl Drop for PinnedRecord<'_> {
|
|
fn drop(&mut self) {
|
|
if !self.released && self.release_inner().is_ok() {
|
|
self.released = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Producer reservation for a complete framed record. Payload bytes are
|
|
/// written directly into the ring and remain invisible until `commit`.
|
|
pub struct WritableRecord<'a> {
|
|
endpoint: &'a mut Endpoint,
|
|
reservation: Option<Reservation>,
|
|
payload_start: u64,
|
|
payload_len: u64,
|
|
}
|
|
|
|
impl std::fmt::Debug for WritableRecord<'_> {
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
f.debug_struct("WritableRecord")
|
|
.field("payload_len", &self.payload_len)
|
|
.finish_non_exhaustive()
|
|
}
|
|
}
|
|
|
|
impl WritableRecord<'_> {
|
|
pub fn len(&self) -> usize {
|
|
self.payload_len as usize
|
|
}
|
|
|
|
pub fn is_empty(&self) -> bool {
|
|
self.payload_len == 0
|
|
}
|
|
|
|
pub fn spans_mut(&mut self) -> (&mut [u8], &mut [u8]) {
|
|
self.endpoint
|
|
.write_spans(self.payload_start, self.payload_len)
|
|
}
|
|
|
|
pub fn commit(mut self) -> Result<(), FlowError> {
|
|
let reservation = self
|
|
.reservation
|
|
.take()
|
|
.expect("writable record commits at most once");
|
|
self.endpoint.commit(reservation)
|
|
}
|
|
}
|
|
|
|
impl RecordKind {
|
|
pub fn to_byte(self) -> u8 {
|
|
match self {
|
|
Self::Data => 1,
|
|
Self::Eof => 2,
|
|
Self::Fault => 3,
|
|
}
|
|
}
|
|
|
|
pub fn from_byte(byte: u8) -> Option<Self> {
|
|
match byte {
|
|
1 => Some(Self::Data),
|
|
2 => Some(Self::Eof),
|
|
3 => Some(Self::Fault),
|
|
_ => None,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug)]
|
|
pub enum InstallError {
|
|
ZeroCapacity,
|
|
ZeroGeneration,
|
|
ZeroAlignment,
|
|
LeaseRejected(RingLeaseRejection),
|
|
LeaseQueued,
|
|
UnexpectedLeaseOutcome,
|
|
Io(std::io::Error),
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub enum HeaderError {
|
|
BadMagic {
|
|
found: u32,
|
|
},
|
|
UnsupportedVersion {
|
|
found: u16,
|
|
supported: u16,
|
|
},
|
|
ReservedBytesNotZero {
|
|
at: u64,
|
|
},
|
|
CapacityMismatch {
|
|
header: u64,
|
|
handle: u64,
|
|
},
|
|
GenerationMismatch {
|
|
header: u64,
|
|
handle: u64,
|
|
},
|
|
CommitBelowConsume {
|
|
commit: u64,
|
|
consume: u64,
|
|
},
|
|
ReadableExceedsCapacity {
|
|
commit: u64,
|
|
consume: u64,
|
|
capacity: u64,
|
|
},
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub enum AttachError {
|
|
/// The handle itself points outside the arena.
|
|
OutOfBounds {
|
|
end: u64,
|
|
arena_len: u64,
|
|
},
|
|
Header(HeaderError),
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub enum RecordError {
|
|
LengthExceedsCapacity { len: u64, capacity: u64 },
|
|
InvalidKind(u8),
|
|
LengthExceedsWire { len: u64 },
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub enum FlowError {
|
|
InsufficientSpace {
|
|
requested: u64,
|
|
free: u64,
|
|
},
|
|
BeyondCommitted {
|
|
requested: u64,
|
|
readable: u64,
|
|
},
|
|
/// The reservation's generation no longer matches the ring header.
|
|
StaleReservation {
|
|
reservation: u64,
|
|
ring: u64,
|
|
},
|
|
/// Operation reserved for the other role (property P3).
|
|
RoleViolation {
|
|
operation: &'static str,
|
|
role: Role,
|
|
},
|
|
/// The header stopped validating mid-protocol (property P5).
|
|
Corrupt(HeaderError),
|
|
PinnedRecordMoved {
|
|
expected: u64,
|
|
found: u64,
|
|
},
|
|
BadRecord(RecordError),
|
|
InvalidPeerTermination {
|
|
found: u64,
|
|
},
|
|
Io,
|
|
}
|
|
|
|
/// One side of an installed ring. Owns its access path into the mapping so
|
|
/// endpoints can cross threads without sharing the arena manager.
|
|
pub struct Endpoint {
|
|
header: NonNull<u8>,
|
|
info: RingHandle,
|
|
role: Role,
|
|
}
|
|
|
|
#[derive(Clone, Copy)]
|
|
pub struct RingProbe {
|
|
header: NonNull<u8>,
|
|
info: RingHandle,
|
|
}
|
|
|
|
unsafe impl Send for RingProbe {}
|
|
unsafe impl Sync for RingProbe {}
|
|
|
|
impl RingProbe {
|
|
fn atomic(&self, off: u64) -> &AtomicU64 {
|
|
// SAFETY: probes are created only from a successfully attached,
|
|
// aligned endpoint and remain bounded by that ring's live lease.
|
|
unsafe { &*(self.header.as_ptr().add(off as usize) as *const AtomicU64) }
|
|
}
|
|
|
|
pub fn positions(&self) -> Result<(u64, u64), FlowError> {
|
|
let generation = self.atomic(OFF_GENERATION).load(Ordering::Relaxed);
|
|
if generation != self.info.generation {
|
|
return Err(FlowError::StaleReservation {
|
|
reservation: self.info.generation,
|
|
ring: generation,
|
|
});
|
|
}
|
|
let commit = self.atomic(OFF_COMMIT).load(Ordering::Acquire);
|
|
let consume = self.atomic(OFF_CONSUME).load(Ordering::Acquire);
|
|
if commit < consume {
|
|
return Err(FlowError::Corrupt(HeaderError::CommitBelowConsume {
|
|
commit,
|
|
consume,
|
|
}));
|
|
}
|
|
if commit - consume > self.info.capacity {
|
|
return Err(FlowError::Corrupt(HeaderError::ReadableExceedsCapacity {
|
|
commit,
|
|
consume,
|
|
capacity: self.info.capacity,
|
|
}));
|
|
}
|
|
Ok((commit, consume))
|
|
}
|
|
|
|
pub fn has_data(&self) -> bool {
|
|
self.positions()
|
|
.is_ok_and(|(commit, consume)| commit > consume)
|
|
}
|
|
|
|
pub fn has_capacity(&self) -> bool {
|
|
self.positions()
|
|
.is_ok_and(|(commit, consume)| commit - consume < self.info.capacity)
|
|
}
|
|
}
|
|
|
|
// SAFETY: an endpoint touches only its role-owned cursor field and the data
|
|
// region under the single-writer protocol (properties P3/P4); the raw
|
|
// pointer is never dereferenced outside `[header, header + DATA_OFFSET +
|
|
// capacity)`.
|
|
unsafe impl Send for Endpoint {}
|
|
|
|
impl std::fmt::Debug for Endpoint {
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
f.debug_struct("Endpoint")
|
|
.field("info", &self.info)
|
|
.field("role", &self.role)
|
|
.finish_non_exhaustive()
|
|
}
|
|
}
|
|
|
|
/// Lease a region, write the ring header, and zero the data region.
|
|
///
|
|
/// All writes complete before the handle is returned, so the peer may
|
|
/// `attach` at any time afterwards.
|
|
pub fn install(arena: &mut ArenaManager, spec: ByteRingSpec) -> Result<RingHandle, InstallError> {
|
|
if spec.capacity == 0 {
|
|
return Err(InstallError::ZeroCapacity);
|
|
}
|
|
if spec.generation == 0 {
|
|
return Err(InstallError::ZeroGeneration);
|
|
}
|
|
if spec.alignment == 0 {
|
|
return Err(InstallError::ZeroAlignment);
|
|
}
|
|
|
|
let lease = lease_ring(
|
|
arena,
|
|
spec.request_id,
|
|
RingSpec {
|
|
header_bytes: DATA_OFFSET,
|
|
data_bytes: spec.capacity,
|
|
alignment: spec.alignment,
|
|
},
|
|
)?;
|
|
let zeros = [0_u8; ZERO_CHUNK];
|
|
let data_start = lease.layout.start_offset + DATA_OFFSET;
|
|
let data_end = lease.layout.end_offset;
|
|
let mut offset = data_start;
|
|
while offset < data_end {
|
|
let take = ((data_end - offset) as usize).min(ZERO_CHUNK);
|
|
arena
|
|
.write_arena(offset, &zeros[..take])
|
|
.map_err(InstallError::Io)?;
|
|
offset += take as u64;
|
|
}
|
|
|
|
let mut header = [0_u8; DATA_OFFSET as usize];
|
|
header[0..4].copy_from_slice(&RING_MAGIC.to_le_bytes());
|
|
header[4..6].copy_from_slice(&RING_VERSION.to_le_bytes());
|
|
header[8..16].copy_from_slice(&spec.capacity.to_le_bytes());
|
|
header[16..24].copy_from_slice(&spec.generation.to_le_bytes());
|
|
// commit and consume stay zero.
|
|
arena
|
|
.write_arena(lease.layout.start_offset, &header)
|
|
.map_err(InstallError::Io)?;
|
|
|
|
Ok(RingHandle {
|
|
offset: lease.layout.start_offset,
|
|
capacity: spec.capacity,
|
|
generation: spec.generation,
|
|
lease_id: lease.ring_id.0,
|
|
})
|
|
}
|
|
|
|
/// Validate the header against `handle` and the arena bounds, then open one
|
|
/// side of the ring.
|
|
pub fn attach(
|
|
arena: &ArenaManager,
|
|
handle: RingHandle,
|
|
role: Role,
|
|
) -> Result<Endpoint, AttachError> {
|
|
let end = handle
|
|
.offset
|
|
.checked_add(DATA_OFFSET)
|
|
.and_then(|v| v.checked_add(handle.capacity))
|
|
.ok_or(AttachError::OutOfBounds {
|
|
end: u64::MAX,
|
|
arena_len: arena.arena_len(),
|
|
})?;
|
|
if end > arena.arena_len() {
|
|
return Err(AttachError::OutOfBounds {
|
|
end,
|
|
arena_len: arena.arena_len(),
|
|
});
|
|
}
|
|
let header = arena
|
|
.region_ptr(handle.offset, DATA_OFFSET + handle.capacity)
|
|
.ok_or(AttachError::OutOfBounds {
|
|
end,
|
|
arena_len: arena.arena_len(),
|
|
})?;
|
|
|
|
let endpoint = Endpoint {
|
|
header,
|
|
info: handle,
|
|
role,
|
|
};
|
|
endpoint.validate_fixed().map_err(AttachError::Header)?;
|
|
endpoint
|
|
.validate_generation(handle.generation)
|
|
.map_err(AttachError::Header)?;
|
|
endpoint.validate_cursors().map_err(AttachError::Header)?;
|
|
Ok(endpoint)
|
|
}
|
|
|
|
/// Attach through a child/host process-local mapping of the same arena.
|
|
pub fn attach_mapped(
|
|
arena: &crate::mapped_arena::MappedArena,
|
|
handle: RingHandle,
|
|
role: Role,
|
|
) -> Result<Endpoint, AttachError> {
|
|
let total = DATA_OFFSET
|
|
.checked_add(handle.capacity)
|
|
.ok_or(AttachError::OutOfBounds {
|
|
end: u64::MAX,
|
|
arena_len: arena.len() as u64,
|
|
})?;
|
|
let range =
|
|
arena
|
|
.checked_range(handle.offset, total)
|
|
.map_err(|_| AttachError::OutOfBounds {
|
|
end: handle.offset.saturating_add(total),
|
|
arena_len: arena.len() as u64,
|
|
})?;
|
|
let endpoint = Endpoint {
|
|
header: arena.ptr_at(range.start),
|
|
info: handle,
|
|
role,
|
|
};
|
|
endpoint.validate_fixed().map_err(AttachError::Header)?;
|
|
endpoint
|
|
.validate_generation(handle.generation)
|
|
.map_err(AttachError::Header)?;
|
|
endpoint.validate_cursors().map_err(AttachError::Header)?;
|
|
Ok(endpoint)
|
|
}
|
|
|
|
pub fn mark_peer_terminated_mapped(
|
|
arena: &crate::mapped_arena::MappedArena,
|
|
handle: RingHandle,
|
|
termination: PeerTermination,
|
|
) -> Result<(), AttachError> {
|
|
let total = DATA_OFFSET
|
|
.checked_add(handle.capacity)
|
|
.ok_or(AttachError::OutOfBounds {
|
|
end: u64::MAX,
|
|
arena_len: arena.len() as u64,
|
|
})?;
|
|
let range =
|
|
arena
|
|
.checked_range(handle.offset, total)
|
|
.map_err(|_| AttachError::OutOfBounds {
|
|
end: handle.offset.saturating_add(total),
|
|
arena_len: arena.len() as u64,
|
|
})?;
|
|
let endpoint = Endpoint {
|
|
header: arena.ptr_at(range.start),
|
|
info: handle,
|
|
role: Role::Producer,
|
|
};
|
|
endpoint.validate_fixed().map_err(AttachError::Header)?;
|
|
endpoint
|
|
.validate_generation(handle.generation)
|
|
.map_err(AttachError::Header)?;
|
|
let marker = match termination {
|
|
PeerTermination::Closed => 1,
|
|
PeerTermination::PathReplaced => 2,
|
|
};
|
|
// Cursor fields change independently while a stream is active. Validating
|
|
// a multi-atomic cursor snapshot here can spuriously reject the terminal
|
|
// marker; only fixed identity is required for this host-owned field.
|
|
endpoint
|
|
.atomic(OFF_TERMINAL)
|
|
.fetch_max(marker, Ordering::Release);
|
|
Ok(())
|
|
}
|
|
|
|
fn lease_ring(
|
|
arena: &mut ArenaManager,
|
|
request_id: u64,
|
|
ring_spec: RingSpec,
|
|
) -> Result<RingLease, InstallError> {
|
|
let request = LeaseRing {
|
|
request_id: LeaseRequestId(request_id),
|
|
ring_spec,
|
|
};
|
|
let mut events = arena.request(ArenaRequest::LeaseRing(request));
|
|
match (events.len(), events.pop()) {
|
|
(1, Some(ArenaEvent::RingLeased { lease })) => Ok(lease),
|
|
(1, Some(ArenaEvent::RingLeaseRejected { reason, .. })) => {
|
|
Err(InstallError::LeaseRejected(reason))
|
|
}
|
|
|
|
(1, Some(ArenaEvent::RingLeaseQueued { .. })) => Err(InstallError::LeaseQueued),
|
|
_ => Err(InstallError::UnexpectedLeaseOutcome),
|
|
}
|
|
}
|
|
|
|
impl Endpoint {
|
|
pub fn capacity(&self) -> u64 {
|
|
self.info.capacity
|
|
}
|
|
|
|
pub fn role(&self) -> Role {
|
|
self.role
|
|
}
|
|
|
|
pub fn peer_termination(&self) -> Result<Option<PeerTermination>, FlowError> {
|
|
self.validate_fixed().map_err(FlowError::Corrupt)?;
|
|
self.validate_generation(self.info.generation)
|
|
.map_err(FlowError::Corrupt)?;
|
|
match self.atomic(OFF_TERMINAL).load(Ordering::Acquire) {
|
|
0 => Ok(None),
|
|
1 => Ok(Some(PeerTermination::Closed)),
|
|
2 => Ok(Some(PeerTermination::PathReplaced)),
|
|
found => Err(FlowError::InvalidPeerTermination { found }),
|
|
}
|
|
}
|
|
|
|
/// Current published producer and consumer positions. This role-neutral
|
|
/// observation is used only to wait for an already-committed clean close
|
|
/// to enter the downstream bounded transport.
|
|
pub fn positions(&self) -> Result<(u64, u64), FlowError> {
|
|
self.validate_fixed().map_err(FlowError::Corrupt)?;
|
|
self.validate_generation(self.info.generation)
|
|
.map_err(FlowError::Corrupt)?;
|
|
self.validate_cursors().map_err(FlowError::Corrupt)?;
|
|
Ok((self.commit_cursor(), self.consume_cursor()))
|
|
}
|
|
|
|
pub fn probe(&self) -> RingProbe {
|
|
RingProbe {
|
|
header: self.header,
|
|
info: self.info,
|
|
}
|
|
}
|
|
|
|
// ─── field access ────────────────────────────────────────────────────────
|
|
|
|
/// SAFETY: callers keep `self` alive; the pointer is bounds-checked at
|
|
/// attach against the lease and the arena.
|
|
fn field_u32(&self, off: u64) -> u32 {
|
|
// SAFETY: fixed fields sit inside the bounds-checked lease.
|
|
let bytes =
|
|
unsafe { std::slice::from_raw_parts(self.header.as_ptr(), DATA_OFFSET as usize) };
|
|
u32::from_le_bytes(bytes[off as usize..off as usize + 4].try_into().unwrap())
|
|
}
|
|
|
|
fn field_u16(&self, off: u64) -> u16 {
|
|
// SAFETY: as `field_u32`.
|
|
let bytes =
|
|
unsafe { std::slice::from_raw_parts(self.header.as_ptr(), DATA_OFFSET as usize) };
|
|
u16::from_le_bytes(bytes[off as usize..off as usize + 2].try_into().unwrap())
|
|
}
|
|
|
|
/// Relaxed load of a fixed-at-install field.
|
|
fn field_u64(&self, off: u64) -> u64 {
|
|
self.atomic(off).load(Ordering::Relaxed)
|
|
}
|
|
|
|
/// The cursor words live at 8-aligned offsets inside a 64-aligned lease.
|
|
fn atomic(&self, off: u64) -> &AtomicU64 {
|
|
// SAFETY: attach bounds-checked the lease and arena placement
|
|
// guarantees 64-byte alignment, so `header + off` is 8-aligned and
|
|
// inside the mapping for the cursor offsets.
|
|
unsafe { &*(self.header.as_ptr().add(off as usize) as *const AtomicU64) }
|
|
}
|
|
|
|
fn commit_cursor(&self) -> u64 {
|
|
self.atomic(OFF_COMMIT).load(Ordering::Acquire)
|
|
}
|
|
|
|
fn consume_cursor(&self) -> u64 {
|
|
self.atomic(OFF_CONSUME).load(Ordering::Acquire)
|
|
}
|
|
|
|
// ─── validation (P5: never trust the header) ─────────────────────────────
|
|
|
|
fn validate_fixed(&self) -> Result<(), HeaderError> {
|
|
let magic = self.field_u32(OFF_MAGIC);
|
|
if magic != RING_MAGIC {
|
|
return Err(HeaderError::BadMagic { found: magic });
|
|
}
|
|
let version = self.field_u16(OFF_VERSION);
|
|
if version != RING_VERSION {
|
|
return Err(HeaderError::UnsupportedVersion {
|
|
found: version,
|
|
supported: RING_VERSION,
|
|
});
|
|
}
|
|
let reserved = self.field_u16(OFF_RESERVED);
|
|
if reserved != 0 {
|
|
return Err(HeaderError::ReservedBytesNotZero { at: OFF_RESERVED });
|
|
}
|
|
let capacity = self.field_u64(OFF_CAPACITY);
|
|
if capacity != self.info.capacity {
|
|
return Err(HeaderError::CapacityMismatch {
|
|
header: capacity,
|
|
handle: self.info.capacity,
|
|
});
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn validate_generation(&self, expected: u64) -> Result<(), HeaderError> {
|
|
let generation = self.field_u64(OFF_GENERATION);
|
|
if generation != expected {
|
|
return Err(HeaderError::GenerationMismatch {
|
|
header: generation,
|
|
handle: expected,
|
|
});
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn validate_cursors(&self) -> Result<(), HeaderError> {
|
|
let (commit, consume) = (self.commit_cursor(), self.consume_cursor());
|
|
if commit < consume {
|
|
return Err(HeaderError::CommitBelowConsume { commit, consume });
|
|
}
|
|
if commit - consume > self.info.capacity {
|
|
return Err(HeaderError::ReadableExceedsCapacity {
|
|
commit,
|
|
consume,
|
|
capacity: self.info.capacity,
|
|
});
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Every protocol operation revalidates before touching memory.
|
|
fn check(&self, operation: &'static str, role: Role) -> Result<(), FlowError> {
|
|
if self.role != role {
|
|
return Err(FlowError::RoleViolation {
|
|
operation,
|
|
role: self.role,
|
|
});
|
|
}
|
|
self.validate_fixed().map_err(FlowError::Corrupt)?;
|
|
self.validate_cursors().map_err(FlowError::Corrupt)?;
|
|
Ok(())
|
|
}
|
|
|
|
// ─── wrapped copies (the only wrap arithmetic in the protocol) ───────────
|
|
|
|
fn data_ptr(&self) -> *mut u8 {
|
|
// SAFETY: attach bounds-checked `[offset, offset + DATA_OFFSET +
|
|
// capacity)` against the arena.
|
|
unsafe { self.header.as_ptr().add(DATA_OFFSET as usize) }
|
|
}
|
|
|
|
fn copy_into(&self, stream_pos: u64, bytes: &[u8]) {
|
|
let capacity = self.info.capacity as usize;
|
|
let start = (stream_pos % self.info.capacity) as usize;
|
|
let first = bytes.len().min(capacity - start);
|
|
// SAFETY: both slices are in-bounds by construction (lease checked,
|
|
// `first` splits the wrap).
|
|
unsafe {
|
|
std::ptr::copy_nonoverlapping(bytes.as_ptr(), self.data_ptr().add(start), first);
|
|
std::ptr::copy_nonoverlapping(
|
|
bytes.as_ptr().add(first),
|
|
self.data_ptr(),
|
|
bytes.len() - first,
|
|
);
|
|
}
|
|
}
|
|
|
|
fn read_spans(&self, stream_pos: u64, len: u64) -> (&[u8], &[u8]) {
|
|
let capacity = self.info.capacity as usize;
|
|
let start = (stream_pos % self.info.capacity) as usize;
|
|
let len = len as usize;
|
|
let first = len.min(capacity - start);
|
|
// SAFETY: `stream_pos` and `len` describe a validated committed
|
|
// record no larger than the ring. The split ranges do not overlap.
|
|
unsafe {
|
|
(
|
|
std::slice::from_raw_parts(self.data_ptr().add(start), first),
|
|
std::slice::from_raw_parts(self.data_ptr(), len - first),
|
|
)
|
|
}
|
|
}
|
|
|
|
fn write_spans(&mut self, stream_pos: u64, len: u64) -> (&mut [u8], &mut [u8]) {
|
|
let capacity = self.info.capacity as usize;
|
|
let start = (stream_pos % self.info.capacity) as usize;
|
|
let len = len as usize;
|
|
let first = len.min(capacity - start);
|
|
// SAFETY: the producer exclusively owns this uncommitted reservation;
|
|
// it is no larger than the ring and the split ranges do not overlap.
|
|
unsafe {
|
|
(
|
|
std::slice::from_raw_parts_mut(self.data_ptr().add(start), first),
|
|
std::slice::from_raw_parts_mut(self.data_ptr(), len - first),
|
|
)
|
|
}
|
|
}
|
|
|
|
fn copy_into_slice(&self, stream_pos: u64, destination: &mut [u8]) {
|
|
let capacity = self.info.capacity as usize;
|
|
let start = (stream_pos % self.info.capacity) as usize;
|
|
let first = destination.len().min(capacity - start);
|
|
// SAFETY: the caller validated the source range against committed bytes.
|
|
unsafe {
|
|
std::ptr::copy_nonoverlapping(
|
|
self.data_ptr().add(start),
|
|
destination.as_mut_ptr(),
|
|
first,
|
|
);
|
|
std::ptr::copy_nonoverlapping(
|
|
self.data_ptr(),
|
|
destination.as_mut_ptr().add(first),
|
|
destination.len() - first,
|
|
);
|
|
}
|
|
}
|
|
|
|
fn copy_out(&self, stream_pos: u64, len: u64) -> Vec<u8> {
|
|
let capacity = self.info.capacity as usize;
|
|
let start = (stream_pos % self.info.capacity) as usize;
|
|
let len = len as usize;
|
|
let first = len.min(capacity - start);
|
|
let mut bytes = vec![0_u8; len];
|
|
// SAFETY: as `copy_into`.
|
|
unsafe {
|
|
std::ptr::copy_nonoverlapping(self.data_ptr().add(start), bytes.as_mut_ptr(), first);
|
|
std::ptr::copy_nonoverlapping(
|
|
self.data_ptr(),
|
|
bytes.as_mut_ptr().add(first),
|
|
len - first,
|
|
);
|
|
}
|
|
bytes
|
|
}
|
|
|
|
// ─── producer side ───────────────────────────────────────────────────────
|
|
|
|
/// Free space computed from the consumer's published cursor.
|
|
pub fn reserve(&self, len: u64) -> Result<Reservation, FlowError> {
|
|
self.check("reserve", Role::Producer)?;
|
|
self.validate_generation(self.info.generation)
|
|
.map_err(FlowError::Corrupt)?;
|
|
let (commit, consume) = (self.commit_cursor(), self.consume_cursor());
|
|
let used = commit - consume;
|
|
let free = self.info.capacity - used;
|
|
if len > free {
|
|
return Err(FlowError::InsufficientSpace {
|
|
requested: len,
|
|
free,
|
|
});
|
|
}
|
|
Ok(Reservation {
|
|
start: commit,
|
|
len,
|
|
generation: self.info.generation,
|
|
})
|
|
}
|
|
|
|
pub fn writable_payload_capacity(&self) -> Result<u64, FlowError> {
|
|
self.check("writable_payload_capacity", Role::Producer)?;
|
|
self.validate_generation(self.info.generation)
|
|
.map_err(FlowError::Corrupt)?;
|
|
let used = self.commit_cursor() - self.consume_cursor();
|
|
Ok((self.info.capacity - used).saturating_sub(RECORD_HEADER_LEN))
|
|
}
|
|
|
|
/// Copy bytes into the reserved (producer-side) span.
|
|
pub fn write(&self, reservation: &Reservation, bytes: &[u8]) -> Result<(), FlowError> {
|
|
self.check("write", Role::Producer)?;
|
|
self.stale_check(reservation)?;
|
|
assert_eq!(
|
|
reservation.len as usize,
|
|
bytes.len(),
|
|
"reservation length must match the payload"
|
|
);
|
|
self.copy_into(reservation.start, bytes);
|
|
Ok(())
|
|
}
|
|
|
|
/// Publish reserved bytes to the consumer (release-ordered).
|
|
pub fn commit(&mut self, reservation: Reservation) -> Result<(), FlowError> {
|
|
self.check("commit", Role::Producer)?;
|
|
// Generation fence first (P10): a replaced ring rejects stale work.
|
|
self.stale_check(&reservation)?;
|
|
let commit = self.commit_cursor();
|
|
assert_eq!(
|
|
reservation.start, commit,
|
|
"reservations must commit in stream order"
|
|
);
|
|
self.atomic(OFF_COMMIT)
|
|
.store(commit + reservation.len, Ordering::Release);
|
|
Ok(())
|
|
}
|
|
|
|
fn stale_check(&self, reservation: &Reservation) -> Result<(), FlowError> {
|
|
let generation = self.field_u64(OFF_GENERATION);
|
|
if generation != reservation.generation {
|
|
return Err(FlowError::StaleReservation {
|
|
reservation: reservation.generation,
|
|
ring: generation,
|
|
});
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
// ─── consumer side ───────────────────────────────────────────────────────
|
|
|
|
/// Committed-but-unconsumed byte count.
|
|
pub fn readable(&self) -> Result<u64, FlowError> {
|
|
self.check("readable", Role::Consumer)?;
|
|
Ok(self.commit_cursor() - self.consume_cursor())
|
|
}
|
|
|
|
/// Copy committed bytes out without advancing the cursor.
|
|
pub fn read(&self, len: u64) -> Result<Vec<u8>, FlowError> {
|
|
self.check("read", Role::Consumer)?;
|
|
let (commit, consume) = (self.commit_cursor(), self.consume_cursor());
|
|
let readable = commit - consume;
|
|
if len > readable {
|
|
return Err(FlowError::BeyondCommitted {
|
|
requested: len,
|
|
readable,
|
|
});
|
|
}
|
|
Ok(self.copy_out(consume, len))
|
|
}
|
|
|
|
/// Release consumed bytes back to the producer.
|
|
pub fn consume(&mut self, len: u64) -> Result<(), FlowError> {
|
|
self.check("consume", Role::Consumer)?;
|
|
let (commit, consume) = (self.commit_cursor(), self.consume_cursor());
|
|
let readable = commit - consume;
|
|
if len > readable {
|
|
return Err(FlowError::BeyondCommitted {
|
|
requested: len,
|
|
readable,
|
|
});
|
|
}
|
|
self.atomic(OFF_CONSUME)
|
|
.store(consume + len, Ordering::Release);
|
|
Ok(())
|
|
}
|
|
|
|
// ─── records (P9) ────────────────────────────────────────────────────────
|
|
|
|
/// Reserve one complete record for direct producer-side payload writes.
|
|
pub fn reserve_record(
|
|
&mut self,
|
|
kind: RecordKind,
|
|
len: u64,
|
|
) -> Result<WritableRecord<'_>, FlowError> {
|
|
if len > u64::from(u32::MAX) {
|
|
return Err(FlowError::BadRecord(RecordError::LengthExceedsWire { len }));
|
|
}
|
|
if RECORD_HEADER_LEN + len > self.info.capacity {
|
|
return Err(FlowError::BadRecord(RecordError::LengthExceedsCapacity {
|
|
len,
|
|
capacity: self.info.capacity,
|
|
}));
|
|
}
|
|
let reservation = self.reserve(RECORD_HEADER_LEN + len)?;
|
|
let mut prefix = [0_u8; RECORD_HEADER_LEN as usize];
|
|
prefix[0] = kind.to_byte();
|
|
prefix[1..5].copy_from_slice(&(len as u32).to_le_bytes());
|
|
self.copy_into(reservation.start, &prefix);
|
|
let payload_start = reservation.start + RECORD_HEADER_LEN;
|
|
Ok(WritableRecord {
|
|
endpoint: self,
|
|
reservation: Some(reservation),
|
|
payload_start,
|
|
payload_len: len,
|
|
})
|
|
}
|
|
/// Frame and commit one record in a single step.
|
|
pub fn send_record(&mut self, kind: RecordKind, bytes: &[u8]) -> Result<(), FlowError> {
|
|
let len = bytes.len() as u64;
|
|
if RECORD_HEADER_LEN + len > self.info.capacity {
|
|
return Err(FlowError::BadRecord(RecordError::LengthExceedsCapacity {
|
|
len,
|
|
capacity: self.info.capacity,
|
|
}));
|
|
}
|
|
let reservation = self.reserve(RECORD_HEADER_LEN + len)?;
|
|
let mut prefix = [0_u8; RECORD_HEADER_LEN as usize];
|
|
prefix[0] = kind.to_byte();
|
|
prefix[1..5].copy_from_slice(&(len as u32).to_le_bytes());
|
|
self.copy_into(reservation.start, &prefix);
|
|
self.copy_into(reservation.start + RECORD_HEADER_LEN, bytes);
|
|
self.commit(reservation)
|
|
}
|
|
|
|
/// Inspect the next complete committed record without pinning or consuming
|
|
/// it. Used by local direct transfer to reserve destination capacity before
|
|
/// borrowing the source payload.
|
|
pub fn next_record_meta(&self) -> Result<Option<RecordMeta>, FlowError> {
|
|
self.check("next_record_meta", Role::Consumer)?;
|
|
self.validate_generation(self.info.generation)
|
|
.map_err(FlowError::Corrupt)?;
|
|
let (commit, consume) = (self.commit_cursor(), self.consume_cursor());
|
|
let readable = commit - consume;
|
|
if readable < RECORD_HEADER_LEN {
|
|
return Ok(None);
|
|
}
|
|
let prefix = self.copy_out(consume, RECORD_HEADER_LEN);
|
|
let kind = RecordKind::from_byte(prefix[0])
|
|
.ok_or(FlowError::BadRecord(RecordError::InvalidKind(prefix[0])))?;
|
|
let len = u64::from(u32::from_le_bytes(prefix[1..5].try_into().unwrap()));
|
|
if RECORD_HEADER_LEN + len > self.info.capacity {
|
|
return Err(FlowError::BadRecord(RecordError::LengthExceedsCapacity {
|
|
len,
|
|
capacity: self.info.capacity,
|
|
}));
|
|
}
|
|
if readable < RECORD_HEADER_LEN + len {
|
|
return Ok(None);
|
|
}
|
|
Ok(Some(RecordMeta { kind, len }))
|
|
}
|
|
/// Borrow one complete committed record without copying its payload.
|
|
/// The record remains pinned until the returned view is released or
|
|
/// dropped.
|
|
pub fn peek_record(&mut self) -> Result<Option<PinnedRecord<'_>>, FlowError> {
|
|
self.check("peek_record", Role::Consumer)?;
|
|
self.validate_generation(self.info.generation)
|
|
.map_err(FlowError::Corrupt)?;
|
|
let (commit, consume) = (self.commit_cursor(), self.consume_cursor());
|
|
let readable = commit - consume;
|
|
if readable < RECORD_HEADER_LEN {
|
|
return Ok(None);
|
|
}
|
|
let prefix = self.copy_out(consume, RECORD_HEADER_LEN);
|
|
let kind = RecordKind::from_byte(prefix[0])
|
|
.ok_or(FlowError::BadRecord(RecordError::InvalidKind(prefix[0])))?;
|
|
let len = u64::from(u32::from_le_bytes(prefix[1..5].try_into().unwrap()));
|
|
if RECORD_HEADER_LEN + len > self.info.capacity {
|
|
return Err(FlowError::BadRecord(RecordError::LengthExceedsCapacity {
|
|
len,
|
|
capacity: self.info.capacity,
|
|
}));
|
|
}
|
|
let record_len = RECORD_HEADER_LEN + len;
|
|
if readable < record_len {
|
|
return Ok(None);
|
|
}
|
|
let generation = self.info.generation;
|
|
Ok(Some(PinnedRecord {
|
|
endpoint: self,
|
|
kind,
|
|
payload_start: consume + RECORD_HEADER_LEN,
|
|
payload_len: len,
|
|
record_start: consume,
|
|
record_len,
|
|
generation,
|
|
released: false,
|
|
}))
|
|
}
|
|
|
|
/// Acquire an owned cursor for the next complete record. The cursor does
|
|
/// not release capacity and remains valid only while that record is first.
|
|
pub fn record_cursor(&self) -> Result<Option<RecordCursor>, FlowError> {
|
|
self.check("record_cursor", Role::Consumer)?;
|
|
self.validate_generation(self.info.generation)
|
|
.map_err(FlowError::Corrupt)?;
|
|
let (commit, consume) = (self.commit_cursor(), self.consume_cursor());
|
|
let readable = commit - consume;
|
|
if readable < RECORD_HEADER_LEN {
|
|
return Ok(None);
|
|
}
|
|
let mut prefix = [0_u8; RECORD_HEADER_LEN as usize];
|
|
self.copy_into_slice(consume, &mut prefix);
|
|
let kind = RecordKind::from_byte(prefix[0])
|
|
.ok_or(FlowError::BadRecord(RecordError::InvalidKind(prefix[0])))?;
|
|
let payload_len = u64::from(u32::from_le_bytes(prefix[1..5].try_into().unwrap()));
|
|
let record_len = RECORD_HEADER_LEN + payload_len;
|
|
if record_len > self.info.capacity {
|
|
return Err(FlowError::BadRecord(RecordError::LengthExceedsCapacity {
|
|
len: payload_len,
|
|
capacity: self.info.capacity,
|
|
}));
|
|
}
|
|
if readable < record_len {
|
|
return Ok(None);
|
|
}
|
|
Ok(Some(RecordCursor {
|
|
kind,
|
|
payload_start: consume + RECORD_HEADER_LEN,
|
|
payload_len,
|
|
record_start: consume,
|
|
record_len,
|
|
generation: self.info.generation,
|
|
}))
|
|
}
|
|
|
|
pub fn copy_record_range(
|
|
&self,
|
|
cursor: RecordCursor,
|
|
offset: u64,
|
|
destination: &mut [u8],
|
|
) -> Result<usize, FlowError> {
|
|
self.check("copy_record_range", Role::Consumer)?;
|
|
let generation = self.field_u64(OFF_GENERATION);
|
|
if generation != cursor.generation {
|
|
return Err(FlowError::StaleReservation {
|
|
reservation: cursor.generation,
|
|
ring: generation,
|
|
});
|
|
}
|
|
let consume = self.consume_cursor();
|
|
if consume != cursor.record_start {
|
|
return Err(FlowError::PinnedRecordMoved {
|
|
expected: cursor.record_start,
|
|
found: consume,
|
|
});
|
|
}
|
|
if offset > cursor.payload_len {
|
|
return Err(FlowError::BeyondCommitted {
|
|
requested: offset,
|
|
readable: cursor.payload_len,
|
|
});
|
|
}
|
|
let count = destination
|
|
.len()
|
|
.min(usize::try_from(cursor.payload_len - offset).unwrap_or(usize::MAX));
|
|
self.copy_into_slice(cursor.payload_start + offset, &mut destination[..count]);
|
|
Ok(count)
|
|
}
|
|
|
|
pub fn release_record_cursor(&mut self, cursor: RecordCursor) -> Result<(), FlowError> {
|
|
self.check("release_record_cursor", Role::Consumer)?;
|
|
let generation = self.field_u64(OFF_GENERATION);
|
|
if generation != cursor.generation {
|
|
return Err(FlowError::StaleReservation {
|
|
reservation: cursor.generation,
|
|
ring: generation,
|
|
});
|
|
}
|
|
let consume = self.consume_cursor();
|
|
if consume != cursor.record_start {
|
|
return Err(FlowError::PinnedRecordMoved {
|
|
expected: cursor.record_start,
|
|
found: consume,
|
|
});
|
|
}
|
|
self.atomic(OFF_CONSUME)
|
|
.store(cursor.record_start + cursor.record_len, Ordering::Release);
|
|
Ok(())
|
|
}
|
|
|
|
/// Receive one complete record; `Ok(None)` when nothing (or only a
|
|
/// torn, uncommitted prefix) is readable.
|
|
pub fn recv_record(&mut self) -> Result<Option<(RecordKind, Vec<u8>)>, FlowError> {
|
|
self.check("recv_record", Role::Consumer)?;
|
|
let (commit, consume) = (self.commit_cursor(), self.consume_cursor());
|
|
let readable = commit - consume;
|
|
if readable < RECORD_HEADER_LEN {
|
|
return Ok(None);
|
|
}
|
|
let prefix = self.copy_out(consume, RECORD_HEADER_LEN);
|
|
let kind = RecordKind::from_byte(prefix[0])
|
|
.ok_or(FlowError::BadRecord(RecordError::InvalidKind(prefix[0])))?;
|
|
let len = u32::from_le_bytes(prefix[1..5].try_into().unwrap()) as u64;
|
|
if RECORD_HEADER_LEN + len > self.info.capacity {
|
|
return Err(FlowError::BadRecord(RecordError::LengthExceedsCapacity {
|
|
len,
|
|
capacity: self.info.capacity,
|
|
}));
|
|
}
|
|
if readable < RECORD_HEADER_LEN + len {
|
|
// Torn tail: the producer has not committed the whole record.
|
|
return Ok(None);
|
|
}
|
|
let payload = self.copy_out(consume + RECORD_HEADER_LEN, len);
|
|
self.atomic(OFF_CONSUME)
|
|
.store(consume + RECORD_HEADER_LEN + len, Ordering::Release);
|
|
Ok(Some((kind, payload)))
|
|
}
|
|
}
|