Zachery Aaron Shores-Chmielewski
3598b8a359
Add actor-owned contextual process provisioning, descriptor bootstrap, scoped namespace operations, Python bindings, Myelin orchestration, and adversarial end-to-end coverage. Remove the superseded job runner, pipeline APIs, images, entrypoints, and specifications.
1338 lines
44 KiB
Rust
1338 lines
44 KiB
Rust
use std::collections::BTreeMap;
|
|
use std::path::{Path, PathBuf};
|
|
use std::sync::Arc;
|
|
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
|
use std::time::{Duration, Instant};
|
|
|
|
use data_plane::namespace::{
|
|
DataDirectoryActor, DataDirectoryIn, DirectoryClient, EntryKind, NamespaceClient,
|
|
NamespaceClientActor, NamespaceClientIn, NamespaceDiscovery, NamespaceError, OperationId,
|
|
RetirementRetry, SourceRecovery, StreamRole,
|
|
};
|
|
use data_plane::path::DataPath;
|
|
use futures_lite::future;
|
|
use parking_lot::RwLock;
|
|
use proptest::prelude::*;
|
|
use swactor::actor::ActorAddress;
|
|
use swactor::config::RuntimeConfig;
|
|
use swactor::runtime::{Runtime, RuntimeParts};
|
|
use swactor_engine::{Engine, TokioBackend, TokioConfig};
|
|
|
|
static NEXT_TEMP: AtomicU64 = AtomicU64::new(1);
|
|
|
|
struct TempState {
|
|
root: PathBuf,
|
|
}
|
|
|
|
impl TempState {
|
|
fn new(label: &str) -> Self {
|
|
let sequence = NEXT_TEMP.fetch_add(1, Ordering::Relaxed);
|
|
let root = std::env::temp_dir().join(format!(
|
|
"swactor-namespace-{label}-{}-{sequence}",
|
|
std::process::id()
|
|
));
|
|
std::fs::create_dir_all(&root).expect("create temporary namespace directory");
|
|
Self { root }
|
|
}
|
|
|
|
fn store(&self) -> PathBuf {
|
|
self.root.join("namespace.json")
|
|
}
|
|
}
|
|
|
|
impl Drop for TempState {
|
|
fn drop(&mut self) {
|
|
let _ = std::fs::remove_dir_all(&self.root);
|
|
}
|
|
}
|
|
|
|
struct DirectoryHarness {
|
|
engine: Engine,
|
|
runtime: Runtime,
|
|
client: DirectoryClient,
|
|
}
|
|
|
|
fn path(value: &str) -> DataPath {
|
|
DataPath::parse(value).expect("test path")
|
|
}
|
|
|
|
fn source(byte: u8) -> ActorAddress {
|
|
ActorAddress([byte; 32])
|
|
}
|
|
|
|
fn recovery(actor: ActorAddress) -> SourceRecovery {
|
|
SourceRecovery::Actor { actor }
|
|
}
|
|
fn spawn_directory(store: &Path) -> DirectoryHarness {
|
|
let parts = RuntimeParts::new(RuntimeConfig {
|
|
worker_count: 1,
|
|
..RuntimeConfig::default()
|
|
});
|
|
let runtime = parts.runtime().clone();
|
|
let engine = Engine::new(
|
|
parts,
|
|
TokioBackend::new(TokioConfig {
|
|
worker_threads: 1,
|
|
..TokioConfig::default()
|
|
})
|
|
.expect("tokio backend"),
|
|
)
|
|
.expect("directory engine");
|
|
let actor = DataDirectoryActor::recover(
|
|
store,
|
|
Some(RetirementRetry::new(
|
|
engine.handle(),
|
|
runtime.create_sender(),
|
|
Duration::from_millis(50),
|
|
)),
|
|
|record, _length| match record {
|
|
SourceRecovery::Actor { actor } => Ok(*actor),
|
|
SourceRecovery::File { path } => Err(NamespaceError::SourceRecovery(format!(
|
|
"test cannot recover file source {}",
|
|
path.display()
|
|
))),
|
|
},
|
|
)
|
|
.expect("recover directory");
|
|
let directory = runtime.spawn(actor).expect("spawn directory actor");
|
|
DirectoryHarness {
|
|
engine,
|
|
runtime: runtime.clone(),
|
|
client: DirectoryClient::new(runtime, directory),
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn namespace_mutations_are_linearizable_and_durable() {
|
|
let state = TempState::new("linearizable");
|
|
let directory = spawn_directory(&state.store());
|
|
let logical = path("/models/tiny-linear/weights");
|
|
let first = source(1);
|
|
let second = source(2);
|
|
|
|
future::block_on(async {
|
|
let registered = directory
|
|
.client
|
|
.register(
|
|
logical.clone(),
|
|
first,
|
|
24,
|
|
recovery(first),
|
|
OperationId::from_u128(1),
|
|
)
|
|
.await
|
|
.expect("register first source");
|
|
assert_eq!(registered.revision, 1);
|
|
let node = directory
|
|
.client
|
|
.lookup(logical.clone())
|
|
.await
|
|
.expect("lookup first blob node");
|
|
assert_eq!(node.kind, EntryKind::Blob);
|
|
assert_eq!(node.revision, registered.revision);
|
|
|
|
let selected_first = directory
|
|
.client
|
|
.resolve(logical.clone())
|
|
.await
|
|
.expect("resolve first source");
|
|
assert_eq!(selected_first.source, first);
|
|
assert_eq!(selected_first.length, 24);
|
|
assert_eq!(selected_first.revision, 1);
|
|
|
|
let replaced = directory
|
|
.client
|
|
.register(
|
|
logical.clone(),
|
|
second,
|
|
32,
|
|
recovery(second),
|
|
OperationId::from_u128(2),
|
|
)
|
|
.await
|
|
.expect("replace source");
|
|
assert_eq!(replaced.revision, 2);
|
|
|
|
let selected_second = directory
|
|
.client
|
|
.resolve(logical.clone())
|
|
.await
|
|
.expect("resolve replacement");
|
|
assert_eq!(selected_second.source, second);
|
|
assert_eq!(selected_second.length, 32);
|
|
assert_eq!(selected_second.revision, 2);
|
|
|
|
// A completed resolve is a binding snapshot. Later replacement cannot
|
|
// mutate the source selected by the earlier logical read.
|
|
assert_eq!(selected_first.source, first);
|
|
assert_eq!(selected_first.revision, 1);
|
|
|
|
let removed = directory
|
|
.client
|
|
.unregister(logical.clone(), OperationId::from_u128(3))
|
|
.await
|
|
.expect("unregister source");
|
|
assert_eq!(removed.revision, 3);
|
|
assert!(matches!(
|
|
directory.client.resolve(logical.clone()).await,
|
|
Err(NamespaceError::PathNotFound(found)) if found == logical
|
|
));
|
|
});
|
|
|
|
// The acknowledged removal is recovered from a fresh actor instance.
|
|
let recovered = spawn_directory(&state.store());
|
|
assert!(matches!(
|
|
future::block_on(recovered.client.resolve(logical.clone())),
|
|
Err(NamespaceError::PathNotFound(found)) if found == logical
|
|
));
|
|
}
|
|
|
|
#[test]
|
|
fn rename_is_atomic_replayable_and_revisioned() {
|
|
let state = TempState::new("rename");
|
|
let directory = spawn_directory(&state.store());
|
|
let source_path = path("/rename/source");
|
|
let destination_path = path("/rename/destination");
|
|
let source_actor = source(7);
|
|
let destination_actor = source(8);
|
|
|
|
future::block_on(async {
|
|
directory
|
|
.client
|
|
.register(
|
|
source_path.clone(),
|
|
source_actor,
|
|
7,
|
|
recovery(source_actor),
|
|
OperationId::from_u128(100),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
directory
|
|
.client
|
|
.register(
|
|
destination_path.clone(),
|
|
destination_actor,
|
|
8,
|
|
recovery(destination_actor),
|
|
OperationId::from_u128(101),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
|
|
assert!(matches!(
|
|
directory
|
|
.client
|
|
.rename(
|
|
source_path.clone(),
|
|
destination_path.clone(),
|
|
false,
|
|
OperationId::from_u128(102),
|
|
)
|
|
.await,
|
|
Err(NamespaceError::PathExists(path)) if path == destination_path
|
|
));
|
|
assert_eq!(
|
|
directory
|
|
.client
|
|
.resolve(source_path.clone())
|
|
.await
|
|
.unwrap()
|
|
.source,
|
|
source_actor
|
|
);
|
|
assert_eq!(
|
|
directory
|
|
.client
|
|
.resolve(destination_path.clone())
|
|
.await
|
|
.unwrap()
|
|
.source,
|
|
destination_actor
|
|
);
|
|
|
|
let renamed = directory
|
|
.client
|
|
.rename(
|
|
source_path.clone(),
|
|
destination_path.clone(),
|
|
true,
|
|
OperationId::from_u128(103),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(renamed.revision, 3);
|
|
let replayed = directory
|
|
.client
|
|
.rename(
|
|
source_path.clone(),
|
|
destination_path.clone(),
|
|
true,
|
|
OperationId::from_u128(103),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(replayed, renamed);
|
|
assert!(matches!(
|
|
directory.client.lookup(source_path.clone()).await,
|
|
Err(NamespaceError::PathNotFound(path)) if path == source_path
|
|
));
|
|
let resolved = directory
|
|
.client
|
|
.resolve(destination_path.clone())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resolved.source, source_actor);
|
|
assert_eq!(resolved.revision, renamed.revision);
|
|
});
|
|
|
|
let recovered = spawn_directory(&state.store());
|
|
let resolved = future::block_on(recovered.client.resolve(destination_path)).unwrap();
|
|
assert_eq!(resolved.source, source_actor);
|
|
assert_eq!(resolved.revision, 3);
|
|
}
|
|
|
|
#[test]
|
|
fn mutation_rejections_are_sticky_across_reservation_lifecycle() {
|
|
let state = TempState::new("sticky-rejections");
|
|
let directory = spawn_directory(&state.store());
|
|
let source_path = path("/sticky/source");
|
|
let reserved_path = path("/sticky/reserved");
|
|
let unbound_path = path("/sticky/unbound");
|
|
let stream_path = path("/sticky/stream");
|
|
let source_actor = source(9);
|
|
|
|
future::block_on(async {
|
|
directory
|
|
.client
|
|
.register(
|
|
source_path.clone(),
|
|
source_actor,
|
|
5,
|
|
recovery(source_actor),
|
|
OperationId::from_u128(200),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
|
|
// A concurrent blob upload holds a reservation on the rename target.
|
|
directory
|
|
.client
|
|
.reserve_blob(reserved_path.clone(), OperationId::from_u128(201))
|
|
.await
|
|
.unwrap();
|
|
assert!(matches!(
|
|
directory
|
|
.client
|
|
.rename(
|
|
source_path.clone(),
|
|
reserved_path.clone(),
|
|
true,
|
|
OperationId::from_u128(202),
|
|
)
|
|
.await,
|
|
Err(NamespaceError::PathExists(path)) if path == reserved_path
|
|
));
|
|
// The upload is abandoned and its reservation released. Retrying the
|
|
// SAME operation id must replay the rejection: an id the client saw
|
|
// rejected may never commit later, whatever else changed.
|
|
directory
|
|
.client
|
|
.release_blob_reservation(reserved_path.clone(), OperationId::from_u128(201))
|
|
.await
|
|
.unwrap();
|
|
assert!(matches!(
|
|
directory
|
|
.client
|
|
.rename(
|
|
source_path.clone(),
|
|
reserved_path.clone(),
|
|
true,
|
|
OperationId::from_u128(202),
|
|
)
|
|
.await,
|
|
Err(NamespaceError::PathExists(path)) if path == reserved_path
|
|
));
|
|
assert_eq!(
|
|
directory
|
|
.client
|
|
.resolve(source_path.clone())
|
|
.await
|
|
.unwrap()
|
|
.source,
|
|
source_actor
|
|
);
|
|
|
|
// Unregister against a reserved-but-unbound path is sticky the same way.
|
|
directory
|
|
.client
|
|
.reserve_blob(unbound_path.clone(), OperationId::from_u128(203))
|
|
.await
|
|
.unwrap();
|
|
assert!(matches!(
|
|
directory
|
|
.client
|
|
.unregister(unbound_path.clone(), OperationId::from_u128(204))
|
|
.await,
|
|
Err(NamespaceError::PathExists(path)) if path == unbound_path
|
|
));
|
|
directory
|
|
.client
|
|
.release_blob_reservation(unbound_path.clone(), OperationId::from_u128(203))
|
|
.await
|
|
.unwrap();
|
|
assert!(matches!(
|
|
directory
|
|
.client
|
|
.unregister(unbound_path.clone(), OperationId::from_u128(204))
|
|
.await,
|
|
Err(NamespaceError::PathExists(path)) if path == unbound_path
|
|
));
|
|
|
|
// Stream binding against a reserved path replays its rejection too.
|
|
directory
|
|
.client
|
|
.reserve_blob(stream_path.clone(), OperationId::from_u128(205))
|
|
.await
|
|
.unwrap();
|
|
assert!(matches!(
|
|
directory
|
|
.client
|
|
.replace_with_stream(
|
|
stream_path.clone(),
|
|
StreamRole::Source,
|
|
source(4),
|
|
OperationId::from_u128(206),
|
|
)
|
|
.await,
|
|
Err(NamespaceError::PathExists(path)) if path == stream_path
|
|
));
|
|
directory
|
|
.client
|
|
.release_blob_reservation(stream_path.clone(), OperationId::from_u128(205))
|
|
.await
|
|
.unwrap();
|
|
assert!(matches!(
|
|
directory
|
|
.client
|
|
.replace_with_stream(
|
|
stream_path.clone(),
|
|
StreamRole::Source,
|
|
source(4),
|
|
OperationId::from_u128(206),
|
|
)
|
|
.await,
|
|
Err(NamespaceError::PathExists(path)) if path == stream_path
|
|
));
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
fn committed_mutations_replay_even_when_path_is_reserved_again() {
|
|
let state = TempState::new("replay-across-reservation");
|
|
let directory = spawn_directory(&state.store());
|
|
let logical = path("/replay/reserved");
|
|
let blob = source(3);
|
|
|
|
future::block_on(async {
|
|
directory
|
|
.client
|
|
.register(
|
|
logical.clone(),
|
|
blob,
|
|
12,
|
|
recovery(blob),
|
|
OperationId::from_u128(300),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let removed = directory
|
|
.client
|
|
.unregister(logical.clone(), OperationId::from_u128(301))
|
|
.await
|
|
.unwrap();
|
|
|
|
// A later upload reserves the freed path. A duplicate delivery of the
|
|
// already-acknowledged unregister must still replay its receipt
|
|
// instead of failing against the unrelated reservation.
|
|
directory
|
|
.client
|
|
.reserve_blob(logical.clone(), OperationId::from_u128(302))
|
|
.await
|
|
.unwrap();
|
|
let replayed = directory
|
|
.client
|
|
.unregister(logical.clone(), OperationId::from_u128(301))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(replayed, removed);
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
fn active_stream_rename_is_typed_and_quiescent_stream_rename_succeeds() {
|
|
let state = TempState::new("stream-rename");
|
|
let directory = spawn_directory(&state.store());
|
|
let source_path = path("/rename/stream-source");
|
|
let destination_path = path("/rename/stream-destination");
|
|
|
|
future::block_on(async {
|
|
let mut source_open = Box::pin(directory.client.open_stream(
|
|
source_path.clone(),
|
|
StreamRole::Source,
|
|
source(30),
|
|
OperationId::from_u128(110),
|
|
));
|
|
assert!(future::poll_once(source_open.as_mut()).await.is_none());
|
|
let sink_match = directory
|
|
.client
|
|
.open_stream(
|
|
source_path.clone(),
|
|
StreamRole::Sink,
|
|
source(31),
|
|
OperationId::from_u128(111),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let source_match = source_open.await.unwrap();
|
|
assert_eq!(source_match, sink_match);
|
|
let missing_source = path("/rename/missing-source");
|
|
assert!(matches!(
|
|
directory
|
|
.client
|
|
.rename(
|
|
missing_source.clone(),
|
|
source_path.clone(),
|
|
true,
|
|
OperationId::from_u128(114),
|
|
)
|
|
.await,
|
|
Err(NamespaceError::PathNotFound(path)) if path == missing_source
|
|
));
|
|
|
|
assert!(matches!(
|
|
directory
|
|
.client
|
|
.rename(
|
|
source_path.clone(),
|
|
destination_path.clone(),
|
|
false,
|
|
OperationId::from_u128(112),
|
|
)
|
|
.await,
|
|
Err(NamespaceError::StreamActive(path)) if path == source_path
|
|
));
|
|
directory
|
|
.client
|
|
.close_stream(source_path.clone(), source_match.incarnation)
|
|
.await
|
|
.unwrap();
|
|
let renamed = directory
|
|
.client
|
|
.rename(
|
|
source_path.clone(),
|
|
destination_path.clone(),
|
|
false,
|
|
OperationId::from_u128(113),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let node = directory
|
|
.client
|
|
.lookup(destination_path.clone())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(node.kind, EntryKind::Stream);
|
|
assert_eq!(node.revision, renamed.revision);
|
|
assert!(matches!(
|
|
directory.client.lookup(source_path.clone()).await,
|
|
Err(NamespaceError::PathNotFound(path)) if path == source_path
|
|
));
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
fn stream_rendezvous_is_symmetric_and_incarnations_are_isolated() {
|
|
let state = TempState::new("stream-rendezvous");
|
|
let directory = spawn_directory(&state.store());
|
|
let logical = path("/runs/7/results");
|
|
|
|
future::block_on(async {
|
|
let mut source_open = Box::pin(directory.client.open_stream(
|
|
logical.clone(),
|
|
StreamRole::Source,
|
|
source(10),
|
|
OperationId::from_u128(10),
|
|
));
|
|
assert!(future::poll_once(source_open.as_mut()).await.is_none());
|
|
let node = directory
|
|
.client
|
|
.lookup(logical.clone())
|
|
.await
|
|
.expect("lookup ensured stream node");
|
|
assert_eq!(node.kind, EntryKind::Stream);
|
|
|
|
let sink_match = directory
|
|
.client
|
|
.open_stream(
|
|
logical.clone(),
|
|
StreamRole::Sink,
|
|
source(11),
|
|
OperationId::from_u128(11),
|
|
)
|
|
.await
|
|
.expect("sink matches source");
|
|
let source_match = source_open.await.expect("source matches sink");
|
|
assert_eq!(source_match, sink_match);
|
|
assert_eq!(source_match.source, source(10));
|
|
assert_eq!(source_match.sink, source(11));
|
|
|
|
directory
|
|
.client
|
|
.close_stream(logical.clone(), source_match.incarnation)
|
|
.await
|
|
.expect("close first incarnation");
|
|
|
|
assert!(matches!(
|
|
directory
|
|
.client
|
|
.open_stream(
|
|
logical.clone(),
|
|
StreamRole::Source,
|
|
source(10),
|
|
OperationId::from_u128(10),
|
|
)
|
|
.await,
|
|
Err(NamespaceError::StaleIncarnation { .. })
|
|
));
|
|
assert!(matches!(
|
|
directory
|
|
.client
|
|
.open_stream(
|
|
logical.clone(),
|
|
StreamRole::Sink,
|
|
source(11),
|
|
OperationId::from_u128(11),
|
|
)
|
|
.await,
|
|
Err(NamespaceError::StaleIncarnation { .. })
|
|
));
|
|
|
|
let mut sink_open = Box::pin(directory.client.open_stream(
|
|
logical.clone(),
|
|
StreamRole::Sink,
|
|
source(12),
|
|
OperationId::from_u128(12),
|
|
));
|
|
assert!(future::poll_once(sink_open.as_mut()).await.is_none());
|
|
assert!(matches!(
|
|
directory
|
|
.client
|
|
.open_stream(
|
|
logical.clone(),
|
|
StreamRole::Source,
|
|
source(10),
|
|
OperationId::from_u128(10),
|
|
)
|
|
.await,
|
|
Err(NamespaceError::StaleIncarnation { .. })
|
|
));
|
|
let second_source = directory
|
|
.client
|
|
.open_stream(
|
|
logical,
|
|
StreamRole::Source,
|
|
source(13),
|
|
OperationId::from_u128(13),
|
|
)
|
|
.await
|
|
.expect("source matches waiting sink");
|
|
let second_sink = sink_open.await.expect("sink matches source");
|
|
assert_eq!(second_source, second_sink);
|
|
assert_ne!(source_match.incarnation, second_source.incarnation);
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
fn typed_paths_require_explicit_rebinding() {
|
|
let state = TempState::new("typed-path");
|
|
let directory = spawn_directory(&state.store());
|
|
let logical = path("/typed/value");
|
|
let blob_source = source(20);
|
|
|
|
future::block_on(async {
|
|
directory
|
|
.client
|
|
.register(
|
|
logical.clone(),
|
|
blob_source,
|
|
4,
|
|
recovery(blob_source),
|
|
OperationId::from_u128(20),
|
|
)
|
|
.await
|
|
.expect("register blob");
|
|
|
|
assert!(matches!(
|
|
directory
|
|
.client
|
|
.open_stream(
|
|
logical.clone(),
|
|
StreamRole::Source,
|
|
source(21),
|
|
OperationId::from_u128(21),
|
|
)
|
|
.await,
|
|
Err(NamespaceError::WrongEntryType {
|
|
expected: EntryKind::Stream,
|
|
found: EntryKind::Blob,
|
|
..
|
|
})
|
|
));
|
|
|
|
let mut source_open = Box::pin(directory.client.replace_with_stream(
|
|
logical.clone(),
|
|
StreamRole::Source,
|
|
source(22),
|
|
OperationId::from_u128(22),
|
|
));
|
|
assert!(future::poll_once(source_open.as_mut()).await.is_none());
|
|
assert!(matches!(
|
|
directory.client.resolve(logical.clone()).await,
|
|
Err(NamespaceError::WrongEntryType {
|
|
expected: EntryKind::Blob,
|
|
found: EntryKind::Stream,
|
|
..
|
|
})
|
|
));
|
|
let sink_match = directory
|
|
.client
|
|
.open_stream(
|
|
logical,
|
|
StreamRole::Sink,
|
|
source(23),
|
|
OperationId::from_u128(23),
|
|
)
|
|
.await
|
|
.expect("match rebound stream");
|
|
assert_eq!(
|
|
source_open.await.expect("rebound source matched"),
|
|
sink_match
|
|
);
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
fn replacing_waiting_stream_displaces_old_open() {
|
|
let state = TempState::new("stream-displacement");
|
|
let directory = spawn_directory(&state.store());
|
|
let logical = path("/replace/waiting");
|
|
|
|
future::block_on(async {
|
|
let mut old_open = Box::pin(directory.client.open_stream(
|
|
logical.clone(),
|
|
StreamRole::Source,
|
|
source(30),
|
|
OperationId::from_u128(30),
|
|
));
|
|
assert!(future::poll_once(old_open.as_mut()).await.is_none());
|
|
|
|
let mut replacement = Box::pin(directory.client.replace_with_stream(
|
|
logical.clone(),
|
|
StreamRole::Source,
|
|
source(31),
|
|
OperationId::from_u128(31),
|
|
));
|
|
assert!(future::poll_once(replacement.as_mut()).await.is_none());
|
|
assert!(matches!(
|
|
old_open.await,
|
|
Err(NamespaceError::PathReplaced(found)) if found == logical
|
|
));
|
|
|
|
let sink_match = directory
|
|
.client
|
|
.open_stream(
|
|
logical,
|
|
StreamRole::Sink,
|
|
source(32),
|
|
OperationId::from_u128(32),
|
|
)
|
|
.await
|
|
.expect("sink matches replacement");
|
|
assert_eq!(
|
|
replacement.await.expect("replacement source matched"),
|
|
sink_match
|
|
);
|
|
assert_eq!(sink_match.source, source(31));
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
fn duplicate_stream_role_fails_without_replacing_the_waiter() {
|
|
let state = TempState::new("duplicate-stream-role");
|
|
let directory = spawn_directory(&state.store());
|
|
let logical = path("/duplicate/source");
|
|
|
|
future::block_on(async {
|
|
let mut first = Box::pin(directory.client.open_stream(
|
|
logical.clone(),
|
|
StreamRole::Source,
|
|
source(51),
|
|
OperationId::from_u128(51),
|
|
));
|
|
assert!(future::poll_once(first.as_mut()).await.is_none());
|
|
assert!(matches!(
|
|
directory
|
|
.client
|
|
.open_stream(
|
|
logical.clone(),
|
|
StreamRole::Source,
|
|
source(52),
|
|
OperationId::from_u128(52),
|
|
)
|
|
.await,
|
|
Err(NamespaceError::DuplicateStreamRole {
|
|
role: StreamRole::Source,
|
|
..
|
|
})
|
|
));
|
|
let matched = directory
|
|
.client
|
|
.open_stream(
|
|
logical,
|
|
StreamRole::Sink,
|
|
source(53),
|
|
OperationId::from_u128(53),
|
|
)
|
|
.await
|
|
.expect("sink matches original source");
|
|
assert_eq!(matched.source, source(51));
|
|
assert_eq!(first.await.expect("original source survives"), matched);
|
|
});
|
|
}
|
|
#[test]
|
|
fn committed_mutation_retry_has_at_most_once_effect() {
|
|
let state = TempState::new("idempotent");
|
|
let logical = path("/models/a");
|
|
let actor = source(7);
|
|
let operation = OperationId::from_u128(99);
|
|
|
|
let first_runtime = spawn_directory(&state.store());
|
|
let first_receipt = future::block_on(first_runtime.client.register(
|
|
logical.clone(),
|
|
actor,
|
|
8,
|
|
recovery(actor),
|
|
operation,
|
|
))
|
|
.expect("initial registration");
|
|
assert_eq!(first_receipt.revision, 1);
|
|
drop(first_runtime);
|
|
|
|
let recovered = spawn_directory(&state.store());
|
|
let retried = future::block_on(recovered.client.register(
|
|
logical.clone(),
|
|
actor,
|
|
8,
|
|
recovery(actor),
|
|
operation,
|
|
))
|
|
.expect("retry committed registration");
|
|
assert_eq!(retried, first_receipt);
|
|
assert_eq!(
|
|
future::block_on(recovered.client.resolve(logical.clone()))
|
|
.expect("resolve recovered binding")
|
|
.revision,
|
|
1
|
|
);
|
|
|
|
let conflict = future::block_on(recovered.client.register(
|
|
logical,
|
|
source(8),
|
|
9,
|
|
recovery(source(8)),
|
|
operation,
|
|
));
|
|
assert!(matches!(conflict, Err(NamespaceError::OperationConflict(id)) if id == operation));
|
|
}
|
|
|
|
struct StaticDiscovery {
|
|
directory: Arc<RwLock<Option<ActorAddress>>>,
|
|
}
|
|
|
|
impl NamespaceDiscovery for StaticDiscovery {
|
|
fn current_directory(&self) -> Option<ActorAddress> {
|
|
*self.directory.read()
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn unresolved_request_waits_for_recovered_authority() {
|
|
let state = TempState::new("restart");
|
|
let harness = spawn_directory(&state.store());
|
|
let logical = path("/models/restartable");
|
|
let source = source(44);
|
|
future::block_on(harness.client.register(
|
|
logical.clone(),
|
|
source,
|
|
16,
|
|
recovery(source),
|
|
OperationId::from_u128(500),
|
|
))
|
|
.expect("seed durable binding");
|
|
let old_directory = harness.client.directory();
|
|
harness.runtime.stop_actor(old_directory).unwrap();
|
|
|
|
let discovered = Arc::new(RwLock::new(None));
|
|
let proxy = harness
|
|
.runtime
|
|
.spawn(NamespaceClientActor::new(
|
|
harness.engine.handle(),
|
|
harness.runtime.create_sender(),
|
|
Arc::new(StaticDiscovery {
|
|
directory: Arc::clone(&discovered),
|
|
}),
|
|
Duration::from_millis(5),
|
|
))
|
|
.expect("spawn namespace client");
|
|
let client = NamespaceClient::new(harness.runtime.clone(), proxy);
|
|
|
|
future::block_on(async {
|
|
let mut resolving = Box::pin(client.resolve(logical.clone()));
|
|
assert!(
|
|
future::poll_once(resolving.as_mut()).await.is_none(),
|
|
"resolve must remain pending while authority is absent"
|
|
);
|
|
|
|
let recovered = DataDirectoryActor::recover(
|
|
state.store(),
|
|
Some(RetirementRetry::new(
|
|
harness.engine.handle(),
|
|
harness.runtime.create_sender(),
|
|
Duration::from_millis(50),
|
|
)),
|
|
|record, _length| match record {
|
|
SourceRecovery::Actor { actor } => Ok(*actor),
|
|
SourceRecovery::File { path } => Err(NamespaceError::SourceRecovery(format!(
|
|
"test cannot recover file source {}",
|
|
path.display()
|
|
))),
|
|
},
|
|
)
|
|
.expect("recover directory state");
|
|
let recovered = harness
|
|
.runtime
|
|
.spawn(recovered)
|
|
.expect("spawn recovered directory");
|
|
*discovered.write() = Some(recovered);
|
|
harness
|
|
.runtime
|
|
.send_to(proxy, NamespaceClientIn::Retry)
|
|
.expect("trigger rediscovery retry");
|
|
|
|
let binding = resolving.await.expect("resolve after authority recovery");
|
|
assert_eq!(binding.source, source);
|
|
assert_eq!(binding.length, 16);
|
|
assert_eq!(binding.revision, 1);
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
fn unresolved_request_fails_bounded_when_authority_is_lost() {
|
|
let discovered = Arc::new(RwLock::new(None));
|
|
let parts = RuntimeParts::new(RuntimeConfig {
|
|
worker_count: 1,
|
|
..RuntimeConfig::default()
|
|
});
|
|
let runtime = parts.runtime().clone();
|
|
let engine = Engine::new(
|
|
parts,
|
|
TokioBackend::new(TokioConfig {
|
|
worker_threads: 1,
|
|
..TokioConfig::default()
|
|
})
|
|
.expect("tokio backend"),
|
|
)
|
|
.expect("client engine");
|
|
let proxy = runtime
|
|
.spawn(NamespaceClientActor::new_with_deadline(
|
|
engine.handle(),
|
|
runtime.create_sender(),
|
|
Arc::new(StaticDiscovery {
|
|
directory: Arc::clone(&discovered),
|
|
}),
|
|
Duration::from_millis(5),
|
|
Duration::from_millis(150),
|
|
))
|
|
.expect("spawn namespace client");
|
|
let client = NamespaceClient::new(runtime.clone(), proxy);
|
|
|
|
let outcome = future::block_on(async {
|
|
let bound = Instant::now() + Duration::from_secs(5);
|
|
let mut resolving = Box::pin(client.resolve(path("/models/lost")));
|
|
loop {
|
|
if let Some(outcome) = future::poll_once(resolving.as_mut()).await {
|
|
return outcome;
|
|
}
|
|
assert!(
|
|
Instant::now() < bound,
|
|
"authority loss must fail the request within the deadline, not hang"
|
|
);
|
|
std::thread::yield_now();
|
|
}
|
|
});
|
|
assert!(
|
|
matches!(outcome, Err(NamespaceError::DirectoryUnavailable(_))),
|
|
"expected a bounded directory-unavailable failure, got {outcome:?}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn namespace_process_restart_helper() {
|
|
let Ok(mode) = std::env::var("SWACTOR_NAMESPACE_HELPER_MODE") else {
|
|
return;
|
|
};
|
|
let store = PathBuf::from(
|
|
std::env::var_os("SWACTOR_NAMESPACE_HELPER_STORE").expect("helper store path"),
|
|
);
|
|
let directory = spawn_directory(&store);
|
|
let logical = path("/models/process-restart");
|
|
match mode.as_str() {
|
|
"write" => {
|
|
let actor = source(55);
|
|
let receipt = future::block_on(directory.client.register(
|
|
logical,
|
|
actor,
|
|
32,
|
|
recovery(actor),
|
|
OperationId::from_u128(900),
|
|
))
|
|
.expect("helper durable registration");
|
|
assert_eq!(receipt.revision, 1);
|
|
}
|
|
"read" => {
|
|
let binding =
|
|
future::block_on(directory.client.resolve(logical)).expect("helper recovery");
|
|
assert_eq!(binding.source, source(55));
|
|
assert_eq!(binding.length, 32);
|
|
assert_eq!(binding.revision, 1);
|
|
}
|
|
other => panic!("unknown namespace helper mode {other}"),
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn namespace_client_routes_remote_transfer_failures_to_local_binding() {
|
|
let state = TempState::new("transfer-failure");
|
|
let harness = spawn_directory(&state.store());
|
|
let proxy = harness
|
|
.runtime
|
|
.spawn(NamespaceClientActor::new(
|
|
harness.engine.handle(),
|
|
harness.runtime.create_sender(),
|
|
Arc::new(StaticDiscovery {
|
|
directory: Arc::new(RwLock::new(Some(harness.client.directory()))),
|
|
}),
|
|
Duration::from_millis(5),
|
|
))
|
|
.unwrap();
|
|
let destination = harness
|
|
.runtime
|
|
.new_inbox::<data_plane::blob_transfer::BlobTransferEvent>()
|
|
.unwrap();
|
|
let transfer_id = data_plane::blob_transfer::BlobTransferId(71);
|
|
harness
|
|
.runtime
|
|
.send_to(
|
|
proxy,
|
|
NamespaceClientIn::TransferFailed {
|
|
destination: *destination.addr(),
|
|
transfer_id,
|
|
reason: "injected source failure".to_owned(),
|
|
},
|
|
)
|
|
.unwrap();
|
|
assert_eq!(
|
|
future::block_on(destination.recv()),
|
|
data_plane::blob_transfer::BlobTransferEvent::Failed {
|
|
transfer_id,
|
|
reason: "injected source failure".to_owned(),
|
|
}
|
|
);
|
|
}
|
|
|
|
#[derive(Clone, Debug)]
|
|
struct ModelBinding {
|
|
source: ActorAddress,
|
|
length: u64,
|
|
revision: u64,
|
|
}
|
|
|
|
#[derive(Clone, Debug)]
|
|
enum TypedModelEntry {
|
|
Blob(ModelBinding),
|
|
Stream(Option<data_plane::namespace::StreamMatch>),
|
|
}
|
|
|
|
proptest! {
|
|
#![proptest_config(ProptestConfig {
|
|
cases: 8,
|
|
max_shrink_iters: 128,
|
|
..ProptestConfig::default()
|
|
})]
|
|
|
|
#[test]
|
|
fn legal_action_sequences_preserve_namespace_guarantees(actions in prop::collection::vec(any::<u8>(), 1..48)) {
|
|
let state = TempState::new("stateful");
|
|
let directory = spawn_directory(&state.store());
|
|
let paths = [path("/models/a"), path("/models/b"), path("/runs/7/result")];
|
|
let mut model = BTreeMap::<DataPath, ModelBinding>::new();
|
|
let mut next_revision = 1_u64;
|
|
let mut next_operation = 1_u128;
|
|
|
|
for (step, action) in actions.into_iter().enumerate() {
|
|
let logical = paths[usize::from(action) % paths.len()].clone();
|
|
match action % 4 {
|
|
0 | 1 => {
|
|
let actor = source(action.wrapping_add(step as u8).wrapping_add(1));
|
|
let length = u64::from(action) + 1;
|
|
let operation = OperationId::from_u128(next_operation);
|
|
next_operation += 1;
|
|
let receipt = future::block_on(directory.client.register(
|
|
logical.clone(),
|
|
actor,
|
|
length,
|
|
recovery(actor),
|
|
operation,
|
|
)).expect("model registration");
|
|
prop_assert_eq!(receipt.revision, next_revision);
|
|
model.insert(logical, ModelBinding { source: actor, length, revision: next_revision });
|
|
next_revision += 1;
|
|
}
|
|
2 if model.contains_key(&logical) => {
|
|
let operation = OperationId::from_u128(next_operation);
|
|
next_operation += 1;
|
|
let receipt = future::block_on(directory.client.unregister(logical.clone(), operation))
|
|
.expect("model unregistration");
|
|
prop_assert_eq!(receipt.revision, next_revision);
|
|
model.remove(&logical);
|
|
next_revision += 1;
|
|
}
|
|
_ => {
|
|
let observed = future::block_on(directory.client.resolve(logical.clone()));
|
|
match model.get(&logical) {
|
|
Some(expected) => {
|
|
let observed = observed.expect("model binding must resolve");
|
|
prop_assert_eq!(observed.source, expected.source);
|
|
prop_assert_eq!(observed.length, expected.length);
|
|
prop_assert_eq!(observed.revision, expected.revision);
|
|
}
|
|
None => prop_assert!(matches!(
|
|
observed,
|
|
Err(NamespaceError::PathNotFound(found)) if found == logical
|
|
)),
|
|
}
|
|
}
|
|
}
|
|
|
|
for (path, expected) in &model {
|
|
let observed = future::block_on(directory.client.resolve(path.clone()))
|
|
.expect("all model bindings remain resolvable");
|
|
prop_assert_eq!(observed.source, expected.source);
|
|
|
|
prop_assert_eq!(observed.length, expected.length);
|
|
prop_assert_eq!(observed.revision, expected.revision);
|
|
}
|
|
}
|
|
}
|
|
#[test]
|
|
fn typed_binding_action_strings_match_reference_model(actions in prop::collection::vec(any::<u8>(), 1..64)) {
|
|
let state = TempState::new("typed-stateful");
|
|
let directory = spawn_directory(&state.store());
|
|
let paths = [path("/state/a"), path("/state/b"), path("/state/c")];
|
|
let mut model = BTreeMap::<DataPath, TypedModelEntry>::new();
|
|
let mut next_operation = 10_000_u128;
|
|
|
|
for (step, action) in actions.into_iter().enumerate() {
|
|
let logical = paths[usize::from(action) % paths.len()].clone();
|
|
match action % 4 {
|
|
0 => {
|
|
let actor = source(action.wrapping_add(step as u8).wrapping_add(1));
|
|
let length = u64::from(action) + 1;
|
|
let receipt = future::block_on(directory.client.register(
|
|
logical.clone(),
|
|
actor,
|
|
length,
|
|
recovery(actor),
|
|
OperationId::from_u128(next_operation),
|
|
)).expect("blob rebind");
|
|
next_operation += 1;
|
|
model.insert(logical, TypedModelEntry::Blob(ModelBinding {
|
|
source: actor,
|
|
length,
|
|
revision: receipt.revision,
|
|
}));
|
|
}
|
|
1 => {
|
|
let source_actor = source(action.wrapping_add(41));
|
|
let sink_actor = source(action.wrapping_add(97));
|
|
let mut source_open = Box::pin(directory.client.replace_with_stream(
|
|
logical.clone(),
|
|
StreamRole::Source,
|
|
source_actor,
|
|
OperationId::from_u128(next_operation),
|
|
));
|
|
next_operation += 1;
|
|
prop_assert!(future::block_on(future::poll_once(source_open.as_mut())).is_none());
|
|
let sink_match = future::block_on(directory.client.open_stream(
|
|
logical.clone(),
|
|
StreamRole::Sink,
|
|
sink_actor,
|
|
OperationId::from_u128(next_operation),
|
|
)).expect("sink match");
|
|
next_operation += 1;
|
|
let source_match = future::block_on(source_open).expect("source match");
|
|
prop_assert_eq!(&source_match, &sink_match);
|
|
model.insert(logical, TypedModelEntry::Stream(Some(sink_match)));
|
|
}
|
|
2 => {
|
|
if let Some(TypedModelEntry::Stream(binding)) = model.get_mut(&logical)
|
|
&& let Some(binding) = binding.take()
|
|
{
|
|
future::block_on(directory.client.close_stream(
|
|
logical.clone(),
|
|
binding.incarnation,
|
|
)).expect("close current stream");
|
|
}
|
|
}
|
|
_ => {
|
|
let observed = future::block_on(directory.client.resolve(logical.clone()));
|
|
match model.get(&logical) {
|
|
Some(TypedModelEntry::Blob(expected)) => {
|
|
let observed = observed.expect("blob resolves");
|
|
prop_assert_eq!(observed.source, expected.source);
|
|
prop_assert_eq!(observed.length, expected.length);
|
|
prop_assert_eq!(observed.revision, expected.revision);
|
|
}
|
|
Some(TypedModelEntry::Stream(_)) => {
|
|
let wrong_type = matches!(
|
|
observed,
|
|
Err(NamespaceError::WrongEntryType {
|
|
expected: EntryKind::Blob,
|
|
found: EntryKind::Stream,
|
|
..
|
|
})
|
|
);
|
|
prop_assert!(wrong_type, "blob lookup must reject a stream binding");
|
|
}
|
|
None => prop_assert!(matches!(
|
|
observed,
|
|
Err(NamespaceError::PathNotFound(found)) if found == logical
|
|
)),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn retirement_retry_redrives_until_acknowledged() {
|
|
struct RetireProbe {
|
|
retire_count: Arc<AtomicU64>,
|
|
acknowledge: Arc<AtomicBool>,
|
|
}
|
|
|
|
impl swactor::actor::ActorInterface for RetireProbe {
|
|
type Incoming = data_plane::source::BlobSourceIn;
|
|
type Response = ();
|
|
|
|
fn handle(&mut self, ctx: &swactor::actor::Ctx<'_>, message: Self::Incoming) {
|
|
if let data_plane::source::BlobSourceIn::Retire { reply_to } = message
|
|
&& let Some(reply_to) = reply_to
|
|
{
|
|
self.retire_count.fetch_add(1, Ordering::SeqCst);
|
|
if self.acknowledge.load(Ordering::SeqCst) {
|
|
let _ = ctx.send(
|
|
reply_to,
|
|
DataDirectoryIn::SourceRetired {
|
|
source: ctx.self_addr(),
|
|
},
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
let state = TempState::new("retire-retry");
|
|
let harness = spawn_directory(&state.store());
|
|
let retire_count = Arc::new(AtomicU64::new(0));
|
|
let acknowledge = Arc::new(AtomicBool::new(false));
|
|
let probe = harness
|
|
.runtime
|
|
.spawn(RetireProbe {
|
|
retire_count: Arc::clone(&retire_count),
|
|
acknowledge: Arc::clone(&acknowledge),
|
|
})
|
|
.expect("spawn retire probe");
|
|
|
|
let logical = path("/models/retire-retry");
|
|
future::block_on(harness.client.register(
|
|
logical.clone(),
|
|
probe,
|
|
8,
|
|
recovery(probe),
|
|
OperationId::from_u128(1),
|
|
))
|
|
.expect("register probe source");
|
|
future::block_on(
|
|
harness
|
|
.client
|
|
.unregister(logical.clone(), OperationId::from_u128(2)),
|
|
)
|
|
.expect("unregister probe source");
|
|
|
|
// No other directory traffic follows the unregister: only the periodic
|
|
// retry tick may re-deliver Retire. Poll past several tick periods.
|
|
let redrive_deadline = Instant::now() + Duration::from_secs(2);
|
|
let mut redriven = 0;
|
|
while retire_count.load(Ordering::SeqCst) < 2 {
|
|
assert!(
|
|
Instant::now() < redrive_deadline,
|
|
"retirement was not re-driven without directory traffic (observed {redriven})"
|
|
);
|
|
redriven = retire_count.load(Ordering::SeqCst);
|
|
std::thread::yield_now();
|
|
}
|
|
|
|
// Once the source acknowledges, retries must stop. A tick can already
|
|
// be in flight when the acknowledgement lands, so require the count to
|
|
// stay unchanged for a full quiet window rather than stopping at the
|
|
// first acknowledgement.
|
|
acknowledge.store(true, Ordering::SeqCst);
|
|
let settle_deadline = Instant::now() + Duration::from_secs(2);
|
|
let mut last = retire_count.load(Ordering::SeqCst);
|
|
while last == redriven {
|
|
assert!(
|
|
Instant::now() < settle_deadline,
|
|
"acknowledged Retire was never observed after enabling replies"
|
|
);
|
|
last = retire_count.load(Ordering::SeqCst);
|
|
std::thread::yield_now();
|
|
}
|
|
loop {
|
|
let quiet_until = Instant::now() + Duration::from_millis(300);
|
|
while Instant::now() < quiet_until {
|
|
std::thread::yield_now();
|
|
}
|
|
let observed = retire_count.load(Ordering::SeqCst);
|
|
if observed == last {
|
|
break;
|
|
}
|
|
last = observed;
|
|
assert!(
|
|
Instant::now() < settle_deadline,
|
|
"retirement retries never stopped after SourceRetired (observed {observed})"
|
|
);
|
|
}
|
|
}
|