zacheryasc/swactor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor: prune tests according to spec-defined behavior

Browse source
This commit is contained in:
Zachery Aaron Shores-Chmielewski 2026-07-29 14:34:33 +04:00
parent f0c12531ff
commit 73cfaad5d6
40 changed files with 2006 additions and 11073 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
crates/mvp-system/Cargo.toml
View file

@ -45,7 +45,3 @@ path = "src/bin/orchestrator.rs"
name = "mvp-chat"
path = "src/bin/mvp_chat.rs"
[[test]]
name = "mvp_chat_mock"
path = "tests/mvp_chat_mock.rs"
harness = false

334
crates/mvp-system/src/chat/node_image.rs
View file

@ -1150,337 +1150,3 @@ impl ImageName {
format!("{}:{tag}", self.repository)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[derive(Default)]
struct CollectProgress {
events: Vec<NodeImageProgressEvent>,
}
impl NodeImageProgressSink for CollectProgress {
fn emit(&mut self, event: NodeImageProgressEvent) {
self.events.push(event);
}
}
#[derive(Default)]
struct DryImageCommandRunner {
commands: Vec<(String, Vec<String>, String, Option<String>)>,
labels: BTreeMap<String, BTreeMap<String, String>>,
existing_images: BTreeSet<String>,
manifests: BTreeSet<String>,
image_tags: Vec<(String, String)>,
containers: BTreeSet<String>,
removed_images: Vec<String>,
}
impl ImageCommandRunner for DryImageCommandRunner {
fn run_status(
&mut self,
_root: &Path,
program: &str,
args: &[String],
label: &str,
image_ref: Option<&str>,
progress: &mut Option<&mut dyn NodeImageProgressSink>,
) -> Result<(), String> {
self.commands.push((
program.to_owned(),
args.to_vec(),
label.to_owned(),
image_ref.map(str::to_owned),
));
emit_command_progress(
progress,
label,
image_ref,
0,
NodeImageProgressEventKind::CommandStarted {
program: program.to_owned(),
args: args.to_vec(),
},
);
emit_command_progress(
progress,
label,
image_ref,
1,
NodeImageProgressEventKind::CommandStdout {
line: format!("{label} stdout"),
},
);
emit_command_progress(
progress,
label,
image_ref,
2,
NodeImageProgressEventKind::CommandStderr {
line: format!("{label} stderr"),
},
);
emit_command_progress(
progress,
label,
image_ref,
3,
NodeImageProgressEventKind::CommandExited {
status: "exit status: 0".to_owned(),
code: Some(0),
success: true,
},
);
Ok(())
}
fn docker_image_exists(&mut self, _root: &Path, image_ref: &str) -> bool {
self.existing_images.contains(image_ref)
}
fn docker_image_labels(
&mut self,
_root: &Path,
image_ref: &str,
) -> Result<Option<BTreeMap<String, String>>, String> {
Ok(self.labels.get(image_ref).cloned())
}
fn docker_manifest_exists(&mut self, _root: &Path, image_ref: &str) -> bool {
self.manifests.contains(image_ref)
}
fn docker_image_has_container(&mut self, _root: &Path, image_ref: &str) -> bool {
self.containers.contains(image_ref)
}
fn docker_image_tags(
&mut self,
_root: &Path,
_repository: &str,
) -> Result<Vec<(String, String)>, String> {
Ok(self.image_tags.clone())
}
fn docker_image_remove(&mut self, _root: &Path, image_ref: &str) -> Result<(), String> {
self.removed_images.push(image_ref.to_owned());
Ok(())
}
}
#[test]
fn fake_child_command_emits_structured_progress_and_duration() {
let mut runner = RealImageCommandRunner;
let mut progress = CollectProgress::default();
let mut sink: Option<&mut dyn NodeImageProgressSink> = Some(&mut progress);
runner
.run_status(
Path::new("."),
"sh",
&[
"-c".to_owned(),
"printf 'stdout line\\n'; printf 'stderr line\\n' >&2".to_owned(),
],
"fake child progress",
Some("docker.io/acme/node:test"),
&mut sink,
)
.expect("fake child exits successfully");
assert!(progress.events.iter().any(|event| matches!(
&event.kind,
NodeImageProgressEventKind::CommandStarted { program, .. } if program == "sh"
)));
assert!(progress.events.iter().any(|event| matches!(
&event.kind,
NodeImageProgressEventKind::CommandStdout { line } if line == "stdout line"
)));
assert!(progress.events.iter().any(|event| matches!(
&event.kind,
NodeImageProgressEventKind::CommandStderr { line } if line == "stderr line"
)));
let exit = progress
.events
.iter()
.find(|event| matches!(event.kind, NodeImageProgressEventKind::CommandExited { .. }))
.expect("exit progress event");
assert_eq!(exit.command_label.as_deref(), Some("fake child progress"));
assert_eq!(exit.image_ref.as_deref(), Some("docker.io/acme/node:test"));
assert!(exit.elapsed_ms.is_some());
assert!(matches!(
exit.kind,
NodeImageProgressEventKind::CommandExited {
success: true,
code: Some(0),
..
}
));
}
#[test]
fn fake_child_failure_preserves_command_label_and_status() {
let mut runner = RealImageCommandRunner;
let mut progress = CollectProgress::default();
let mut sink: Option<&mut dyn NodeImageProgressSink> = Some(&mut progress);
let error = runner
.run_status(
Path::new("."),
"sh",
&["-c".to_owned(), "exit 7".to_owned()],
"failing fake child",
None,
&mut sink,
)
.expect_err("fake child failure propagates");
assert!(error.contains("failing fake child"), "{error}");
let exit = progress
.events
.iter()
.find_map(|event| match &event.kind {
NodeImageProgressEventKind::CommandExited {
status,
code,
success,
} => Some((event, status, code, success)),
_ => None,
})
.expect("exit progress event");
assert_eq!(exit.0.command_label.as_deref(), Some("failing fake child"));
assert!(exit.1.contains("exit status"), "{}", exit.1);
assert_eq!(*exit.2, Some(7));
assert!(!*exit.3);
}
#[test]
fn prepare_node_image_with_dry_runner_returns_expected_image_and_progress() {
let node_bin = std::env::current_exe().expect("test binary path resolves");
let mut runner = DryImageCommandRunner::default();
let mut progress = CollectProgress::default();
let mut sink: Option<&mut dyn NodeImageProgressSink> = Some(&mut progress);
let prepared = prepare_node_image_inner(
NodeImageRequest {
requested_image: "docker.io/acme/mvp-node:latest".to_owned(),
base_image: "swactor-mvp-node-base:cuda12.6".to_owned(),
node_bin,
provider: NodeImageProvider::Docker,
extra_tag: Some("smoke".to_owned()),
push: false,
force_refresh: false,
enabled: true,
},
&mut sink,
&mut runner,
)
.expect("dry image preparation succeeds");
assert_eq!(
prepared.image_ref,
format!("docker.io/acme/mvp-node:{}", prepared.tag)
);
assert!(prepared.built);
assert!(!prepared.pushed);
assert!(runner.commands.iter().any(|(_, _, label, image_ref)| {
label == "build mvp node image"
&& image_ref.as_deref() == Some(prepared.image_ref.as_str())
}));
assert!(progress.events.iter().any(|event| matches!(
&event.kind,
NodeImageProgressEventKind::ImageReference { role, image_ref }
if role == "resolved" && image_ref == &prepared.image_ref
)));
assert!(progress.events.iter().any(|event| matches!(
&event.kind,
NodeImageProgressEventKind::CommandStdout { line }
if line == "build mvp node image stdout"
)));
}
#[test]
fn node_image_content_hash_tracks_node_payload_not_unrelated_files() {
let root = workspace_root().expect("workspace root resolves");
let scratch = root
.join("target/node-image-hash-test")
.join(std::process::id().to_string());
fs::create_dir_all(&scratch).expect("scratch dir is writable");
let node_bin = scratch.join("mvp-worker-node");
fs::write(&node_bin, b"worker binary v1").expect("node bin fixture is writable");
let initial =
node_image_content_hash(&root, &node_bin, "base-v1").expect("initial hash succeeds");
fs::write(scratch.join("unrelated.txt"), b"not part of the image")
.expect("unrelated fixture is writable");
let after_unrelated =
node_image_content_hash(&root, &node_bin, "base-v1").expect("unrelated hash succeeds");
fs::write(&node_bin, b"worker binary v2").expect("node bin fixture update is writable");
let after_node_bin =
node_image_content_hash(&root, &node_bin, "base-v1").expect("node bin hash succeeds");
let after_base =
node_image_content_hash(&root, &node_bin, "base-v2").expect("base hash succeeds");
let _ = fs::remove_dir_all(&scratch);
assert_eq!(initial, after_unrelated);
assert_ne!(initial, after_node_bin);
assert_ne!(after_node_bin, after_base);
}
#[test]
fn image_name_splits_tag_after_last_slash() {
let image = ImageName::parse("localhost:5000/team/mvp-node:trial").unwrap();
assert_eq!(image.repository, "localhost:5000/team/mvp-node");
assert_eq!(image.requested_tag.as_deref(), Some("trial"));
assert_eq!(
image.ref_for_tag("git-abcdef"),
"localhost:5000/team/mvp-node:git-abcdef"
);
}
#[test]
fn image_name_keeps_registry_port_without_tag() {
let image = ImageName::parse("localhost:5000/team/mvp-node").unwrap();
assert_eq!(image.repository, "localhost:5000/team/mvp-node");
assert_eq!(image.requested_tag, None);
}
#[test]
fn alias_tags_include_requested_and_extra_without_version_duplicate() {
let image = ImageName::parse("ghcr.io/team/mvp-node:latest").unwrap();
let aliases = alias_tags(&image, Some("smoke"), "dirty-1234").unwrap();
assert_eq!(
aliases.into_iter().collect::<Vec<_>>(),
vec!["latest".to_owned(), "smoke".to_owned()]
);
}
#[test]
fn docker_build_context_path_makes_worker_binary_relative_to_workspace() {
let root = Path::new("/workspace/swactor");
assert_eq!(
docker_build_context_path(
root,
Path::new("/workspace/swactor/target/debug/mvp-worker-node")
)
.expect("absolute workspace path is valid"),
"target/debug/mvp-worker-node"
);
assert_eq!(
docker_build_context_path(root, Path::new("target/debug/mvp-worker-node"))
.expect("relative workspace path is valid"),
"target/debug/mvp-worker-node"
);
assert!(
docker_build_context_path(root, Path::new("/tmp/mvp-worker-node")).is_err(),
"Docker COPY inputs must stay inside the build context"
);
}
}

790
crates/mvp-system/src/chat/runtime.rs
View file

@ -2511,22 +2511,6 @@ mod tests {
}
}
#[test]
fn observability_server_launch_contract_enables_orchestrator_dashboard() {
let build_args = mvp_orchestrator_build_args();
assert!(
build_args
.windows(2)
.any(|pair| pair[0] == "--features" && pair[1] == "dashboard"),
"{build_args:?}"
);
let config = base_config(provider_kind::process());
let args = config.orchestrator_cli_args("resolved-image");
assert!(args.iter().any(|arg| arg == "--dashboard"), "{args:?}");
assert!(!args.iter().any(|arg| arg == "--no-dashboard"), "{args:?}");
}
fn valid_vastai() -> ResolvedVastAiConfig {
ResolvedVastAiConfig {
api_key: "secret".to_owned(),
@ -2583,48 +2567,6 @@ mod tests {
.collect()
}
#[test]
fn benchmark_observability_chat_progress_records_include_run_id_and_stamp() {
let temp = TempDir::new("chat-progress-archive");
let archive_path = temp.path().join("frames.ndjson");
let mut progress = ChatDatastream::new(77, Some(archive_path.clone()))
.expect("chat datastream constructs");
progress.emit(
CHAT_RUNTIME_CHANNEL,
"unit_phase",
"ready",
serde_json::json!({"ok": true}),
);
progress.archive_pending().expect("archive pending frames");
let archive = fs::read_to_string(&archive_path).expect("read archive");
let line = archive.lines().next().expect("archive line");
let outer: serde_json::Value = serde_json::from_str(line).expect("outer archive JSON");
let inner_text = outer
.get("payload")
.and_then(|payload| payload.get("value"))
.and_then(serde_json::Value::as_str)
.expect("inner event text");
let inner: serde_json::Value = serde_json::from_str(inner_text).expect("inner event JSON");
assert_eq!(
inner.get("type").and_then(serde_json::Value::as_str),
Some("ChatProgress")
);
assert_eq!(
inner.get("run_id").and_then(serde_json::Value::as_u64),
Some(77)
);
assert_eq!(
inner
.get("benchmark")
.and_then(|benchmark| benchmark.get("schema"))
.and_then(serde_json::Value::as_u64),
Some(1)
);
}
#[test]
fn parsed_args_accepts_public_flags() {
let parsed = ParsedArgs::parse(strings(&[
@ -2662,85 +2604,6 @@ mod tests {
assert_eq!(alias.pipeline_stages, Some(4));
}
#[test]
fn config_gpu_flag_selects_in_process_gpu_run() {
let temp = TempDir::new("gpu-flag-config");
let cache = temp.path().join(REPO_MODEL_CACHE_DIR);
fs::create_dir_all(&cache).expect("create model cache");
let cached_path = cache.join("default.gguf");
fs::write(&cached_path, b"cached model").expect("write cached model");
with_process_state(&[], Some(temp.path()), || {
let config = Config::from_args(strings(&["--gpu", "--skip-rebuild"]))
.expect("gpu config resolves");
assert!(config.gpu_run);
assert_eq!(config.orchestrator_launch_mode(), "in_process_actor");
assert_eq!(
config
.cached_model
.as_ref()
.map(|model| model.host_path.clone()),
Some(cached_path.canonicalize().expect("canonical cached model"))
);
});
}
#[test]
fn benchmark_observability_parsed_args_accepts_run_id_and_forwards_to_orchestrator() {
let parsed = ParsedArgs::parse(strings(&["--run-id", "123"])).expect("run id parses");
assert_eq!(parsed.run_id, Some(123));
let temp = TempDir::new("run-id-config");
with_process_state(&[], Some(temp.path()), || {
let config =
Config::from_args(strings(&["--run-id", "123"])).expect("config resolves run id");
assert_eq!(config.run_id, 123);
let args = config.orchestrator_cli_args("resolved-image");
let run_id_arg = args
.windows(2)
.find(|pair| pair[0] == "--run-id")
.map(|pair| pair[1].as_str());
assert_eq!(run_id_arg, Some("123"), "{args:?}");
});
}
#[test]
fn benchmark_observability_parsed_args_rejects_zero_run_id() {
let error =
ParsedArgs::parse(strings(&["--run-id", "0"])).expect_err("zero run id should fail");
assert_eq!(error, "--run-id must be greater than 0");
}
#[test]
fn parsed_args_accepts_cached_model_path() {
let parsed = ParsedArgs::parse(strings(&["--cached-model=/tmp/model.gguf"]))
.expect("cached model path parses");
assert_eq!(
parsed.cached_model,
Some(CachedModelSource::Path(PathBuf::from("/tmp/model.gguf")))
);
}
#[test]
fn parsed_args_accepts_cached_model_equals_path_with_dash_prefix() {
let parsed = ParsedArgs::parse(strings(&["--cached-model=-model.gguf"]))
.expect("cached model path parses");
assert_eq!(
parsed.cached_model,
Some(CachedModelSource::Path(PathBuf::from("-model.gguf")))
);
}
#[test]
fn parsed_args_accepts_dump_logs_equals_path_with_dash_prefix() {
let parsed = ParsedArgs::parse(strings(&["--dump-logs=-logs.ndjson"]))
.expect("dump log path parses");
assert!(parsed.dump_logs);
assert_eq!(parsed.dump_log_path, Some(PathBuf::from("-logs.ndjson")));
}
#[test]
fn parsed_args_rejects_conflicts_and_pruned_inputs() {
for args in [
@ -2939,202 +2802,6 @@ kind = "mock"
});
}
#[test]
fn vastai_config_requires_secret_relay_bootstrap_and_remote_image() {
let missing_secret = TempDir::new("vastai-missing-secret");
let missing_secret_config = write_config(
&missing_secret,
"chat.toml",
r#"
[provider]
kind = "vastai"
[image]
node = "docker.io/acme/node:latest"
[vastai]
relay_url = "https://relay.example"
bootstrap_command = "boot"
"#,
);
with_process_state(&[], Some(missing_secret.path()), || {
let config_arg = missing_secret_config.to_string_lossy().into_owned();
assert!(Config::from_args(strings(&["--config", config_arg.as_str()])).is_err());
});
let missing_relay = TempDir::new("vastai-missing-relay");
let missing_relay_config = write_config(
&missing_relay,
"chat.toml",
r#"
[provider]
kind = "vastai"
[image]
node = "docker.io/acme/node:latest"
[vastai]
bootstrap_command = "boot"
"#,
);
with_process_state(
&[("VAST_API_KEY", Some("secret"))],
Some(missing_relay.path()),
|| {
let config_arg = missing_relay_config.to_string_lossy().into_owned();
assert!(Config::from_args(strings(&["--config", config_arg.as_str()])).is_err());
},
);
let local_image = TempDir::new("vastai-local-image");
let local_image_config = write_config(
&local_image,
"chat.toml",
r#"
[provider]
kind = "vastai"
[image]
node = "local-node:latest"
[vastai]
relay_url = "https://relay.example"
bootstrap_command = "boot"
"#,
);
with_process_state(
&[("VAST_API_KEY", Some("secret"))],
Some(local_image.path()),
|| {
let config_arg = local_image_config.to_string_lossy().into_owned();
assert!(Config::from_args(strings(&["--config", config_arg.as_str()])).is_err());
},
);
let valid = TempDir::new("vastai-valid");
let valid_config = write_config(
&valid,
"chat.toml",
r#"
[provider]
kind = "vastai"
[image]
node = "docker.io/acme/node:latest"
[vastai]
relay_url = "https://relay.example"
bootstrap_command = "boot"
blacklist_hosts = [155385, 546483]
"#,
);
with_process_state(
&[("VAST_API_KEY", Some("secret"))],
Some(valid.path()),
|| {
let config_arg = valid_config.to_string_lossy().into_owned();
let config = Config::from_args(strings(&["--config", config_arg.as_str()]))
.expect("valid Vast.ai config resolves");
let vastai = config.vastai.as_ref().expect("resolved Vast.ai config");
assert_eq!(vastai.api_key, "secret");
assert_eq!(vastai.relay_url, "https://relay.example");
assert_eq!(vastai.bootstrap_command, "boot");
assert_eq!(vastai.image, "docker.io/acme/node:latest");
assert_eq!(vastai.blacklist_hosts, vec![155385, 546483]);
let args = config.orchestrator_cli_args("docker.io/acme/node:latest");
assert!(
!args
.iter()
.any(|arg| arg == "--vastai-api-key" || arg == "secret"),
"Vast.ai API key must not be exposed in orchestrator argv: {args:?}"
);
assert!(
args.windows(2)
.any(|pair| pair == ["--vastai-bootstrap-command", "boot"]),
"non-secret Vast.ai config should still be forwarded"
);
assert!(
args.windows(2)
.any(|pair| pair == ["--vastai-blacklist-host", "155385"])
&& args
.windows(2)
.any(|pair| pair == ["--vastai-blacklist-host", "546483"]),
"Vast.ai host blacklist must be forwarded to orchestrator argv: {args:?}"
);
},
);
}
#[test]
fn relay_only_endpoint_mask_requires_and_forwards_relay_url() {
let missing = TempDir::new("relay-mask-missing-url");
let missing_config = write_config(
&missing,
"chat.toml",
r#"
[provider]
kind = "docker"
[image]
node = "docker.io/acme/node:latest"
"#,
);
with_process_state(&[], Some(missing.path()), || {
let config_arg = missing_config.to_string_lossy().into_owned();
let error = match Config::from_args(strings(&[
"--config",
config_arg.as_str(),
"--endpoint-addr-mask",
"relay-only",
])) {
Ok(_) => panic!("relay-only mask without relay URL should fail"),
Err(error) => error,
};
assert!(
error.contains("requires [relay].url, --relay-url, or [vastai].relay_url"),
"{error}"
);
});
let fallback = TempDir::new("relay-mask-vastai-fallback");
let fallback_config = write_config(
&fallback,
"chat.toml",
r#"
[provider]
kind = "docker"
[image]
node = "docker.io/acme/node:latest"
[relay]
endpoint_addr_mask = "relay-only"
[vastai]
relay_url = "https://relay.example"
"#,
);
with_process_state(&[], Some(fallback.path()), || {
let config_arg = fallback_config.to_string_lossy().into_owned();
let config = Config::from_args(strings(&["--config", config_arg.as_str()]))
.expect("relay-only mask uses Vast.ai relay fallback");
assert_eq!(config.provider, provider_kind::docker());
assert_eq!(config.relay_mode.as_deref(), Some("default"));
assert_eq!(config.relay_url.as_deref(), Some("https://relay.example"));
assert_eq!(config.endpoint_addr_mask, EndpointAddrMask::RelayOnly);
let args = config.orchestrator_cli_args("docker.io/acme/node:latest");
assert!(
args.windows(2)
.any(|pair| pair == ["--relay-url", "https://relay.example"])
);
assert!(
args.windows(2)
.any(|pair| pair == ["--endpoint-addr-mask", "relay-only"])
);
});
}
struct MockApproval {
terminal: bool,
answer: Result<bool, String>,
@ -3150,191 +2817,6 @@ relay_url = "https://relay.example"
}
}
#[test]
fn parse_approval_accepts_only_yes_variants() {
for value in ["y", "Y", " yes \n", "YeS"] {
assert!(parse_approval(value), "{value:?} should approve");
}
for value in ["", "n", "no", "yep", " yes please"] {
assert!(!parse_approval(value), "{value:?} should decline");
}
}
#[test]
fn vastai_approval_is_used_only_when_required() {
let process = base_config(provider_kind::process());
let mut approval = MockApproval {
terminal: false,
answer: Err("should not ask".to_owned()),
};
confirm_vastai_if_needed_with_approval(&process, &mut approval)
.expect("non-Vast.ai skips approval");
let mut yes_config = base_config(provider_kind::vastai());
yes_config.vastai = Some(valid_vastai());
yes_config.vastai_yes = true;
let mut approval = MockApproval {
terminal: false,
answer: Err("should not ask".to_owned()),
};
confirm_vastai_if_needed_with_approval(&yes_config, &mut approval)
.expect("--yes skips approval prompt");
let mut non_terminal = base_config(provider_kind::vastai());
non_terminal.vastai = Some(valid_vastai());
let mut approval = MockApproval {
terminal: false,
answer: Err("should not ask".to_owned()),
};
assert!(confirm_vastai_if_needed_with_approval(&non_terminal, &mut approval).is_err());
let mut accepted = base_config(provider_kind::vastai());
accepted.vastai = Some(valid_vastai());
let mut approval = MockApproval {
terminal: true,
answer: Ok(true),
};
confirm_vastai_if_needed_with_approval(&accepted, &mut approval)
.expect("interactive approval accepts");
let mut declined = base_config(provider_kind::vastai());
declined.vastai = Some(valid_vastai());
let mut approval = MockApproval {
terminal: true,
answer: Ok(false),
};
assert!(confirm_vastai_if_needed_with_approval(&declined, &mut approval).is_err());
}
#[test]
fn cached_model_discovery_selects_first_sorted_gguf_file() {
let temp = TempDir::new("cached-model-selects");
let cache_dir = temp.path().join(".model-cache");
fs::create_dir_all(&cache_dir).expect("create cache dir");
fs::write(cache_dir.join("z.gguf"), b"z").expect("write z model");
fs::write(cache_dir.join("a.gguf"), b"a").expect("write a model");
fs::write(cache_dir.join("ignored.txt"), b"ignored").expect("write ignored file");
fs::create_dir(cache_dir.join("0.gguf")).expect("create ignored directory");
with_process_state(&[], Some(temp.path()), || {
let cached = CachedModelConfig::discover().expect("cached model discovered");
assert_eq!(cached.host_path.file_name(), Some(OsStr::new("a.gguf")));
});
}
#[test]
fn cached_model_discovery_errors_when_no_usable_model_exists() {
let missing = TempDir::new("cached-model-missing");
with_process_state(&[], Some(missing.path()), || {
assert!(CachedModelConfig::discover().is_err());
});
let empty = TempDir::new("cached-model-empty");
let cache_dir = empty.path().join(".model-cache");
fs::create_dir_all(&cache_dir).expect("create cache dir");
fs::write(cache_dir.join("ignored.txt"), b"ignored").expect("write ignored file");
fs::create_dir(cache_dir.join("not-a-file.gguf")).expect("create ignored directory");
with_process_state(&[], Some(empty.path()), || {
assert!(CachedModelConfig::discover().is_err());
});
}
#[test]
fn cached_model_path_resolves_regular_gguf_file() {
let temp = TempDir::new("cached-model-path");
let model = temp.path().join("chosen.gguf");
fs::write(&model, b"model").expect("write chosen model");
let model_arg = model.to_string_lossy().into_owned();
with_process_state(&[], Some(temp.path()), || {
let parsed = ParsedArgs::parse(strings(&[&format!("--cached-model={model_arg}")]))
.expect("cached model path parses");
assert_eq!(
parsed.cached_model,
Some(CachedModelSource::Path(PathBuf::from(model_arg.as_str())))
);
let config = Config::from_args(strings(&[&format!("--cached-model={model_arg}")]))
.expect("cached model path resolves");
assert_eq!(
config.cached_model.unwrap().host_path.file_name(),
Some(OsStr::new("chosen.gguf"))
);
let upper_model = temp.path().join("upper.GGUF");
fs::write(&upper_model, b"model").expect("write uppercase model");
let upper = CachedModelConfig::from_path(upper_model)
.expect("uppercase cached model extension resolves");
assert_eq!(upper.host_path.file_name(), Some(OsStr::new("upper.GGUF")));
});
}
#[test]
fn vastai_cached_pipeline_model_selects_matching_remote_gguf() {
let temp = TempDir::new("vastai-cached-pipeline-model");
let cached_path = temp.path().join(DEFAULT_PIPELINE_CACHED_MODEL_FILE);
fs::write(&cached_path, b"cached model").expect("write cached model");
let config_path = write_config(
&temp,
"chat.toml",
r#"
[provider]
kind = "vastai"
[image]
node = "docker.io/acme/node:latest"
[model]
id = "qwen2.5-7b-instruct-q4-k-m"
gguf_repo = "bartowski/Qwen2.5-7B-Instruct-GGUF"
gguf_file = "Qwen2.5-7B-Instruct-Q4_K_M.gguf"
max_context = 512
[vastai]
relay_url = "https://relay.example"
bootstrap_command = "boot"
"#,
);
let cached_arg = format!("--cached-model={}", cached_path.display());
with_process_state(&[("VAST_API_KEY", Some("secret"))], None, || {
let config_arg = config_path.to_string_lossy().into_owned();
let config = Config::from_args(strings(&[
"--config",
config_arg.as_str(),
cached_arg.as_str(),
"--yes",
]))
.expect("VastAI cached pipeline model resolves");
let args = config.orchestrator_cli_args("docker.io/acme/node:prepared");
assert_eq!(
config
.cached_model
.as_ref()
.and_then(|model| model.host_path.file_name()),
Some(OsStr::new(DEFAULT_PIPELINE_CACHED_MODEL_FILE))
);
assert!(
args.windows(2)
.any(|pair| pair == ["--model-id", DEFAULT_PIPELINE_CACHED_MODEL_ID])
);
assert!(
args.windows(2)
.any(|pair| pair == ["--gguf-repo", DEFAULT_PIPELINE_CACHED_MODEL_REPO])
);
assert!(
args.windows(2)
.any(|pair| pair == ["--gguf-file", DEFAULT_PIPELINE_CACHED_MODEL_FILE])
);
let expected_context = DEFAULT_PIPELINE_CACHED_MODEL_MAX_CONTEXT.to_string();
assert!(
args.windows(2)
.any(|pair| pair == ["--max-context", expected_context.as_str()])
);
});
}
fn panic_prepare_node_image(_: NodeImageRequest) -> Result<PreparedNodeImage, String> {
panic!("image preparer must not be called when --skip-rebuild is set")
}
@ -3445,278 +2927,6 @@ bootstrap_command = "boot"
Err("build mvp node image failed with exit status: 42".to_owned())
}
#[test]
fn skip_rebuild_requires_existing_artifacts_and_skips_image_preparation() {
let temp = TempDir::new("skip-rebuild");
let orch_bin = temp.path().join("mvp-orchestrator");
let worker_bin = temp.path().join("mvp-worker-node");
let mut config = base_config(provider_kind::docker());
config.skip_rebuild = true;
config.orch_bin = orch_bin.clone();
config.worker_bin = worker_bin.clone();
config.node_image = "docker.io/acme/node:latest".to_owned();
assert!(prepare_runtime_with(&config, panic_prepare_node_image).is_err());
fs::write(&orch_bin, b"orch").expect("write orchestrator artifact");
assert!(prepare_runtime_with(&config, panic_prepare_node_image).is_err());
fs::write(&worker_bin, b"worker").expect("write worker artifact");
let image_ref = prepare_runtime_with(&config, panic_prepare_node_image)
.expect("skip rebuild uses existing artifacts");
assert_eq!(image_ref, "docker.io/acme/node:latest");
}
#[test]
fn prepare_runtime_progress_records_local_prep_details() {
let temp = TempDir::new("prep-progress");
let archive_path = temp.path().join("frames.ndjson");
let orch_bin = temp.path().join("mvp-orchestrator");
let worker_bin = temp.path().join("mvp-worker-node");
fs::write(&orch_bin, b"orch").expect("write orchestrator artifact");
fs::write(&worker_bin, b"worker").expect("write worker artifact");
let mut config = base_config(provider_kind::docker());
config.skip_rebuild = true;
config.orch_bin = orch_bin;
config.worker_bin = worker_bin;
config.node_image = "docker.io/acme/node:latest".to_owned();
let mut progress =
ChatDatastream::new(91, Some(archive_path.clone())).expect("datastream constructs");
prepare_runtime_with_progress(
&config,
panic_prepare_node_image_with_progress,
Some(&mut progress),
)
.expect("skip rebuild uses existing artifacts");
progress.archive_pending().expect("archive prep frames");
let events = fs::read_to_string(&archive_path).expect("read archive");
let inner_events = events
.lines()
.filter_map(|line| {
let outer: serde_json::Value = serde_json::from_str(line).ok()?;
if outer.get("channel").and_then(serde_json::Value::as_str)
!= Some(CHAT_RUNTIME_CHANNEL)
{
return None;
}
let inner = outer
.get("payload")?
.get("value")?
.as_str()
.and_then(|text| serde_json::from_str::<serde_json::Value>(text).ok())?;
Some(inner)
})
.collect::<Vec<_>>();
let ensure_ready = inner_events
.iter()
.find(|event| {
event.get("phase").and_then(serde_json::Value::as_str) == Some("ensure_orch_binary")
&& event.get("status").and_then(serde_json::Value::as_str) == Some("ready")
})
.expect("ensure_orch_binary ready event");
assert_eq!(
ensure_ready
.pointer("/detail/command_label")
.and_then(serde_json::Value::as_str),
Some("ensure_orch_binary")
);
assert!(
ensure_ready
.pointer("/detail/elapsed_ms")
.and_then(serde_json::Value::as_u64)
.is_some()
);
let image_skip = inner_events
.iter()
.find(|event| {
event.get("phase").and_then(serde_json::Value::as_str) == Some("prepare_node_image")
&& event.get("status").and_then(serde_json::Value::as_str) == Some("skipped")
})
.expect("prepare_node_image skipped event");
assert_eq!(
image_skip
.pointer("/detail/reason")
.and_then(serde_json::Value::as_str),
Some("skip_rebuild")
);
}
#[test]
fn prepare_runtime_streams_node_image_command_progress() {
let temp = TempDir::new("node-image-command-progress");
let archive_path = temp.path().join("frames.ndjson");
let mut config = base_config(provider_kind::docker());
config.skip_rebuild = false;
config.gpu_run = true;
config.node_image = "docker.io/acme/node:latest".to_owned();
let mut progress =
ChatDatastream::new(92, Some(archive_path.clone())).expect("datastream constructs");
let image_ref = prepare_runtime_with_progress(
&config,
fake_prepare_node_image_with_progress,
Some(&mut progress),
)
.expect("fake image preparation succeeds");
progress.archive_pending().expect("archive prep frames");
assert_eq!(image_ref, "docker.io/acme/node:prepared");
let events = runtime_events(&archive_path);
assert!(events.iter().any(|event| {
event.get("phase").and_then(serde_json::Value::as_str) == Some("prepare_node_image")
&& event.get("status").and_then(serde_json::Value::as_str) == Some("image_ref")
&& event
.pointer("/detail/image_ref")
.and_then(serde_json::Value::as_str)
== Some("docker.io/acme/node:prepared")
}));
assert!(events.iter().any(|event| {
event.get("phase").and_then(serde_json::Value::as_str) == Some("node_image_command")
&& event.get("status").and_then(serde_json::Value::as_str) == Some("started")
&& event
.pointer("/detail/command_label")
.and_then(serde_json::Value::as_str)
== Some("build mvp node image")
&& event
.pointer("/detail/program")
.and_then(serde_json::Value::as_str)
== Some("fake-docker")
}));
assert!(events.iter().any(|event| {
event.get("status").and_then(serde_json::Value::as_str) == Some("stdout")
&& event
.pointer("/detail/line")
.and_then(serde_json::Value::as_str)
== Some("building layer")
}));
assert!(events.iter().any(|event| {
event.get("status").and_then(serde_json::Value::as_str) == Some("stderr")
&& event
.pointer("/detail/line")
.and_then(serde_json::Value::as_str)
== Some("pushing metadata")
}));
assert!(events.iter().any(|event| {
event.get("status").and_then(serde_json::Value::as_str) == Some("exited")
&& event
.pointer("/detail/command_status")
.and_then(serde_json::Value::as_str)
== Some("exit status: 0")
&& event.pointer("/detail/duration_ms").is_some()
}));
}
#[test]
fn prepare_runtime_command_failure_preserves_label_and_status() {
let temp = TempDir::new("node-image-command-failure");
let archive_path = temp.path().join("frames.ndjson");
let mut config = base_config(provider_kind::docker());
config.skip_rebuild = false;
config.gpu_run = true;
let mut progress =
ChatDatastream::new(93, Some(archive_path.clone())).expect("datastream constructs");
let error = prepare_runtime_with_progress(
&config,
failing_prepare_node_image_with_progress,
Some(&mut progress),
)
.expect_err("fake image preparation failure propagates");
progress.archive_pending().expect("archive prep frames");
assert!(error.contains("build mvp node image"), "{error}");
let events = runtime_events(&archive_path);
let failure = events
.iter()
.find(|event| {
event.get("phase").and_then(serde_json::Value::as_str) == Some("node_image_command")
&& event.get("status").and_then(serde_json::Value::as_str) == Some("failed")
})
.expect("failed command progress event");
assert_eq!(
failure
.pointer("/detail/command_label")
.and_then(serde_json::Value::as_str),
Some("build mvp node image")
);
assert_eq!(
failure
.pointer("/detail/command_status")
.and_then(serde_json::Value::as_str),
Some("exit status: 42")
);
}
#[test]
fn vastai_skip_rebuild_uses_remote_image_without_worker_artifact() {
let temp = TempDir::new("vastai-skip-rebuild");
let orch_bin = temp.path().join("mvp-orchestrator");
let worker_bin = temp.path().join("mvp-worker-node");
let mut config = base_config(provider_kind::vastai());
config.skip_rebuild = true;
config.orch_bin = orch_bin.clone();
config.worker_bin = worker_bin;
config.node_image = "docker.io/acme/node:latest".to_owned();
assert!(prepare_runtime_with(&config, panic_prepare_node_image).is_err());
fs::write(&orch_bin, b"orch").expect("write orchestrator artifact");
let image_ref = prepare_runtime_with(&config, panic_prepare_node_image)
.expect("VastAI skip rebuild reuses remote image");
assert_eq!(image_ref, "docker.io/acme/node:latest");
}
#[test]
fn artifact_roots_use_current_directory() {
let temp = TempDir::new("artifact-root");
with_process_state(&[], Some(temp.path()), || {
let path = default_orch_bin().expect("default orchestrator path resolves");
assert!(path.starts_with(temp.path()), "{path:?}");
assert!(path.ends_with("target/debug/mvp-orchestrator"), "{path:?}");
});
}
#[cfg(target_os = "linux")]
#[test]
fn orch_child_shutdown_sends_sigterm_to_process_group() {
let temp = TempDir::new("orch-shutdown");
let flag_path = temp.path().join("term.flag");
let mut command = Command::new("sh");
command
.args([
"-c",
"trap 'echo term > \"$1\"; exit 0' TERM; while true; do sleep 1; done",
"sh",
])
.arg(&flag_path)
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null());
unsafe {
command.pre_exec(|| {
if libc::setpgid(0, 0) == 0 {
Ok(())
} else {
Err(std::io::Error::last_os_error())
}
});
}
let child = command.spawn().expect("spawn signal test child");
thread::sleep(Duration::from_millis(100));
let mut orch = OrchChild {
child,
cleaned: false,
shutdown_grace: Duration::from_millis(ORCH_SHUTDOWN_GRACE_MS),
};
orch.shutdown();
assert!(flag_path.exists(), "SIGTERM trap should write flag");
}
#[test]
fn prompt_loop_exits_cleanly_and_ignores_empty_prompts() {
let mut rpc_writer = Vec::new();

10
crates/mvp-system/src/lib.rs
View file

@ -62,13 +62,3 @@ mod transport;
#[cfg(test)]
mod tests;
#[cfg(test)]
#[path = "tests/shard_fetch_guarantees.rs"]
mod shard_fetch_guarantees;
#[cfg(test)]
#[path = "tests/shard_weight_lifecycle_guarantees.rs"]
mod shard_weight_lifecycle_guarantees;
#[cfg(test)]
#[path = "tests/weight_shards_guarantees.rs"]
mod weight_shards_guarantees;

94
crates/mvp-system/src/node/actor.rs
View file

@ -755,97 +755,3 @@ pub fn register_codecs(registry: &mut CodecRegistry) {
registry.register::<NodeAgentMsg, _>(JsonCodec::<NodeAgentMsg>::default());
registry.register::<NodeAgentReport, _>(JsonCodec::<NodeAgentReport>::default());
}
#[cfg(test)]
mod tests {
use super::*;
use iroh::SecretKey;
use swactor::config::RuntimeConfig;
use swactor::runtime::Runtime;
#[test]
fn node_agent_runtime_loaded_reports_orchestrator() {
let runtime = Runtime::new(RuntimeConfig::default());
let orchestrator_inbox = runtime
.new_inbox::<OrchestratorMsg>()
.expect("orchestrator inbox");
let orchestrator = *orchestrator_inbox.addr();
let node_actor = ActorAddress::new_random();
let datastream_publisher = ActorAddress::new_random();
let endpoint = EndpointAddr::new(SecretKey::from_bytes(&[9; 32]).public());
let actor = runtime
.spawn(NodeAgentActor::new(stage::NodeId(11), orchestrator, None))
.expect("spawn node agent");
runtime
.send_to(
actor,
NodeAgentMsg::RuntimeLoaded {
run_id: 7,
node_id: 11,
stage_index: 3,
endpoint: endpoint.clone(),
node_actor,
datastream_publisher,
readiness_id: 99,
},
)
.expect("send runtime loaded");
runtime.tick();
assert_eq!(
orchestrator_inbox.try_recv(),
Some(OrchestratorMsg::ObserveNodeRuntimeReady {
run_id: 7,
node_id: 11,
stage_index: 3,
endpoint,
node_actor,
datastream_publisher,
readiness_id: 99,
})
);
}
#[test]
fn node_agent_runtime_ready_ack_reports_worker_loop() {
let runtime = Runtime::new(RuntimeConfig::default());
let orchestrator_inbox = runtime
.new_inbox::<OrchestratorMsg>()
.expect("orchestrator inbox");
let reports = runtime
.new_inbox::<NodeAgentReport>()
.expect("node report inbox");
let actor = runtime
.spawn(NodeAgentActor::new(
stage::NodeId(11),
*orchestrator_inbox.addr(),
Some(*reports.addr()),
))
.expect("spawn node agent");
runtime
.send_to(
actor,
NodeAgentMsg::RuntimeReadyAck {
run_id: 7,
node_id: 11,
stage_index: 3,
readiness_id: 99,
},
)
.expect("send runtime ready ack");
runtime.tick();
assert_eq!(
reports.try_recv(),
Some(NodeAgentReport::RuntimeReadyAck {
run_id: 7,
node_id: 11,
stage_index: 3,
readiness_id: 99,
})
);
assert_eq!(reports.try_recv(), None);
}
}

522
crates/mvp-system/src/node/worker_node_runtime.rs
View file

@ -4815,525 +4815,3 @@ fn tokenizer_from_env() -> TokenizerSource {
.map(TokenizerSource::LocalPath)
.unwrap_or(TokenizerSource::EmbeddedGguf)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::orchestration::actor::OrchestratorMsg;
use distribution::swim::actor::MembershipChanged;
use distribution::swim::node::{SwimObservation, SwimObserver};
fn endpoint(seed: u8) -> EndpointAddr {
EndpointAddr::new(iroh::SecretKey::from_bytes(&[seed; 32]).public())
}
fn test_config(coordinator_endpoint: Option<EndpointAddr>) -> DeploymentConfig {
DeploymentConfig {
run_id: 7,
logical_node_id: 11,
stage_index: 3,
coordinator_endpoint,
orchestrator_actor: Some(ActorAddress::new_random()),
datastream_frame_log: None,
debug_join_socket: None,
relay_mode: iroh::RelayMode::Disabled,
endpoint_addr_mask: EndpointAddrMask::Full,
worker_script: DEFAULT_WORKER_SCRIPT.to_owned(),
device: DEFAULT_DEVICE.to_owned(),
model_id: DEFAULT_MODEL_ID.to_owned(),
gguf_source: GgufSource::LocalPath("/tmp/model.gguf".to_owned()),
tokenizer: TokenizerSource::EmbeddedGguf,
self_test_prompt: None,
self_test_layer_end: 16,
self_test_max_tokens: 1,
arena_bytes: DEFAULT_ARENA_BYTES,
arena_alignment: DEFAULT_ARENA_ALIGNMENT,
}
}
fn test_stack() -> DistributionRuntimeStack {
DistributionRuntimeStack::new(DistNodeId([1; 32]), DistributedNodeConfig::default())
}
#[test]
fn benchmark_observability_node_event_payload_includes_stamp() {
let config = test_config(None);
let payload = node_event_payload(&config, "phase", "ready", json!({"ok": true}));
assert_eq!(payload.get("run_id").and_then(Value::as_u64), Some(7));
assert_eq!(payload.get("node_id").and_then(Value::as_u64), Some(11));
assert_eq!(payload.get("stage_index").and_then(Value::as_u64), Some(3));
assert_eq!(
payload
.get("benchmark")
.and_then(|benchmark| benchmark.get("schema"))
.and_then(Value::as_u64),
Some(1)
);
let producer_instance_id = payload
.get("producer_instance_id")
.and_then(Value::as_str)
.expect("producer instance id");
assert!(producer_instance_id.starts_with("mvp-worker-node:7:node-11:pid-"));
assert_eq!(
payload
.get("benchmark")
.and_then(|benchmark| benchmark.get("producer_instance_id"))
.and_then(Value::as_str),
Some(producer_instance_id)
);
}
#[test]
fn worker_swim_telemetry_emits_probe_records_to_datastream() {
let stack = test_stack();
let peer = DistNodeId([9; 32]);
stack.swim_telemetry.observe(SwimObservation::ProbeSent {
target: peer,
sequence: 41,
kind: "direct",
});
stack
.swim_telemetry
.observe(SwimObservation::ProbeTimedOut {
target: peer,
sequence: 41,
kind: "direct",
budget_ticks: 15_000,
});
let mut datastream = NodeDatastream::new(&test_config(None));
emit_swim_telemetry(&mut datastream, &stack, "unit_phase");
let frames = datastream.endpoint.mux().drain();
let probe_records = frames
.iter()
.filter(|frame| {
datastream
.by_id
.get(&frame.channel)
.is_some_and(|channel| channel == SwimProbeEvent::CHANNEL)
})
.map(|frame| SwimProbeEvent::decode(&frame.payload).expect("probe record decodes"))
.collect::<Vec<_>>();
assert_eq!(probe_records.len(), 2);
assert_eq!(probe_records[0].event, "sent");
assert_eq!(probe_records[0].sequence, 41);
assert_eq!(probe_records[0].local_phase, "unit_phase");
assert_eq!(probe_records[0].probe_timeout_ms, 15_000);
assert_eq!(probe_records[1].event, "timed_out");
assert_eq!(probe_records[1].budget_ms, Some(15_000));
assert_eq!(probe_records[1].consecutive_timeouts, 1);
}
#[test]
fn debug_join_client_serializes_endpoint_from_stdin() {
let secret = iroh::SecretKey::from_bytes(&[7; 32]);
let endpoint = EndpointAddr::new(secret.public()).with_relay_url(
"http://relay.example.com"
.parse::<iroh::RelayUrl>()
.unwrap(),
);
let line = debug_join_request_line(endpoint).expect("serialize debug join request");
let request: DebugJoinRequestWire =
serde_json::from_str(&line).expect("deserialize debug join request");
match request {
DebugJoinRequestWire::JoinEndpoint { endpoint } => {
assert_eq!(
endpoint.relay_urls().next().map(ToString::to_string),
Some("http://relay.example.com/".to_owned())
);
}
}
}
#[test]
fn debug_join_listener_queues_join_endpoint() {
let root = std::env::temp_dir().join(format!(
"mvp-worker-debug-join-test-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("system time after epoch")
.as_nanos()
));
std::fs::create_dir(&root).expect("create debug join test temp dir");
let socket_path = root.join("debug-join.sock");
let runtime = tokio::runtime::Runtime::new().expect("create tokio runtime");
let mut commands = spawn_debug_join_listener(runtime.handle().clone(), socket_path.clone())
.expect("spawn debug join listener");
let secret = iroh::SecretKey::from_bytes(&[8; 32]);
let endpoint = EndpointAddr::new(secret.public()).with_relay_url(
"http://relay.example.com"
.parse::<iroh::RelayUrl>()
.unwrap(),
);
let request = DebugJoinRequestWire::JoinEndpoint { endpoint };
let mut request_line =
serde_json::to_string(&request).expect("serialize debug join request");
request_line.push('\n');
runtime.block_on(async {
let mut stream = tokio::net::UnixStream::connect(&socket_path)
.await
.expect("connect to debug join listener");
stream
.write_all(request_line.as_bytes())
.await
.expect("write debug join request");
stream.flush().await.expect("flush debug join request");
let DebugJoinCommand::JoinEndpoint { endpoint, reply } =
commands.recv().await.expect("receive debug join command");
assert_eq!(
endpoint.relay_urls().next().map(ToString::to_string),
Some("http://relay.example.com/".to_owned())
);
let peer_node_id = endpoint.id.to_string();
assert!(
reply
.send(DebugJoinResponseWire::JoinQueued {
peer_node_id,
has_relay: true,
direct_addr_count: 0,
})
.is_ok()
);
let mut reader = tokio::io::BufReader::new(stream);
let mut response_line = String::new();
reader
.read_line(&mut response_line)
.await
.expect("read debug join response");
let response: DebugJoinResponseWire =
serde_json::from_str(&response_line).expect("deserialize debug join response");
match response {
DebugJoinResponseWire::JoinQueued { has_relay, .. } => {
assert!(has_relay);
}
DebugJoinResponseWire::JoinRejected { error, detail } => {
panic!("debug join was rejected: {error}: {detail}");
}
}
});
drop(commands);
let _ = std::fs::remove_file(&socket_path);
std::fs::remove_dir(&root).expect("remove debug join test temp dir");
}
#[test]
fn runtime_ready_retry_waits_for_swim() {
let stack = test_stack();
let node_actor = ActorAddress::new_random();
let mut pending = PendingRuntimeReady::new(
&test_config(None),
endpoint(3),
node_actor,
ActorAddress::new_random(),
);
pending.coordinator = Some(DistNodeId([2; 32]));
assert_eq!(pending.maybe_send(&stack, node_actor), Ok(false));
assert_eq!(pending.attempts, 0);
}
#[test]
fn runtime_ready_retry_stops_after_matching_ack() {
let stack = test_stack();
let node_actor = ActorAddress::new_random();
let mut pending = PendingRuntimeReady::new(
&test_config(None),
endpoint(4),
node_actor,
ActorAddress::new_random(),
);
assert!(pending.observe_ack(
pending.run_id,
pending.node_id,
pending.stage_index,
pending.readiness_id,
));
assert!(pending.acked);
assert_eq!(pending.maybe_send(&stack, node_actor), Ok(false));
}
#[test]
fn runtime_ready_retry_backoff_caps() {
let stack = test_stack();
let orchestrator_inbox = stack
.runtime
.new_inbox::<OrchestratorMsg>()
.expect("orchestrator inbox");
let node_actor = stack
.runtime
.spawn(NodeAgentActor::new(
stage::NodeId(11),
*orchestrator_inbox.addr(),
None,
))
.expect("spawn node agent");
let coordinator = DistNodeId([2; 32]);
stack
.runtime
.send_to(
stack.actors.membership_fanout,
MembershipChanged {
node_id: coordinator,
state: MemberState::Alive,
incarnation: 1,
},
)
.expect("send membership change");
stack.pump_runtime_once();
let mut pending = PendingRuntimeReady::new(
&test_config(None),
endpoint(5),
node_actor,
ActorAddress::new_random(),
);
pending.coordinator = Some(coordinator);
for expected_attempts in 1..=4 {
pending.next_attempt_at = Instant::now();
assert!(
pending
.maybe_send(&stack, node_actor)
.expect("runtime ready send")
);
assert_eq!(pending.attempts, expected_attempts);
assert!(pending.backoff <= RUNTIME_READY_RETRY_MAX);
}
}
fn fast_helper_wait_config() -> HelperCommandWaitConfig {
HelperCommandWaitConfig {
poll_interval: Duration::from_millis(5),
telemetry_interval: Duration::from_millis(10),
}
}
fn drain_json_frames(
datastream: &NodeDatastream,
subscription: &DatastreamSubscription,
) -> Vec<(String, Value)> {
datastream.endpoint.tick();
subscription
.drain_available()
.into_iter()
.filter_map(|event| {
let DatastreamEvent::Frame(frame) = event else {
return None;
};
let channel = datastream.by_id.get(&frame.channel.channel)?.clone();
let value = serde_json::from_slice(&frame.payload).ok()?;
Some((channel, value))
})
.collect()
}
#[test]
fn helper_wait_pumps_and_emits_busy_telemetry_during_quiet_stdout() {
let config = test_config(None);
let mut datastream = NodeDatastream::new(&config);
let channel = datastream.channel_by_name("mvp.worker.weights");
let subscription = datastream.endpoint.subscribe_all("helper-wait-test");
let (tx, rx) = mpsc::channel();
thread::spawn(move || {
thread::sleep(Duration::from_millis(35));
tx.send(HelperStdoutEvent::Line(
json!({"type":"WeightsLoaded","model_id":"fake"}).to_string() + "\n",
))
.expect("send fake helper event");
});
let mut pump_count = 0_u32;
let event = wait_for_helper_event(
&rx,
None,
"WeightsLoaded",
"LoadWeights",
&config,
&mut datastream,
channel,
"mvp.worker.weights",
fast_helper_wait_config(),
&mut || {
pump_count = pump_count.saturating_add(1);
},
)
.expect("quiet helper eventually returns expected event");
assert_eq!(
event.get("type").and_then(Value::as_str),
Some("WeightsLoaded")
);
assert!(pump_count >= 2, "pump_count={pump_count}");
let frames = drain_json_frames(&datastream, &subscription);
assert!(frames.iter().any(|(channel, value)| {
channel == "mvp.worker.weights"
&& value.get("type").and_then(Value::as_str) == Some("WeightsLoaded")
}));
assert!(frames.iter().any(|(channel, value)| {
channel == NODE_WORKER_CHANNEL
&& value.get("phase").and_then(Value::as_str) == Some("worker_command_wait")
&& value.get("status").and_then(Value::as_str) == Some("waiting")
&& value
.get("detail")
.and_then(|detail| detail.get("state"))
.and_then(Value::as_str)
== Some("busy_waiting_for_helper_stdout")
}));
}
#[test]
fn helper_wait_errors_on_worker_fatal_event() {
let config = test_config(None);
let mut datastream = NodeDatastream::new(&config);
let channel = datastream.channel_by_name("mvp.worker.weights");
let (tx, rx) = mpsc::channel();
tx.send(HelperStdoutEvent::Line(
json!({"type":"WorkerFatal","error":"boom"}).to_string() + "\n",
))
.expect("send fatal event");
let mut pump_count = 0_u32;
let error = wait_for_helper_event(
&rx,
None,
"WeightsLoaded",
"LoadWeights",
&config,
&mut datastream,
channel,
"mvp.worker.weights",
fast_helper_wait_config(),
&mut || {
pump_count = pump_count.saturating_add(1);
},
)
.expect_err("worker fatal must fail command wait");
assert!(error.contains("worker fatal"));
assert_eq!(pump_count, 1);
}
#[test]
fn helper_wait_errors_on_closed_stdout_event() {
let config = test_config(None);
let mut datastream = NodeDatastream::new(&config);
let channel = datastream.channel_by_name("mvp.worker.weights");
let (tx, rx) = mpsc::channel();
tx.send(HelperStdoutEvent::Closed)
.expect("send closed stdout event");
let mut pump_count = 0_u32;
let error = wait_for_helper_event(
&rx,
None,
"WeightsLoaded",
"LoadWeights",
&config,
&mut datastream,
channel,
"mvp.worker.weights",
fast_helper_wait_config(),
&mut || {
pump_count = pump_count.saturating_add(1);
},
)
.expect_err("closed stdout must fail command wait");
assert!(error.contains("stdout closed"));
assert_eq!(pump_count, 0);
}
#[test]
fn sampler_health_start_and_no_sample_records_are_json_events() {
let config = test_config(None);
let mut datastream = NodeDatastream::new(&config);
let health_channel = datastream.channel_by_name(NODE_SAMPLER_CHANNEL);
let subscription = datastream.endpoint.subscribe_all("sampler-health-test");
let context = SamplerHealthContext::from_config(&config);
for (sampler, sample_channel) in [
("gpu", datastream::hardware::gpu::HOST_GPU_CHANNEL),
("cpu", datastream::hardware::cpu::HOST_CPU_CHANNEL),
("net", datastream::hardware::net::HOST_NET_CHANNEL),
] {
submit_sampler_started(
&datastream.producer,
health_channel,
context,
sampler,
sample_channel,
Duration::from_secs(1),
);
}
submit_sampler_sample_health(
&datastream.producer,
health_channel,
context,
"gpu",
datastream::hardware::gpu::HOST_GPU_CHANNEL,
0,
Some("nvidia-smi unavailable"),
);
let frames = drain_json_frames(&datastream, &subscription);
let sampler_events = frames
.iter()
.filter(|(channel, _)| channel == NODE_SAMPLER_CHANNEL)
.map(|(_, value)| value)
.collect::<Vec<_>>();
assert_eq!(sampler_events.len(), 7);
assert!(sampler_events.iter().all(|value| {
value
.get("producer_instance_id")
.and_then(Value::as_str)
.is_some_and(|id| id.starts_with("mvp-worker-node:7:node-11:pid-"))
}));
assert!(sampler_events.iter().all(|value| {
value
.get("benchmark")
.and_then(|benchmark| benchmark.get("producer_instance_id"))
== value.get("producer_instance_id")
}));
for sampler in ["gpu", "cpu", "net"] {
assert!(sampler_events.iter().any(|value| {
value.get("type").and_then(Value::as_str) == Some("SamplerHealth")
&& value.get("schema").and_then(Value::as_str)
== Some("mvp.node.sampler.health.v1")
&& value.get("run_id").and_then(Value::as_u64) == Some(7)
&& value.get("node_id").and_then(Value::as_u64) == Some(11)
&& value.get("stage_index").and_then(Value::as_u64) == Some(3)
&& value.get("sampler").and_then(Value::as_str) == Some(sampler)
&& value.get("status").and_then(Value::as_str) == Some("started")
}));
assert!(sampler_events.iter().any(|value| {
value.get("sampler").and_then(Value::as_str) == Some(sampler)
&& value.get("status").and_then(Value::as_str) == Some("waiting")
&& value
.get("detail")
.and_then(|detail| detail.get("state"))
.and_then(Value::as_str)
== Some("no_sample_yet")
}));
}
assert!(sampler_events.iter().any(|value| {
value.get("sampler").and_then(Value::as_str) == Some("gpu")
&& value.get("status").and_then(Value::as_str) == Some("failed")
&& value
.get("detail")
.and_then(|detail| detail.get("error"))
.and_then(Value::as_str)
== Some("nvidia-smi unavailable")
}));
}
}

60
crates/mvp-system/src/orchestration/actor.rs
View file

@ -489,63 +489,3 @@ impl From<&core::TokenObjectPayload> for TokenObjectPayloadWire {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use iroh::SecretKey;
use swactor::config::RuntimeConfig;
use swactor::runtime::Runtime;
#[test]
fn orchestrator_actor_reports_node_runtime_ready() {
let runtime = Runtime::new(RuntimeConfig::default());
let reports = runtime
.new_inbox::<OrchestratorReport>()
.expect("orchestrator report inbox");
let report_to = *reports.addr();
let actor = runtime
.spawn(OrchestratorActor::new(
core::RunConfig {
run_id: core::RunId(7),
max_tokens: 1,
prompt: Vec::new(),
},
Some(report_to),
))
.expect("spawn orchestrator actor");
let endpoint = EndpointAddr::new(SecretKey::from_bytes(&[8; 32]).public());
let node_actor = ActorAddress::new_random();
let datastream_publisher = ActorAddress::new_random();
runtime
.send_to(
actor,
OrchestratorMsg::ObserveNodeRuntimeReady {
run_id: 7,
node_id: 11,
stage_index: 3,
endpoint: endpoint.clone(),
node_actor,
datastream_publisher,
readiness_id: 99,
},
)
.expect("send runtime ready");
runtime.tick();
assert_eq!(
reports.try_recv(),
Some(OrchestratorReport::NodeRuntimeReady {
run_id: 7,
node_id: 11,
stage_index: 3,
endpoint,
node_actor,
datastream_publisher,
readiness_id: 99,
})
);
assert_eq!(reports.try_recv(), None);
}
}

3215
crates/mvp-system/src/orchestration/app.rs
View file

File diff suppressed because it is too large Load diff

284
crates/mvp-system/src/orchestration/config.rs
View file

@ -192,287 +192,3 @@ impl TomlConfigOverlay {
toml::from_str(text)
}
}
#[cfg(test)]
mod tests {
use super::*;
use super::{ResolvedVastAiConfig, looks_remote_image};
use std::sync::atomic::{AtomicU64, Ordering};
static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0);
fn valid_resolved_vastai_config() -> ResolvedVastAiConfig {
ResolvedVastAiConfig {
api_key: "vast-key".to_owned(),
relay_url: "https://relay.example.com".to_owned(),
image: "ghcr.io/swactor/mvp-node:latest".to_owned(),
bootstrap_command: "/usr/local/bin/mvp-node".to_owned(),
disk_gb: Some(80),
gpu_name: Some("RTX 4090".to_owned()),
min_gpu_ram_mb: Some(16_000),
min_down_mbps: Some(100.0),
min_up_mbps: Some(25.0),
max_dph_total: Some(0.10),
min_reliability: Some(0.98),
require_verified: Some(true),
blacklist_hosts: vec![155385],
onstart: None,
ssh_identity: Some("~/.ssh/swactor_vastai_ed25519".to_owned()),
}
}
#[test]
fn config_toml_parses_explicit_chat_and_vastai_fields() {
let config = TomlConfigOverlay::from_str(
r#"
[runtime]
profile = "deploy"
run_id = 42
node_id = 7
stage_index = 2
layer_end_exclusive = 24
pipeline_stages = 3
[provider]
kind = "vastai"
[image]
node = "ghcr.io/swactor/mvp-node:latest"
tag = "trial"
build = false
push = true
force_refresh = true
[relay]
mode = "disabled"
url = "https://relay.example.com"
[prompt]
rpc_addr = "127.0.0.1:19000"
max_tokens = 128
dashboard = false
[model]
id = "smollm2-135m-instruct-q4"
gguf_local_path = "/models/local.gguf"
gguf_repo = "QuantFactory/SmolLM2-135M-Instruct-GGUF"
gguf_file = "SmolLM2-135M-Instruct.Q4_K_M.gguf"
gguf_revision = "main"
tokenizer_local_path = "/models/tokenizer.json"
max_context = 512
[docker]
gpus = "all"
cached_model_host_path = "/cache/model.gguf"
[observability]
datastream_frame_log = "/tmp/frames.jsonl"
[vastai]
api_key = "vast-key"
image = "registry.example.com/team/mvp-node:latest"
bootstrap_command = "/opt/mvp/node --join"
disk_gb = 80
ssh_user = "ubuntu"
confirm_lease = true
gpu_name = "RTX 4090"
min_gpu_ram_mb = 24000
min_down_mbps = 250.5
min_up_mbps = 50.25
max_dph_total = 0.10
min_reliability = 0.99
require_verified = true
blacklist_hosts = [155385, 59017]
onstart = "echo preparing"
ssh_identity = "~/.ssh/swactor_vastai_ed25519"
poll_interval_secs = 30
"#,
)
.expect("explicit config TOML parses");
assert_eq!(config.runtime.profile.as_deref(), Some("deploy"));
assert_eq!(config.runtime.run_id, Some(42));
assert_eq!(config.runtime.node_id, Some(7));
assert_eq!(config.runtime.stage_index, Some(2));
assert_eq!(config.runtime.layer_end_exclusive, Some(24));
assert_eq!(config.runtime.pipeline_stages, Some(3));
assert_eq!(config.provider.kind.as_deref(), Some("vastai"));
assert_eq!(
config.image.node.as_deref(),
Some("ghcr.io/swactor/mvp-node:latest")
);
assert_eq!(config.image.tag.as_deref(), Some("trial"));
assert_eq!(config.image.build, Some(false));
assert_eq!(config.image.push, Some(true));
assert_eq!(config.image.force_refresh, Some(true));
assert_eq!(config.relay.mode.as_deref(), Some("disabled"));
assert_eq!(
config.relay.url.as_deref(),
Some("https://relay.example.com")
);
assert_eq!(config.prompt.rpc_addr.as_deref(), Some("127.0.0.1:19000"));
assert_eq!(config.prompt.max_tokens, Some(128));
assert_eq!(config.prompt.dashboard, Some(false));
assert_eq!(config.model.id.as_deref(), Some("smollm2-135m-instruct-q4"));
assert_eq!(
config.model.gguf_local_path.as_deref(),
Some("/models/local.gguf")
);
assert_eq!(
config.model.gguf_repo.as_deref(),
Some("QuantFactory/SmolLM2-135M-Instruct-GGUF")
);
assert_eq!(
config.model.gguf_file.as_deref(),
Some("SmolLM2-135M-Instruct.Q4_K_M.gguf")
);
assert_eq!(config.model.gguf_revision.as_deref(), Some("main"));
assert_eq!(
config.model.tokenizer_local_path.as_deref(),
Some("/models/tokenizer.json")
);
assert_eq!(config.model.max_context, Some(512));
assert_eq!(config.docker.gpus.as_deref(), Some("all"));
assert_eq!(
config.docker.cached_model_host_path.as_deref(),
Some("/cache/model.gguf")
);
assert_eq!(
config.observability.datastream_frame_log.as_deref(),
Some("/tmp/frames.jsonl")
);
assert_eq!(config.vastai.api_key.as_deref(), Some("vast-key"));
assert_eq!(
config.vastai.image.as_deref(),
Some("registry.example.com/team/mvp-node:latest")
);
assert_eq!(
config.vastai.bootstrap_command.as_deref(),
Some("/opt/mvp/node --join")
);
assert_eq!(config.vastai.disk_gb, Some(80));
assert_eq!(config.vastai.ssh_user.as_deref(), Some("ubuntu"));
assert_eq!(config.vastai.confirm_lease, Some(true));
assert_eq!(config.vastai.gpu_name.as_deref(), Some("RTX 4090"));
assert_eq!(config.vastai.min_gpu_ram_mb, Some(24_000));
assert_eq!(config.vastai.min_down_mbps, Some(250.5));
assert_eq!(config.vastai.min_up_mbps, Some(50.25));
assert_eq!(config.vastai.max_dph_total, Some(0.10));
assert_eq!(config.vastai.min_reliability, Some(0.99));
assert_eq!(config.vastai.require_verified, Some(true));
assert_eq!(config.vastai.blacklist_hosts, vec![155385, 59017]);
assert_eq!(config.vastai.poll_interval_secs, Some(30));
assert_eq!(config.vastai.onstart.as_deref(), Some("echo preparing"));
assert_eq!(
config.vastai.ssh_identity.as_deref(),
Some("~/.ssh/swactor_vastai_ed25519")
);
}
#[test]
fn config_toml_rejects_non_numeric_pipeline_stages() {
let error = TomlConfigOverlay::from_str(
r#"
[runtime]
pipeline_stages = "many"
"#,
)
.expect_err("non-numeric pipeline_stages must not parse");
assert!(
error.to_string().contains("pipeline_stages"),
"error should identify pipeline_stages: {error}"
);
}
#[test]
fn explicit_config_path_error_includes_the_requested_path() {
let counter = TEMP_COUNTER.fetch_add(1, Ordering::Relaxed);
let path = std::env::temp_dir().join(format!(
"mvp-system-missing-config-{}-{counter}.toml",
std::process::id()
));
let _ = fs::remove_file(&path);
let error = TomlConfigOverlay::load_required(&path)
.expect_err("missing explicit config path errors");
assert!(
error.contains(&path.display().to_string()),
"error {error:?} must include requested path {}",
path.display()
);
}
#[test]
fn looks_remote_image_accepts_registry_refs_and_rejects_local_refs() {
for image in [
"ghcr.io/swactor/mvp-node:latest",
"registry.example.com:5000/team/mvp-node@sha256:abcdef",
"localhost:5000/team/mvp-node:latest",
] {
assert!(looks_remote_image(image), "{image:?} should be remote");
}
for image in [
"swactor-mvp-node:latest",
"team/mvp-node:latest",
"mvp-node@sha256:abcdef",
] {
assert!(!looks_remote_image(image), "{image:?} should be local");
}
}
#[test]
fn resolved_vastai_config_validate_rejects_missing_required_fields() {
let cases = [
(
"VAST_API_KEY",
ResolvedVastAiConfig {
api_key: " ".to_owned(),
..valid_resolved_vastai_config()
},
),
(
"relay.url",
ResolvedVastAiConfig {
relay_url: "\t".to_owned(),
..valid_resolved_vastai_config()
},
),
(
"vastai.image",
ResolvedVastAiConfig {
image: String::new(),
..valid_resolved_vastai_config()
},
),
(
"vastai.bootstrap_command",
ResolvedVastAiConfig {
bootstrap_command: "\n".to_owned(),
..valid_resolved_vastai_config()
},
),
(
"vastai.ssh_identity",
ResolvedVastAiConfig {
ssh_identity: Some(" ".to_owned()),
..valid_resolved_vastai_config()
},
),
];
for (label, config) in cases {
let error = config
.validate()
.expect_err("missing field must be rejected");
assert!(
error.contains(label),
"error {error:?} must identify missing {label}"
);
}
}
}

38
crates/mvp-system/src/orchestration/distribution_stack.rs
View file

@ -260,41 +260,3 @@ impl ActorInterface for MembershipFanout {
let _ = ctx.send(self.directory, DirectoryIn::Membership(change));
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn distribution_stack_reports_member_state_and_route_owner() {
let stack = DistributionRuntimeStack::new(
distribution::types::NodeId([1; 32]),
DistributedNodeConfig::default(),
);
let remote = distribution::types::NodeId([2; 32]);
stack
.runtime
.send_to(
stack.actors.membership_fanout,
distribution::swim::actor::MembershipChanged {
node_id: remote,
state: MemberState::Alive,
incarnation: 1,
},
)
.expect("send membership change");
stack.pump_runtime_once();
assert_eq!(stack.member_state(remote), Some(MemberState::Alive));
let actor = ActorAddress::new_random();
{
let mut route_view = match stack.route_view.write() {
Ok(route_view) => route_view,
Err(poisoned) => poisoned.into_inner(),
};
route_view.insert(actor, remote);
}
assert_eq!(stack.route_owner(actor), Some(remote));
}
}

365
crates/mvp-system/src/orchestration/provider_adapters/vastai/mod.rs
View file

@ -1743,368 +1743,3 @@ where
result
}
}
#[cfg(test)]
mod tests {
use super::*;
struct NoopLeaseClient;
impl VastAiLeaseClient for NoopLeaseClient {
fn provision_one(
&mut self,
_request: ProvisionRequest,
) -> Result<ProvisionedInstance, String> {
panic!("build_request tests must not provision a real Vast.ai lease")
}
fn ssh_endpoint(
&mut self,
_contract_id: u64,
_label: &str,
_lifecycle: &LifecyclePolicy,
_ssh_user: &str,
) -> Result<VastAiSshEndpoint, String> {
panic!("build_request tests must not query a real Vast.ai endpoint")
}
fn destroy_contract(&mut self, _contract_id: u64) -> Result<(), String> {
panic!("build_request tests must not destroy a real Vast.ai lease")
}
}
struct NoopBootstrapLauncher;
impl VastAiBootstrapLauncher for NoopBootstrapLauncher {
type Handle = ();
fn start_bootstrap(
&mut self,
_spec: NodeProvisionSpec,
_endpoint: VastAiSshEndpoint,
_sink: PluginSink,
_producer: Option<DatastreamProducer>,
_lifecycle: LifecyclePolicy,
) -> Result<Self::Handle, String> {
panic!("build_request tests must not start SSH bootstrap")
}
fn stop_bootstrap(&mut self, _handle: &mut Self::Handle, _reason: BootstrapStopReason) {
panic!("build_request tests must not stop SSH bootstrap")
}
}
fn node_spec_with_bootstrap_args() -> NodeProvisionSpec {
NodeProvisionSpec {
run_id: 9,
node_id: 11,
stage_index: Some(2),
image: "registry.example.com/mvp-worker:latest".to_owned(),
env: vec![("EXISTING".to_owned(), "1".to_owned())],
args: vec!["python".to_owned(), "worker.py".to_owned()],
mounts: Vec::new(),
}
}
fn plugin_with_onstart(
onstart: Option<String>,
) -> VastAiProvisioningPlugin<NoopLeaseClient, NoopBootstrapLauncher> {
let config = VastAiProvisioningConfig {
onstart,
..VastAiProvisioningConfig::default()
};
VastAiProvisioningPlugin::new(NoopLeaseClient, NoopBootstrapLauncher, config)
}
#[derive(Clone, Default)]
struct ObservationSink {
observations: Arc<Mutex<Vec<PluginObservation>>>,
}
impl crate::provisioning::PluginObservationSink for ObservationSink {
fn observe(&self, observation: PluginObservation) {
self.observations.lock().push(observation);
}
}
#[derive(Clone)]
struct RecordingLeaseClient {
destroyed_contracts: Arc<Mutex<Vec<u64>>>,
}
impl VastAiLeaseClient for RecordingLeaseClient {
fn provision_one(
&mut self,
_request: ProvisionRequest,
) -> Result<ProvisionedInstance, String> {
Ok(ProvisionedInstance {
index: 0,
contract_id: 42,
offer_id: 7,
host_id: Some(99),
gpu_name: "RTX 4060".to_owned(),
gpu_ram: Some(8_192.0),
dph_total: 0.064,
})
}
fn ssh_endpoint(
&mut self,
_contract_id: u64,
_label: &str,
_lifecycle: &LifecyclePolicy,
ssh_user: &str,
) -> Result<VastAiSshEndpoint, String> {
Ok(VastAiSshEndpoint {
host: "ssh5.vast.ai".to_owned(),
port: 22_017,
user: ssh_user.to_owned(),
})
}
fn destroy_contract(&mut self, contract_id: u64) -> Result<(), String> {
self.destroyed_contracts.lock().push(contract_id);
Ok(())
}
}
#[derive(Clone)]
struct RecordingBootstrapLauncher {
stop_reasons: Arc<Mutex<Vec<BootstrapStopReason>>>,
}
impl VastAiBootstrapLauncher for RecordingBootstrapLauncher {
type Handle = u64;
fn start_bootstrap(
&mut self,
spec: NodeProvisionSpec,
_endpoint: VastAiSshEndpoint,
_sink: PluginSink,
_producer: Option<DatastreamProducer>,
_lifecycle: LifecyclePolicy,
) -> Result<Self::Handle, String> {
Ok(spec.node_id)
}
fn stop_bootstrap(&mut self, _handle: &mut Self::Handle, reason: BootstrapStopReason) {
self.stop_reasons.lock().push(reason);
}
}
#[test]
fn vastai_provisioning_build_request_keeps_bootstrap_args_out_of_onstart() {
let plugin = plugin_with_onstart(None);
let request =
plugin.build_request(&node_spec_with_bootstrap_args(), "test-label".to_owned());
assert_eq!(request.onstart, None);
assert_eq!(request.label.as_deref(), Some("test-label"));
assert_eq!(request.env.get("EXISTING").map(String::as_str), Some("1"));
}
#[test]
fn vastai_provisioning_build_request_uses_explicit_onstart() {
let plugin = plugin_with_onstart(Some("echo explicit setup".to_owned()));
let request =
plugin.build_request(&node_spec_with_bootstrap_args(), "test-label".to_owned());
assert_eq!(request.onstart.as_deref(), Some("echo explicit setup"));
}
#[test]
fn vastai_complete_bootstrap_keeps_optional_log_tail_until_node_stop() {
let destroyed_contracts = Arc::new(Mutex::new(Vec::new()));
let stop_reasons = Arc::new(Mutex::new(Vec::new()));
let sink = PluginSink::new(Arc::new(ObservationSink::default()));
let mut plugin = VastAiProvisioningPlugin::new(
RecordingLeaseClient {
destroyed_contracts: destroyed_contracts.clone(),
},
RecordingBootstrapLauncher {
stop_reasons: stop_reasons.clone(),
},
VastAiProvisioningConfig::default(),
);
let handle = plugin
.start_node(node_spec_with_bootstrap_args(), sink)
.expect("VastAI node starts");
plugin
.complete_bootstrap(&handle)
.expect("runtime-ready bootstrap completion succeeds");
assert!(
stop_reasons.lock().is_empty(),
"runtime-ready keeps the SSH log tail alive for post-bootstrap diagnostics"
);
plugin.stop_node(&handle).expect("VastAI node stops");
assert_eq!(*stop_reasons.lock(), vec![BootstrapStopReason::NodeStop]);
assert_eq!(*destroyed_contracts.lock(), vec![42]);
}
#[test]
fn vastai_provisioning_next_ssh_backoff_doubles_until_thirty_second_cap() {
for (current, expected) in [
(Duration::from_secs(1), Duration::from_secs(2)),
(Duration::from_secs(15), Duration::from_secs(30)),
(Duration::from_secs(20), Duration::from_secs(30)),
(Duration::from_secs(30), Duration::from_secs(30)),
] {
assert_eq!(
next_ssh_backoff(current),
expected,
"backoff from {current:?}"
);
}
}
#[test]
fn ssh_bootstrap_observation_classifies_auth_and_transport_failures() {
for (line, expected) in [
("Permission denied (publickey).", Some("auth_denied")),
(
"ssh: connect to host ssh5.vast.ai port 22017: Connection refused",
Some("refused"),
),
("ssh: connect timed out", Some("timeout")),
("debug1: permanently_set_uid", None),
] {
assert_eq!(classify_ssh_observation(line), expected, "{line}");
}
}
#[test]
fn ssh_bootstrap_args_include_verbose_flag_and_identity_when_configured() {
let endpoint = VastAiSshEndpoint {
host: "ssh5.vast.ai".to_owned(),
port: 22017,
user: "ubuntu".to_owned(),
};
let args = ssh_bootstrap_args(&endpoint, "python worker.py", Some(Path::new("/tmp/key")));
assert_eq!(
args,
vec![
"-v",
"-p",
"22017",
"-o",
"BatchMode=yes",
"-o",
"StrictHostKeyChecking=accept-new",
"-i",
"/tmp/key",
"-o",
"IdentitiesOnly=yes",
"ubuntu@ssh5.vast.ai",
"python worker.py",
]
);
assert_eq!(args.iter().filter(|arg| arg.as_str() == "-v").count(), 1);
}
#[test]
fn ssh_bootstrap_args_keep_identity_absent_when_not_configured() {
let endpoint = VastAiSshEndpoint {
host: "ssh5.vast.ai".to_owned(),
port: 22017,
user: "ubuntu".to_owned(),
};
let args = ssh_bootstrap_args(&endpoint, "python worker.py", None);
assert_eq!(
args,
vec![
"-v",
"-p",
"22017",
"-o",
"BatchMode=yes",
"-o",
"StrictHostKeyChecking=accept-new",
"ubuntu@ssh5.vast.ai",
"python worker.py",
]
);
assert!(!args.iter().any(|arg| arg == "-i"));
assert!(!args.iter().any(|arg| arg == "IdentitiesOnly=yes"));
}
#[test]
fn provider_terminal_start_error_ignores_package_names_while_loading() {
let package_log = "#7 1.745 libevent-core-2.1-7t64 liberror-perl libglib2.0-data";
assert_eq!(
provider_terminal_start_error(46132050, "loading", "running", Some(package_log)),
None
);
}
#[test]
fn provider_terminal_start_error_reports_standalone_failure_words() {
assert_eq!(
provider_terminal_start_error(
46132050,
"loading",
"running",
Some("ERROR: build failed")
),
Some("instance 46132050 error: ERROR: build failed".to_owned())
);
}
#[test]
fn ssh_bootstrap_actor_stop_kills_child() {
let runtime = Arc::new(swactor::runtime::Runtime::new(
swactor::config::RuntimeConfig::default(),
));
let child = std::process::Command::new("sleep")
.arg("30")
.spawn()
.expect("spawn sleep child");
let pid = child.id();
let bridge = BootstrapDatastreamBridge::new(
node_spec_with_bootstrap_args(),
PluginSink::new(Arc::new(ObservationSink::default())),
None,
);
let actor = runtime
.spawn(SshBootstrapActor {
bridge,
endpoint: VastAiSshEndpoint {
host: "127.0.0.1".to_owned(),
port: 22,
user: "ubuntu".to_owned(),
},
ssh_identity: None,
sender: runtime.create_sender(),
child: Some(child),
stdout_reader: None,
stderr_reader: None,
stdout_closed: true,
stderr_closed: true,
pending_status: None,
pending_wait_error: None,
attempt: 1,
backoff: Duration::from_secs(1),
observation_class: None,
stopped: false,
start_on_boot: false,
})
.expect("spawn ssh bootstrap actor");
runtime
.send_to(actor, SshBootstrapMsg::Stop)
.expect("send stop");
runtime.tick();
#[cfg(target_os = "linux")]
assert!(
!std::path::Path::new(&format!("/proc/{pid}")).exists(),
"child process should be reaped"
);
}
}

290
crates/mvp-system/src/orchestration/provisioning.rs
View file

@ -583,293 +583,3 @@ fn spawn_stderr_reader(
) {
BootstrapDatastreamBridge::new(spec, sink, None).spawn_stderr_reader(stderr);
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Mutex;
use std::time::{Duration, Instant};
#[cfg(unix)]
use std::os::unix::fs::PermissionsExt;
#[derive(Default)]
struct RecordingSink {
observations: Mutex<Vec<PluginObservation>>,
}
impl RecordingSink {
fn observations(&self) -> Vec<PluginObservation> {
self.observations
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.clone()
}
}
impl PluginObservationSink for RecordingSink {
fn observe(&self, observation: PluginObservation) {
self.observations
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.push(observation);
}
}
fn recording_sink() -> (Arc<RecordingSink>, PluginSink) {
let recorder = Arc::new(RecordingSink::default());
(Arc::clone(&recorder), PluginSink::new(recorder))
}
fn wait_for_observation(
recorder: &RecordingSink,
predicate: impl Fn(&PluginObservation) -> bool,
) -> PluginObservation {
let deadline = Instant::now() + Duration::from_secs(5);
loop {
if let Some(observation) = recorder
.observations()
.into_iter()
.find(|observation| predicate(observation))
{
return observation;
}
assert!(
Instant::now() < deadline,
"timed out waiting for observation; observed: {:?}",
recorder.observations()
);
thread::sleep(Duration::from_millis(20));
}
}
fn process_spec(node_id: u64, args: Vec<String>) -> NodeProvisionSpec {
NodeProvisionSpec {
run_id: 17,
node_id,
stage_index: Some(2),
image: "unused-for-local-process".to_owned(),
env: vec![("MVP_RUN_ID".to_owned(), "17".to_owned())],
args,
mounts: Vec::new(),
}
}
#[cfg(unix)]
struct TempScript {
root: PathBuf,
path: PathBuf,
}
#[cfg(unix)]
impl TempScript {
fn new(name: &str, content: &str) -> Self {
let root = std::env::temp_dir().join(format!(
"mvp-local-process-plugin-{name}-{}-{}",
std::process::id(),
thread::current().name().unwrap_or("unnamed")
));
let _ = fs::remove_dir_all(&root);
fs::create_dir_all(&root).expect("create temp script directory");
let path = root.join("helper.sh");
fs::write(&path, content).expect("write local process helper");
let mut permissions = fs::metadata(&path)
.expect("stat local process helper")
.permissions();
permissions.set_mode(0o755);
fs::set_permissions(&path, permissions).expect("chmod local process helper");
Self { root, path }
}
}
#[cfg(unix)]
impl Drop for TempScript {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.root);
}
}
#[cfg(unix)]
fn process_alive(pid: u32) -> bool {
unsafe {
if libc::kill(pid as i32, 0) == 0 {
true
} else {
std::io::Error::last_os_error().raw_os_error() != Some(libc::ESRCH)
}
}
}
fn spec_with_mounts(mounts: Vec<ProviderMount>) -> NodeProvisionSpec {
NodeProvisionSpec {
run_id: 17,
node_id: 23,
stage_index: Some(2),
image: "swactor-mvp-node:test".to_owned(),
env: vec![("MVP_RUN_ID".to_owned(), "17".to_owned())],
args: vec!["--serve".to_owned()],
mounts,
}
}
#[test]
fn node_provision_spec_serde_preserves_readonly_mounts() {
let spec = spec_with_mounts(vec![ProviderMount {
host_path: "/cache/models/model.gguf".to_owned(),
container_path: "/models/cached/model.gguf".to_owned(),
readonly: true,
}]);
let json = serde_json::to_string(&spec).expect("serialize mounted node spec");
let decoded: NodeProvisionSpec =
serde_json::from_str(&json).expect("deserialize mounted node spec");
assert_eq!(decoded, spec);
}
#[test]
fn node_provision_spec_serde_omits_empty_mounts_and_accepts_missing_mounts() {
let spec = spec_with_mounts(Vec::new());
let json = serde_json::to_value(&spec).expect("serialize unmounted node spec");
assert_eq!(json.get("mounts"), None);
let decoded: NodeProvisionSpec = serde_json::from_value(serde_json::json!({
"run_id": 17,
"node_id": 23,
"stage_index": 2,
"image": "swactor-mvp-node:test",
"env": [["MVP_RUN_ID", "17"]],
"args": ["--serve"]
}))
.expect("deserialize node spec written before mounts existed");
assert!(decoded.mounts.is_empty());
}
#[test]
fn docker_mount_arg_uses_bind_src_dst_and_readonly_flag() {
let mount = ProviderMount {
host_path: "/cache/models/model.gguf".to_owned(),
container_path: "/models/cached/model.gguf".to_owned(),
readonly: true,
};
assert_eq!(
docker_mount_arg(&mount),
"type=bind,src=/cache/models/model.gguf,dst=/models/cached/model.gguf,readonly"
);
let writable_mount = ProviderMount {
readonly: false,
..mount
};
assert_eq!(
docker_mount_arg(&writable_mount),
"type=bind,src=/cache/models/model.gguf,dst=/models/cached/model.gguf"
);
}
#[cfg(unix)]
#[test]
fn local_process_plugin_observes_stdio_frames_and_graceful_shutdown() {
let helper = TempScript::new(
"graceful",
r#"#!/bin/sh
printf '%s\n' '{"mvp_stdio_event":1,"kind":"datastream_frame","channel":"mvp.node.bootstrap","payload":{"type":"TestFrame","status":"ready"}}'
printf '%s\n' 'local-process-stderr' >&2
while IFS= read -r line; do
if [ "$line" = shutdown ]; then
exit 0
fi
done
exit 0
"#,
);
let mut plugin = LocalProcessPlugin::new("/bin/sh");
let (recorder, sink) = recording_sink();
let spec = process_spec(23, vec![helper.path.to_string_lossy().to_string()]);
let handle = plugin
.start_node(spec, sink)
.expect("start local process node");
assert!(handle.provider_process_id.is_some());
wait_for_observation(&recorder, |observation| {
matches!(
observation,
PluginObservation::DatastreamFrame {
channel,
payload,
..
} if channel == "mvp.node.bootstrap"
&& payload.contains("\"TestFrame\"")
&& payload.contains("\"ready\"")
)
});
wait_for_observation(&recorder, |observation| {
matches!(
observation,
PluginObservation::StderrLine { line, .. } if line == "local-process-stderr"
)
});
plugin.stop_node(&handle).expect("stop local process node");
wait_for_observation(&recorder, |observation| {
matches!(
observation,
PluginObservation::Exited {
node_id: 23,
status: Some(0),
..
}
)
});
}
#[cfg(unix)]
#[test]
fn local_process_plugin_kills_and_reaps_unresponsive_child() {
let helper = TempScript::new(
"unresponsive",
r#"#!/bin/sh
while :; do
sleep 1
done
"#,
);
let mut plugin = LocalProcessPlugin::new("/bin/sh");
let (recorder, sink) = recording_sink();
let spec = process_spec(24, vec![helper.path.to_string_lossy().to_string()]);
let handle = plugin
.start_node(spec, sink)
.expect("start unresponsive local process node");
let pid = handle
.provider_process_id
.expect("local process handle exposes child pid");
plugin
.stop_node(&handle)
.expect("stop unresponsive local process node");
wait_for_observation(&recorder, |observation| {
matches!(
observation,
PluginObservation::Exited {
node_id: 24,
status,
..
} if *status != Some(0)
)
});
let deadline = Instant::now() + Duration::from_secs(1);
while process_alive(pid) && Instant::now() < deadline {
thread::sleep(Duration::from_millis(20));
}
assert!(
!process_alive(pid),
"local process child {pid} is still live"
);
}
}

44
crates/mvp-system/src/prompt/rpc.rs
View file

@ -98,47 +98,3 @@ pub fn read_submit_prompt(reader: &mut impl BufRead) -> Result<Option<SubmitProm
.map(Some)
.map_err(|e| format!("parse prompt request: {e}"))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn zero_request_limits_take_loop_defaults() {
let request = SubmitPrompt {
request_id: 7,
prompt_text: "hello".to_owned(),
max_tokens: 0,
}
.with_defaults(32);
assert_eq!(request.max_tokens, 32);
}
#[test]
fn event_terminal_state_is_explicit() {
assert!(
!PromptEvent::TextDelta {
request_id: 1,
text: "a".to_owned(),
}
.is_terminal()
);
assert!(
PromptEvent::Done {
request_id: 1,
final_text: "a".to_owned(),
tokens_generated: 1,
elapsed_ms: 2,
}
.is_terminal()
);
assert!(
PromptEvent::Fault {
request_id: 1,
error: "boom".to_owned(),
}
.is_terminal()
);
}
}

178
crates/mvp-system/src/staging/gguf_metadata.rs
View file

@ -365,181 +365,3 @@ fn read_i64<R: Read>(reader: &mut R) -> Result<i64, String> {
.map_err(|e| format!("read i64: {e}"))?;
Ok(i64::from_le_bytes(bytes))
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Cursor;
#[test]
fn reads_planning_metadata_from_minimal_gguf_header() {
let mut bytes = Vec::new();
bytes.extend_from_slice(b"GGUF");
bytes.extend_from_slice(&3_u32.to_le_bytes());
bytes.extend_from_slice(&0_u64.to_le_bytes());
bytes.extend_from_slice(&6_u64.to_le_bytes());
push_string_kv(&mut bytes, "general.architecture", "llama");
push_string_kv(&mut bytes, "general.name", "fixture");
push_u32_kv(&mut bytes, "llama.block_count", 30);
push_u32_kv(&mut bytes, "llama.embedding_length", 576);
push_u32_kv(&mut bytes, "llama.context_length", 8192);
push_u32_kv(&mut bytes, "tokenizer.ggml.eos_token_id", 2);
let metadata = read_gguf_planning_metadata_from_reader(Cursor::new(bytes)).unwrap();
assert_eq!(metadata.version, 3);
assert_eq!(metadata.architecture, "llama");
assert_eq!(metadata.name.as_deref(), Some("fixture"));
assert_eq!(metadata.num_layers, 30);
assert_eq!(metadata.hidden_dim, 576);
assert_eq!(metadata.context_length, 8192);
assert_eq!(metadata.eos_token_id, 2);
}
#[test]
fn model_facts_clamp_effective_context_to_gguf_context() {
let metadata = GgufPlanningMetadata {
version: 3,
architecture: "llama".to_owned(),
name: None,
num_layers: 30,
hidden_dim: 576,
context_length: 256,
eos_token_id: 2,
};
let facts = metadata
.to_model_facts(
"fixture",
GgufSource::LocalPath("/models/fixture.gguf".to_owned()),
TokenizerSource::EmbeddedGguf,
Some(512),
)
.unwrap();
assert_eq!(facts.max_seq_len, 256);
assert_eq!(facts.dtype_family, DTypeFamily::BFloat);
assert_eq!(facts.dtype_width_bytes, 2);
}
#[test]
fn model_facts_use_explicit_context_when_it_fits_inside_gguf_context() {
let metadata = GgufPlanningMetadata {
version: 3,
architecture: "llama".to_owned(),
name: Some("fixture".to_owned()),
num_layers: 30,
hidden_dim: 576,
context_length: 8192,
eos_token_id: 2,
};
let facts = metadata
.to_model_facts(
"fixture-model",
GgufSource::LocalPath("/models/fixture.gguf".to_owned()),
TokenizerSource::EmbeddedGguf,
Some(384),
)
.unwrap();
assert_eq!(facts.model_id, "fixture-model");
assert_eq!(
facts.gguf_source,
GgufSource::LocalPath("/models/fixture.gguf".to_owned())
);
assert_eq!(facts.num_layers, 30);
assert_eq!(facts.hidden_dim, 576);
assert_eq!(facts.dtype_family, DTypeFamily::BFloat);
assert_eq!(facts.dtype_width_bytes, 2);
assert_eq!(facts.max_seq_len, 384);
assert_eq!(facts.eos_token_id, 2);
assert_eq!(facts.tokenizer, TokenizerSource::EmbeddedGguf);
}
#[test]
fn reader_rejects_missing_required_planning_metadata() {
for (missing_key, expected_error) in [
("general.architecture", "missing general.architecture"),
(
"llama.block_count",
"missing layer count key llama.block_count",
),
(
"llama.embedding_length",
"missing hidden dimension key llama.embedding_length",
),
(
"llama.context_length",
"missing context length key llama.context_length",
),
(
"tokenizer.ggml.eos_token_id",
"missing EOS token id key tokenizer.ggml.eos_token_id",
),
] {
let error = read_gguf_planning_metadata_from_reader(Cursor::new(minimal_gguf_without(
missing_key,
)))
.expect_err("metadata with a missing planning field must reject");
assert!(
error.contains(expected_error),
"missing {missing_key} error {error:?} should contain {expected_error:?}"
);
}
}
fn minimal_gguf_without(missing_key: &str) -> Vec<u8> {
let fields = [
"general.architecture",
"general.name",
"llama.block_count",
"llama.embedding_length",
"llama.context_length",
"tokenizer.ggml.eos_token_id",
];
let mut bytes = Vec::new();
bytes.extend_from_slice(b"GGUF");
bytes.extend_from_slice(&3_u32.to_le_bytes());
bytes.extend_from_slice(&0_u64.to_le_bytes());
bytes.extend_from_slice(
&(fields.iter().filter(|field| **field != missing_key).count() as u64).to_le_bytes(),
);
if missing_key != "general.architecture" {
push_string_kv(&mut bytes, "general.architecture", "llama");
}
if missing_key != "general.name" {
push_string_kv(&mut bytes, "general.name", "fixture");
}
if missing_key != "llama.block_count" {
push_u32_kv(&mut bytes, "llama.block_count", 30);
}
if missing_key != "llama.embedding_length" {
push_u32_kv(&mut bytes, "llama.embedding_length", 576);
}
if missing_key != "llama.context_length" {
push_u32_kv(&mut bytes, "llama.context_length", 8192);
}
if missing_key != "tokenizer.ggml.eos_token_id" {
push_u32_kv(&mut bytes, "tokenizer.ggml.eos_token_id", 2);
}
bytes
}
fn push_string_kv(bytes: &mut Vec<u8>, key: &str, value: &str) {
push_string(bytes, key);
bytes.extend_from_slice(&8_u32.to_le_bytes());
push_string(bytes, value);
}
fn push_u32_kv(bytes: &mut Vec<u8>, key: &str, value: u32) {
push_string(bytes, key);
bytes.extend_from_slice(&4_u32.to_le_bytes());
bytes.extend_from_slice(&value.to_le_bytes());
}
fn push_string(bytes: &mut Vec<u8>, value: &str) {
bytes.extend_from_slice(&(value.len() as u64).to_le_bytes());
bytes.extend_from_slice(value.as_bytes());
}
}

386
crates/mvp-system/src/staging/gguf_shard.rs
View file

@ -991,389 +991,3 @@ fn read_i64<R: Read>(reader: &mut R) -> Result<i64, String> {
.map_err(|e| format!("read i64: {e}"))?;
Ok(i64::from_le_bytes(bytes))
}
#[cfg(test)]
mod tests {
use super::*;
use parking_lot::Mutex;
use std::io::{Read, Write};
use std::net::{TcpListener, TcpStream};
use std::sync::Arc;
use std::thread;
#[test]
fn stage_plans_select_only_assigned_layer_tensors_and_boundaries() {
let fixture = SyntheticGguf::new(8);
let source = GgufSource::HuggingFaceGguf {
repo: "org/repo".to_owned(),
file: "model file.gguf".to_owned(),
revision: Some("abc123".to_owned()),
};
let stage0 = plan_stage_shard(&fixture.path, source.clone(), 0, 4, 0, 2).unwrap();
let stage1 = plan_stage_shard(&fixture.path, source.clone(), 1, 4, 2, 4).unwrap();
let stage3 = plan_stage_shard(&fixture.path, source.clone(), 3, 4, 6, 8).unwrap();
assert_eq!(
names(&stage0),
vec![
"token_embd.weight",
"blk.0.attn_q.weight",
"blk.0.ffn_up.weight",
"blk.1.attn_q.weight",
"blk.1.ffn_up.weight",
]
);
assert_eq!(
names(&stage1),
vec![
"blk.2.attn_q.weight",
"blk.2.ffn_up.weight",
"blk.3.attn_q.weight",
"blk.3.ffn_up.weight",
]
);
assert_eq!(
names(&stage3),
vec![
"blk.6.attn_q.weight",
"blk.6.ffn_up.weight",
"blk.7.attn_q.weight",
"blk.7.ffn_up.weight",
"output_norm.weight",
"output.weight",
]
);
assert_eq!(
stage0.source_url().unwrap(),
"https://huggingface.co/org/repo/resolve/abc123/model%20file.gguf"
);
assert!(
stage1
.merged_tensor_ranges
.iter()
.all(|range| range.start >= stage1.data_start)
);
assert!(
stage1
.merged_tensor_ranges
.iter()
.map(|range| range.len)
.sum::<u64>()
< fixture.bytes_len
);
}
#[test]
fn tied_output_final_stage_includes_token_embedding_when_output_weight_is_missing() {
let fixture = SyntheticGguf::without_output_weight(4);
let source = GgufSource::HuggingFaceGguf {
repo: "org/repo".to_owned(),
file: "model.gguf".to_owned(),
revision: None,
};
let final_stage = plan_stage_shard(&fixture.path, source, 1, 2, 2, 4).unwrap();
assert!(names(&final_stage).contains(&"token_embd.weight"));
assert!(names(&final_stage).contains(&"output_norm.weight"));
assert!(!names(&final_stage).contains(&"output.weight"));
}
#[test]
fn materialized_stage_shard_fetches_only_http_ranges_and_loads_as_gguf() {
let fixture = SyntheticGguf::new(4);
let bytes = std::fs::read(&fixture.path).unwrap();
let server = RangeServer::start(bytes);
let source = GgufSource::HuggingFaceGguf {
repo: "org/repo".to_owned(),
file: "model.gguf".to_owned(),
revision: None,
};
let plan = plan_stage_shard(&fixture.path, source, 1, 2, 2, 4).unwrap();
let output_path = fixture.path.with_file_name("stage-1.gguf");
let mut events = Vec::new();
materialize_stage_shard_from_url(&plan, &server.url, &output_path, |event| {
events.push(event);
})
.unwrap();
let materialized = read_gguf_directory(&output_path).unwrap();
assert_eq!(materialized.tensors.len(), plan.tensors.len());
assert_eq!(
materialized
.tensors
.iter()
.map(|tensor| tensor.name.as_str())
.collect::<Vec<_>>(),
names(&plan)
);
let ready = events.iter().find(|event| {
event.get("type").and_then(serde_json::Value::as_str) == Some("StageShardReady")
});
assert_eq!(
ready
.and_then(|event| event.get("bytes_done"))
.and_then(serde_json::Value::as_u64),
Some(plan.planned_fetch_bytes())
);
assert_eq!(
ready
.and_then(|event| event.get("bytes_total"))
.and_then(serde_json::Value::as_u64),
Some(plan.planned_fetch_bytes())
);
let ranges = server.ranges();
let expected_ranges = std::iter::once((0, plan.metadata_end.saturating_sub(1)))
.chain(plan.merged_tensor_ranges.iter().map(|range| {
(
range.start,
range
.end_exclusive()
.expect("planned range end")
.saturating_sub(1),
)
}))
.collect::<Vec<_>>();
assert_eq!(ranges, expected_ranges);
assert!(
ranges.len() < plan.tensors.len() + 1,
"coalesced tensor ranges should replace one HTTP request per tensor"
);
let range_ready = events
.iter()
.filter(|event| {
event.get("type").and_then(serde_json::Value::as_str)
== Some("StageShardRangeFetchReady")
})
.collect::<Vec<_>>();
assert_eq!(range_ready.len(), plan.planned_range_count());
assert_eq!(
range_ready
.last()
.and_then(|event| event.get("bytes_done"))
.and_then(serde_json::Value::as_u64),
Some(plan.planned_fetch_bytes())
);
assert_eq!(
range_ready
.last()
.and_then(|event| event.get("range_index"))
.and_then(serde_json::Value::as_u64),
Some((plan.planned_range_count() - 1) as u64)
);
let total_requested = ranges
.iter()
.map(|(start, end)| end.saturating_sub(*start) + 1)
.sum::<u64>();
assert_eq!(total_requested, plan.planned_fetch_bytes());
assert!(total_requested < fixture.bytes_len);
validate_stage_shard_cache(&output_path, &plan).unwrap();
}
#[test]
fn corrupted_stage_shard_cache_is_rejected_before_reuse() {
let fixture = SyntheticGguf::new(4);
let source = GgufSource::HuggingFaceGguf {
repo: "org/repo".to_owned(),
file: "model.gguf".to_owned(),
revision: None,
};
let plan = plan_stage_shard(&fixture.path, source, 1, 2, 2, 4).unwrap();
let output_path = fixture.path.with_file_name(plan.cache_file_name());
std::fs::write(&output_path, b"not a gguf").unwrap();
let error = validate_stage_shard_cache(&output_path, &plan).unwrap_err();
assert!(error.contains("invalid cached stage shard"));
}
fn names(plan: &StageShardPlan) -> Vec<&str> {
plan.tensors
.iter()
.map(|tensor| tensor.name.as_str())
.collect()
}
struct RangeServer {
url: String,
ranges: Arc<Mutex<Vec<(u64, u64)>>>,
}
impl RangeServer {
fn start(bytes: Vec<u8>) -> Self {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
let bytes = Arc::new(bytes);
let ranges = Arc::new(Mutex::new(Vec::new()));
let server_ranges = Arc::clone(&ranges);
thread::spawn(move || {
for stream in listener.incoming().flatten() {
handle_range_request(stream, Arc::clone(&bytes), Arc::clone(&server_ranges));
}
});
Self { url, ranges }
}
fn ranges(&self) -> Vec<(u64, u64)> {
self.ranges.lock().clone()
}
}
fn handle_range_request(
mut stream: TcpStream,
bytes: Arc<Vec<u8>>,
ranges: Arc<Mutex<Vec<(u64, u64)>>>,
) {
let mut request = Vec::new();
let mut buffer = [0_u8; 1024];
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
let Ok(read) = stream.read(&mut buffer) else {
return;
};
if read == 0 {
return;
}
request.extend_from_slice(&buffer[..read]);
}
let request = String::from_utf8_lossy(&request);
let Some(range_header) = request
.lines()
.find(|line| line.to_ascii_lowercase().starts_with("range: bytes="))
else {
let body = b"missing range";
let _ = write!(
stream,
"HTTP/1.1 400 Bad Request\r\nContent-Length: {}\r\n\r\n",
body.len()
);
let _ = stream.write_all(body);
return;
};
let range = range_header
.split_once("bytes=")
.map(|(_, range)| range.trim())
.unwrap();
let (start, end) = range.split_once('-').unwrap();
let start = start.parse::<u64>().unwrap();
let end = end.parse::<u64>().unwrap();
let content = &bytes[start as usize..=end as usize];
ranges.lock().push((start, end));
let _ = write!(
stream,
"HTTP/1.1 206 Partial Content\r\nContent-Length: {}\r\nContent-Range: bytes {}-{}/{}\r\nAccept-Ranges: bytes\r\n\r\n",
content.len(),
start,
end,
bytes.len()
);
let _ = stream.write_all(content);
}
struct SyntheticGguf {
path: std::path::PathBuf,
bytes_len: u64,
_dir: std::path::PathBuf,
}
impl SyntheticGguf {
fn new(layers: u32) -> Self {
Self::build(layers, true)
}
fn without_output_weight(layers: u32) -> Self {
Self::build(layers, false)
}
fn build(layers: u32, output_weight: bool) -> Self {
let dir = std::env::temp_dir().join(format!(
"mvp-gguf-shard-test-{}-{:?}",
std::process::id(),
std::thread::current().id()
));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join(if output_weight {
"model.gguf"
} else {
"tied.gguf"
});
let bytes = synthetic_gguf(layers, output_weight);
std::fs::File::create(&path)
.unwrap()
.write_all(&bytes)
.unwrap();
Self {
path,
bytes_len: bytes.len() as u64,
_dir: dir,
}
}
}
fn synthetic_gguf(layers: u32, output_weight: bool) -> Vec<u8> {
let mut tensors = vec!["token_embd.weight".to_owned()];
for layer in 0..layers {
tensors.push(format!("blk.{layer}.attn_q.weight"));
tensors.push(format!("blk.{layer}.ffn_up.weight"));
}
tensors.push("output_norm.weight".to_owned());
if output_weight {
tensors.push("output.weight".to_owned());
}
let mut metadata = Vec::new();
push_string_kv(&mut metadata, "general.architecture", "llama");
push_u32_kv(&mut metadata, "general.alignment", 32);
push_u32_kv(&mut metadata, "llama.block_count", layers);
push_u32_kv(&mut metadata, "llama.embedding_length", 8);
push_u32_kv(&mut metadata, "llama.context_length", 16);
push_u32_kv(&mut metadata, "tokenizer.ggml.eos_token_id", 2);
let mut tensor_infos = Vec::new();
let mut data = Vec::new();
let mut offset = 0_u64;
for (index, name) in tensors.iter().enumerate() {
push_gguf_string(&mut tensor_infos, name);
tensor_infos.extend_from_slice(&2_u32.to_le_bytes());
tensor_infos.extend_from_slice(&2_u64.to_le_bytes());
tensor_infos.extend_from_slice(&2_u64.to_le_bytes());
tensor_infos.extend_from_slice(&0_u32.to_le_bytes());
tensor_infos.extend_from_slice(&offset.to_le_bytes());
let len = 16 + index as u64;
data.extend(std::iter::repeat_n(index as u8, len as usize));
offset += len;
}
let mut bytes = Vec::new();
bytes.extend_from_slice(b"GGUF");
bytes.extend_from_slice(&3_u32.to_le_bytes());
bytes.extend_from_slice(&(tensors.len() as u64).to_le_bytes());
bytes.extend_from_slice(&6_u64.to_le_bytes());
bytes.extend_from_slice(&metadata);
bytes.extend_from_slice(&tensor_infos);
while bytes.len() % 32 != 0 {
bytes.push(0);
}
bytes.extend_from_slice(&data);
bytes
}
fn push_string_kv(bytes: &mut Vec<u8>, key: &str, value: &str) {
push_gguf_string(bytes, key);
bytes.extend_from_slice(&8_u32.to_le_bytes());
push_gguf_string(bytes, value);
}
fn push_u32_kv(bytes: &mut Vec<u8>, key: &str, value: u32) {
push_gguf_string(bytes, key);
bytes.extend_from_slice(&4_u32.to_le_bytes());
bytes.extend_from_slice(&value.to_le_bytes());
}
fn push_gguf_string(bytes: &mut Vec<u8>, value: &str) {
bytes.extend_from_slice(&(value.len() as u64).to_le_bytes());
bytes.extend_from_slice(value.as_bytes());
}
}

120
crates/mvp-system/src/tests/bootstrap_datastream_guarantees.rs
View file

@ -1,120 +0,0 @@
use std::sync::Arc;
use crate::provisioning::{
NodeProvisionSpec, PluginObservation, PluginObservationSink, PluginSink, ProvisionLogStream,
};
use datastream::{DatastreamEndpoint, Record};
use iroh::{EndpointAddr, SecretKey};
use mvp_system::observability::provisioning_logs::{BootstrapDatastreamBridge, node_stream_id};
use mvp_system::observability::telemetry::MvpProvisionLogRecord;
use parking_lot::Mutex;
use serde_json::json;
#[derive(Default)]
struct RecordingSink {
observations: Mutex<Vec<PluginObservation>>,
}
impl RecordingSink {
fn observations(&self) -> Vec<PluginObservation> {
self.observations.lock().clone()
}
}
impl PluginObservationSink for RecordingSink {
fn observe(&self, observation: PluginObservation) {
self.observations.lock().push(observation);
}
}
fn recording_sink() -> (Arc<RecordingSink>, PluginSink) {
let recording = Arc::new(RecordingSink::default());
(recording.clone(), PluginSink::new(recording))
}
fn spec() -> NodeProvisionSpec {
NodeProvisionSpec {
run_id: 7,
node_id: 42,
stage_index: Some(3),
image: "worker:latest".to_owned(),
env: Vec::new(),
args: Vec::new(),
mounts: Vec::new(),
}
}
#[test]
fn bootstrap_bridge_writes_node_stream_and_forwards_plugin_observations() {
let endpoint = DatastreamEndpoint::new(node_stream_id(7, 42));
let subscription = endpoint.subscribe_all("test");
let (recording, sink) = recording_sink();
let bridge = BootstrapDatastreamBridge::new(spec(), sink, Some(endpoint.producer()));
bridge.observe_stdout_line("ssh stdout diagnostic");
bridge.observe_stderr_line("debug1: ssh stderr diagnostic");
endpoint.tick();
let observations = recording.observations();
assert_eq!(
observations,
vec![
PluginObservation::StdoutLine {
run_id: 7,
node_id: 42,
line: "ssh stdout diagnostic".to_owned(),
},
PluginObservation::StderrLine {
run_id: 7,
node_id: 42,
line: "debug1: ssh stderr diagnostic".to_owned(),
},
]
);
let deliveries = subscription.drain_available();
let logs = deliveries
.iter()
.filter_map(|event| match event {
datastream::DatastreamEvent::Frame(delivery)
if delivery.channel.stream == node_stream_id(7, 42) =>
{
MvpProvisionLogRecord::decode(&delivery.payload).ok()
}
_ => None,
})
.collect::<Vec<_>>();
assert_eq!(logs.len(), 2, "{deliveries:?}");
let stdout = &logs[0];
let stderr = &logs[1];
assert_eq!(stdout.line.stream, ProvisionLogStream::Stdout);
assert_eq!(stdout.line.line, "ssh stdout diagnostic");
assert_eq!(stderr.line.stream, ProvisionLogStream::Stderr);
assert_eq!(stderr.line.line, "debug1: ssh stderr diagnostic");
}
#[test]
fn stdout_ready_json_is_log_only() {
let (recording, sink) = recording_sink();
let bridge = BootstrapDatastreamBridge::new(spec(), sink, None);
let line = serde_json::to_string(&json!({
"type": "ready",
"endpoint": EndpointAddr::new(SecretKey::from_bytes(&[7; 32]).public()),
"node_actor": "ignored-by-stdout-bridge",
"logical_node_id": 42,
"stage_index": 3,
}))
.unwrap();
bridge.observe_stdout_line(line.clone());
assert_eq!(
recording.observations(),
vec![PluginObservation::StdoutLine {
run_id: 7,
node_id: 42,
line,
}]
);
}

200
crates/mvp-system/src/tests/engine_builder_guarantees.rs
View file

@ -1,200 +0,0 @@
//! Black-box contract tests for the pool-based engine builder.
//!
//! The builder contract is topology/lifecycle only: it acquires a neutral node
//! pool, launches the same node image, waits for readiness/convergence, lets a
//! planner assign roles, and stays agnostic to workload input semantics.
use mvp_system::orchestration::engine_builder as engine;
use mvp_system::orchestration::engine_builder::WorkloadAdapter;
fn model() -> engine::ModelSpec {
engine::ModelSpec::mvp_tiny_open_llm_fixture()
}
fn image() -> engine::NodeImageSpec {
engine::NodeImageSpec::new("mvp-node:cuda").worker_runtime(
engine::WorkerRuntimeSpec::TinygradCuda {
worker_script: "/opt/mvp/mvp_tinygrad_worker.py".to_owned(),
device_env: "CUDA".to_owned(),
},
)
}
fn full_pool() -> engine::StaticPoolProvider {
engine::StaticPoolProvider::new(vec![
engine::NodeLease::new(
"coordinator",
engine::NodeId(900),
[engine::NodeCapability::Coordinator],
)
.resources(engine::ResourceFacts::cpu_only(2, 2 << 30)),
engine::NodeLease::new(
"worker-0",
engine::NodeId(11),
[engine::NodeCapability::Worker],
)
.resources(engine::ResourceFacts::cuda(1, 8 << 30, 4, 8 << 30)),
engine::NodeLease::new(
"worker-1",
engine::NodeId(12),
[engine::NodeCapability::Worker],
)
.resources(engine::ResourceFacts::cuda(1, 8 << 30, 4, 8 << 30)),
])
}
#[test]
fn builder_launches_pool_converges_and_assigns_planned_roles() {
let cluster = engine::ClusterBuilder::new("cluster-a", model())
.run_id(77)
.image(image())
.pool_provider(full_pool())
.launcher(engine::StaticNodeLauncher)
.planner(engine::FixedLinearPipelinePlanner::new(2))
.launch()
.expect("launch cluster");
let summaries = cluster.node_summaries();
let coordinator = summaries
.iter()
.find(|node| node.node_id == engine::NodeId(900))
.expect("coordinator summary");
assert_eq!(coordinator.roles, vec![engine::RoleKind::Coordinator]);
let worker0 = summaries
.iter()
.find(|node| node.node_id == engine::NodeId(11))
.expect("worker 0 summary");
assert_eq!(
worker0.roles,
vec![engine::RoleKind::StageWorker { stage_index: 0 }]
);
let worker1 = summaries
.iter()
.find(|node| node.node_id == engine::NodeId(12))
.expect("worker 1 summary");
assert_eq!(
worker1.roles,
vec![engine::RoleKind::StageWorker { stage_index: 1 }]
);
let plan = cluster.role_plan();
assert_eq!(plan.run_plan.stages.len(), 2);
assert_eq!(plan.run_plan.edges.len(), 3);
assert_eq!(plan.run_plan.stages[0].layer_start, 0);
assert_eq!(plan.run_plan.stages[0].layer_end_exclusive, 2);
assert_eq!(plan.run_plan.stages[1].layer_start, 2);
assert_eq!(plan.run_plan.stages[1].layer_end_exclusive, 4);
assert!(
cluster
.events()
.contains(&engine::EngineEvent::EngineReady {
cluster_id: "cluster-a".to_owned()
})
);
let shutdown_events = cluster.shutdown().expect("shutdown cluster");
assert!(
shutdown_events.contains(&engine::EngineEvent::ShutdownComplete {
cluster_id: "cluster-a".to_owned()
})
);
}
#[test]
fn planner_rejects_when_pool_cannot_supply_stage_workers() {
let short_pool = engine::StaticPoolProvider::new(vec![
engine::NodeLease::new(
"coordinator",
engine::NodeId(900),
[engine::NodeCapability::Coordinator],
),
engine::NodeLease::new(
"worker-0",
engine::NodeId(11),
[engine::NodeCapability::Worker],
),
engine::NodeLease::new(
"observer",
engine::NodeId(901),
[engine::NodeCapability::Coordinator],
),
]);
let result = engine::ClusterBuilder::new("cluster-short", model())
.run_id(77)
.image(image())
.pool_provider(short_pool)
.launcher(engine::StaticNodeLauncher)
.planner(engine::FixedLinearPipelinePlanner::new(2))
.launch();
match result {
Err(engine::EngineBuildError::Planning(engine::PlanningError::InsufficientWorkers {
required,
available,
})) => {
assert_eq!(required, 2);
assert_eq!(available, 1);
}
Ok(_) => panic!("expected insufficient worker planning error, got launched cluster"),
Err(other) => panic!("unexpected error: {other:?}"),
}
}
struct ProbeWorkload;
impl engine::WorkloadAdapter for ProbeWorkload {
type Input = Vec<&'static str>;
type Output = usize;
type Error = std::convert::Infallible;
fn submit(
&self,
_cluster: &mut engine::ClusterHandle,
input: Self::Input,
) -> Result<Self::Output, Self::Error> {
Ok(input.len())
}
}
#[test]
fn workload_input_semantics_live_outside_the_cluster_builder() {
let mut cluster = engine::ClusterBuilder::new("cluster-opaque", model())
.run_id(77)
.image(image())
.pool_provider(full_pool())
.launcher(engine::StaticNodeLauncher)
.planner(engine::FixedLinearPipelinePlanner::new(2))
.launch()
.expect("launch cluster");
let observed = ProbeWorkload
.submit(&mut cluster, vec!["not", "tokens"])
.expect("submit probe workload");
assert_eq!(observed, 2);
cluster.shutdown().expect("shutdown cluster");
}
#[test]
fn runtime_node_builds_the_reusable_iroh_swactor_stack() {
let mut node = engine::RuntimeNode::start_default().expect("start runtime node");
let orchestrator = node
.spawn_orchestrator_actor(
crate::run_fsm::RunConfig {
run_id: crate::run_fsm::RunId(77),
max_tokens: 1,
prompt: vec![1, 2, 3],
},
None,
)
.expect("spawn orchestrator actor");
let worker = node
.spawn_node_agent_actor(mvp_system::staging::NodeId(11), orchestrator, None)
.expect("spawn node agent actor");
node.wait_for_routes(&[orchestrator, worker])
.expect("local actor routes");
assert_eq!(node.node_id().0, *node.endpoint_addr().id.as_bytes());
}

2
crates/mvp-system/src/tests/local_mock_pipeline_integration.rs → crates/mvp-system/src/tests/local_e2e_guarantees.rs
View file

@ -1,4 +1,4 @@
//! Local mock integration coverage for the MVP system lifecycle.
//! Local end-to-end behavior guarantee for the MVP system lifecycle.
//!
//! This test composes the crate's MVP contract harnesses through one in-process
//! mock environment. It deliberately avoids Docker, real SWIM, real iroh, GGUF,

20
crates/mvp-system/src/tests/mod.rs
View file

@ -1,14 +1,8 @@
mod bootstrap_datastream_guarantees;
mod engine_builder_guarantees;
mod local_e2e_guarantees;
mod local_mock;
mod local_mock_pipeline_integration;
mod observability_surface_guarantees;
mod orchestrator_run_fsm_guarantees;
mod relay_provisioning_guarantees;
mod run_plan_guarantees;
mod shared_ring_helper_abi_guarantees;
mod stage_controller_guarantees;
mod telemetry_guarantees;
mod tx_rx_edge_actor_guarantees;
mod weight_lifecycle_guarantees;
mod worker_edge_adapter_guarantees;
mod node_guarantees;
mod observability_guarantees;
mod orchestration_guarantees;
mod prompt_guarantees;
mod staging_guarantees;
mod transport_guarantees;

95
crates/mvp-system/src/tests/node_guarantees.rs Normal file
View file

@ -0,0 +1,95 @@
//! Behavior guarantees for the `node` module.
use crate::node_actor::{NodeAgentActor, NodeAgentMsg, NodeAgentReport};
use crate::orchestration::actor::OrchestratorMsg;
use iroh::{EndpointAddr, SecretKey};
use mvp_system::staging as stage;
use swactor::actor::ActorAddress;
use swactor::config::RuntimeConfig;
use swactor::runtime::Runtime;
#[test]
fn node_agent_runtime_loaded_reports_orchestrator() {
let runtime = Runtime::new(RuntimeConfig::default());
let orchestrator_inbox = runtime
.new_inbox::<OrchestratorMsg>()
.expect("orchestrator inbox");
let orchestrator = *orchestrator_inbox.addr();
let node_actor = ActorAddress::new_random();
let datastream_publisher = ActorAddress::new_random();
let endpoint = EndpointAddr::new(SecretKey::from_bytes(&[9; 32]).public());
let actor = runtime
.spawn(NodeAgentActor::new(stage::NodeId(11), orchestrator, None))
.expect("spawn node agent");
runtime
.send_to(
actor,
NodeAgentMsg::RuntimeLoaded {
run_id: 7,
node_id: 11,
stage_index: 3,
endpoint: endpoint.clone(),
node_actor,
datastream_publisher,
readiness_id: 99,
},
)
.expect("send runtime loaded");
runtime.tick();
assert_eq!(
orchestrator_inbox.try_recv(),
Some(OrchestratorMsg::ObserveNodeRuntimeReady {
run_id: 7,
node_id: 11,
stage_index: 3,
endpoint,
node_actor,
datastream_publisher,
readiness_id: 99,
})
);
}
#[test]
fn node_agent_runtime_ready_ack_reports_worker_loop() {
let runtime = Runtime::new(RuntimeConfig::default());
let orchestrator_inbox = runtime
.new_inbox::<OrchestratorMsg>()
.expect("orchestrator inbox");
let reports = runtime
.new_inbox::<NodeAgentReport>()
.expect("node report inbox");
let actor = runtime
.spawn(NodeAgentActor::new(
stage::NodeId(11),
*orchestrator_inbox.addr(),
Some(*reports.addr()),
))
.expect("spawn node agent");
runtime
.send_to(
actor,
NodeAgentMsg::RuntimeReadyAck {
run_id: 7,
node_id: 11,
stage_index: 3,
readiness_id: 99,
},
)
.expect("send runtime ready ack");
runtime.tick();
assert_eq!(
reports.try_recv(),
Some(NodeAgentReport::RuntimeReadyAck {
run_id: 7,
node_id: 11,
stage_index: 3,
readiness_id: 99,
})
);
assert_eq!(reports.try_recv(), None);
}

30
crates/mvp-system/src/tests/observability_surface_guarantees.rs → crates/mvp-system/src/tests/observability_guarantees.rs
View file

@ -1,3 +1,5 @@
//! Behavior guarantees for the `observability` module.
//! Black-box contract tests for the MVP observability surface.
//!
//! These tests intentionally know only the public event stream surface:
@ -347,31 +349,3 @@ fn event_ordering_reflects_component_contracts_and_one_terminal_outcome() {
.count();
assert_eq!(terminal_count, 1);
}
// This proves observability tests are independent of transport, storage, and
// batching policy by asserting the same event facts after batching is changed.
#[test]
fn event_contract_survives_transport_storage_and_batching_policy() {
// Build the same logical events under two batching policies.
let unbatched =
obs::EventSubscriberHarness::collect(successful_run_trace(), obs::Batching::None);
let batched =
obs::EventSubscriberHarness::collect(successful_run_trace(), obs::Batching::Fixed(8));
// Flattened public event facts must match as an ordered stream.
let unbatched_kinds = unbatched
.flattened_events()
.iter()
.map(|event| event.kind())
.collect::<Vec<_>>();
let batched_kinds = batched
.flattened_events()
.iter()
.map(|event| event.kind())
.collect::<Vec<_>>();
assert_eq!(batched_kinds, unbatched_kinds);
// Neither subscriber depends on transport or storage implementation names.
assert!(!unbatched.used_transport_specific_assertions());
assert!(!batched.used_storage_specific_assertions());
}

1074
crates/mvp-system/src/tests/orchestration_guarantees.rs Normal file
View file

File diff suppressed because it is too large Load diff

520
crates/mvp-system/src/tests/orchestrator_run_fsm_guarantees.rs
View file

@ -1,520 +0,0 @@
//! Black-box contract tests for the MVP orchestrator run FSM.
//!
//! These tests intentionally know only the public orchestrator surface:
//!
//! - pool, plan, stage, endpoint, token, fault, and stop events in
//! - commands, lifecycle events, and terminal outcome out
//!
//! They assert the guarantees in
//! `specs/mvp_system/orchestrator_run_fsm_contract.md`.
use crate::run_fsm as fsm;
// A three-stage plan proves multi-stage provisioning and readiness without
// making tests depend on any placement heuristic. The plan is already valid;
// these tests are about how the orchestrator consumes it.
fn committed_plan() -> fsm::RunPlan {
fsm::RunPlan::test_linear(
fsm::RunId(7),
vec![
fsm::StageRef {
stage_index: 0,
node_id: fsm::NodeId(10),
},
fsm::StageRef {
stage_index: 1,
node_id: fsm::NodeId(11),
},
fsm::StageRef {
stage_index: 2,
node_id: fsm::NodeId(12),
},
],
)
}
// The harness is the black-box public boundary for the run FSM. It accepts
// observable events and records emitted commands/events; tests never inspect an
// internal FSM enum or private readiness counter.
fn new_run() -> fsm::OrchestratorHarness {
fsm::OrchestratorHarness::new(fsm::RunConfig {
run_id: fsm::RunId(7),
max_tokens: 4,
prompt: vec![101, 102, 103],
})
}
// Stage readiness events are generated from the committed plan so the tests
// prove readiness by stage identity instead of relying on command ordering.
fn stage_ready_events(plan: &fsm::RunPlan) -> Vec<fsm::RunEvent> {
plan.stages
.iter()
.map(|stage| fsm::RunEvent::StageReady {
run_id: plan.run_id,
stage_index: stage.stage_index,
})
.collect()
}
// Transcript positions turn ordering claims into proofs over observable output.
// If an event is missing, the test fails at the boundary where users and other
// components would also lose the guarantee.
fn position_of(events: &[fsm::LifecycleEvent], needle: &fsm::LifecycleEvent) -> usize {
events
.iter()
.position(|event| event == needle)
.expect("expected lifecycle event missing")
}
// This proves planning and provisioning are gated by PoolReady, and that the
// orchestrator provisions exactly the committed stages and local token endpoints
// from a valid RunPlan.
#[test]
fn planning_and_provisioning_start_only_after_pool_ready() {
// Start the run and give it a valid plan, but no PoolReady event.
let plan = committed_plan();
let mut harness = new_run();
harness.observe(fsm::RunEvent::PlanAvailable(plan.clone()));
// Without PoolReady, provisioning must not begin.
assert!(
!harness
.commands()
.iter()
.any(|command| { matches!(command, fsm::RunCommand::ProvisionStage { .. }) })
);
// Once PoolReady is observed, the committed plan may be provisioned.
harness.observe(fsm::RunEvent::PoolReady {
nodes: plan.stage_nodes(),
});
// Every planned stage gets exactly one provision command.
let provisioned = harness
.commands()
.iter()
.filter_map(|command| match command {
fsm::RunCommand::ProvisionStage { provision } => Some(provision.stage_index),
_ => None,
})
.collect::<std::collections::BTreeSet<_>>();
let expected = plan
.stages
.iter()
.map(|stage| stage.stage_index)
.collect::<std::collections::BTreeSet<_>>();
assert_eq!(provisioned, expected);
// Provisioning must not mention nodes outside the committed plan.
let plan_nodes = plan
.stage_nodes()
.into_iter()
.collect::<std::collections::BTreeSet<_>>();
for command in harness.commands() {
if let fsm::RunCommand::ProvisionStage { provision } = command {
assert!(plan_nodes.contains(&provision.node_id));
}
}
// Token endpoints are created locally from the same committed plan.
assert!(harness.commands().iter().any(|command| {
matches!(command, fsm::RunCommand::CreateTokenInEndpoint { run_id } if *run_id == fsm::RunId(7))
}));
assert!(harness.commands().iter().any(|command| {
matches!(command, fsm::RunCommand::CreateTokenOutEndpoint { run_id } if *run_id == fsm::RunId(7))
}));
}
// This proves prompt injection is blocked until every planned stage and both
// local token endpoints are ready. Duplicate readiness must not count as a
// missing stage, and foreign readiness must fault or reject.
#[test]
fn readiness_barrier_controls_prompt_injection() {
// Provision a valid plan after PoolReady.
let plan = committed_plan();
let mut harness = new_run();
harness.observe(fsm::RunEvent::PoolReady {
nodes: plan.stage_nodes(),
});
harness.observe(fsm::RunEvent::PlanAvailable(plan.clone()));
// A duplicate StageReady for stage 0 cannot satisfy stage 1 or 2.
harness.observe(fsm::RunEvent::StageReady {
run_id: fsm::RunId(7),
stage_index: 0,
});
harness.observe(fsm::RunEvent::StageReady {
run_id: fsm::RunId(7),
stage_index: 0,
});
harness.observe(fsm::RunEvent::TokenInEndpointReady);
harness.observe(fsm::RunEvent::TokenOutEndpointReady);
assert!(
!harness
.commands()
.iter()
.any(|command| { matches!(command, fsm::RunCommand::InjectTokenObject { .. }) })
);
// Complete the remaining stage readiness facts.
for event in stage_ready_events(&plan).into_iter().skip(1) {
harness.observe(event);
}
// Prompt injection is the public start signal after the full barrier.
assert!(harness.commands().iter().any(|command| {
matches!(
command,
fsm::RunCommand::InjectTokenObject {
run_id: fsm::RunId(7),
object: fsm::TokenObjectInjection {
sequence: 0,
payload: fsm::TokenObjectPayload::Prompt { tokens },
},
} if tokens.as_slice() == [101, 102, 103]
)
}));
// Unknown stage readiness must not silently advance another run.
let mut invalid = new_run();
invalid.observe(fsm::RunEvent::PoolReady {
nodes: plan.stage_nodes(),
});
invalid.observe(fsm::RunEvent::PlanAvailable(plan));
invalid.observe(fsm::RunEvent::StageReady {
run_id: fsm::RunId(7),
stage_index: 99,
});
assert!(invalid.events().iter().any(|event| {
matches!(event, fsm::LifecycleEvent::RunFaulted { .. })
|| matches!(event, fsm::LifecycleEvent::RunRejected { .. })
}));
}
// This proves execution has one start signal and advances by the token feedback
// rule: inject sequence 0 first, then inject k + 1 only after consuming k.
#[test]
fn execution_injects_next_sequence_only_after_consuming_previous_token() {
// Drive a run through the complete readiness barrier.
let plan = committed_plan();
let mut harness = new_run();
harness.observe(fsm::RunEvent::PoolReady {
nodes: plan.stage_nodes(),
});
harness.observe(fsm::RunEvent::PlanAvailable(plan.clone()));
harness.observe(fsm::RunEvent::TokenInEndpointReady);
harness.observe(fsm::RunEvent::TokenOutEndpointReady);
for event in stage_ready_events(&plan) {
harness.observe(event);
}
// Sequence 0 must be injected first as a prompt token object.
let initial_objects = harness
.commands()
.iter()
.filter_map(|command| match command {
fsm::RunCommand::InjectTokenObject { object, .. } => Some(object),
_ => None,
})
.collect::<Vec<_>>();
assert_eq!(initial_objects.len(), 1);
assert_eq!(
*initial_objects[0],
fsm::TokenObjectInjection {
sequence: 0,
payload: fsm::TokenObjectPayload::Prompt {
tokens: vec![101, 102, 103],
},
}
);
assert_eq!(harness.injected_sequences(), vec![0]);
// Consuming token 0 permits injecting sequence 1.
harness.observe(fsm::RunEvent::TokenReceived {
sequence: 0,
token_id: 201,
eos: false,
});
assert_eq!(harness.injected_sequences(), vec![0, 1]);
let decode_object = harness
.commands()
.iter()
.filter_map(|command| match command {
fsm::RunCommand::InjectTokenObject { object, .. } => Some(object),
_ => None,
})
.last()
.expect("decode injection must be recorded");
assert_eq!(
*decode_object,
fsm::TokenObjectInjection {
sequence: 1,
payload: fsm::TokenObjectPayload::Decode {
token_id: 201,
sampling: fsm::SamplingData { source_sequence: 0 },
},
}
);
// No additional injection may happen without consuming sequence 1.
harness.advance_time_ms(10);
assert_eq!(harness.injected_sequences(), vec![0, 1]);
// EOS stops further injection after the consumed sequence.
harness.observe(fsm::RunEvent::TokenReceived {
sequence: 1,
token_id: 2,
eos: true,
});
assert_eq!(harness.injected_sequences(), vec![0, 1]);
}
// This proves every run-level fault source records one terminal fault, and the
// first failure reason is retained if later failures arrive.
#[test]
fn first_run_fault_reason_is_terminal_and_sticky() {
// Prepare an executing run so both setup and execution-time faults would be
// meaningful if observed.
let plan = committed_plan();
let mut harness = new_run();
harness.observe(fsm::RunEvent::PoolReady {
nodes: plan.stage_nodes(),
});
harness.observe(fsm::RunEvent::PlanAvailable(plan.clone()));
harness.observe(fsm::RunEvent::TokenInEndpointReady);
harness.observe(fsm::RunEvent::TokenOutEndpointReady);
for event in stage_ready_events(&plan) {
harness.observe(event);
}
// Inject the first failure source.
harness.observe(fsm::RunEvent::StageFault {
run_id: fsm::RunId(7),
stage_index: 1,
reason: fsm::StageFaultReason::WorkerCrashed,
});
// Inject later failures that must not replace the terminal reason.
harness.observe(fsm::RunEvent::EndpointFault {
run_id: fsm::RunId(7),
endpoint: fsm::EndpointKind::TokenOut,
});
harness.observe(fsm::RunEvent::MembershipLost {
run_id: fsm::RunId(7),
node_id: fsm::NodeId(11),
});
// Exactly one terminal fault is recorded.
let faults = harness
.events()
.iter()
.filter_map(|event| match event {
fsm::LifecycleEvent::RunFaulted { reason, .. } => Some(reason),
_ => None,
})
.collect::<Vec<_>>();
assert_eq!(faults.len(), 1);
assert_eq!(
*faults[0],
fsm::RunFaultReason::StageFault {
stage_index: 1,
reason: fsm::StageFaultReason::WorkerCrashed,
}
);
}
#[test]
fn membership_loss_faults_run() {
let plan = committed_plan();
let mut membership_lost = new_run();
membership_lost.observe(fsm::RunEvent::PoolReady {
nodes: plan.stage_nodes(),
});
membership_lost.observe(fsm::RunEvent::PlanAvailable(plan.clone()));
membership_lost.observe(fsm::RunEvent::MembershipLost {
run_id: fsm::RunId(7),
node_id: fsm::NodeId(12),
});
assert!(membership_lost.events().iter().any(|event| {
matches!(
event,
fsm::LifecycleEvent::RunFaulted {
reason: fsm::RunFaultReason::MembershipLost {
node_id: fsm::NodeId(12)
},
..
}
)
}));
}
// This proves terminal outcomes are mutually exclusive, reject new work, and
// always lead into teardown for success, fault, and operator stop.
#[test]
fn terminal_outcome_is_single_and_requires_teardown() {
// Complete a run by reaching EOS.
let plan = committed_plan();
let mut harness = new_run();
harness.observe(fsm::RunEvent::PoolReady {
nodes: plan.stage_nodes(),
});
harness.observe(fsm::RunEvent::PlanAvailable(plan.clone()));
harness.observe(fsm::RunEvent::TokenInEndpointReady);
harness.observe(fsm::RunEvent::TokenOutEndpointReady);
for event in stage_ready_events(&plan) {
harness.observe(event);
}
harness.observe(fsm::RunEvent::TokenReceived {
sequence: 0,
token_id: 2,
eos: true,
});
// Completed and Faulted are mutually exclusive public outcomes.
let completed = harness
.events()
.iter()
.filter(|event| matches!(event, fsm::LifecycleEvent::RunCompleted { .. }))
.count();
let faulted = harness
.events()
.iter()
.filter(|event| matches!(event, fsm::LifecycleEvent::RunFaulted { .. }))
.count();
assert_eq!(completed, 1);
assert_eq!(faulted, 0);
// New token work after terminal outcome begins must be rejected.
let before = harness.injected_sequences();
harness.observe(fsm::RunEvent::TokenReceived {
sequence: 99,
token_id: 333,
eos: false,
});
assert_eq!(harness.injected_sequences(), before);
// Teardown commands must be emitted for every provisioned stage and local
// endpoint after the terminal outcome.
let stopped_stages = harness
.commands()
.iter()
.filter_map(|command| match command {
fsm::RunCommand::StopRun { stage_index, .. } => Some(*stage_index),
_ => None,
})
.collect::<std::collections::BTreeSet<_>>();
let expected_stages = plan
.stages
.iter()
.map(|stage| stage.stage_index)
.collect::<std::collections::BTreeSet<_>>();
assert_eq!(stopped_stages, expected_stages);
assert!(
harness
.commands()
.iter()
.any(|command| { matches!(command, fsm::RunCommand::TearDownTokenEndpoints { .. }) })
);
let mut stopped = new_run();
stopped.observe(fsm::RunEvent::PoolReady {
nodes: plan.stage_nodes(),
});
stopped.observe(fsm::RunEvent::PlanAvailable(plan));
stopped.observe(fsm::RunEvent::OperatorStop {
run_id: fsm::RunId(7),
});
let stopped_count = stopped
.events()
.iter()
.filter(|event| matches!(event, fsm::LifecycleEvent::RunOperatorStopped { .. }))
.count();
let stopped_faults = stopped
.events()
.iter()
.filter(|event| matches!(event, fsm::LifecycleEvent::RunFaulted { .. }))
.count();
assert_eq!(stopped_count, 1);
assert_eq!(stopped_faults, 0);
assert!(
stopped
.commands()
.iter()
.any(|command| { matches!(command, fsm::RunCommand::TearDownTokenEndpoints { .. }) })
);
}
// This proves run_torn_down is emitted exactly once and only after teardown
// observes every planned stage stop and local endpoint stop.
#[test]
fn run_torn_down_is_emitted_once_after_teardown_terminal_state() {
// Fault a provisioned run so teardown is required.
let plan = committed_plan();
let mut harness = new_run();
harness.observe(fsm::RunEvent::PoolReady {
nodes: plan.stage_nodes(),
});
harness.observe(fsm::RunEvent::PlanAvailable(plan.clone()));
harness.observe(fsm::RunEvent::StageFault {
run_id: fsm::RunId(7),
stage_index: 0,
reason: fsm::StageFaultReason::WorkerCrashed,
});
// StageStopped from only a prefix of stages is not enough to finish
// teardown.
harness.observe(fsm::RunEvent::StageStopped {
run_id: fsm::RunId(7),
stage_index: 0,
});
assert!(
!harness
.events()
.iter()
.any(|event| { matches!(event, fsm::LifecycleEvent::RunTornDown { .. }) })
);
// StageStopped for every stage still is not enough until local endpoints stop.
harness.observe(fsm::RunEvent::StageStopped {
run_id: fsm::RunId(7),
stage_index: 1,
});
harness.observe(fsm::RunEvent::StageStopped {
run_id: fsm::RunId(7),
stage_index: 2,
});
assert!(
!harness
.events()
.iter()
.any(|event| { matches!(event, fsm::LifecycleEvent::RunTornDown { .. }) })
);
harness.observe(fsm::RunEvent::TokenEndpointsStopped);
// The final event may now appear, exactly once.
let torn_down_count = harness
.events()
.iter()
.filter(|event| matches!(event, fsm::LifecycleEvent::RunTornDown { .. }))
.count();
assert_eq!(torn_down_count, 1);
// Ordering is proven over the lifecycle transcript.
let fault_pos = position_of(
harness.events(),
&fsm::LifecycleEvent::RunFaulted {
run_id: fsm::RunId(7),
reason: fsm::RunFaultReason::StageFault {
stage_index: 0,
reason: fsm::StageFaultReason::WorkerCrashed,
},
},
);
let torn_down_pos = position_of(
harness.events(),
&fsm::LifecycleEvent::RunTornDown {
run_id: fsm::RunId(7),
},
);
assert!(fault_pos < torn_down_pos);
}

42
crates/mvp-system/src/tests/prompt_guarantees.rs Normal file
View file

@ -0,0 +1,42 @@
//! Behavior guarantees for the `prompt` module.
use mvp_system::prompt::rpc::{PromptEvent, SubmitPrompt};
#[test]
fn zero_request_limits_take_loop_defaults() {
let request = SubmitPrompt {
request_id: 7,
prompt_text: "hello".to_owned(),
max_tokens: 0,
}
.with_defaults(32);
assert_eq!(request.max_tokens, 32);
}
#[test]
fn event_terminal_state_is_explicit() {
assert!(
!PromptEvent::TextDelta {
request_id: 1,
text: "a".to_owned(),
}
.is_terminal()
);
assert!(
PromptEvent::Done {
request_id: 1,
final_text: "a".to_owned(),
tokens_generated: 1,
elapsed_ms: 2,
}
.is_terminal()
);
assert!(
PromptEvent::Fault {
request_id: 1,
error: "boom".to_owned(),
}
.is_terminal()
);
}

171
crates/mvp-system/src/tests/relay_provisioning_guarantees.rs
View file

@ -1,171 +0,0 @@
use std::ffi::OsString;
use std::sync::Mutex;
use iroh::{RelayMode, RelayUrl};
use mvp_system::orchestration::provider_adapters::relay::{
LocalShimRelayProvider, MVP_IROH_RELAY_MODE_ENV, MVP_IROH_RELAY_URL_ENV, RelayProvider,
RelayProviderKind, RelayProvisionRequest, RelayPurpose, SWACTOR_IROH_RELAY_URL_ENV,
StaticRelayProvider, relay_runtime_config_from_env,
};
static ENV_LOCK: Mutex<()> = Mutex::new(());
const RELAY_ENV_KEYS: &[&str] = &[
MVP_IROH_RELAY_MODE_ENV,
MVP_IROH_RELAY_URL_ENV,
SWACTOR_IROH_RELAY_URL_ENV,
];
struct RestoreEnv {
saved: Vec<(&'static str, Option<OsString>)>,
}
impl Drop for RestoreEnv {
fn drop(&mut self) {
for (key, value) in &self.saved {
match value {
Some(value) => unsafe { std::env::set_var(key, value) },
None => unsafe { std::env::remove_var(key) },
}
}
}
}
fn with_relay_env<T>(settings: &[(&'static str, &'static str)], test: impl FnOnce() -> T) -> T {
let _lock = ENV_LOCK
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let saved = RELAY_ENV_KEYS
.iter()
.map(|&key| (key, std::env::var_os(key)))
.collect::<Vec<_>>();
for key in RELAY_ENV_KEYS {
unsafe { std::env::remove_var(key) };
}
for (key, value) in settings {
assert!(
RELAY_ENV_KEYS.contains(key),
"test env key {key} must be restored"
);
unsafe { std::env::set_var(key, value) };
}
let _restore = RestoreEnv { saved };
test()
}
fn provision_request(run_id: u64) -> RelayProvisionRequest {
RelayProvisionRequest {
run_id,
purpose: RelayPurpose::Combined,
}
}
fn canonical_relay_url(raw: &str) -> String {
raw.parse::<RelayUrl>()
.expect("fixture relay URL parses")
.to_string()
}
fn assert_custom_relay_mode(mode: RelayMode, expected_url: &str) {
let expected_url = expected_url
.parse::<RelayUrl>()
.expect("fixture relay URL parses");
match mode {
RelayMode::Custom(relay_map) => {
assert_eq!(relay_map.len(), 1, "custom relay map must contain one URL");
assert!(
relay_map.contains(&expected_url),
"custom relay map must contain {expected_url}"
);
}
other => panic!("expected custom relay mode for {expected_url}, got {other:?}"),
}
}
#[test]
fn local_shim_provisions_disabled_lease_without_endpoints() {
let mut provider = LocalShimRelayProvider;
let lease = provider
.provision_relay(provision_request(77))
.expect("local shim provisioning succeeds");
assert_eq!(lease.endpoints, Vec::new());
assert!(matches!(
provider
.relay_mode(&lease)
.expect("local shim relay mode resolves"),
RelayMode::Disabled
));
}
#[test]
fn static_provider_provisions_one_endpoint_and_custom_relay_mode() {
const RELAY_URL: &str = "https://relay-static.example.com";
let mut provider =
StaticRelayProvider::from_url_str(RELAY_URL).expect("static relay URL parses");
let expected_url = canonical_relay_url(RELAY_URL);
let lease = provider
.provision_relay(provision_request(88))
.expect("static relay provisioning succeeds");
assert_eq!(lease.endpoints.len(), 1);
assert_eq!(lease.endpoints[0].url, expected_url);
assert_eq!(lease.endpoints[0].provider, RelayProviderKind::Static);
assert_custom_relay_mode(
provider
.relay_mode(&lease)
.expect("static relay mode resolves"),
&expected_url,
);
}
#[test]
fn default_relay_mode_uses_configured_mvp_or_swactor_relay_url() {
const MVP_URL: &str = "https://relay-mvp.example.com";
const SWACTOR_URL: &str = "https://relay-swactor.example.com";
for (name, settings, expected_url) in [
(
"mvp relay URL",
[
(MVP_IROH_RELAY_MODE_ENV, "default"),
(MVP_IROH_RELAY_URL_ENV, MVP_URL),
],
MVP_URL,
),
(
"swactor relay URL fallback",
[
(MVP_IROH_RELAY_MODE_ENV, "default"),
(SWACTOR_IROH_RELAY_URL_ENV, SWACTOR_URL),
],
SWACTOR_URL,
),
] {
with_relay_env(&settings, || {
let config = relay_runtime_config_from_env(901).unwrap_or_else(|error| {
panic!("{name} should resolve custom relay config: {error}")
});
let expected_url = canonical_relay_url(expected_url);
assert_eq!(config.url.as_deref(), Some(expected_url.as_str()));
assert_custom_relay_mode(config.mode, &expected_url);
});
}
}
#[test]
fn disabled_relay_mode_ignores_configured_url() {
with_relay_env(
&[
(MVP_IROH_RELAY_MODE_ENV, "disabled"),
(MVP_IROH_RELAY_URL_ENV, "https://ignored-relay.example.com"),
],
|| {
let config = relay_runtime_config_from_env(902).expect("disabled relay mode resolves");
assert!(matches!(config.mode, RelayMode::Disabled));
assert_eq!(config.url, None);
},
);
}

821
crates/mvp-system/src/tests/run_plan_guarantees.rs
View file

@ -1,821 +0,0 @@
//! Black-box contract tests for MVP RunPlan formation.
//!
//! These tests intentionally know only the public planning surface:
//!
//! - `plan_run(input) -> Result<RunPlan, PlanRejection>`
//! - `derive_stage_provision(&plan, stage_index) -> Result<ProvisionStage, ProjectionRejection>`
//!
//! They assert the guarantees in `specs/mvp_system/run_plan_contract.md`.
//! The planner implementation, placement heuristic, helper APIs, internal graph
//! representation, and allocation strategy are not observable here.
use crate::run_plan as plan;
// Local aliases keep the test prose readable while the file imports only the
// public planning module. The aliases do not grant access to planner internals.
type DTypeFamily = plan::DTypeFamily;
type EdgeEndpoint = plan::EdgeEndpoint;
type EdgeId = plan::EdgeId;
type EdgeKind = plan::EdgeKind;
type EdgePlan = plan::EdgePlan;
type GgufSource = plan::GgufSource;
type HostPinning = plan::HostPinning;
type InboundEdgeProvision = plan::InboundEdgeProvision;
type ModelFacts = plan::ModelFacts;
use plan::NodeId;
type LayoutRule = plan::LayoutRule;
type ObjectKind = plan::ObjectKind;
type OutboundEdgeProvision = plan::OutboundEdgeProvision;
type PromptSource = plan::PromptSource;
type PlacementInput = plan::PlacementInput;
type PlanRejectionKind = plan::PlanRejectionKind;
type PlannerInput = plan::PlannerInput;
type RingSpec = plan::RingSpec;
type RingDirection = plan::RingDirection;
type RunPlan = plan::RunPlan;
type RuntimeConfig = plan::RuntimeConfig;
type SamplingPolicy = plan::SamplingPolicy;
type SequencePolicy = plan::SequencePolicy;
type ShapeRule = plan::ShapeRule;
type StagePlacement = plan::StagePlacement;
type TokenOutputPolicy = plan::TokenOutputPolicy;
type TokenizerSource = plan::TokenizerSource;
type WakeCoalescing = plan::WakeCoalescing;
// Keep test node ids small and readable. The concrete identity mechanism is
// outside this contract; these ids exist only so assertions can name topology
// facts without depending on any address or discovery machinery.
fn node(id: u64) -> NodeId {
NodeId(id)
}
// The candidate pool is deliberately larger than some test placements. That
// lets the tests distinguish "known to the orchestrator" from "assigned to a
// stage", which is one of the planner authority boundaries.
fn valid_nodes() -> Vec<NodeId> {
vec![node(10), node(11), node(12), node(13)]
}
// Fixed linear placement is the smallest placement input that still exercises
// the contract. It supplies stage-to-node intent, while the planner remains
// responsible for validating it and minting the full RunPlan topology.
fn linear_placement(stage_count: u32) -> PlacementInput {
PlacementInput::FixedLinear(
(0..stage_count)
.map(|stage_index| StagePlacement {
stage_index,
node_id: node(10 + u64::from(stage_index)),
})
.collect(),
)
}
// This is the canonical valid fixture for RunPlan guarantees. Each test tweaks
// only the fact it is trying to prove, so a failure points at the violated
// contract instead of at accidental fixture drift.
fn valid_input(stage_count: u32, num_layers: u32) -> PlannerInput {
PlannerInput {
run_id: 7.into(),
orchestrator_node_id: node(99),
model: ModelFacts {
model_id: "test-gguf".into(),
gguf_source: GgufSource::LocalPath("/models/test-gguf.gguf".into()),
num_layers,
hidden_dim: 4096,
dtype_family: DTypeFamily::BFloat,
dtype_width_bytes: 2,
max_seq_len: 2048,
eos_token_id: 2,
tokenizer: TokenizerSource::LocalPath("/tokenizers/test-gguf.json".into()),
},
runtime: RuntimeConfig {
max_tokens: 4,
prompt: PromptSource::Inline("hello from planner input".into()),
sampling: SamplingPolicy {
temperature_millis: 125,
top_k: 7,
},
token_output_policy: TokenOutputPolicy::EmitAll,
},
candidate_pool: valid_nodes(),
stage_count,
placement: linear_placement(stage_count),
activation_ring: RingSpec::test_default_activation(),
token_ring: RingSpec::test_default_token(),
}
}
// Tests frequently need to compare a provisioned edge id back to the canonical
// edge record in the RunPlan. This helper makes that lookup explicit without
// giving tests access to any planner-private index.
fn plan_edges_by_id(plan: &RunPlan) -> std::collections::BTreeMap<EdgeId, &EdgePlan> {
plan.edges
.iter()
.map(|edge| (edge.edge_id, edge))
.collect::<std::collections::BTreeMap<_, _>>()
}
// Edge endpoints can be orchestrator or stage endpoints. Tests use this helper
// when they care only about stage adjacency and want orchestrator endpoints to
// remain visibly outside the stage index space.
fn edge_stage_index(endpoint: &EdgeEndpoint) -> Option<u32> {
match endpoint {
EdgeEndpoint::Orchestrator { .. } => None,
EdgeEndpoint::Stage { stage_index, .. } => Some(*stage_index),
}
}
// Provisioning sends concrete node ids across the data-flow boundary. This
// helper extracts the observable node id from either endpoint shape so tests
// can compare projection output to plan topology.
fn edge_node_id(endpoint: &EdgeEndpoint) -> NodeId {
match endpoint {
EdgeEndpoint::Orchestrator { node_id } => *node_id,
EdgeEndpoint::Stage { node_id, .. } => *node_id,
}
}
// This proves RunPlan formation is a total public boundary for valid input:
// the caller observes one complete plan, not hidden follow-up topology work or
// a partially initialized result.
#[test]
fn valid_input_emits_one_complete_plan() {
// Build one ordinary valid planning request.
let input = valid_input(3, 36);
// Planning valid input must produce a usable plan, not a deferred partial.
let plan = plan::plan_run(input).expect("valid input must emit a plan");
// The plan-level identifiers and counts must be complete immediately.
assert_eq!(plan.run_id, 7.into());
assert_eq!(plan.stages.len(), 3);
assert_eq!(plan.edges.len(), 4);
assert_eq!(plan.max_tokens, 4);
assert_eq!(plan.model.model_id, "test-gguf");
assert_eq!(
plan.model.gguf_source,
GgufSource::LocalPath("/models/test-gguf.gguf".into())
);
assert_eq!(plan.model.num_layers, 36);
assert_eq!(plan.model.hidden_dim, 4096);
assert_eq!(plan.model.dtype_family, DTypeFamily::BFloat);
assert_eq!(plan.model.dtype_width_bytes, 2);
assert_eq!(plan.model.max_seq_len, 2048);
assert_eq!(plan.model.eos_token_id, 2);
assert_eq!(
plan.model.tokenizer,
TokenizerSource::LocalPath("/tokenizers/test-gguf.json".into())
);
assert_eq!(
plan.runtime.sampling,
SamplingPolicy {
temperature_millis: 125,
top_k: 7,
}
);
assert_eq!(
plan.runtime.prompt,
PromptSource::Inline("hello from planner input".into())
);
assert_eq!(plan.runtime.token_output_policy, TokenOutputPolicy::EmitAll);
// Every stage must be bound to this run and know the run's stage count.
for stage in &plan.stages {
assert_eq!(stage.run_id, plan.run_id);
assert_eq!(stage.stage_count, 3);
assert_eq!(stage.gguf_source, plan.model.gguf_source);
}
// Every edge must also be bound to this run; no edge can be a loose fact.
for edge in &plan.edges {
assert_eq!(edge.run_id, plan.run_id);
}
}
// This proves planning does not mutate the candidate pool supplied by the
// caller. The only topology facts the caller can use after planning are the
// facts emitted in the RunPlan itself.
#[test]
fn planner_does_not_mutate_candidate_pool() {
// Keep a copy of the caller-owned pool before the planner sees it.
let input = valid_input(3, 36);
let original_pool = input.candidate_pool.clone();
// Run planning through the public API only.
let _ = plan::plan_run(input.clone()).expect("valid input must emit a plan");
// The input pool remains the caller's fact; topology facts must be in the
// returned plan, not back-written into the input.
assert_eq!(input.candidate_pool, original_pool);
}
// This proves layer assignment is a contiguous, non-overlapping partition of
// the intended GGUF block range, with one non-empty range per stage.
#[test]
fn stage_ranges_partition_the_model_layers() {
// Exercise several deterministic sizes so the check covers one-stage and
// multi-stage partitioning without relying on random generation.
for (stage_count, num_layers) in [(1, 12), (2, 24), (3, 36), (4, 40)] {
// Produce the plan from public inputs.
let plan = plan::plan_run(valid_input(stage_count, num_layers)).unwrap();
// Read only the public stage assignments and sort by stage index.
let mut ranges = plan
.stages
.iter()
.map(|stage| {
(
stage.stage_index,
stage.layer_start,
stage.layer_end_exclusive,
)
})
.collect::<Vec<_>>();
ranges.sort_by_key(|(stage_index, _, _)| *stage_index);
// Walk the sorted ranges as a proof of contiguity. The next start must
// equal the previous end, and every range must consume at least one
// layer inside the model range.
let mut expected_start = 0;
for (_, start, end) in ranges {
assert_eq!(start, expected_start, "range gap or overlap");
assert!(end > start, "stage range must be non-empty");
assert!(end <= num_layers, "stage range exceeds model layer range");
expected_start = end;
}
// The final end must cover the whole intended block range.
assert_eq!(expected_start, num_layers);
}
}
// This proves stage indices are exactly the dense range required by the
// contract. Missing, duplicate, or out-of-range stage indices are observable in
// the returned RunPlan and fail this check.
#[test]
fn stage_indices_are_exactly_zero_to_stage_count_minus_one() {
// Check several stage counts so the dense-index guarantee is not tied to
// the canonical three-stage fixture.
for stage_count in 1..=4 {
// Produce a valid plan and observe only its public stage indices.
let plan = plan::plan_run(valid_input(stage_count, stage_count * 8)).unwrap();
let observed = plan
.stages
.iter()
.map(|stage| stage.stage_index)
.collect::<std::collections::BTreeSet<_>>();
// Compare against the contract's exact dense index set.
let expected = (0..stage_count).collect::<std::collections::BTreeSet<_>>();
assert_eq!(observed, expected);
}
}
// This proves every edge has one public producer and one public consumer, and
// that the returned edge graph is exactly the MVP linear pipeline.
#[test]
fn edge_graph_is_exactly_the_linear_pipeline() {
// Use four stages so the activation chain has multiple interior edges.
let stage_count = 4;
let plan = plan::plan_run(valid_input(stage_count, 40)).unwrap();
// Token-in must be unique and must enter stage 0 from the orchestrator.
let token_in = plan
.edges
.iter()
.filter(|edge| edge.kind == EdgeKind::TokenIn)
.collect::<Vec<_>>();
assert_eq!(token_in.len(), 1);
assert!(matches!(
token_in[0].producer,
EdgeEndpoint::Orchestrator { node_id } if node_id == node(99)
));
assert_eq!(
token_in[0].consumer,
EdgeEndpoint::Stage {
node_id: node(10),
stage_index: 0,
}
);
// Activation edges must be the only stage-to-stage edges, one per adjacent
// stage pair.
let activation_edges = plan
.edges
.iter()
.filter(|edge| edge.kind == EdgeKind::Activation)
.collect::<Vec<_>>();
assert_eq!(activation_edges.len(), (stage_count - 1) as usize);
// Each activation edge produced by stage i must be consumed by stage i+1.
for stage_index in 0..stage_count - 1 {
let edge = activation_edges
.iter()
.find(|edge| edge_stage_index(&edge.producer) == Some(stage_index))
.expect("activation edge produced by stage");
assert_eq!(
edge.consumer,
EdgeEndpoint::Stage {
node_id: node(11 + u64::from(stage_index)),
stage_index: stage_index + 1,
}
);
assert_ne!(edge.producer, edge.consumer, "self-edge is forbidden");
}
// Token-out must be unique and must leave the final stage for the
// orchestrator.
let token_out = plan
.edges
.iter()
.filter(|edge| edge.kind == EdgeKind::TokenOut)
.collect::<Vec<_>>();
assert_eq!(token_out.len(), 1);
assert_eq!(
edge_stage_index(&token_out[0].producer),
Some(stage_count - 1)
);
assert!(matches!(
token_out[0].consumer,
EdgeEndpoint::Orchestrator { node_id } if node_id == node(99)
));
}
// This proves edge ids are run-unique and that stage plans refer only to edge
// ids present in the returned RunPlan, so stages receive assigned ids rather
// than deriving data-flow identity themselves.
#[test]
fn edge_ids_are_unique_and_stage_references_resolve_to_plan_edges() {
// Produce a plan with enough edges to make duplicate ids observable.
let plan = plan::plan_run(valid_input(4, 40)).unwrap();
// Insert every public edge id into a set; a duplicate shrinks the set.
let edge_ids = plan
.edges
.iter()
.map(|edge| edge.edge_id)
.collect::<std::collections::BTreeSet<_>>();
assert_eq!(edge_ids.len(), plan.edges.len(), "duplicate edge id");
// Stage plans may reference only ids that the RunPlan itself assigned.
for stage in &plan.stages {
assert!(
edge_ids.contains(&stage.inbound_edge),
"stage inbound edge id must come from RunPlan edges"
);
assert!(
edge_ids.contains(&stage.outbound_edge),
"stage outbound edge id must come from RunPlan edges"
);
}
}
// This proves deriving ProvisionStage is deterministic and stage-local:
// repeated projection returns the same value, and the projected layer range is
// exactly the range assigned to that stage in the RunPlan.
#[test]
fn provision_stage_projection_is_deterministic_and_stage_local() {
// Start from one committed plan; projection is a pure public view of it.
let plan = plan::plan_run(valid_input(3, 36)).unwrap();
// Check every stage projection, not only one representative stage.
for stage_index in 0..3 {
// Derive twice to prove projection does not depend on hidden mutable
// state or call order.
let first = plan::derive_stage_provision(&plan, stage_index).unwrap();
let second = plan::derive_stage_provision(&plan, stage_index).unwrap();
// Find the corresponding public stage assignment in the plan.
let stage = plan
.stages
.iter()
.find(|stage| stage.stage_index == stage_index)
.unwrap();
// The projected message must be stable and expose only that stage's
// assigned run position and layer range.
assert_eq!(first, second);
assert_eq!(first.stage_index, stage_index);
assert_eq!(first.stage_count, 3);
assert_eq!(first.layer_start, stage.layer_start);
assert_eq!(first.layer_end_exclusive, stage.layer_end_exclusive);
assert_eq!(first.gguf_source, stage.gguf_source);
assert_eq!(first.tokenizer, plan.model.tokenizer);
assert_eq!(first.model.model_id, plan.model.model_id);
assert_eq!(first.model.hidden_dim, plan.model.hidden_dim);
assert_eq!(first.model.dtype_family, plan.model.dtype_family);
assert_eq!(first.model.dtype_width_bytes, plan.model.dtype_width_bytes);
assert_eq!(first.model.max_seq_len, plan.model.max_seq_len);
assert_eq!(first.runtime.role_id, plan::RoleId(u64::from(stage_index)));
assert_eq!(first.runtime.input_port, plan::PortId("input".into()));
assert_eq!(first.runtime.output_port, plan::PortId("output".into()));
let expected_sampling = if stage_index + 1 == stage.stage_count {
Some(plan.runtime.sampling)
} else {
None
};
assert_eq!(first.runtime.sampling, expected_sampling);
}
}
// This proves each stage receives exactly one inbound and one outbound edge
// provision, and that the provisioned ids are the ids assigned to that stage by
// the RunPlan.
#[test]
fn provision_stage_contains_exactly_the_assigned_inbound_and_outbound_edges() {
// Build a valid plan and test projection for every stage in it.
let plan = plan::plan_run(valid_input(3, 36)).unwrap();
for stage in &plan.stages {
// Derive the public provisioning message for this stage.
let provision = plan::derive_stage_provision(&plan, stage.stage_index).unwrap();
// The message exposes exactly the inbound and outbound ids assigned in
// that stage's StagePlan.
assert_eq!(provision.inbound.edge_id, stage.inbound_edge);
assert_eq!(provision.outbound.edge_id, stage.outbound_edge);
}
}
// This proves the data-flow addressing contract of provisioning. The outbound
// side carries the consumer node id; exhaustive struct destructuring also makes
// remote actor-address fields a compile-time contract violation.
#[test]
fn provision_stage_uses_node_id_addressing_not_remote_actor_addresses() {
// Build one plan and an edge lookup using only public edge records.
let plan = plan::plan_run(valid_input(3, 36)).unwrap();
let edges = plan_edges_by_id(&plan);
for stage in &plan.stages {
// Project the stage-local provisioning message.
let provision = plan::derive_stage_provision(&plan, stage.stage_index).unwrap();
// Destructure the inbound provision exhaustively. If a remote actor
// address becomes part of this public type, this test must be updated
// consciously instead of silently accepting it.
let InboundEdgeProvision {
edge_id: inbound_edge_id,
kind: _,
object_spec: _,
ring_spec: _,
} = provision.inbound.clone();
// Destructure the outbound provision exhaustively. The only remote
// routing fact it may expose is the consumer node id.
let OutboundEdgeProvision {
edge_id: outbound_edge_id,
kind: _,
consumer_node_id,
object_spec: _,
ring_spec: _,
} = provision.outbound.clone();
assert_eq!(inbound_edge_id, stage.inbound_edge);
assert_eq!(outbound_edge_id, stage.outbound_edge);
// The provisioned consumer node id must match the consumer endpoint of
// the canonical RunPlan edge.
let outbound_edge = edges.get(&outbound_edge_id).unwrap();
assert_eq!(consumer_node_id, edge_node_id(&outbound_edge.consumer));
}
}
// This proves every edge carries object and ring specs, activation capacity is
// derived from model facts, and edge kind selects the correct object kind.
#[test]
fn object_and_ring_specs_are_present_and_match_edge_kind() {
// Use the canonical model facts so the expected activation capacity is
// known directly from the public input.
let plan = plan::plan_run(valid_input(3, 36)).unwrap();
let expected_activation_extent = 2048 * 4096 * 2;
// Every edge must carry complete movement specs; no worker or transport may
// invent these later.
for edge in &plan.edges {
assert!(edge.object_spec.max_extent > 0);
assert!(edge.ring_spec.data_capacity > 0);
assert!(edge.object_spec.alignment > 0);
assert_eq!(edge.object_spec.layout, LayoutRule::Contiguous);
assert_eq!(edge.object_spec.sequence_policy, SequencePolicy::Ordered);
assert!(edge.ring_spec.alignment > 0);
assert_eq!(edge.ring_spec.direction, RingDirection::Egress);
assert_eq!(edge.ring_spec.host_pinning, HostPinning::Pageable);
assert_eq!(edge.ring_spec.wake_coalescing, WakeCoalescing::PendingBit);
// Edge kind selects the object kind, and activation capacity is derived
// from model facts.
match edge.kind {
EdgeKind::Activation => {
assert_eq!(edge.object_spec.kind, ObjectKind::Activation);
assert_eq!(edge.object_spec.max_extent, expected_activation_extent);
assert_eq!(edge.object_spec.dtype_family, DTypeFamily::BFloat);
assert_eq!(edge.object_spec.dtype_width_bytes, 2);
assert_eq!(
edge.object_spec.shape,
ShapeRule::ActivationRows {
max_seq_len: 2048,
hidden_dim: 4096,
}
);
}
EdgeKind::TokenIn | EdgeKind::TokenOut => {
assert_eq!(edge.object_spec.kind, ObjectKind::Token);
assert_eq!(edge.object_spec.dtype_width_bytes, 4);
assert_eq!(edge.object_spec.shape, ShapeRule::TokenIds);
}
}
}
}
#[test]
fn token_object_alignment_is_token_width_not_ring_alignment() {
let mut input = valid_input(3, 36);
input.token_ring.alignment = 64;
let plan = plan::plan_run(input).unwrap();
for edge in plan
.edges
.iter()
.filter(|edge| matches!(edge.kind, EdgeKind::TokenIn | EdgeKind::TokenOut))
{
assert_eq!(edge.object_spec.dtype_width_bytes, 4);
assert_eq!(edge.object_spec.alignment, 4);
assert_eq!(edge.ring_spec.alignment, 64);
}
}
#[test]
fn activation_object_alignment_is_dtype_width_not_ring_alignment() {
let mut input = valid_input(3, 36);
input.model.hidden_dim = 13;
input.activation_ring.alignment = 64;
let plan = plan::plan_run(input).unwrap();
for edge in plan
.edges
.iter()
.filter(|edge| matches!(edge.kind, EdgeKind::Activation))
{
assert_eq!(edge.object_spec.dtype_width_bytes, 2);
assert_eq!(edge.object_spec.alignment, 2);
assert_eq!(edge.ring_spec.alignment, 64);
}
}
// This proves prompt-token objects are sized from the effective model context,
// not from a small fixed constant, and that token rings can carry one complete
// MO01 record for that object.
#[test]
fn token_edges_size_extent_from_context_and_expand_ring_capacity() {
let mut input = valid_input(3, 36);
input.model.max_seq_len = 512;
input.token_ring.data_capacity = 64;
let plan = plan::plan_run(input).expect("valid input must emit a plan");
let expected_token_extent = 512 * 4;
let expected_token_record_capacity = plan::MO01_HEADER_BYTES + expected_token_extent;
let token_edges = plan
.edges
.iter()
.filter(|edge| matches!(edge.kind, EdgeKind::TokenIn | EdgeKind::TokenOut))
.collect::<Vec<_>>();
assert_eq!(token_edges.len(), 2);
for edge in token_edges {
assert_eq!(edge.object_spec.kind, ObjectKind::Token);
assert_eq!(edge.object_spec.max_extent, expected_token_extent);
assert_eq!(edge.object_spec.dtype_width_bytes, 4);
assert_eq!(edge.object_spec.shape, ShapeRule::TokenIds);
assert!(
edge.ring_spec.data_capacity >= expected_token_record_capacity,
"token ring must carry MO01 header plus max token payload"
);
assert_eq!(edge.ring_spec.data_capacity, expected_token_record_capacity);
}
}
// This proves object and ring specs are projected consistently into every stage
// provision. A stage provision narrows the ring direction to the local role
// while preserving the edge's capacity, alignment, pinning, and wake policy.
#[test]
fn object_and_ring_specs_are_copied_consistently_into_stage_provisions() {
// Build a canonical plan and index its public edge records.
let plan = plan::plan_run(valid_input(3, 36)).unwrap();
let edges = plan_edges_by_id(&plan);
for stage in &plan.stages {
// Project a stage-local provisioning message.
let provision = plan::derive_stage_provision(&plan, stage.stage_index).unwrap();
// The inbound edge spec must preserve the plan edge facts and mark the
// local ring as ingress.
let inbound_edge = edges.get(&provision.inbound.edge_id).unwrap();
let mut expected_inbound_ring = inbound_edge.ring_spec;
expected_inbound_ring.direction = RingDirection::Ingress;
assert_eq!(provision.inbound.kind, inbound_edge.kind);
assert_eq!(provision.inbound.object_spec, inbound_edge.object_spec);
assert_eq!(provision.inbound.ring_spec, expected_inbound_ring);
// The outbound edge spec must preserve the plan edge facts and mark the
// local ring as egress.
let outbound_edge = edges.get(&provision.outbound.edge_id).unwrap();
let mut expected_outbound_ring = outbound_edge.ring_spec;
expected_outbound_ring.direction = RingDirection::Egress;
assert_eq!(provision.outbound.kind, outbound_edge.kind);
assert_eq!(provision.outbound.object_spec, outbound_edge.object_spec);
assert_eq!(provision.outbound.ring_spec, expected_outbound_ring);
}
}
// This proves typed rejection is the public behavior for invalid authority and
// topology inputs. No invalid case is allowed to emit a partial plan.
#[test]
fn invalid_authority_and_topology_inputs_reject_without_plan() {
// Each case changes one authority/topology fact from the valid fixture and
// names the typed rejection the planner must expose.
let cases = [
(
invalid_unknown_node(),
PlanRejectionKind::UnknownNode,
"unknown node id",
),
(
invalid_duplicate_stage_assignment(),
PlanRejectionKind::DuplicateStageAssignment,
"duplicate stage assignment",
),
(
invalid_missing_stage_assignment(),
PlanRejectionKind::MissingStage,
"missing stage",
),
(
invalid_zero_stage_count(),
PlanRejectionKind::InvalidStageCount,
"invalid stage count",
),
(
invalid_model_stage_layout(),
PlanRejectionKind::ModelStageLayoutMismatch,
"model/stage layout mismatch",
),
];
// Invalid inputs must not produce a partial plan. The observable result is
// a typed rejection kind.
for (input, expected, label) in cases {
let err = plan::plan_run(input).expect_err(label);
assert_eq!(err.kind(), expected, "{label}");
}
}
// This proves invalid object and ring spec facts reject before provisioning.
// The planner may choose the exact diagnostic payload, but the rejection kind
// must be typed and no RunPlan may be emitted.
#[test]
fn invalid_object_or_ring_specs_reject_without_plan() {
// Each case changes one object/ring fact from the valid fixture and names
// the typed rejection expected at the planning boundary.
let cases = [
(
invalid_zero_activation_extent(),
PlanRejectionKind::InvalidObjectSpec,
"zero activation extent",
),
(
invalid_dtype_width(),
PlanRejectionKind::InvalidObjectSpec,
"invalid dtype width",
),
(
invalid_unsupported_shape_or_layout(),
PlanRejectionKind::UnsupportedShapeOrLayout,
"unsupported shape/layout",
),
(
invalid_ring_alignment(),
PlanRejectionKind::InvalidRingSpec,
"invalid ring alignment",
),
];
// Rejection happens before provisioning: the only public output is the
// typed error, never a RunPlan with invalid specs.
for (input, expected, label) in cases {
let err = plan::plan_run(input).expect_err(label);
assert_eq!(err.kind(), expected, "{label}");
}
}
// Unknown-node rejection needs the placement to name a node outside the
// orchestrator's candidate pool. The rest of the input stays valid so the
// expected rejection is isolated to authority over node identity.
fn invalid_unknown_node() -> PlannerInput {
let mut input = valid_input(3, 36);
input.placement = PlacementInput::FixedLinear(vec![
StagePlacement {
stage_index: 0,
node_id: node(10),
},
StagePlacement {
stage_index: 1,
node_id: node(404),
},
StagePlacement {
stage_index: 2,
node_id: node(12),
},
]);
input
}
// Duplicate-stage rejection is observable when two placement entries claim the
// same stage index. This checks that the planner does not silently pick one and
// continue with ambiguous authority.
fn invalid_duplicate_stage_assignment() -> PlannerInput {
let mut input = valid_input(3, 36);
input.placement = PlacementInput::FixedLinear(vec![
StagePlacement {
stage_index: 0,
node_id: node(10),
},
StagePlacement {
stage_index: 1,
node_id: node(11),
},
StagePlacement {
stage_index: 1,
node_id: node(12),
},
]);
input
}
// Missing-stage rejection is observable when placement skips an index inside
// `0..stage_count`. This checks that the planner does not invent hidden stage
// ownership to patch an incomplete placement.
fn invalid_missing_stage_assignment() -> PlannerInput {
let mut input = valid_input(3, 36);
input.placement = PlacementInput::FixedLinear(vec![
StagePlacement {
stage_index: 0,
node_id: node(10),
},
StagePlacement {
stage_index: 2,
node_id: node(12),
},
]);
input
}
// Zero stages cannot form the MVP pipeline. This fixture isolates the invalid
// stage-count path without adding any other contradictory facts.
fn invalid_zero_stage_count() -> PlannerInput {
valid_input(0, 36)
}
// The current contract requires one non-empty layer range per stage. Fewer
// layers than stages forces an empty range unless explicitly allowed, so this
// fixture should reject at the model/stage-layout boundary.
fn invalid_model_stage_layout() -> PlannerInput {
let mut input = valid_input(4, 3);
input.placement = linear_placement(4);
input
}
// Zero sequence length makes activation capacity zero. The planner must reject
// before creating edges whose object specs cannot carry an activation.
fn invalid_zero_activation_extent() -> PlannerInput {
let mut input = valid_input(3, 36);
input.model.max_seq_len = 0;
input
}
// Dtype width participates directly in activation extent and object layout.
// A zero width is not a valid dtype fact and must reject before planning.
fn invalid_dtype_width() -> PlannerInput {
let mut input = valid_input(3, 36);
input.model.dtype_width_bytes = 0;
input
}
// Hidden dimension participates directly in activation shape. A zero hidden
// dimension represents an unsupported shape/layout fact for the MVP contract.
fn invalid_unsupported_shape_or_layout() -> PlannerInput {
let mut input = valid_input(3, 36);
input.model.hidden_dim = 0;
input
}
// Ring alignment must be a usable alignment contract for shared memory and
// device copy boundaries. A non-power-of-two alignment makes the ring spec
// invalid before any edge can be provisioned.
fn invalid_ring_alignment() -> PlannerInput {
let mut input = valid_input(3, 36);
input.activation_ring.alignment = 3;
input
}

168
crates/mvp-system/src/tests/shard_fetch_guarantees.rs
View file

@ -1,168 +0,0 @@
use std::collections::BTreeMap;
use mvp_system::staging::shard_fetch as fetch;
use mvp_system::staging::weight_shards as shards;
fn assignment(stage_index: u32) -> shards::ShardAssignment {
let model_ref =
shards::ModelArtifactRef::parse("hf://org/repo@abcdef123456/model.gguf").unwrap();
let split_scheme = shards::SplitScheme::GgufLayerContiguousV1;
let split_id = shards::SplitId::derive(&model_ref, split_scheme);
shards::ShardAssignment::new(
model_ref,
split_id,
split_scheme,
stage_index,
8,
shards::LayerRange::new(stage_index * 4, stage_index * 4 + 4).unwrap(),
)
.unwrap()
}
fn fetched_for(assignment: &shards::ShardAssignment) -> fetch::FetchedShard {
fetch::FetchedShard::new(
format!("/cache/stage-{:05}.gguf", assignment.stage_index),
shards::ShardManifest::for_assignment(
assignment,
shards::ContentHash::literal(format!("sha256:stage-{}", assignment.stage_index))
.unwrap(),
),
)
}
#[derive(Default)]
struct MemoryCache {
entries: BTreeMap<String, fetch::FetchedShard>,
inserts: usize,
}
impl fetch::ShardCache for MemoryCache {
fn get(&self, cache_key: &str) -> Option<fetch::FetchedShard> {
self.entries.get(cache_key).cloned()
}
fn insert(&mut self, cache_key: String, shard: fetch::FetchedShard) {
self.inserts += 1;
self.entries.insert(cache_key, shard);
}
}
struct RecordingFetcher {
calls: Vec<fetch::FetchShard>,
result: Result<fetch::FetchedShard, fetch::FetchError>,
}
impl fetch::ShardFetcher for RecordingFetcher {
fn fetch(
&mut self,
request: &fetch::FetchShard,
) -> Result<fetch::FetchedShard, fetch::FetchError> {
self.calls.push(request.clone());
self.result.clone()
}
}
#[test]
fn shard_locator_derives_expected_uri() {
let assignment = assignment(3);
let location = fetch::ShardLocator::locate(&assignment);
assert_eq!(
location.uri,
format!(
"hf://org/repo@abcdef123456/shards/{}/stage-00003.gguf",
assignment.split_id.as_str()
)
);
assert!(location.cache_key.contains(assignment.split_id.as_str()));
assert!(location.cache_key.ends_with(":00003"));
}
#[test]
fn cache_key_depends_on_model_split_and_stage() {
let stage_three = assignment(3);
let stage_four = assignment(4);
let different_model_ref =
shards::ModelArtifactRef::parse("hf://other/repo@abcdef123456/model.gguf").unwrap();
let split_scheme = shards::SplitScheme::GgufLayerContiguousV1;
let different_model = shards::ShardAssignment::new(
different_model_ref.clone(),
shards::SplitId::derive(&different_model_ref, split_scheme),
split_scheme,
3,
8,
shards::LayerRange::new(12, 16).unwrap(),
)
.unwrap();
let different_split = shards::ShardAssignment::new(
stage_three.model_ref.clone(),
shards::SplitId::literal("split-other").unwrap(),
stage_three.split_scheme,
3,
8,
stage_three.layer_range,
)
.unwrap();
let keys = [
fetch::ShardLocator::locate(&stage_three).cache_key,
fetch::ShardLocator::locate(&stage_four).cache_key,
fetch::ShardLocator::locate(&different_model).cache_key,
fetch::ShardLocator::locate(&different_split).cache_key,
];
assert_eq!(
keys.iter().collect::<std::collections::BTreeSet<_>>().len(),
keys.len()
);
}
#[test]
fn coordinator_downloads_and_caches_missing_shard() {
let assignment = assignment(2);
let expected = fetched_for(&assignment);
let mut cache = MemoryCache::default();
let mut fetcher = RecordingFetcher {
calls: Vec::new(),
result: Ok(expected.clone()),
};
let outcome =
fetch::ShardFetchCoordinator::get_or_fetch(&assignment, &mut cache, &mut fetcher).unwrap();
assert_eq!(outcome.status, fetch::ShardFetchStatus::Downloaded);
assert_eq!(outcome.shard, expected);
assert_eq!(fetcher.calls.len(), 1);
assert_eq!(fetcher.calls[0].location, outcome.location);
assert_eq!(cache.inserts, 1);
assert_eq!(
cache.entries.get(&outcome.location.cache_key),
Some(&expected)
);
}
#[test]
fn coordinator_uses_cache_hit_without_fetching() {
let assignment = assignment(5);
let location = fetch::ShardLocator::locate(&assignment);
let cached = fetched_for(&assignment);
let mut cache = MemoryCache::default();
cache
.entries
.insert(location.cache_key.clone(), cached.clone());
let mut fetcher = RecordingFetcher {
calls: Vec::new(),
result: Err(fetch::FetchError::Unavailable),
};
let outcome =
fetch::ShardFetchCoordinator::get_or_fetch(&assignment, &mut cache, &mut fetcher).unwrap();
assert_eq!(outcome.status, fetch::ShardFetchStatus::CacheHit);
assert_eq!(outcome.location, location);
assert_eq!(outcome.shard, cached);
assert!(fetcher.calls.is_empty());
assert_eq!(cache.inserts, 0);
}

308
crates/mvp-system/src/tests/shard_weight_lifecycle_guarantees.rs
View file

@ -1,308 +0,0 @@
use std::collections::BTreeMap;
use mvp_system::staging::shard_fetch as fetch;
use mvp_system::staging::shard_weight_lifecycle as lifecycle;
use mvp_system::staging::weight_shards as shards;
fn assignment() -> shards::ShardAssignment {
let model_ref =
shards::ModelArtifactRef::parse("hf://org/repo@abcdef123456/model.gguf").unwrap();
let split_scheme = shards::SplitScheme::GgufLayerContiguousV1;
let split_id = shards::SplitId::derive(&model_ref, split_scheme);
shards::ShardAssignment::new(
model_ref,
split_id,
split_scheme,
3,
8,
shards::LayerRange::new(12, 16).unwrap(),
)
.unwrap()
}
fn fetched_for(assignment: &shards::ShardAssignment) -> fetch::FetchedShard {
fetch::FetchedShard::new(
"/cache/stage-00003.gguf",
shards::ShardManifest::for_assignment(
assignment,
shards::ContentHash::literal("sha256:stage-3").unwrap(),
),
)
}
fn event_kinds(events: &[lifecycle::ShardLifecycleEvent]) -> Vec<&'static str> {
events
.iter()
.map(|event| match event {
lifecycle::ShardLifecycleEvent::Assigned { .. } => "assigned",
lifecycle::ShardLifecycleEvent::Located { .. } => "located",
lifecycle::ShardLifecycleEvent::Fetching { .. } => "fetching",
lifecycle::ShardLifecycleEvent::CacheHit { .. } => "cache-hit",
lifecycle::ShardLifecycleEvent::Fetched { .. } => "fetched",
lifecycle::ShardLifecycleEvent::Validated { .. } => "validated",
lifecycle::ShardLifecycleEvent::Binding { .. } => "binding",
lifecycle::ShardLifecycleEvent::Ready { .. } => "ready",
lifecycle::ShardLifecycleEvent::Faulted { .. } => "faulted",
})
.collect()
}
#[derive(Default)]
struct MemoryCache {
entries: BTreeMap<String, fetch::FetchedShard>,
inserts: usize,
}
impl fetch::ShardCache for MemoryCache {
fn get(&self, cache_key: &str) -> Option<fetch::FetchedShard> {
self.entries.get(cache_key).cloned()
}
fn insert(&mut self, cache_key: String, shard: fetch::FetchedShard) {
self.inserts += 1;
self.entries.insert(cache_key, shard);
}
}
struct RecordingFetcher {
calls: Vec<fetch::FetchShard>,
result: Result<fetch::FetchedShard, fetch::FetchError>,
}
impl fetch::ShardFetcher for RecordingFetcher {
fn fetch(
&mut self,
request: &fetch::FetchShard,
) -> Result<fetch::FetchedShard, fetch::FetchError> {
self.calls.push(request.clone());
self.result.clone()
}
}
struct RecordingBinder {
calls: Vec<shards::ValidatedShard>,
result: Result<(), lifecycle::BindError>,
}
impl lifecycle::WorkerShardBinder for RecordingBinder {
fn bind(&mut self, shard: &shards::ValidatedShard) -> Result<(), lifecycle::BindError> {
self.calls.push(shard.clone());
self.result.clone()
}
}
#[test]
fn lifecycle_happy_path_reaches_ready() {
let assignment = assignment();
let fetched = fetched_for(&assignment);
let mut cache = MemoryCache::default();
let mut fetcher = RecordingFetcher {
calls: Vec::new(),
result: Ok(fetched.clone()),
};
let mut binder = RecordingBinder {
calls: Vec::new(),
result: Ok(()),
};
let mut lifecycle = lifecycle::ShardWeightLifecycle::new();
lifecycle.load(assignment.clone(), &mut cache, &mut fetcher, &mut binder);
assert_eq!(lifecycle.state(), lifecycle::ShardLifecycleState::Ready);
assert_eq!(
event_kinds(lifecycle.events()),
vec![
"assigned",
"located",
"fetching",
"fetched",
"validated",
"binding",
"ready"
]
);
assert_eq!(fetcher.calls.len(), 1);
assert_eq!(cache.inserts, 1);
assert_eq!(binder.calls.len(), 1);
assert_eq!(binder.calls[0].assignment, assignment);
assert_eq!(binder.calls[0].local_path, fetched.local_path);
}
#[test]
fn lifecycle_cache_hit_skips_fetch_but_still_validates_and_binds() {
let assignment = assignment();
let fetched = fetched_for(&assignment);
let location = fetch::ShardLocator::locate(&assignment);
let mut cache = MemoryCache::default();
cache.entries.insert(location.cache_key.clone(), fetched);
let mut fetcher = RecordingFetcher {
calls: Vec::new(),
result: Err(fetch::FetchError::Unavailable),
};
let mut binder = RecordingBinder {
calls: Vec::new(),
result: Ok(()),
};
let mut lifecycle = lifecycle::ShardWeightLifecycle::new();
lifecycle.load(assignment, &mut cache, &mut fetcher, &mut binder);
assert_eq!(lifecycle.state(), lifecycle::ShardLifecycleState::Ready);
assert_eq!(
event_kinds(lifecycle.events()),
vec![
"assigned",
"located",
"fetching",
"cache-hit",
"validated",
"binding",
"ready"
]
);
assert!(fetcher.calls.is_empty());
assert_eq!(cache.inserts, 0);
assert_eq!(binder.calls.len(), 1);
}
#[test]
fn lifecycle_fetch_failure_faults_without_validation_or_bind() {
let assignment = assignment();
let mut cache = MemoryCache::default();
let mut fetcher = RecordingFetcher {
calls: Vec::new(),
result: Err(fetch::FetchError::NotFound),
};
let mut binder = RecordingBinder {
calls: Vec::new(),
result: Ok(()),
};
let mut lifecycle = lifecycle::ShardWeightLifecycle::new();
lifecycle.load(assignment, &mut cache, &mut fetcher, &mut binder);
assert_eq!(lifecycle.state(), lifecycle::ShardLifecycleState::Faulted);
assert_eq!(fetcher.calls.len(), 1);
assert!(binder.calls.is_empty());
assert!(matches!(
lifecycle.events().last(),
Some(lifecycle::ShardLifecycleEvent::Faulted {
reason: lifecycle::ShardLifecycleFault::Fetch(fetch::FetchError::NotFound)
})
));
}
#[test]
fn lifecycle_validation_failure_faults_without_bind() {
let assignment = assignment();
let mut bad_manifest = shards::ShardManifest::for_assignment(
&assignment,
shards::ContentHash::literal("sha256:stage-3").unwrap(),
);
bad_manifest.stage_index = 4;
let mut cache = MemoryCache::default();
let mut fetcher = RecordingFetcher {
calls: Vec::new(),
result: Ok(fetch::FetchedShard::new(
"/cache/bad-stage.gguf",
bad_manifest,
)),
};
let mut binder = RecordingBinder {
calls: Vec::new(),
result: Ok(()),
};
let mut lifecycle = lifecycle::ShardWeightLifecycle::new();
lifecycle.load(assignment, &mut cache, &mut fetcher, &mut binder);
assert_eq!(lifecycle.state(), lifecycle::ShardLifecycleState::Faulted);
assert!(binder.calls.is_empty());
assert!(matches!(
lifecycle.events().last(),
Some(lifecycle::ShardLifecycleEvent::Faulted {
reason: lifecycle::ShardLifecycleFault::Validation(
shards::ShardValidationError::StageIndexMismatch
)
})
));
}
#[test]
fn lifecycle_bind_failure_faults_after_validation() {
let assignment = assignment();
let fetched = fetched_for(&assignment);
let mut cache = MemoryCache::default();
let mut fetcher = RecordingFetcher {
calls: Vec::new(),
result: Ok(fetched),
};
let mut binder = RecordingBinder {
calls: Vec::new(),
result: Err(lifecycle::BindError::WorkerRejected),
};
let mut lifecycle = lifecycle::ShardWeightLifecycle::new();
lifecycle.load(assignment, &mut cache, &mut fetcher, &mut binder);
assert_eq!(lifecycle.state(), lifecycle::ShardLifecycleState::Faulted);
assert_eq!(binder.calls.len(), 1);
assert!(event_kinds(lifecycle.events()).contains(&"validated"));
assert!(matches!(
lifecycle.events().last(),
Some(lifecycle::ShardLifecycleEvent::Faulted {
reason: lifecycle::ShardLifecycleFault::Bind(lifecycle::BindError::WorkerRejected)
})
));
}
#[test]
fn ready_terminal_state_is_idempotent() {
let assignment = assignment();
let fetched = fetched_for(&assignment);
let mut cache = MemoryCache::default();
let mut fetcher = RecordingFetcher {
calls: Vec::new(),
result: Ok(fetched.clone()),
};
let mut binder = RecordingBinder {
calls: Vec::new(),
result: Ok(()),
};
let mut lifecycle = lifecycle::ShardWeightLifecycle::new();
lifecycle.load(assignment.clone(), &mut cache, &mut fetcher, &mut binder);
let event_count = lifecycle.events().len();
lifecycle.load(assignment, &mut cache, &mut fetcher, &mut binder);
assert_eq!(lifecycle.state(), lifecycle::ShardLifecycleState::Ready);
assert_eq!(lifecycle.events().len(), event_count);
assert_eq!(fetcher.calls.len(), 1);
assert_eq!(binder.calls.len(), 1);
}
#[test]
fn fault_terminal_state_is_idempotent() {
let assignment = assignment();
let fetched = fetched_for(&assignment);
let mut cache = MemoryCache::default();
let mut fetcher = RecordingFetcher {
calls: Vec::new(),
result: Err(fetch::FetchError::Unavailable),
};
let mut binder = RecordingBinder {
calls: Vec::new(),
result: Ok(()),
};
let mut lifecycle = lifecycle::ShardWeightLifecycle::new();
lifecycle.load(assignment.clone(), &mut cache, &mut fetcher, &mut binder);
let event_count = lifecycle.events().len();
fetcher.result = Ok(fetched);
lifecycle.load(assignment, &mut cache, &mut fetcher, &mut binder);
assert_eq!(lifecycle.state(), lifecycle::ShardLifecycleState::Faulted);
assert_eq!(lifecycle.events().len(), event_count);
assert_eq!(fetcher.calls.len(), 1);
assert!(binder.calls.is_empty());
}

235
crates/mvp-system/src/tests/shared_ring_helper_abi_guarantees.rs
View file

@ -1,235 +0,0 @@
//! Black-box contract tests for the MVP shared ring helper ABI.
//!
//! These tests intentionally know only the public helper surface:
//!
//! - helper-created bounded SPSC rings
//! - reserve, write, publish, read, consume, and wake operations
//! - cursor snapshots and wake hints observed through the helper
//!
//! They assert the guarantees in
//! `specs/mvp_system/shared_ring_helper_abi_contract.md`.
use data_plane::ring;
// A small ring forces wraparound and full/empty transitions quickly. The helper
// still owns the actual shared-memory atomics and process-local address math.
fn new_ring() -> ring::RingHelperHarness {
ring::RingHelperHarness::create(ring::RingConfig {
node_id: ring::NodeId(10),
ring_id: ring::RingId(7000),
capacity: 8,
producer: ring::EndpointId("producer".into()),
consumer: ring::EndpointId("consumer".into()),
})
}
// This helper reads all currently committed bytes through the consumer API.
// It proves visibility through acquire reads instead of peeking at backing
// memory directly.
fn drain_committed(helper: &mut ring::RingHelperHarness) -> Vec<u8> {
let readable = helper.consumer_readable();
let bytes = helper.consumer_read(readable);
helper.consumer_consume(readable);
bytes
}
// Wake hints must remain hints. This helper checks the public wake enum without
// depending on scheduler internals.
fn assert_wake_is_payload_free(wake: &ring::WakeHint) {
match wake {
ring::WakeHint::RingReadable { ring_id } | ring::WakeHint::RingWritable { ring_id } => {
assert_eq!(*ring_id, ring::RingId(7000));
}
}
}
// This proves ring identity is bounded SPSC, has one producer and one consumer,
// uses unique RingId values, and does not allow stale wakes to alias a
// replacement ring.
#[test]
fn ring_identity_is_unique_bounded_spsc_and_stale_wakes_do_not_alias() {
// Create one ring and inspect its public identity.
let mut helper = new_ring();
let identity = helper.identity();
assert_eq!(identity.ring_id, ring::RingId(7000));
assert_eq!(identity.capacity, 8);
assert_eq!(identity.producer_count, 1);
assert_eq!(identity.consumer_count, 1);
// Retire it and create a replacement with the same numeric id but a new
// generation. Stale wake from the retired generation must not wake the new
// ring.
let stale_wake = helper.retire_and_capture_stale_wake();
let mut replacement = ring::RingHelperHarness::create_replacement(identity.ring_id);
replacement.deliver_wake(stale_wake);
assert!(!replacement.wake_log().iter().any(|wake| wake.was_accepted));
}
// This proves commit and consume are monotonic logical byte positions, writes
// are invisible before commit, and physical wrap uses cursor modulo capacity.
#[test]
fn cursors_are_monotonic_and_wrap_by_modulo_capacity() {
// Reserve and write without publishing.
let mut helper = new_ring();
let reservation = helper.producer_reserve(6).expect("space must exist");
helper.producer_write(&reservation, b"abcdef");
// Producer-local write is not readable before commit.
assert_eq!(helper.consumer_readable(), 0);
// Publishing makes the prefix readable and advances commit.
helper.producer_commit(reservation);
assert_eq!(helper.cursor_snapshot().commit, 6);
assert_eq!(drain_committed(&mut helper), b"abcdef");
assert_eq!(helper.cursor_snapshot().consume, 6);
// Wrap the physical index while logical cursors keep increasing.
let wrapped = helper
.producer_reserve(5)
.expect("space must exist after consume");
helper.producer_write(&wrapped, b"ghijk");
helper.producer_commit(wrapped);
assert_eq!(helper.cursor_snapshot().commit, 11);
assert_eq!(
helper.cursor_snapshot().commit % helper.identity().capacity,
3
);
assert_eq!(drain_committed(&mut helper), b"ghijk");
assert_eq!(helper.cursor_snapshot().consume, 11);
}
// This proves the producer computes free space from acquired consume, never
// reserves beyond capacity, writes before publishing commit, and emits readable
// wake hints after publication.
#[test]
fn producer_respects_free_space_and_publishes_after_writing() {
// Reserve the full ring and publish it.
let mut helper = new_ring();
let reservation = helper
.producer_reserve(8)
.expect("full ring reservation fits");
helper.producer_write(&reservation, b"12345678");
helper.producer_commit(reservation);
// With no consumed bytes, producer cannot reserve additional space.
assert!(matches!(
helper.producer_reserve(1),
Err(ring::ReserveError::InsufficientSpace)
));
// The readable wake must be a payload-free hint.
let wake = helper
.wake_hints()
.iter()
.find(|wake| matches!(wake, ring::WakeHint::RingReadable { .. }))
.expect("readable wake must be emitted");
assert_wake_is_payload_free(wake);
// Consumer sees the written bytes, proving commit was not published before
// the payload became valid.
assert_eq!(drain_committed(&mut helper), b"12345678");
}
// This proves the consumer computes readable bytes from acquired commit, never
// reads beyond committed data, advances consume only after release, and emits
// writable wake hints after freeing space.
#[test]
fn consumer_reads_only_committed_bytes_and_releases_after_safe_consume() {
// Publish three committed bytes.
let mut helper = new_ring();
let reservation = helper.producer_reserve(3).expect("space must exist");
helper.producer_write(&reservation, b"abc");
helper.producer_commit(reservation);
// The consumer cannot read beyond the committed prefix.
assert_eq!(helper.consumer_readable(), 3);
assert!(matches!(
helper.consumer_try_read(4),
Err(ring::ReadError::BeyondCommittedBytes)
));
// Reading alone does not release bytes.
assert_eq!(helper.consumer_read(3), b"abc");
assert_eq!(helper.cursor_snapshot().consume, 0);
// Consuming releases space and emits a writable hint.
helper.consumer_consume(3);
assert_eq!(helper.cursor_snapshot().consume, 3);
let wake = helper
.wake_hints()
.iter()
.find(|wake| matches!(wake, ring::WakeHint::RingWritable { .. }))
.expect("writable wake must be emitted");
assert_wake_is_payload_free(wake);
}
// This proves wake hints carry no byte ranges, counts, pointers, or credits,
// and coalescing cannot hide the only readable or writable transition.
#[test]
fn wake_hints_are_edge_hints_without_hiding_transitions() {
// Create an empty ring and publish one byte, causing empty-to-readable.
let mut helper = new_ring();
let reservation = helper.producer_reserve(1).expect("space must exist");
helper.producer_write(&reservation, b"x");
helper.producer_commit(reservation);
// The readable transition must be discoverable even if duplicate wakes are
// coalesced.
helper.coalesce_duplicate_wakes();
assert!(
helper
.scheduler_state()
.readable_rings
.contains(&ring::RingId(7000))
);
// Fill then release space to cause full-to-writable.
let _ = drain_committed(&mut helper);
helper.coalesce_duplicate_wakes();
assert!(
helper
.scheduler_state()
.writable_rings
.contains(&ring::RingId(7000))
);
// Every wake remains a payload-free hint.
for wake in helper.wake_hints() {
assert_wake_is_payload_free(wake);
}
}
// This proves Python-facing helper operations own shared atomics and wrap math,
// while returned pointers are process-local addresses derived from arena base
// plus arena offsets.
#[test]
fn helper_abi_owns_atomics_wrap_math_and_process_local_pointers() {
// Ask the helper for a process-local view of a layout.
let helper = new_ring();
let view = helper.map_process_local_view(ring::ArenaBase(0x1000));
// The helper returns process-local addresses derived from offsets.
assert_eq!(
view.data_pointer,
ring::ProcessLocalPointer::from_base_plus_offset(
ring::ArenaBase(0x1000),
view.layout.data_offset
)
);
// Python operations use helper calls for cursor and wrap behavior instead
// of implementing atomics directly.
for operation in helper.python_visible_operations() {
match operation {
ring::PythonOperation::ReserveViaHelper { .. }
| ring::PythonOperation::CommitViaHelper { .. }
| ring::PythonOperation::ReadableViaHelper { .. }
| ring::PythonOperation::ConsumeViaHelper { .. }
| ring::PythonOperation::MapPointerViaHelper { .. } => {}
ring::PythonOperation::DirectAtomicAccess { .. }
| ring::PythonOperation::DirectWrapArithmetic { .. } => {
panic!("Python operation bypassed helper ABI: {operation:?}")
}
}
}
}

545
crates/mvp-system/src/tests/stage_controller_guarantees.rs
View file

@ -1,545 +0,0 @@
//! Black-box contract tests for MVP StageController behavior.
//!
//! These tests intentionally know only the public stage-controller surface:
//!
//! - `ProvisionStage`, worker, edge, object, stop, and fault events in
//! - worker commands, lifecycle events, and teardown events out
//!
//! They assert the guarantees in
//! `specs/mvp_system/stage_controller_contract.md`.
use mvp_system::staging as stage;
// This provision fixture represents a single middle stage. It has one inbound
// and one outbound edge so tests can prove the controller uses assigned edges
// without relying on endpoint internals.
fn valid_provision() -> stage::ProvisionStage {
stage::ProvisionStage {
run_id: stage::RunId(7),
authorized_orchestrator: stage::NodeId(99),
node_id: stage::NodeId(11),
stage_index: 1,
stage_count: 3,
layer_range: stage::LayerRange {
start: 12,
end_exclusive: 24,
},
inbound: stage::EdgeProvision::inbound(stage::EdgeId(7001)),
outbound: stage::EdgeProvision::outbound(stage::EdgeId(7002)),
weight_source: stage::WeightSource::embedded_gguf("model", "model.gguf"),
shard_plan: None,
}
}
// The harness exposes only public messages. Tests intentionally do not inspect
// private controller states such as "Preparing" or "Executing"; they infer
// controller behavior from emitted commands and lifecycle events.
fn new_controller() -> stage::StageControllerHarness {
stage::StageControllerHarness::new(stage::NodeId(11))
}
// Preparation readiness has four independent prerequisites. Listing them as
// public observations lets tests prove StageReady is a barrier across worker,
// weights, inbound edge, and outbound edge readiness.
fn preparation_ready_events() -> Vec<stage::StageEvent> {
vec![
stage::StageEvent::WorkerReady,
stage::StageEvent::WeightsReady,
stage::StageEvent::InboundEdgeReady {
edge_id: stage::EdgeId(7001),
},
stage::StageEvent::OutboundEdgeReady {
edge_id: stage::EdgeId(7002),
},
]
}
// This helper provisions and readies a stage through public events. Tests that
// focus on execution use it to avoid duplicating setup while still going through
// the same observable path as production.
fn ready_stage() -> stage::StageControllerHarness {
let mut harness = new_controller();
harness.observe(stage::StageEvent::ProvisionStage {
from: stage::NodeId(99),
provision: valid_provision(),
});
for event in preparation_ready_events() {
harness.observe(event);
}
harness
}
// This proves provisioning is authorized, validated before setup, and does not
// allow a stage to rewire its assigned inbound or outbound edge.
#[test]
fn provisioning_validates_authority_and_assigned_shape_before_setup() {
// Send a valid provision from the authorized orchestrator.
let mut harness = new_controller();
harness.observe(stage::StageEvent::ProvisionStage {
from: stage::NodeId(99),
provision: valid_provision(),
});
// Setup commands should be derived from the provided assignment.
assert!(harness.commands().iter().any(|command| {
matches!(
command,
stage::StageCommand::EstablishInboundEdge {
edge_id: stage::EdgeId(7001),
..
}
)
}));
assert!(harness.commands().iter().any(|command| {
matches!(
command,
stage::StageCommand::EstablishOutboundEdge {
edge_id: stage::EdgeId(7002),
..
}
)
}));
// The controller must not emit any command that replaces the provisioned
// edge ids with a locally chosen edge.
assert!(
!harness
.commands()
.iter()
.any(|command| { matches!(command, stage::StageCommand::RewireEdge { .. }) })
);
// An unauthorized provision attempt must fault before setup can begin.
let mut unauthorized = new_controller();
unauthorized.observe(stage::StageEvent::ProvisionStage {
from: stage::NodeId(123),
provision: valid_provision(),
});
assert!(unauthorized.events().iter().any(|event| {
matches!(
event,
stage::StageLifecycleEvent::StageFault {
reason: stage::StageFaultReason::UnauthorizedProvision,
..
}
)
}));
assert!(
!unauthorized
.commands()
.iter()
.any(|command| { matches!(command, stage::StageCommand::ConfigureWorkerRole { .. }) })
);
}
#[test]
fn duplicate_identical_provision_is_idempotent() {
let mut harness = new_controller();
let provision = valid_provision();
harness.observe(stage::StageEvent::ProvisionStage {
from: stage::NodeId(99),
provision: provision.clone(),
});
let command_count = harness.commands().len();
let event_count = harness.events().len();
harness.observe(stage::StageEvent::ProvisionStage {
from: stage::NodeId(99),
provision,
});
assert_eq!(harness.commands().len(), command_count);
assert_eq!(harness.events().len(), event_count);
}
// This proves StageReady is emitted only after worker readiness, weight
// readiness, inbound edge readiness, and outbound edge readiness are all
// observed.
#[test]
fn stage_ready_waits_for_worker_weights_and_both_edges() {
// Provision the stage so preparation can begin.
let mut harness = new_controller();
harness.observe(stage::StageEvent::ProvisionStage {
from: stage::NodeId(99),
provision: valid_provision(),
});
// Feed every readiness event except the final one and prove no prefix is
// enough for StageReady.
let mut events = preparation_ready_events();
let final_event = events.pop().expect("fixture has final setup event");
for event in events {
harness.observe(event);
assert!(
!harness
.events()
.iter()
.any(|event| { matches!(event, stage::StageLifecycleEvent::StageReady { .. }) })
);
}
// The final prerequisite crosses the barrier.
harness.observe(final_event);
// StageReady appears exactly once for the provisioned stage.
let ready_count = harness
.events()
.iter()
.filter(|event| {
matches!(
event,
stage::StageLifecycleEvent::StageReady {
run_id: stage::RunId(7),
stage_index: 1,
}
)
})
.count();
assert_eq!(ready_count, 1);
}
// This proves a ready stage admits work only from inbound ObjectLoaded, issues
// one ExecuteStep per accepted object, and binds output with the same sequence.
#[test]
fn accepted_inbound_object_creates_one_same_sequence_execute_step() {
// Bring the stage to ready state through public setup events.
let mut harness = ready_stage();
// Deliver the first inbound object, sequence 0.
harness.observe(stage::StageEvent::ObjectLoaded {
edge_id: stage::EdgeId(7001),
object_id: stage::ObjectId(9000),
sequence: 0,
handle: stage::DeviceHandle::new_current(42),
});
// Exactly one ExecuteStep command must result from that accepted object.
let execute_steps = harness
.commands()
.iter()
.filter_map(|command| match command {
stage::StageCommand::ExecuteStep(step) => Some(step),
_ => None,
})
.collect::<Vec<_>>();
assert_eq!(execute_steps.len(), 1);
// The output binding must preserve the input sequence.
assert_eq!(execute_steps[0].input.sequence, 0);
assert_eq!(execute_steps[0].outputs[0].sequence, 0);
// A second object while the first step is active must not create another
// active ExecuteStep in the MVP.
harness.observe(stage::StageEvent::ObjectLoaded {
edge_id: stage::EdgeId(7001),
object_id: stage::ObjectId(9001),
sequence: 1,
handle: stage::DeviceHandle::new_current(43),
});
let active_steps = harness
.commands()
.iter()
.filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_)))
.count();
assert_eq!(active_steps, 1);
}
// This proves the sequence contract: sequence 0 is accepted as prefill, decode
// sequences must strictly increase, and duplicate, skipped, or out-of-order
// inputs fault the stage.
#[test]
fn duplicate_skipped_and_out_of_order_sequences_fault() {
// Each invalid trace starts from a freshly readied stage.
let invalid_traces = vec![vec![0, 0], vec![0, 2], vec![0, 1, 0]];
for trace in invalid_traces {
// Accept the first object and complete its step when needed so the next
// object is admitted through the normal public path.
let mut harness = ready_stage();
for (i, sequence) in trace.iter().enumerate() {
harness.observe(stage::StageEvent::ObjectLoaded {
edge_id: stage::EdgeId(7001),
object_id: stage::ObjectId(9000 + i as u64),
sequence: *sequence,
handle: stage::DeviceHandle::new_current(100 + i as u64),
});
if i + 1 < trace.len() {
harness.observe(stage::StageEvent::StepCompleted {
step_id: stage::StepId(i as u64),
});
}
}
// The transcript must contain a sequence fault for the invalid trace.
assert!(harness.events().iter().any(|event| {
matches!(
event,
stage::StageLifecycleEvent::StageFault {
reason: stage::StageFaultReason::SequenceViolation,
..
}
)
}));
}
}
// This proves compute completion is observed only after the worker reports
// StepCompleted, and completion returns the stage to ready-for-next-object.
#[test]
fn step_completed_releases_input_and_admits_next_object() {
// Start one accepted step.
let mut harness = ready_stage();
harness.observe(stage::StageEvent::ObjectLoaded {
edge_id: stage::EdgeId(7001),
object_id: stage::ObjectId(9000),
sequence: 0,
handle: stage::DeviceHandle::new_current(42),
});
// Before worker completion, no compute-complete lifecycle event is allowed.
assert!(!harness.events().iter().any(|event| {
matches!(
event,
stage::StageLifecycleEvent::StepAccepted { sequence: 1, .. }
)
}));
// Worker StepCompleted is the public completion signal.
harness.observe(stage::StageEvent::StepCompleted {
step_id: stage::StepId(0),
});
// The controller releases per-step input according to policy.
assert!(harness.commands().iter().any(|command| {
matches!(
command,
stage::StageCommand::ReleaseInputHandle {
object_id: stage::ObjectId(9000),
..
}
)
}));
// The next sequence is now admissible.
harness.observe(stage::StageEvent::ObjectLoaded {
edge_id: stage::EdgeId(7001),
object_id: stage::ObjectId(9001),
sequence: 1,
handle: stage::DeviceHandle::new_current(43),
});
let execute_count = harness
.commands()
.iter()
.filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_)))
.count();
assert_eq!(execute_count, 2);
}
// This proves worker, object, output-edge, edge, and step failures fault the
// stage with stable public reasons, and after fault no new run work is accepted
// until StopRun.
#[test]
fn stage_failures_map_to_stable_fault_reasons_and_reject_new_work() {
let cases = vec![
(
stage::StageEvent::WorkerCrashed,
stage::StageFaultReason::WorkerCrashed,
),
(
stage::StageEvent::StepFailed {
step_id: stage::StepId(0),
},
stage::StageFaultReason::StepFailed,
),
(
stage::StageEvent::ObjectFailed {
edge_id: stage::EdgeId(7001),
object_id: Some(stage::ObjectId(9000)),
},
stage::StageFaultReason::ObjectFailed,
),
(
stage::StageEvent::OutputFault {
edge_id: stage::EdgeId(7002),
},
stage::StageFaultReason::OutputFault,
),
(
stage::StageEvent::EdgeFault {
edge_id: stage::EdgeId(7002),
},
stage::StageFaultReason::EdgeFault,
),
];
for (fault_event, expected_reason) in cases {
let mut harness = ready_stage();
harness.observe(fault_event);
assert!(harness.events().iter().any(|event| {
matches!(
event,
stage::StageLifecycleEvent::StageFault {
reason,
..
} if *reason == expected_reason
)
}));
let before = harness
.commands()
.iter()
.filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_)))
.count();
harness.observe(stage::StageEvent::ObjectLoaded {
edge_id: stage::EdgeId(7001),
object_id: stage::ObjectId(9999),
sequence: 0,
handle: stage::DeviceHandle::new_current(77),
});
let after = harness
.commands()
.iter()
.filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_)))
.count();
assert_eq!(after, before);
harness.observe(stage::StageEvent::StopRun {
run_id: stage::RunId(7),
});
assert!(
harness
.commands()
.iter()
.any(|command| { matches!(command, stage::StageCommand::StopLocalEdges { .. }) })
);
assert!(
!harness
.events()
.iter()
.any(|event| { matches!(event, stage::StageLifecycleEvent::StageStopped { .. }) })
);
harness.observe(stage::StageEvent::LocalEdgesStopped {
run_id: stage::RunId(7),
});
harness.observe(stage::StageEvent::WorkerRingsQuiesced {
run_id: stage::RunId(7),
});
harness.observe(stage::StageEvent::DeviceObjectsReleased {
run_id: stage::RunId(7),
});
harness.observe(stage::StageEvent::WorkerRoleReset {
run_id: stage::RunId(7),
});
assert!(
harness
.events()
.iter()
.any(|event| { matches!(event, stage::StageLifecycleEvent::StageStopped { .. }) })
);
}
}
// This proves StopRun starts teardown but StageStopped is held back until local
// edges are stopped, worker rings are quiesced, device objects are released, and
// the worker role reset completes.
#[test]
fn stop_run_waits_for_local_teardown_completion_before_stage_stopped() {
let mut harness = ready_stage();
harness.observe(stage::StageEvent::StopRun {
run_id: stage::RunId(7),
});
assert!(
harness
.commands()
.iter()
.any(|command| { matches!(command, stage::StageCommand::StopLocalEdges { .. }) })
);
assert!(
!harness
.events()
.iter()
.any(|event| { matches!(event, stage::StageLifecycleEvent::StageStopped { .. }) })
);
assert!(
!harness.commands().iter().any(|command| {
matches!(command, stage::StageCommand::ReleaseRunDeviceObjects { .. })
})
);
let execute_before_stopping_work = harness
.commands()
.iter()
.filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_)))
.count();
harness.observe(stage::StageEvent::ObjectLoaded {
edge_id: stage::EdgeId(7001),
object_id: stage::ObjectId(9999),
sequence: 0,
handle: stage::DeviceHandle::new_current(77),
});
let execute_after_stopping_work = harness
.commands()
.iter()
.filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_)))
.count();
assert_eq!(execute_after_stopping_work, execute_before_stopping_work);
harness.observe(stage::StageEvent::LocalEdgesStopped {
run_id: stage::RunId(7),
});
assert!(
!harness
.events()
.iter()
.any(|event| { matches!(event, stage::StageLifecycleEvent::StageStopped { .. }) })
);
assert!(
!harness.commands().iter().any(|command| {
matches!(command, stage::StageCommand::ReleaseRunDeviceObjects { .. })
})
);
harness.observe(stage::StageEvent::WorkerRingsQuiesced {
run_id: stage::RunId(7),
});
assert!(harness.commands().iter().any(|command| {
matches!(
command,
stage::StageCommand::ReleaseRunDeviceObjects {
run_id: stage::RunId(7)
}
)
}));
assert!(
!harness
.events()
.iter()
.any(|event| { matches!(event, stage::StageLifecycleEvent::StageStopped { .. }) })
);
harness.observe(stage::StageEvent::DeviceObjectsReleased {
run_id: stage::RunId(7),
});
assert!(
!harness
.events()
.iter()
.any(|event| { matches!(event, stage::StageLifecycleEvent::StageStopped { .. }) })
);
harness.observe(stage::StageEvent::WorkerRoleReset {
run_id: stage::RunId(7),
});
assert!(harness.events().iter().any(|event| {
matches!(
event,
stage::StageLifecycleEvent::StageStopped {
run_id: stage::RunId(7),
stage_index: 1,
}
)
}));
}

747
crates/mvp-system/src/tests/staging_guarantees.rs Normal file
View file

@ -0,0 +1,747 @@
//! Behavior guarantees for the `staging` module.
mod stage_controller {
//! Black-box contract tests for MVP StageController behavior.
//!
//! These tests intentionally know only the public stage-controller surface:
//!
//! - `ProvisionStage`, worker, edge, object, stop, and fault events in
//! - worker commands, lifecycle events, and teardown events out
//!
//! They assert the guarantees in
//! `specs/BEHAVIOR_GUARANTEES.md`.
use mvp_system::staging as stage;
// This provision fixture represents a single middle stage. It has one inbound
// and one outbound edge so tests can prove the controller uses assigned edges
// without relying on endpoint internals.
fn valid_provision() -> stage::ProvisionStage {
stage::ProvisionStage {
run_id: stage::RunId(7),
authorized_orchestrator: stage::NodeId(99),
node_id: stage::NodeId(11),
stage_index: 1,
stage_count: 3,
layer_range: stage::LayerRange {
start: 12,
end_exclusive: 24,
},
inbound: stage::EdgeProvision::inbound(stage::EdgeId(7001)),
outbound: stage::EdgeProvision::outbound(stage::EdgeId(7002)),
weight_source: stage::WeightSource::embedded_gguf("model", "model.gguf"),
shard_plan: None,
}
}
// The harness exposes only public messages. Tests intentionally do not inspect
// private controller states such as "Preparing" or "Executing"; they infer
// controller behavior from emitted commands and lifecycle events.
fn new_controller() -> stage::StageControllerHarness {
stage::StageControllerHarness::new(stage::NodeId(11))
}
// Preparation readiness has four independent prerequisites. Listing them as
// public observations lets tests prove StageReady is a barrier across worker,
// weights, inbound edge, and outbound edge readiness.
fn preparation_ready_events() -> Vec<stage::StageEvent> {
vec![
stage::StageEvent::WorkerReady,
stage::StageEvent::WeightsReady,
stage::StageEvent::InboundEdgeReady {
edge_id: stage::EdgeId(7001),
},
stage::StageEvent::OutboundEdgeReady {
edge_id: stage::EdgeId(7002),
},
]
}
// This helper provisions and readies a stage through public events. Tests that
// focus on execution use it to avoid duplicating setup while still going through
// the same observable path as production.
fn ready_stage() -> stage::StageControllerHarness {
let mut harness = new_controller();
harness.observe(stage::StageEvent::ProvisionStage {
from: stage::NodeId(99),
provision: valid_provision(),
});
for event in preparation_ready_events() {
harness.observe(event);
}
harness
}
// This proves provisioning is authorized, validated before setup, and does not
// allow a stage to rewire its assigned inbound or outbound edge.
#[test]
fn provisioning_validates_authority_and_assigned_shape_before_setup() {
// Send a valid provision from the authorized orchestrator.
let mut harness = new_controller();
harness.observe(stage::StageEvent::ProvisionStage {
from: stage::NodeId(99),
provision: valid_provision(),
});
// Setup commands should be derived from the provided assignment.
assert!(harness.commands().iter().any(|command| {
matches!(
command,
stage::StageCommand::EstablishInboundEdge {
edge_id: stage::EdgeId(7001),
..
}
)
}));
assert!(harness.commands().iter().any(|command| {
matches!(
command,
stage::StageCommand::EstablishOutboundEdge {
edge_id: stage::EdgeId(7002),
..
}
)
}));
// The controller must not emit any command that replaces the provisioned
// edge ids with a locally chosen edge.
assert!(
!harness
.commands()
.iter()
.any(|command| { matches!(command, stage::StageCommand::RewireEdge { .. }) })
);
// An unauthorized provision attempt must fault before setup can begin.
let mut unauthorized = new_controller();
unauthorized.observe(stage::StageEvent::ProvisionStage {
from: stage::NodeId(123),
provision: valid_provision(),
});
assert!(unauthorized.events().iter().any(|event| {
matches!(
event,
stage::StageLifecycleEvent::StageFault {
reason: stage::StageFaultReason::UnauthorizedProvision,
..
}
)
}));
assert!(
!unauthorized.commands().iter().any(|command| {
matches!(command, stage::StageCommand::ConfigureWorkerRole { .. })
})
);
}
// This proves StageReady is emitted only after worker readiness, weight
// readiness, inbound edge readiness, and outbound edge readiness are all
// observed.
#[test]
fn stage_ready_waits_for_worker_weights_and_both_edges() {
// Provision the stage so preparation can begin.
let mut harness = new_controller();
harness.observe(stage::StageEvent::ProvisionStage {
from: stage::NodeId(99),
provision: valid_provision(),
});
// Feed every readiness event except the final one and prove no prefix is
// enough for StageReady.
let mut events = preparation_ready_events();
let final_event = events.pop().expect("fixture has final setup event");
for event in events {
harness.observe(event);
assert!(
!harness.events().iter().any(|event| {
matches!(event, stage::StageLifecycleEvent::StageReady { .. })
})
);
}
// The final prerequisite crosses the barrier.
harness.observe(final_event);
// StageReady appears exactly once for the provisioned stage.
let ready_count = harness
.events()
.iter()
.filter(|event| {
matches!(
event,
stage::StageLifecycleEvent::StageReady {
run_id: stage::RunId(7),
stage_index: 1,
}
)
})
.count();
assert_eq!(ready_count, 1);
}
// This proves a ready stage admits work only from inbound ObjectLoaded, issues
// one ExecuteStep per accepted object, and binds output with the same sequence.
#[test]
fn accepted_inbound_object_creates_one_same_sequence_execute_step() {
// Bring the stage to ready state through public setup events.
let mut harness = ready_stage();
// Deliver the first inbound object, sequence 0.
harness.observe(stage::StageEvent::ObjectLoaded {
edge_id: stage::EdgeId(7001),
object_id: stage::ObjectId(9000),
sequence: 0,
handle: stage::DeviceHandle::new_current(42),
});
// Exactly one ExecuteStep command must result from that accepted object.
let execute_steps = harness
.commands()
.iter()
.filter_map(|command| match command {
stage::StageCommand::ExecuteStep(step) => Some(step),
_ => None,
})
.collect::<Vec<_>>();
assert_eq!(execute_steps.len(), 1);
// The output binding must preserve the input sequence.
assert_eq!(execute_steps[0].input.sequence, 0);
assert_eq!(execute_steps[0].outputs[0].sequence, 0);
// A second object while the first step is active must not create another
// active ExecuteStep in the MVP.
harness.observe(stage::StageEvent::ObjectLoaded {
edge_id: stage::EdgeId(7001),
object_id: stage::ObjectId(9001),
sequence: 1,
handle: stage::DeviceHandle::new_current(43),
});
let active_steps = harness
.commands()
.iter()
.filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_)))
.count();
assert_eq!(active_steps, 1);
}
// This proves the sequence contract: sequence 0 is accepted as prefill, decode
// sequences must strictly increase, and duplicate, skipped, or out-of-order
// inputs fault the stage.
#[test]
fn duplicate_skipped_and_out_of_order_sequences_fault() {
// Each invalid trace starts from a freshly readied stage.
let invalid_traces = vec![vec![0, 0], vec![0, 2], vec![0, 1, 0]];
for trace in invalid_traces {
// Accept the first object and complete its step when needed so the next
// object is admitted through the normal public path.
let mut harness = ready_stage();
for (i, sequence) in trace.iter().enumerate() {
harness.observe(stage::StageEvent::ObjectLoaded {
edge_id: stage::EdgeId(7001),
object_id: stage::ObjectId(9000 + i as u64),
sequence: *sequence,
handle: stage::DeviceHandle::new_current(100 + i as u64),
});
if i + 1 < trace.len() {
harness.observe(stage::StageEvent::StepCompleted {
step_id: stage::StepId(i as u64),
});
}
}
// The transcript must contain a sequence fault for the invalid trace.
assert!(harness.events().iter().any(|event| {
matches!(
event,
stage::StageLifecycleEvent::StageFault {
reason: stage::StageFaultReason::SequenceViolation,
..
}
)
}));
}
}
// This proves compute completion is observed only after the worker reports
// StepCompleted, and completion returns the stage to ready-for-next-object.
#[test]
fn step_completed_releases_input_and_admits_next_object() {
// Start one accepted step.
let mut harness = ready_stage();
harness.observe(stage::StageEvent::ObjectLoaded {
edge_id: stage::EdgeId(7001),
object_id: stage::ObjectId(9000),
sequence: 0,
handle: stage::DeviceHandle::new_current(42),
});
// Before worker completion, no compute-complete lifecycle event is allowed.
assert!(!harness.events().iter().any(|event| {
matches!(
event,
stage::StageLifecycleEvent::StepAccepted { sequence: 1, .. }
)
}));
// Worker StepCompleted is the public completion signal.
harness.observe(stage::StageEvent::StepCompleted {
step_id: stage::StepId(0),
});
// The controller releases per-step input according to policy.
assert!(harness.commands().iter().any(|command| {
matches!(
command,
stage::StageCommand::ReleaseInputHandle {
object_id: stage::ObjectId(9000),
..
}
)
}));
// The next sequence is now admissible.
harness.observe(stage::StageEvent::ObjectLoaded {
edge_id: stage::EdgeId(7001),
object_id: stage::ObjectId(9001),
sequence: 1,
handle: stage::DeviceHandle::new_current(43),
});
let execute_count = harness
.commands()
.iter()
.filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_)))
.count();
assert_eq!(execute_count, 2);
}
// This proves worker, object, output-edge, edge, and step failures fault the
// stage with stable public reasons, and after fault no new run work is accepted
// until StopRun.
#[test]
fn stage_failures_map_to_stable_fault_reasons_and_reject_new_work() {
let cases = vec![
(
stage::StageEvent::WorkerCrashed,
stage::StageFaultReason::WorkerCrashed,
),
(
stage::StageEvent::StepFailed {
step_id: stage::StepId(0),
},
stage::StageFaultReason::StepFailed,
),
(
stage::StageEvent::ObjectFailed {
edge_id: stage::EdgeId(7001),
object_id: Some(stage::ObjectId(9000)),
},
stage::StageFaultReason::ObjectFailed,
),
(
stage::StageEvent::OutputFault {
edge_id: stage::EdgeId(7002),
},
stage::StageFaultReason::OutputFault,
),
(
stage::StageEvent::EdgeFault {
edge_id: stage::EdgeId(7002),
},
stage::StageFaultReason::EdgeFault,
),
];
for (fault_event, expected_reason) in cases {
let mut harness = ready_stage();
harness.observe(fault_event);
assert!(harness.events().iter().any(|event| {
matches!(
event,
stage::StageLifecycleEvent::StageFault {
reason,
..
} if *reason == expected_reason
)
}));
let before = harness
.commands()
.iter()
.filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_)))
.count();
harness.observe(stage::StageEvent::ObjectLoaded {
edge_id: stage::EdgeId(7001),
object_id: stage::ObjectId(9999),
sequence: 0,
handle: stage::DeviceHandle::new_current(77),
});
let after = harness
.commands()
.iter()
.filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_)))
.count();
assert_eq!(after, before);
harness.observe(stage::StageEvent::StopRun {
run_id: stage::RunId(7),
});
assert!(
harness.commands().iter().any(|command| {
matches!(command, stage::StageCommand::StopLocalEdges { .. })
})
);
assert!(
!harness.events().iter().any(|event| {
matches!(event, stage::StageLifecycleEvent::StageStopped { .. })
})
);
harness.observe(stage::StageEvent::LocalEdgesStopped {
run_id: stage::RunId(7),
});
harness.observe(stage::StageEvent::WorkerRingsQuiesced {
run_id: stage::RunId(7),
});
harness.observe(stage::StageEvent::DeviceObjectsReleased {
run_id: stage::RunId(7),
});
harness.observe(stage::StageEvent::WorkerRoleReset {
run_id: stage::RunId(7),
});
assert!(
harness.events().iter().any(|event| {
matches!(event, stage::StageLifecycleEvent::StageStopped { .. })
})
);
}
}
// This proves StopRun starts teardown but StageStopped is held back until local
// edges are stopped, worker rings are quiesced, device objects are released, and
// the worker role reset completes.
#[test]
fn stop_run_waits_for_local_teardown_completion_before_stage_stopped() {
let mut harness = ready_stage();
harness.observe(stage::StageEvent::StopRun {
run_id: stage::RunId(7),
});
assert!(
harness
.commands()
.iter()
.any(|command| { matches!(command, stage::StageCommand::StopLocalEdges { .. }) })
);
assert!(
!harness
.events()
.iter()
.any(|event| { matches!(event, stage::StageLifecycleEvent::StageStopped { .. }) })
);
assert!(!harness.commands().iter().any(|command| {
matches!(command, stage::StageCommand::ReleaseRunDeviceObjects { .. })
}));
let execute_before_stopping_work = harness
.commands()
.iter()
.filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_)))
.count();
harness.observe(stage::StageEvent::ObjectLoaded {
edge_id: stage::EdgeId(7001),
object_id: stage::ObjectId(9999),
sequence: 0,
handle: stage::DeviceHandle::new_current(77),
});
let execute_after_stopping_work = harness
.commands()
.iter()
.filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_)))
.count();
assert_eq!(execute_after_stopping_work, execute_before_stopping_work);
harness.observe(stage::StageEvent::LocalEdgesStopped {
run_id: stage::RunId(7),
});
assert!(
!harness
.events()
.iter()
.any(|event| { matches!(event, stage::StageLifecycleEvent::StageStopped { .. }) })
);
assert!(!harness.commands().iter().any(|command| {
matches!(command, stage::StageCommand::ReleaseRunDeviceObjects { .. })
}));
harness.observe(stage::StageEvent::WorkerRingsQuiesced {
run_id: stage::RunId(7),
});
assert!(harness.commands().iter().any(|command| {
matches!(
command,
stage::StageCommand::ReleaseRunDeviceObjects {
run_id: stage::RunId(7)
}
)
}));
assert!(
!harness
.events()
.iter()
.any(|event| { matches!(event, stage::StageLifecycleEvent::StageStopped { .. }) })
);
harness.observe(stage::StageEvent::DeviceObjectsReleased {
run_id: stage::RunId(7),
});
assert!(
!harness
.events()
.iter()
.any(|event| { matches!(event, stage::StageLifecycleEvent::StageStopped { .. }) })
);
harness.observe(stage::StageEvent::WorkerRoleReset {
run_id: stage::RunId(7),
});
assert!(harness.events().iter().any(|event| {
matches!(
event,
stage::StageLifecycleEvent::StageStopped {
run_id: stage::RunId(7),
stage_index: 1,
}
)
}));
}
}
mod weight_lifecycle {
//! Black-box contract tests for MVP stage-local weight lifecycle.
//!
//! These tests intentionally know only the public weight-work surface:
//!
//! - `ProvisionStage` assignment in
//! - artifact, parse, allocation, binding, and cache outcomes in
//! - `WeightsReady`, `StageReady`, and `StageFault` out
//!
//! They assert the guarantees in
//! `specs/BEHAVIOR_GUARANTEES.md`.
use mvp_system::staging::weight_lifecycle as weights;
// A valid assignment gives the stage exactly one layer range and one source.
// Tests vary only source or failure outcome so the assignment contract remains
// visible.
fn valid_assignment() -> weights::WeightAssignment {
weights::WeightAssignment {
run_id: weights::RunId(7),
stage_index: 1,
plan_layer_range: weights::LayerRange {
start: 12,
end_exclusive: 24,
},
assigned_layer_range: weights::LayerRange {
start: 12,
end_exclusive: 24,
},
source: weights::WeightSource::WholeGguf {
uri: "test://model.gguf".into(),
},
}
}
// Weight loading is intentionally opaque. The harness accepts public loader and
// worker outcomes and records only stage-visible events and commands.
fn new_weight_harness() -> weights::WeightLifecycleHarness {
weights::WeightLifecycleHarness::new(weights::NodeId(11))
}
// The success facts represent the observable prerequisites for WeightsReady:
// artifact bytes exist, the assigned layer range is valid, and the worker has
// loaded or bound that range.
fn successful_load_events() -> Vec<weights::WeightEvent> {
vec![
weights::WeightEvent::ArtifactAvailable {
bytes: weights::ArtifactBytes::Local,
},
weights::WeightEvent::LayerRangeValidated,
weights::WeightEvent::WorkerRangeBound,
]
}
// Each failure case maps one public loader or worker failure to the stable
// stage fault reason expected at the control boundary.
fn failure_cases() -> Vec<(weights::WeightEvent, weights::StageFaultReason)> {
vec![
(
weights::WeightEvent::DownloadFailed,
weights::StageFaultReason::WeightDownloadFailed,
),
(
weights::WeightEvent::ParseFailed,
weights::StageFaultReason::WeightParseFailed,
),
(
weights::WeightEvent::DeviceAllocationFailed,
weights::StageFaultReason::DeviceAllocationFailed,
),
(
weights::WeightEvent::BindingFailed,
weights::StageFaultReason::WeightBindingFailed,
),
(
weights::WeightEvent::InvalidLayerRange,
weights::StageFaultReason::InvalidLayerRange,
),
]
}
// This proves a stage receives weight source and exactly one assigned layer
// range from provisioning, validates it against the plan, and does not claim
// graph-visible ownership outside that range.
#[test]
fn assignment_is_stage_local_and_range_limited() {
// Start weight work from the provisioned assignment.
let mut harness = new_weight_harness();
harness.observe(weights::WeightEvent::Provisioned(valid_assignment()));
// The load command may use the physical source, but its graph-visible layer
// range must be the assigned range.
for command in harness.commands() {
if let weights::WeightCommand::LoadOrBindRange { range, .. } = command {
assert_eq!(
*range,
weights::LayerRange {
start: 12,
end_exclusive: 24,
}
);
}
}
// There must be no command claiming ownership of neighboring layers.
assert!(!harness.commands().iter().any(|command| {
matches!(
command,
weights::WeightCommand::AdvertiseLoadedLayerRange {
range,
..
} if range.start < 12 || range.end_exclusive > 24
)
}));
}
// This proves whole GGUF download, shard download, and cache use are physical
// mechanisms with the same public outcome: WeightsReady or StageFault.
#[test]
fn supported_physical_sources_have_same_visible_success_contract() {
// Exercise every supported source without asserting how bytes are obtained.
let sources = vec![
weights::WeightSource::WholeGguf {
uri: "test://model.gguf".into(),
},
weights::WeightSource::ShardSet {
uris: vec!["test://model.layers.12-24.gguf".into()],
},
weights::WeightSource::CachedArtifact {
cache_key: "model:layers:12-24".into(),
},
];
for source in sources {
// Install the source in an otherwise valid assignment.
let mut assignment = valid_assignment();
assignment.source = source;
let mut harness = new_weight_harness();
harness.observe(weights::WeightEvent::Provisioned(assignment));
// Drive the same public success facts for every source.
for event in successful_load_events() {
harness.observe(event);
}
// The system-visible success outcome is WeightsReady.
assert!(harness.events().iter().any(|event| {
matches!(
event,
weights::WeightLifecycleEvent::WeightsReady {
run_id: weights::RunId(7),
stage_index: 1,
}
)
}));
}
}
// This proves WeightsReady requires artifact availability, layer validation,
// and worker bind/load completion, and that WeightsReady precedes StageReady.
#[test]
fn weights_ready_requires_all_weight_facts_and_precedes_stage_ready() {
// Start from a valid assignment.
let mut harness = new_weight_harness();
harness.observe(weights::WeightEvent::Provisioned(valid_assignment()));
// Feed every success fact except the final one and prove no prefix is
// sufficient for WeightsReady.
let mut events = successful_load_events();
let final_event = events.pop().expect("fixture has final weight event");
for event in events {
harness.observe(event);
assert!(!harness.events().iter().any(|event| {
matches!(event, weights::WeightLifecycleEvent::WeightsReady { .. })
}));
}
// The final weight prerequisite emits WeightsReady.
harness.observe(final_event);
let weights_ready_pos = harness
.events()
.iter()
.position(|event| matches!(event, weights::WeightLifecycleEvent::WeightsReady { .. }))
.expect("WeightsReady must be emitted");
// StageReady may occur only after the StageController observes WeightsReady
// and the other local setup prerequisites.
harness.observe(weights::WeightEvent::OtherStagePrerequisitesReady);
let stage_ready_pos = harness
.events()
.iter()
.position(|event| matches!(event, weights::WeightLifecycleEvent::StageReady { .. }))
.expect("StageReady must be emitted after prerequisites");
assert!(weights_ready_pos < stage_ready_pos);
}
// This proves every weight failure source faults the stage and suppresses both
// WeightsReady and StageReady.
#[test]
fn weight_failures_emit_stage_fault_without_readiness() {
// Each failure source gets an isolated attempt.
for (failure_event, expected_reason) in failure_cases() {
let mut harness = new_weight_harness();
harness.observe(weights::WeightEvent::Provisioned(valid_assignment()));
// Deliver the public failure outcome from weight work.
harness.observe(failure_event);
// The stable fault reason must be observable.
assert!(harness.events().iter().any(|event| {
matches!(
event,
weights::WeightLifecycleEvent::StageFault {
reason,
..
} if *reason == expected_reason
)
}));
// Readiness cannot also be emitted after a faulted weight attempt.
assert!(!harness.events().iter().any(|event| {
matches!(event, weights::WeightLifecycleEvent::WeightsReady { .. })
|| matches!(event, weights::WeightLifecycleEvent::StageReady { .. })
}));
}
}
}

173
crates/mvp-system/src/tests/telemetry_guarantees.rs
View file

@ -1,173 +0,0 @@
use crate::provisioning::{
ProvisionEvent, ProvisionEventKind, ProvisionLogLine, ProvisionLogStream,
};
use datastream::{ChannelId, ChannelKind, Frame, Lifetime, Record, StreamId};
use mvp_system::observability::frame_archive::FrameArchive;
use mvp_system::observability::lifecycle as obs;
use mvp_system::observability::telemetry::{
self, MvpLifecycleRecord, MvpProvisionEventRecord, MvpProvisionLogRecord,
};
#[test]
fn mvp_lifecycle_record_round_trips_on_owned_datastream_channel() {
let record = MvpLifecycleRecord::new(obs::Event::StageScoped {
kind: obs::EventKind::StageFaulted,
run_id: obs::RunId(7),
stage_index: obs::StageIndex(2),
reason: Some(obs::FaultReason::WorkerCrashed),
component: obs::Component::StageController,
});
assert_eq!(MvpLifecycleRecord::CHANNEL, telemetry::MVP_LIFECYCLE);
assert_eq!(MvpLifecycleRecord::channel_name(), "mvp.lifecycle");
assert_eq!(record.kind(), obs::EventKind::StageFaulted);
assert_eq!(
MvpLifecycleRecord::decode(&record.encode()).unwrap(),
record
);
}
#[test]
fn mvp_channel_registry_marks_lifecycle_payloads_as_typed() {
let registry = telemetry::channel_registry();
assert_eq!(
registry.classify_name(MvpLifecycleRecord::channel_name()),
ChannelKind::Typed
);
}
#[test]
fn provisioning_records_round_trip_on_owned_datastream_channels() {
let event = MvpProvisionEventRecord::new(ProvisionEvent {
run_id: 77,
node_id: 11,
kind: ProvisionEventKind::NodeLive,
provider: Some("docker".to_owned()),
message: None,
});
let event_without_provider = MvpProvisionEventRecord::new(ProvisionEvent {
run_id: 77,
node_id: 12,
kind: ProvisionEventKind::ProvisionStart,
provider: None,
message: Some("queued".to_owned()),
});
let log = MvpProvisionLogRecord::new(ProvisionLogLine {
run_id: 77,
node_id: 11,
stream: ProvisionLogStream::Stdout,
line: "{\"type\":\"ready\"}".to_owned(),
});
assert_eq!(
MvpProvisionEventRecord::CHANNEL,
telemetry::MVP_PROVISIONING_EVENTS
);
assert_eq!(
MvpProvisionLogRecord::CHANNEL,
telemetry::MVP_PROVISIONING_LOGS
);
assert_eq!(
MvpProvisionEventRecord::decode(&event.encode()).unwrap(),
event
);
assert_eq!(MvpProvisionLogRecord::decode(&log.encode()).unwrap(), log);
assert_eq!(
MvpProvisionEventRecord::decode(&event_without_provider.encode()).unwrap(),
event_without_provider
);
assert_eq!(
telemetry::mvp_provision_log_channel(11, ProvisionLogStream::Stdout).as_str(),
"mvp.provisioning.logs.node.11.stdout"
);
}
#[test]
fn mvp_channel_registry_marks_provisioning_payloads_as_typed() {
let registry = telemetry::channel_registry();
assert_eq!(
registry.classify_name(MvpProvisionEventRecord::channel_name()),
ChannelKind::Typed
);
assert_eq!(
registry.classify_name(MvpProvisionLogRecord::channel_name()),
ChannelKind::Typed
);
}
#[test]
fn frame_archive_writes_jsonl_records_for_text_and_binary_payloads() {
static NEXT_TEMP_FILE: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
let suffix = NEXT_TEMP_FILE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let path = std::env::temp_dir().join(format!(
"mvp-observability-frame-archive-test-{}-{suffix}.jsonl",
std::process::id()
));
let _ = std::fs::remove_file(&path);
let stream = StreamId::new("test-node", Lifetime(42));
let mut archive = FrameArchive::open(&path).expect("frame archive opens");
archive
.record(
"orchestrator",
&stream,
"stdout",
&Frame::new(
ChannelId(1),
datastream::Position(7),
b"hello \xce\xbb".to_vec(),
),
)
.expect("text frame archives");
archive
.record(
"orchestrator",
&stream,
"stderr",
&Frame::new(
ChannelId(2),
datastream::Position(8),
vec![0xff, 0x00, b'A'],
),
)
.expect("binary frame archives");
drop(archive);
let contents = std::fs::read_to_string(&path).expect("read frame archive jsonl");
let records = contents
.lines()
.map(|line| serde_json::from_str::<serde_json::Value>(line).expect("archive line is json"))
.collect::<Vec<_>>();
let _ = std::fs::remove_file(&path);
assert_eq!(records.len(), 2);
assert_eq!(records[0]["arrival_seq"], serde_json::json!(0));
assert!(
records[0]["arrival_unix_ms"]
.as_u64()
.is_some_and(|value| value > 0)
);
assert_eq!(records[0]["source"], serde_json::json!("orchestrator"));
assert_eq!(records[0]["stream"], serde_json::json!("test-node#42"));
assert_eq!(records[0]["channel"], serde_json::json!("stdout"));
assert_eq!(records[0]["channel_id"], serde_json::json!(1));
assert_eq!(records[0]["position"], serde_json::json!(7));
assert_eq!(
records[0]["payload"],
serde_json::json!({"encoding": "utf8", "value": "hello λ"})
);
assert_eq!(records[1]["arrival_seq"], serde_json::json!(1));
assert_eq!(records[1]["source"], serde_json::json!("orchestrator"));
assert_eq!(records[1]["stream"], serde_json::json!("test-node#42"));
assert_eq!(records[1]["channel"], serde_json::json!("stderr"));
assert_eq!(records[1]["channel_id"], serde_json::json!(2));
assert_eq!(records[1]["position"], serde_json::json!(8));
assert_eq!(
records[1]["payload"],
serde_json::json!({"encoding": "bytes", "value": [255, 0, 65]})
);
}

38
crates/mvp-system/src/tests/transport_guarantees.rs Normal file
View file

@ -0,0 +1,38 @@
//! Behavior guarantees for the `transport` module.
use std::net::{Ipv4Addr, SocketAddr};
use mvp_system::transport::endpoint_advertisement::{EndpointAddrMask, advertised_endpoint};
#[test]
fn relay_only_mask_preserves_relay_urls_and_removes_direct_addresses() {
let relay = "http://relay.example.com"
.parse::<iroh::RelayUrl>()
.expect("relay URL parses");
let endpoint = iroh::EndpointAddr::new(iroh::SecretKey::from_bytes(&[9; 32]).public())
.with_relay_url(relay.clone())
.with_ip_addr(SocketAddr::from((Ipv4Addr::LOCALHOST, 7777)));
let masked = advertised_endpoint(endpoint, EndpointAddrMask::RelayOnly)
.expect("relay-only endpoint builds");
assert_eq!(masked.ip_addrs().count(), 0);
assert_eq!(
masked.relay_urls().next().map(ToString::to_string),
Some(relay.to_string())
);
}
#[test]
fn relay_only_mask_rejects_endpoint_without_relay_url() {
let endpoint = iroh::EndpointAddr::new(iroh::SecretKey::from_bytes(&[7; 32]).public())
.with_ip_addr(SocketAddr::from((Ipv4Addr::LOCALHOST, 7777)));
let error = advertised_endpoint(endpoint, EndpointAddrMask::RelayOnly)
.expect_err("missing relay URL fails");
assert!(
error.contains("requires an endpoint relay URL"),
"unexpected error: {error}"
);
}

276
crates/mvp-system/src/tests/tx_rx_edge_actor_guarantees.rs
View file

@ -1,276 +0,0 @@
//! Black-box contract tests for MVP Tx and Rx edge actors.
//!
//! These tests intentionally know only the public edge-actor surface:
//!
//! - lifecycle, object identity, stop, stream fault, and object fault events in
//! - role-facing lifecycle/object events out
//!
//! They assert the guarantees in
//! `specs/mvp_system/tx_rx_edge_actor_contract.md`.
use data_plane::edge_actor;
// The edge id fixture gives both actors a shared identity while keeping Tx and
// Rx lifecycle tests independent from driver and ring internals.
fn edge_id() -> edge_actor::EdgeId {
edge_actor::EdgeId(7001)
}
#[test]
fn object_id_allocator_is_scoped_to_one_producer_edge() {
let mut stage0_output = edge_actor::ObjectIdAllocator::new(edge_actor::EdgeId(7001));
let mut stage1_output = edge_actor::ObjectIdAllocator::new(edge_actor::EdgeId(7002));
assert_eq!(
stage0_output.alloc(),
edge_actor::ObjectKey::new(edge_actor::EdgeId(7001), edge_actor::ObjectId(1))
);
assert_eq!(
stage1_output.alloc(),
edge_actor::ObjectKey::new(edge_actor::EdgeId(7002), edge_actor::ObjectId(1))
);
assert_eq!(
stage0_output.alloc(),
edge_actor::ObjectKey::new(edge_actor::EdgeId(7001), edge_actor::ObjectId(2))
);
}
// Tx starts in provisioning and represents the producer side of one edge. The
// harness records only actor messages, not bytes or flow-control details.
fn new_tx() -> edge_actor::TxActorHarness {
edge_actor::TxActorHarness::new(edge_actor::TxConfig {
edge_id: edge_id(),
role_port: edge_actor::PortId("out".into()),
})
}
// Rx starts in provisioning and represents the consumer side of one edge. It
// exposes complete object identities and opaque handles to the role layer.
fn new_rx() -> edge_actor::RxActorHarness {
edge_actor::RxActorHarness::new(edge_actor::RxConfig {
edge_id: edge_id(),
role_port: edge_actor::PortId("in".into()),
})
}
// This helper is a compile-time and runtime guard for payload isolation. If the
// public actor message enum grows a payload-bearing variant, this exhaustive
// match has to be updated and the test discussion becomes explicit.
fn assert_actor_message_is_payload_free(message: &edge_actor::ActorMessage) {
match message {
edge_actor::ActorMessage::Lifecycle { .. }
| edge_actor::ActorMessage::ObjectIdentity { .. }
| edge_actor::ActorMessage::OpaqueHandle { .. }
| edge_actor::ActorMessage::CoarseFault { .. } => {}
}
}
// This proves Tx and Rx actors are tied to one edge id, receive lifecycle/object
// events only, and do not traffic payload bytes, pointers, ranges, credits, or
// free-space counts.
#[test]
fn edge_actor_messages_are_lifecycle_identity_and_handle_only() {
// Create one Tx and one Rx actor for the same edge id.
let mut tx = new_tx();
let mut rx = new_rx();
// Drive typical lifecycle and object events.
tx.observe(edge_actor::TxEvent::EdgeReady { edge_id: edge_id() });
tx.observe(edge_actor::TxEvent::ObjectProduced {
edge_id: edge_id(),
object_id: edge_actor::ObjectId(9000),
sequence: 0,
});
rx.observe(edge_actor::RxEvent::EdgeReady { edge_id: edge_id() });
rx.observe(edge_actor::RxEvent::ObjectLoaded {
edge_id: edge_id(),
object_id: edge_actor::ObjectId(9000),
sequence: 0,
handle: edge_actor::OpaqueHandle::new(42),
});
// Every emitted actor message must stay payload-free.
for message in tx.messages().iter().chain(rx.messages()) {
assert_actor_message_is_payload_free(message);
}
// Both actors remain tied to exactly one edge id.
assert!(
tx.messages()
.iter()
.all(|message| message.edge_id() == edge_id())
);
assert!(
rx.messages()
.iter()
.all(|message| message.edge_id() == edge_id())
);
}
// This proves Tx starts in provisioning, becomes ready only after EdgeReady,
// allows producing only after ready, reports produced object identity, and moves
// to faulted on stream or object faults.
#[test]
fn tx_lifecycle_gates_production_and_faults_on_stream_or_object_failure() {
// Before EdgeReady, production is rejected.
let mut tx = new_tx();
tx.observe(edge_actor::TxEvent::ObjectProduced {
edge_id: edge_id(),
object_id: edge_actor::ObjectId(9000),
sequence: 0,
});
assert!(
!tx.messages()
.iter()
.any(|message| { matches!(message, edge_actor::ActorMessage::ObjectIdentity { .. }) })
);
// EdgeReady admits production.
tx.observe(edge_actor::TxEvent::EdgeReady { edge_id: edge_id() });
tx.observe(edge_actor::TxEvent::ObjectProduced {
edge_id: edge_id(),
object_id: edge_actor::ObjectId(9000),
sequence: 0,
});
assert!(tx.messages().iter().any(|message| {
matches!(
message,
edge_actor::ActorMessage::ObjectIdentity {
edge_id: edge_actor::EdgeId(7001),
object_id: edge_actor::ObjectId(9000),
sequence: 0,
..
}
)
}));
// Stream fault moves Tx to faulted and suppresses later production.
tx.observe(edge_actor::TxEvent::StreamFault { edge_id: edge_id() });
let produced_before = tx
.messages()
.iter()
.filter(|message| matches!(message, edge_actor::ActorMessage::ObjectIdentity { .. }))
.count();
tx.observe(edge_actor::TxEvent::ObjectProduced {
edge_id: edge_id(),
object_id: edge_actor::ObjectId(9001),
sequence: 1,
});
let produced_after = tx
.messages()
.iter()
.filter(|message| matches!(message, edge_actor::ActorMessage::ObjectIdentity { .. }))
.count();
assert_eq!(produced_after, produced_before);
}
// This proves Rx starts in provisioning, becomes ready only after EdgeReady,
// exposes loaded objects only after ObjectLoaded, and moves to faulted on stream
// or object faults.
#[test]
fn rx_lifecycle_gates_loaded_objects_and_faults_on_stream_or_object_failure() {
// Before EdgeReady, loaded objects are not exposed to the role layer.
let mut rx = new_rx();
rx.observe(edge_actor::RxEvent::ObjectLoaded {
edge_id: edge_id(),
object_id: edge_actor::ObjectId(9000),
sequence: 0,
handle: edge_actor::OpaqueHandle::new(42),
});
assert!(
!rx.messages()
.iter()
.any(|message| { matches!(message, edge_actor::ActorMessage::OpaqueHandle { .. }) })
);
// EdgeReady admits ObjectLoaded exposure.
rx.observe(edge_actor::RxEvent::EdgeReady { edge_id: edge_id() });
rx.observe(edge_actor::RxEvent::ObjectLoaded {
edge_id: edge_id(),
object_id: edge_actor::ObjectId(9000),
sequence: 0,
handle: edge_actor::OpaqueHandle::new(42),
});
assert!(rx.messages().iter().any(|message| {
matches!(
message,
edge_actor::ActorMessage::OpaqueHandle {
edge_id: edge_actor::EdgeId(7001),
object_id: edge_actor::ObjectId(9000),
sequence: 0,
..
}
)
}));
// Object failure faults Rx and suppresses later loaded objects.
rx.observe(edge_actor::RxEvent::ObjectFailed { edge_id: edge_id() });
let loaded_before = rx
.messages()
.iter()
.filter(|message| matches!(message, edge_actor::ActorMessage::OpaqueHandle { .. }))
.count();
rx.observe(edge_actor::RxEvent::ObjectLoaded {
edge_id: edge_id(),
object_id: edge_actor::ObjectId(9001),
sequence: 1,
handle: edge_actor::OpaqueHandle::new(43),
});
let loaded_after = rx
.messages()
.iter()
.filter(|message| matches!(message, edge_actor::ActorMessage::OpaqueHandle { .. }))
.count();
assert_eq!(loaded_after, loaded_before);
}
// This proves StopEdge moves actors toward stopped, stale events after stop are
// ignored, and mismatched edge ids reject or fault according to policy.
#[test]
fn stop_and_mismatched_edge_events_do_not_create_run_work() {
// Ready Tx and Rx actors, then stop them.
let mut tx = new_tx();
let mut rx = new_rx();
tx.observe(edge_actor::TxEvent::EdgeReady { edge_id: edge_id() });
rx.observe(edge_actor::RxEvent::EdgeReady { edge_id: edge_id() });
tx.observe(edge_actor::TxEvent::StopEdge { edge_id: edge_id() });
rx.observe(edge_actor::RxEvent::StopEdge { edge_id: edge_id() });
// Stale post-stop object events must be ignored.
tx.observe(edge_actor::TxEvent::ObjectProduced {
edge_id: edge_id(),
object_id: edge_actor::ObjectId(9000),
sequence: 0,
});
rx.observe(edge_actor::RxEvent::ObjectLoaded {
edge_id: edge_id(),
object_id: edge_actor::ObjectId(9000),
sequence: 0,
handle: edge_actor::OpaqueHandle::new(42),
});
assert!(
!tx.messages()
.iter()
.any(|message| { matches!(message, edge_actor::ActorMessage::ObjectIdentity { .. }) })
);
assert!(
!rx.messages()
.iter()
.any(|message| { matches!(message, edge_actor::ActorMessage::OpaqueHandle { .. }) })
);
// A mismatched edge id must reject or fault, not create work on this actor.
let mut mismatched = new_tx();
mismatched.observe(edge_actor::TxEvent::EdgeReady {
edge_id: edge_actor::EdgeId(9999),
});
assert!(mismatched.messages().iter().any(|message| {
matches!(
message,
edge_actor::ActorMessage::CoarseFault {
reason: edge_actor::ActorFaultReason::MismatchedEdgeId,
..
}
)
}));
}

227
crates/mvp-system/src/tests/weight_lifecycle_guarantees.rs
View file

@ -1,227 +0,0 @@
//! Black-box contract tests for MVP stage-local weight lifecycle.
//!
//! These tests intentionally know only the public weight-work surface:
//!
//! - `ProvisionStage` assignment in
//! - artifact, parse, allocation, binding, and cache outcomes in
//! - `WeightsReady`, `StageReady`, and `StageFault` out
//!
//! They assert the guarantees in
//! `specs/mvp_system/weight_lifecycle_contract.md`.
use mvp_system::staging::weight_lifecycle as weights;
// A valid assignment gives the stage exactly one layer range and one source.
// Tests vary only source or failure outcome so the assignment contract remains
// visible.
fn valid_assignment() -> weights::WeightAssignment {
weights::WeightAssignment {
run_id: weights::RunId(7),
stage_index: 1,
plan_layer_range: weights::LayerRange {
start: 12,
end_exclusive: 24,
},
assigned_layer_range: weights::LayerRange {
start: 12,
end_exclusive: 24,
},
source: weights::WeightSource::WholeGguf {
uri: "test://model.gguf".into(),
},
}
}
// Weight loading is intentionally opaque. The harness accepts public loader and
// worker outcomes and records only stage-visible events and commands.
fn new_weight_harness() -> weights::WeightLifecycleHarness {
weights::WeightLifecycleHarness::new(weights::NodeId(11))
}
// The success facts represent the observable prerequisites for WeightsReady:
// artifact bytes exist, the assigned layer range is valid, and the worker has
// loaded or bound that range.
fn successful_load_events() -> Vec<weights::WeightEvent> {
vec![
weights::WeightEvent::ArtifactAvailable {
bytes: weights::ArtifactBytes::Local,
},
weights::WeightEvent::LayerRangeValidated,
weights::WeightEvent::WorkerRangeBound,
]
}
// Each failure case maps one public loader or worker failure to the stable
// stage fault reason expected at the control boundary.
fn failure_cases() -> Vec<(weights::WeightEvent, weights::StageFaultReason)> {
vec![
(
weights::WeightEvent::DownloadFailed,
weights::StageFaultReason::WeightDownloadFailed,
),
(
weights::WeightEvent::ParseFailed,
weights::StageFaultReason::WeightParseFailed,
),
(
weights::WeightEvent::DeviceAllocationFailed,
weights::StageFaultReason::DeviceAllocationFailed,
),
(
weights::WeightEvent::BindingFailed,
weights::StageFaultReason::WeightBindingFailed,
),
(
weights::WeightEvent::InvalidLayerRange,
weights::StageFaultReason::InvalidLayerRange,
),
]
}
// This proves a stage receives weight source and exactly one assigned layer
// range from provisioning, validates it against the plan, and does not claim
// graph-visible ownership outside that range.
#[test]
fn assignment_is_stage_local_and_range_limited() {
// Start weight work from the provisioned assignment.
let mut harness = new_weight_harness();
harness.observe(weights::WeightEvent::Provisioned(valid_assignment()));
// The load command may use the physical source, but its graph-visible layer
// range must be the assigned range.
for command in harness.commands() {
if let weights::WeightCommand::LoadOrBindRange { range, .. } = command {
assert_eq!(
*range,
weights::LayerRange {
start: 12,
end_exclusive: 24,
}
);
}
}
// There must be no command claiming ownership of neighboring layers.
assert!(!harness.commands().iter().any(|command| {
matches!(
command,
weights::WeightCommand::AdvertiseLoadedLayerRange {
range,
..
} if range.start < 12 || range.end_exclusive > 24
)
}));
}
// This proves whole GGUF download, shard download, and cache use are physical
// mechanisms with the same public outcome: WeightsReady or StageFault.
#[test]
fn supported_physical_sources_have_same_visible_success_contract() {
// Exercise every supported source without asserting how bytes are obtained.
let sources = vec![
weights::WeightSource::WholeGguf {
uri: "test://model.gguf".into(),
},
weights::WeightSource::ShardSet {
uris: vec!["test://model.layers.12-24.gguf".into()],
},
weights::WeightSource::CachedArtifact {
cache_key: "model:layers:12-24".into(),
},
];
for source in sources {
// Install the source in an otherwise valid assignment.
let mut assignment = valid_assignment();
assignment.source = source;
let mut harness = new_weight_harness();
harness.observe(weights::WeightEvent::Provisioned(assignment));
// Drive the same public success facts for every source.
for event in successful_load_events() {
harness.observe(event);
}
// The system-visible success outcome is WeightsReady.
assert!(harness.events().iter().any(|event| {
matches!(
event,
weights::WeightLifecycleEvent::WeightsReady {
run_id: weights::RunId(7),
stage_index: 1,
}
)
}));
}
}
// This proves WeightsReady requires artifact availability, layer validation,
// and worker bind/load completion, and that WeightsReady precedes StageReady.
#[test]
fn weights_ready_requires_all_weight_facts_and_precedes_stage_ready() {
// Start from a valid assignment.
let mut harness = new_weight_harness();
harness.observe(weights::WeightEvent::Provisioned(valid_assignment()));
// Feed every success fact except the final one and prove no prefix is
// sufficient for WeightsReady.
let mut events = successful_load_events();
let final_event = events.pop().expect("fixture has final weight event");
for event in events {
harness.observe(event);
assert!(
!harness.events().iter().any(|event| {
matches!(event, weights::WeightLifecycleEvent::WeightsReady { .. })
})
);
}
// The final weight prerequisite emits WeightsReady.
harness.observe(final_event);
let weights_ready_pos = harness
.events()
.iter()
.position(|event| matches!(event, weights::WeightLifecycleEvent::WeightsReady { .. }))
.expect("WeightsReady must be emitted");
// StageReady may occur only after the StageController observes WeightsReady
// and the other local setup prerequisites.
harness.observe(weights::WeightEvent::OtherStagePrerequisitesReady);
let stage_ready_pos = harness
.events()
.iter()
.position(|event| matches!(event, weights::WeightLifecycleEvent::StageReady { .. }))
.expect("StageReady must be emitted after prerequisites");
assert!(weights_ready_pos < stage_ready_pos);
}
// This proves every weight failure source faults the stage and suppresses both
// WeightsReady and StageReady.
#[test]
fn weight_failures_emit_stage_fault_without_readiness() {
// Each failure source gets an isolated attempt.
for (failure_event, expected_reason) in failure_cases() {
let mut harness = new_weight_harness();
harness.observe(weights::WeightEvent::Provisioned(valid_assignment()));
// Deliver the public failure outcome from weight work.
harness.observe(failure_event);
// The stable fault reason must be observable.
assert!(harness.events().iter().any(|event| {
matches!(
event,
weights::WeightLifecycleEvent::StageFault {
reason,
..
} if *reason == expected_reason
)
}));
// Readiness cannot also be emitted after a faulted weight attempt.
assert!(!harness.events().iter().any(|event| {
matches!(event, weights::WeightLifecycleEvent::WeightsReady { .. })
|| matches!(event, weights::WeightLifecycleEvent::StageReady { .. })
}));
}
}

134
crates/mvp-system/src/tests/weight_shards_guarantees.rs
View file

@ -1,134 +0,0 @@
use mvp_system::staging::weight_shards as shards;
fn model_ref() -> shards::ModelArtifactRef {
shards::ModelArtifactRef::parse("hf://org/repo@abcdef123456/model.gguf").unwrap()
}
fn assignment() -> shards::ShardAssignment {
let model_ref = model_ref();
let split_scheme = shards::SplitScheme::GgufLayerContiguousV1;
let split_id = shards::SplitId::derive(&model_ref, split_scheme);
shards::ShardAssignment::new(
model_ref,
split_id,
split_scheme,
3,
8,
shards::LayerRange::new(12, 16).unwrap(),
)
.unwrap()
}
fn content_hash() -> shards::ContentHash {
shards::ContentHash::literal("sha256:test-content").unwrap()
}
#[test]
fn model_ref_canonicalization_is_stable() {
let parsed = shards::ModelArtifactRef::parse("hf://org/repo@abcdef123456/model.gguf").unwrap();
let from_parts =
shards::ModelArtifactRef::hugging_face("/org/repo/", "abcdef123456", "/model.gguf")
.unwrap();
assert_eq!(parsed, from_parts);
assert_eq!(parsed.as_str(), "hf://org/repo@abcdef123456/model.gguf");
assert_eq!(parsed.repo(), "org/repo");
assert_eq!(parsed.revision(), "abcdef123456");
assert_eq!(parsed.path(), "model.gguf");
}
#[test]
fn split_id_is_deterministic_and_model_sensitive() {
let first = model_ref();
let same = shards::ModelArtifactRef::parse("hf://org/repo@abcdef123456/model.gguf").unwrap();
let different =
shards::ModelArtifactRef::parse("hf://org/repo@fedcba654321/model.gguf").unwrap();
let scheme = shards::SplitScheme::GgufLayerContiguousV1;
assert_eq!(
shards::SplitId::derive(&first, scheme),
shards::SplitId::derive(&same, scheme)
);
assert_ne!(
shards::SplitId::derive(&first, scheme),
shards::SplitId::derive(&different, scheme)
);
}
#[test]
fn assignment_rejects_invalid_stage_shape_and_ranges() {
let model_ref = model_ref();
let scheme = shards::SplitScheme::GgufLayerContiguousV1;
let split_id = shards::SplitId::derive(&model_ref, scheme);
let range = shards::LayerRange::new(1, 2).unwrap();
assert_eq!(
shards::ShardAssignment::new(model_ref.clone(), split_id.clone(), scheme, 0, 0, range),
Err(shards::ShardAssignmentError::EmptyStageCount)
);
assert_eq!(
shards::ShardAssignment::new(model_ref, split_id, scheme, 2, 2, range),
Err(shards::ShardAssignmentError::StageIndexOutOfRange)
);
assert_eq!(
shards::LayerRange::new(4, 4),
Err(shards::LayerRangeError::EmptyOrInverted)
);
assert_eq!(
shards::LayerRange::new(5, 4),
Err(shards::LayerRangeError::EmptyOrInverted)
);
}
#[test]
fn validator_accepts_matching_manifest() {
let assignment = assignment();
let manifest = shards::ShardManifest::for_assignment(&assignment, content_hash());
assert_eq!(
shards::ShardValidator::validate(&assignment, &manifest),
Ok(())
);
assert!(shards::ValidatedShard::new(assignment, manifest, "/cache/stage-00003.gguf").is_ok());
}
#[test]
fn validator_rejects_mismatched_manifest() {
let assignment = assignment();
let matching = shards::ShardManifest::for_assignment(&assignment, content_hash());
let mut wrong_model = matching.clone();
wrong_model.model_digest = shards::ModelDigest::literal("wrong-model").unwrap();
assert_eq!(
shards::ShardValidator::validate(&assignment, &wrong_model),
Err(shards::ShardValidationError::ModelDigestMismatch)
);
let mut wrong_split = matching.clone();
wrong_split.split_id = shards::SplitId::literal("split-wrong").unwrap();
assert_eq!(
shards::ShardValidator::validate(&assignment, &wrong_split),
Err(shards::ShardValidationError::SplitIdMismatch)
);
let mut wrong_stage = matching.clone();
wrong_stage.stage_index = 4;
assert_eq!(
shards::ShardValidator::validate(&assignment, &wrong_stage),
Err(shards::ShardValidationError::StageIndexMismatch)
);
let mut wrong_count = matching.clone();
wrong_count.stage_count = 9;
assert_eq!(
shards::ShardValidator::validate(&assignment, &wrong_count),
Err(shards::ShardValidationError::StageCountMismatch)
);
let mut wrong_range = matching;
wrong_range.layer_range = shards::LayerRange::new(16, 20).unwrap();
assert_eq!(
shards::ShardValidator::validate(&assignment, &wrong_range),
Err(shards::ShardValidationError::LayerRangeMismatch)
);
}

340
crates/mvp-system/src/tests/worker_edge_adapter_guarantees.rs
View file

@ -1,340 +0,0 @@
use crate::node_actor::{
NodeAgentActor, NodeAgentMsg, NodeAgentReport, StageCommandWire, StageEdgeKindWire,
StageInboundEdgeWire, StageObjectSpecWire, StageOutboundEdgeWire, StageProvisionWire,
StageRingSpecWire,
};
use crate::run_plan::{self, GgufSource, TokenizerSource};
use data_plane::object_record as ingress;
use mvp_system::staging as stage;
use swactor::actor::ActorAddress;
use swactor::config::RuntimeConfig;
use swactor::runtime::Runtime;
fn inbound_edge() -> StageInboundEdgeWire {
StageInboundEdgeWire {
edge_id: 7001,
kind: StageEdgeKindWire::TokenIn,
object_spec: StageObjectSpecWire {
max_extent: 2048,
alignment: 4,
},
ring_spec: StageRingSpecWire {
data_capacity: 2088,
alignment: 64,
},
}
}
fn outbound_edge() -> StageOutboundEdgeWire {
StageOutboundEdgeWire {
edge_id: 7002,
kind: StageEdgeKindWire::Activation,
consumer_node_id: 43,
consumer_endpoint: None,
object_spec: StageObjectSpecWire {
max_extent: 589_824,
alignment: 128,
},
ring_spec: StageRingSpecWire {
data_capacity: 589_864,
alignment: 256,
},
}
}
fn provision_wire() -> StageProvisionWire {
StageProvisionWire {
run_id: 55,
authorized_orchestrator: 9,
node_id: 11,
stage_index: 1,
stage_count: 3,
layer_start: 10,
layer_end_exclusive: 20,
inbound_edge_id: 7001,
outbound_edge_id: 7002,
inbound_edge: Some(inbound_edge()),
outbound_edge: Some(outbound_edge()),
model_id: "smollm2-135m-q4".to_owned(),
gguf_source: GgufSource::LocalPath("/models/smollm.gguf".to_owned()),
tokenizer: TokenizerSource::EmbeddedGguf,
stage_shard_plan: None,
}
}
fn plan_fixture(stage_count: u32) -> run_plan::RunPlan {
let placements = (0..stage_count)
.map(|stage_index| run_plan::StagePlacement {
stage_index,
node_id: run_plan::NodeId(11 + u64::from(stage_index)),
})
.collect::<Vec<_>>();
let candidate_pool = placements
.iter()
.map(|placement| placement.node_id)
.collect::<Vec<_>>();
run_plan::plan_run(run_plan::PlannerInput {
run_id: run_plan::RunId(55),
orchestrator_node_id: run_plan::NodeId(1),
model: run_plan::ModelFacts {
model_id: "fixture-model".to_owned(),
gguf_source: GgufSource::LocalPath("/models/fixture.gguf".to_owned()),
num_layers: 7,
hidden_dim: 13,
dtype_family: run_plan::DTypeFamily::BFloat,
dtype_width_bytes: 2,
max_seq_len: 32,
eos_token_id: 2,
tokenizer: TokenizerSource::EmbeddedGguf,
},
runtime: run_plan::RuntimeConfig::test_default(),
candidate_pool,
stage_count,
placement: run_plan::PlacementInput::FixedLinear(placements),
activation_ring: run_plan::RingSpec {
data_capacity: 4096,
alignment: 64,
direction: run_plan::RingDirection::Egress,
host_pinning: run_plan::HostPinning::Pageable,
wake_coalescing: run_plan::WakeCoalescing::PendingBit,
},
token_ring: run_plan::RingSpec {
data_capacity: 4096,
alignment: 64,
direction: run_plan::RingDirection::Egress,
host_pinning: run_plan::HostPinning::Pageable,
wake_coalescing: run_plan::WakeCoalescing::PendingBit,
},
})
.expect("plan fixture builds")
}
fn provision_wire_from_plan(plan: &run_plan::RunPlan, stage_index: u32) -> StageProvisionWire {
let provision = run_plan::derive_stage_provision(plan, stage_index)
.expect("stage provision derives from plan");
StageProvisionWire {
run_id: provision.run_id.0,
authorized_orchestrator: 9,
node_id: provision.node_id.0,
stage_index: provision.stage_index,
stage_count: provision.stage_count,
layer_start: provision.layer_start,
layer_end_exclusive: provision.layer_end_exclusive,
inbound_edge_id: provision.inbound.edge_id.0,
outbound_edge_id: provision.outbound.edge_id.0,
inbound_edge: Some(StageInboundEdgeWire {
edge_id: provision.inbound.edge_id.0,
kind: edge_kind_wire(provision.inbound.kind),
object_spec: object_spec_wire(provision.inbound.object_spec),
ring_spec: ring_spec_wire(provision.inbound.ring_spec),
}),
outbound_edge: Some(StageOutboundEdgeWire {
edge_id: provision.outbound.edge_id.0,
kind: edge_kind_wire(provision.outbound.kind),
consumer_node_id: provision.outbound.consumer_node_id.0,
consumer_endpoint: None,
object_spec: object_spec_wire(provision.outbound.object_spec),
ring_spec: ring_spec_wire(provision.outbound.ring_spec),
}),
model_id: provision.model.model_id,
gguf_source: provision.gguf_source,
tokenizer: provision.tokenizer,
stage_shard_plan: None,
}
}
fn edge_kind_wire(kind: run_plan::EdgeKind) -> StageEdgeKindWire {
match kind {
run_plan::EdgeKind::TokenIn => StageEdgeKindWire::TokenIn,
run_plan::EdgeKind::Activation => StageEdgeKindWire::Activation,
run_plan::EdgeKind::TokenOut => StageEdgeKindWire::TokenOut,
}
}
fn object_spec_wire(spec: run_plan::ObjectSpec) -> StageObjectSpecWire {
StageObjectSpecWire {
max_extent: spec.max_extent,
alignment: spec.alignment,
}
}
fn ring_spec_wire(spec: run_plan::RingSpec) -> StageRingSpecWire {
StageRingSpecWire {
data_capacity: spec.data_capacity,
alignment: spec.alignment,
}
}
fn token_spec() -> ingress::ObjectSpec {
ingress::ObjectSpec {
max_extent: 16,
alignment: 4,
layout: ingress::ObjectLayout::Token,
}
}
fn record(sequence: u64, extent: u64) -> Vec<u8> {
ingress::ObjectRecordBuilder::new(token_spec())
.object_id(ingress::ObjectId(9000 + sequence))
.sequence(sequence)
.extent(extent)
.payload(vec![sequence as u8; extent as usize])
.encode()
}
#[test]
fn stage_provision_wire_round_trips_edge_object_and_ring_facts() {
let wire = provision_wire();
let encoded = serde_json::to_string(&NodeAgentMsg::ProvisionStage(wire.clone()))
.expect("serialize provision wire");
let decoded: NodeAgentMsg = serde_json::from_str(&encoded).expect("deserialize provision wire");
let NodeAgentMsg::ProvisionStage(decoded) = decoded else {
panic!("decoded message must remain a provision");
};
assert_eq!(decoded.inbound_edge, Some(inbound_edge()));
assert_eq!(decoded.outbound_edge, Some(outbound_edge()));
}
#[test]
fn node_agent_establish_edge_commands_preserve_provisioned_runtime_facts() {
let runtime = Runtime::new(RuntimeConfig::default());
let reports = runtime
.new_inbox::<NodeAgentReport>()
.expect("node report inbox");
let actor = runtime
.spawn(NodeAgentActor::new(
stage::NodeId(11),
ActorAddress::new_random(),
Some(*reports.addr()),
))
.expect("spawn node agent");
runtime
.send_to(actor, NodeAgentMsg::ProvisionStage(provision_wire()))
.expect("send provision");
runtime.tick();
let commands = [
reports.try_recv().expect("inbound command"),
reports.try_recv().expect("outbound command"),
reports.try_recv().expect("configure command"),
reports.try_recv().expect("load command"),
];
assert!(commands.iter().any(|report| {
matches!(
report,
NodeAgentReport::Command(StageCommandWire::EstablishInboundEdge { edge_id: 7001, edge })
if edge == &inbound_edge()
)
}));
assert!(commands.iter().any(|report| {
matches!(
report,
NodeAgentReport::Command(StageCommandWire::EstablishOutboundEdge { edge_id: 7002, edge })
if edge == &outbound_edge()
)
}));
}
#[test]
fn node_agent_commands_preserve_plan_derived_single_stage_edge_contract() {
let plan = plan_fixture(1);
let wire = provision_wire_from_plan(&plan, 0);
assert_eq!(wire.stage_count, 1);
assert_eq!(wire.layer_start, 0);
assert_eq!(wire.layer_end_exclusive, plan.model.num_layers);
assert_eq!(
wire.inbound_edge.as_ref().expect("inbound edge").kind,
StageEdgeKindWire::TokenIn
);
assert_eq!(
wire.outbound_edge.as_ref().expect("outbound edge").kind,
StageEdgeKindWire::TokenOut
);
let runtime = Runtime::new(RuntimeConfig::default());
let reports = runtime
.new_inbox::<NodeAgentReport>()
.expect("node report inbox");
let actor = runtime
.spawn(NodeAgentActor::new(
stage::NodeId(wire.node_id),
ActorAddress::new_random(),
Some(*reports.addr()),
))
.expect("spawn node agent");
runtime
.send_to(actor, NodeAgentMsg::ProvisionStage(wire.clone()))
.expect("send provision");
runtime.tick();
let commands = [
reports.try_recv().expect("inbound command"),
reports.try_recv().expect("outbound command"),
reports.try_recv().expect("configure command"),
reports.try_recv().expect("load command"),
];
assert!(commands.iter().any(|report| {
matches!(
report,
NodeAgentReport::Command(StageCommandWire::EstablishInboundEdge { edge_id, edge })
if *edge_id == wire.inbound_edge_id
&& edge.kind == StageEdgeKindWire::TokenIn
&& edge.object_spec.max_extent == 128
&& edge.object_spec.alignment == 4
)
}));
assert!(commands.iter().any(|report| {
matches!(
report,
NodeAgentReport::Command(StageCommandWire::EstablishOutboundEdge { edge_id, edge })
if *edge_id == wire.outbound_edge_id
&& edge.kind == StageEdgeKindWire::TokenOut
&& edge.object_spec.max_extent == 128
&& edge.object_spec.alignment == 4
)
}));
}
#[test]
fn mo01_reader_uses_total_len_to_split_records_and_faults_truncated_eof() {
let first = record(0, 8);
let second = record(1, 4);
let mut joined = first.clone();
joined.extend_from_slice(&second);
let parsed = ingress::read_object_record(&joined, token_spec(), false)
.expect("joined stream starts with a complete record");
let ingress::ObjectRecordRead::Complete(first_record) = parsed else {
panic!("first record must be complete");
};
assert_eq!(first_record.object_id, ingress::ObjectId(9000));
assert_eq!(first_record.sequence, 0);
assert_eq!(first_record.extent, 8);
assert_eq!(first_record.total_len, first.len());
let parsed_second =
ingress::read_object_record(&joined[first_record.total_len..], token_spec(), true)
.expect("split cursor points at the next complete record");
let ingress::ObjectRecordRead::Complete(second_record) = parsed_second else {
panic!("second record must be complete");
};
assert_eq!(second_record.object_id, ingress::ObjectId(9001));
assert_eq!(second_record.sequence, 1);
assert_eq!(second_record.extent, 4);
assert_eq!(second_record.total_len, second.len());
assert_eq!(
ingress::read_object_record(&joined[..ingress::HEADER_LEN - 1], token_spec(), false),
Ok(ingress::ObjectRecordRead::Incomplete)
);
assert_eq!(
ingress::read_object_record(&joined[..ingress::HEADER_LEN - 1], token_spec(), true),
Err(ingress::ObjectFailureReason::EofBeforeFullPayload)
);
}

40
crates/mvp-system/src/transport/endpoint_advertisement.rs
View file

@ -62,43 +62,3 @@ fn relay_only_endpoint(endpoint: EndpointAddr) -> Result<EndpointAddr, String> {
}
Ok(out)
}
#[cfg(test)]
mod tests {
use std::net::{Ipv4Addr, SocketAddr};
use super::*;
#[test]
fn relay_only_mask_preserves_relay_urls_and_removes_direct_addresses() {
let relay = "http://relay.example.com"
.parse::<iroh::RelayUrl>()
.expect("relay URL parses");
let endpoint = EndpointAddr::new(iroh::SecretKey::from_bytes(&[9; 32]).public())
.with_relay_url(relay.clone())
.with_ip_addr(SocketAddr::from((Ipv4Addr::LOCALHOST, 7777)));
let masked = advertised_endpoint(endpoint, EndpointAddrMask::RelayOnly)
.expect("relay-only endpoint builds");
assert_eq!(masked.ip_addrs().count(), 0);
assert_eq!(
masked.relay_urls().next().map(ToString::to_string),
Some(relay.to_string())
);
}
#[test]
fn relay_only_mask_rejects_endpoint_without_relay_url() {
let endpoint = EndpointAddr::new(iroh::SecretKey::from_bytes(&[7; 32]).public())
.with_ip_addr(SocketAddr::from((Ipv4Addr::LOCALHOST, 7777)));
let error = advertised_endpoint(endpoint, EndpointAddrMask::RelayOnly)
.expect_err("missing relay URL fails");
assert!(
error.contains("requires an endpoint relay URL"),
"unexpected error: {error}"
);
}
}

7
crates/mvp-system/tests/mvp_chat_mock.rs
View file

@ -1,7 +0,0 @@
fn main() -> std::process::ExitCode {
let args: Vec<String> = std::env::args().skip(1).collect();
if args.is_empty() {
return std::process::ExitCode::SUCCESS;
}
mvp_system::run_chat_from_args(args)
}

132
crates/mvp-system/tests/python_worker_protocol.rs
View file

@ -1,132 +0,0 @@
#![cfg(target_os = "linux")]
use std::io::{BufRead, BufReader, Write};
use std::process::{Child, ChildStdin, Command, Stdio};
use serde_json::{Value, json};
struct WorkerProcess {
child: Child,
stdin: ChildStdin,
stdout: BufReader<std::process::ChildStdout>,
}
impl WorkerProcess {
fn spawn() -> Self {
let script = worker_script();
let mut child = Command::new("python3")
.arg(&script)
.env("DEV", "CPU")
.env("MVP_RUN_ID", "9")
.env("MVP_LOGICAL_NODE_ID", "3")
.env("MVP_STAGE_INDEX", "2")
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.unwrap_or_else(|e| panic!("spawn {}: {e}", script.display()));
let stdin = child.stdin.take().expect("worker stdin");
let stdout = BufReader::new(child.stdout.take().expect("worker stdout"));
Self {
child,
stdin,
stdout,
}
}
fn send_collect_until(&mut self, command: Value, expected_type: &str) -> Vec<Value> {
writeln!(self.stdin, "{command}").expect("write worker command");
self.stdin.flush().expect("flush worker command");
let mut events = Vec::new();
loop {
let mut line = String::new();
let read = self.stdout.read_line(&mut line).expect("read worker event");
assert_ne!(read, 0, "worker exited before {expected_type}");
let event: Value = serde_json::from_str(line.trim_end()).expect("worker event JSON");
let actual_type = event.get("type").and_then(Value::as_str).unwrap_or("");
assert_ne!(
actual_type, "WorkerFatal",
"worker fatal while waiting for {expected_type}: {event}"
);
let done = actual_type == expected_type;
events.push(event);
if done {
return events;
}
}
}
}
impl Drop for WorkerProcess {
fn drop(&mut self) {
let _ = self.child.kill();
let _ = self.child.wait();
}
}
#[test]
fn benchmark_observability_worker_ready_includes_stamps_and_identity() {
if !tinygrad_available() {
eprintln!("skipping Python worker protocol check: tinygrad is unavailable");
return;
}
let mut worker = WorkerProcess::spawn();
let events = worker.send_collect_until(
json!({"type":"InitializeWorker","helper_abi_version":1,"backend":{"device":"CPU"}}),
"WorkerReady",
);
assert!(
events
.iter()
.any(|event| event.get("type").and_then(Value::as_str) == Some("TinygradImportStarted")),
"expected tinygrad import milestone in {events:?}"
);
assert!(
events
.iter()
.any(|event| event.get("type").and_then(Value::as_str) == Some("WorkerReady")),
"expected WorkerReady in {events:?}"
);
for event in &events {
assert_eq!(
event
.get("benchmark")
.and_then(|benchmark| benchmark.get("schema"))
.and_then(Value::as_u64),
Some(1),
"{event}"
);
assert_eq!(
event.get("run_id").and_then(Value::as_u64),
Some(9),
"{event}"
);
assert_eq!(
event.get("node_id").and_then(Value::as_u64),
Some(3),
"{event}"
);
assert_eq!(
event.get("stage_index").and_then(Value::as_u64),
Some(2),
"{event}"
);
}
}
fn tinygrad_available() -> bool {
Command::new("python3")
.args(["-c", "import tinygrad"])
.status()
.is_ok_and(|status| status.success())
}
fn worker_script() -> std::path::PathBuf {
std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("../..")
.join("apps/mvp-node/tinygrad_worker.py")
.canonicalize()
.expect("tinygrad worker script exists")
}