From 73cfaad5d6dd6f674112d678e6408d7b544676ef Mon Sep 17 00:00:00 2001 From: Zachery Aaron Shores-Chmielewski Date: Wed, 29 Jul 2026 14:34:33 +0400 Subject: [PATCH] refactor: prune tests according to spec-defined behavior --- crates/mvp-system/Cargo.toml | 4 - crates/mvp-system/src/chat/node_image.rs | 334 -- crates/mvp-system/src/chat/runtime.rs | 790 ---- crates/mvp-system/src/lib.rs | 10 - crates/mvp-system/src/node/actor.rs | 94 - .../src/node/worker_node_runtime.rs | 522 --- crates/mvp-system/src/orchestration/actor.rs | 60 - crates/mvp-system/src/orchestration/app.rs | 3215 ----------------- crates/mvp-system/src/orchestration/config.rs | 284 -- .../src/orchestration/distribution_stack.rs | 38 - .../provider_adapters/vastai/mod.rs | 365 -- .../src/orchestration/provisioning.rs | 290 -- crates/mvp-system/src/prompt/rpc.rs | 44 - .../mvp-system/src/staging/gguf_metadata.rs | 178 - crates/mvp-system/src/staging/gguf_shard.rs | 386 -- .../tests/bootstrap_datastream_guarantees.rs | 120 - .../src/tests/engine_builder_guarantees.rs | 200 - ...integration.rs => local_e2e_guarantees.rs} | 2 +- crates/mvp-system/src/tests/mod.rs | 20 +- .../mvp-system/src/tests/node_guarantees.rs | 95 + ...rantees.rs => observability_guarantees.rs} | 30 +- .../src/tests/orchestration_guarantees.rs | 1074 ++++++ .../tests/orchestrator_run_fsm_guarantees.rs | 520 --- .../mvp-system/src/tests/prompt_guarantees.rs | 42 + .../tests/relay_provisioning_guarantees.rs | 171 - .../src/tests/run_plan_guarantees.rs | 821 ----- .../src/tests/shard_fetch_guarantees.rs | 168 - .../shard_weight_lifecycle_guarantees.rs | 308 -- .../shared_ring_helper_abi_guarantees.rs | 235 -- .../src/tests/stage_controller_guarantees.rs | 545 --- .../src/tests/staging_guarantees.rs | 747 ++++ .../src/tests/telemetry_guarantees.rs | 173 - .../src/tests/transport_guarantees.rs | 38 + .../src/tests/tx_rx_edge_actor_guarantees.rs | 276 -- .../src/tests/weight_lifecycle_guarantees.rs | 227 -- .../src/tests/weight_shards_guarantees.rs | 134 - .../tests/worker_edge_adapter_guarantees.rs | 340 -- .../src/transport/endpoint_advertisement.rs | 40 - crates/mvp-system/tests/mvp_chat_mock.rs | 7 - .../tests/python_worker_protocol.rs | 132 - 40 files changed, 2006 insertions(+), 11073 deletions(-) delete mode 100644 crates/mvp-system/src/tests/bootstrap_datastream_guarantees.rs delete mode 100644 crates/mvp-system/src/tests/engine_builder_guarantees.rs rename crates/mvp-system/src/tests/{local_mock_pipeline_integration.rs => local_e2e_guarantees.rs} (99%) create mode 100644 crates/mvp-system/src/tests/node_guarantees.rs rename crates/mvp-system/src/tests/{observability_surface_guarantees.rs => observability_guarantees.rs} (92%) create mode 100644 crates/mvp-system/src/tests/orchestration_guarantees.rs delete mode 100644 crates/mvp-system/src/tests/orchestrator_run_fsm_guarantees.rs create mode 100644 crates/mvp-system/src/tests/prompt_guarantees.rs delete mode 100644 crates/mvp-system/src/tests/relay_provisioning_guarantees.rs delete mode 100644 crates/mvp-system/src/tests/run_plan_guarantees.rs delete mode 100644 crates/mvp-system/src/tests/shard_fetch_guarantees.rs delete mode 100644 crates/mvp-system/src/tests/shard_weight_lifecycle_guarantees.rs delete mode 100644 crates/mvp-system/src/tests/shared_ring_helper_abi_guarantees.rs delete mode 100644 crates/mvp-system/src/tests/stage_controller_guarantees.rs create mode 100644 crates/mvp-system/src/tests/staging_guarantees.rs delete mode 100644 crates/mvp-system/src/tests/telemetry_guarantees.rs create mode 100644 crates/mvp-system/src/tests/transport_guarantees.rs delete mode 100644 crates/mvp-system/src/tests/tx_rx_edge_actor_guarantees.rs delete mode 100644 crates/mvp-system/src/tests/weight_lifecycle_guarantees.rs delete mode 100644 crates/mvp-system/src/tests/weight_shards_guarantees.rs delete mode 100644 crates/mvp-system/src/tests/worker_edge_adapter_guarantees.rs delete mode 100644 crates/mvp-system/tests/mvp_chat_mock.rs delete mode 100755 crates/mvp-system/tests/python_worker_protocol.rs diff --git a/crates/mvp-system/Cargo.toml b/crates/mvp-system/Cargo.toml index f891d20..1105ddf 100644 --- a/crates/mvp-system/Cargo.toml +++ b/crates/mvp-system/Cargo.toml @@ -45,7 +45,3 @@ path = "src/bin/orchestrator.rs" name = "mvp-chat" path = "src/bin/mvp_chat.rs" -[[test]] -name = "mvp_chat_mock" -path = "tests/mvp_chat_mock.rs" -harness = false diff --git a/crates/mvp-system/src/chat/node_image.rs b/crates/mvp-system/src/chat/node_image.rs index 8fa1280..9b7e1d2 100644 --- a/crates/mvp-system/src/chat/node_image.rs +++ b/crates/mvp-system/src/chat/node_image.rs @@ -1150,337 +1150,3 @@ impl ImageName { format!("{}:{tag}", self.repository) } } - -#[cfg(test)] -mod tests { - use super::*; - - #[derive(Default)] - struct CollectProgress { - events: Vec, - } - - impl NodeImageProgressSink for CollectProgress { - fn emit(&mut self, event: NodeImageProgressEvent) { - self.events.push(event); - } - } - - #[derive(Default)] - struct DryImageCommandRunner { - commands: Vec<(String, Vec, String, Option)>, - labels: BTreeMap>, - existing_images: BTreeSet, - manifests: BTreeSet, - image_tags: Vec<(String, String)>, - containers: BTreeSet, - removed_images: Vec, - } - - impl ImageCommandRunner for DryImageCommandRunner { - fn run_status( - &mut self, - _root: &Path, - program: &str, - args: &[String], - label: &str, - image_ref: Option<&str>, - progress: &mut Option<&mut dyn NodeImageProgressSink>, - ) -> Result<(), String> { - self.commands.push(( - program.to_owned(), - args.to_vec(), - label.to_owned(), - image_ref.map(str::to_owned), - )); - emit_command_progress( - progress, - label, - image_ref, - 0, - NodeImageProgressEventKind::CommandStarted { - program: program.to_owned(), - args: args.to_vec(), - }, - ); - emit_command_progress( - progress, - label, - image_ref, - 1, - NodeImageProgressEventKind::CommandStdout { - line: format!("{label} stdout"), - }, - ); - emit_command_progress( - progress, - label, - image_ref, - 2, - NodeImageProgressEventKind::CommandStderr { - line: format!("{label} stderr"), - }, - ); - emit_command_progress( - progress, - label, - image_ref, - 3, - NodeImageProgressEventKind::CommandExited { - status: "exit status: 0".to_owned(), - code: Some(0), - success: true, - }, - ); - Ok(()) - } - - fn docker_image_exists(&mut self, _root: &Path, image_ref: &str) -> bool { - self.existing_images.contains(image_ref) - } - - fn docker_image_labels( - &mut self, - _root: &Path, - image_ref: &str, - ) -> Result>, String> { - Ok(self.labels.get(image_ref).cloned()) - } - - fn docker_manifest_exists(&mut self, _root: &Path, image_ref: &str) -> bool { - self.manifests.contains(image_ref) - } - - fn docker_image_has_container(&mut self, _root: &Path, image_ref: &str) -> bool { - self.containers.contains(image_ref) - } - - fn docker_image_tags( - &mut self, - _root: &Path, - _repository: &str, - ) -> Result, String> { - Ok(self.image_tags.clone()) - } - - fn docker_image_remove(&mut self, _root: &Path, image_ref: &str) -> Result<(), String> { - self.removed_images.push(image_ref.to_owned()); - Ok(()) - } - } - - #[test] - fn fake_child_command_emits_structured_progress_and_duration() { - let mut runner = RealImageCommandRunner; - let mut progress = CollectProgress::default(); - let mut sink: Option<&mut dyn NodeImageProgressSink> = Some(&mut progress); - - runner - .run_status( - Path::new("."), - "sh", - &[ - "-c".to_owned(), - "printf 'stdout line\\n'; printf 'stderr line\\n' >&2".to_owned(), - ], - "fake child progress", - Some("docker.io/acme/node:test"), - &mut sink, - ) - .expect("fake child exits successfully"); - - assert!(progress.events.iter().any(|event| matches!( - &event.kind, - NodeImageProgressEventKind::CommandStarted { program, .. } if program == "sh" - ))); - assert!(progress.events.iter().any(|event| matches!( - &event.kind, - NodeImageProgressEventKind::CommandStdout { line } if line == "stdout line" - ))); - assert!(progress.events.iter().any(|event| matches!( - &event.kind, - NodeImageProgressEventKind::CommandStderr { line } if line == "stderr line" - ))); - let exit = progress - .events - .iter() - .find(|event| matches!(event.kind, NodeImageProgressEventKind::CommandExited { .. })) - .expect("exit progress event"); - assert_eq!(exit.command_label.as_deref(), Some("fake child progress")); - assert_eq!(exit.image_ref.as_deref(), Some("docker.io/acme/node:test")); - assert!(exit.elapsed_ms.is_some()); - assert!(matches!( - exit.kind, - NodeImageProgressEventKind::CommandExited { - success: true, - code: Some(0), - .. - } - )); - } - - #[test] - fn fake_child_failure_preserves_command_label_and_status() { - let mut runner = RealImageCommandRunner; - let mut progress = CollectProgress::default(); - let mut sink: Option<&mut dyn NodeImageProgressSink> = Some(&mut progress); - - let error = runner - .run_status( - Path::new("."), - "sh", - &["-c".to_owned(), "exit 7".to_owned()], - "failing fake child", - None, - &mut sink, - ) - .expect_err("fake child failure propagates"); - assert!(error.contains("failing fake child"), "{error}"); - - let exit = progress - .events - .iter() - .find_map(|event| match &event.kind { - NodeImageProgressEventKind::CommandExited { - status, - code, - success, - } => Some((event, status, code, success)), - _ => None, - }) - .expect("exit progress event"); - assert_eq!(exit.0.command_label.as_deref(), Some("failing fake child")); - assert!(exit.1.contains("exit status"), "{}", exit.1); - assert_eq!(*exit.2, Some(7)); - assert!(!*exit.3); - } - - #[test] - fn prepare_node_image_with_dry_runner_returns_expected_image_and_progress() { - let node_bin = std::env::current_exe().expect("test binary path resolves"); - let mut runner = DryImageCommandRunner::default(); - let mut progress = CollectProgress::default(); - let mut sink: Option<&mut dyn NodeImageProgressSink> = Some(&mut progress); - - let prepared = prepare_node_image_inner( - NodeImageRequest { - requested_image: "docker.io/acme/mvp-node:latest".to_owned(), - base_image: "swactor-mvp-node-base:cuda12.6".to_owned(), - node_bin, - provider: NodeImageProvider::Docker, - extra_tag: Some("smoke".to_owned()), - push: false, - force_refresh: false, - enabled: true, - }, - &mut sink, - &mut runner, - ) - .expect("dry image preparation succeeds"); - - assert_eq!( - prepared.image_ref, - format!("docker.io/acme/mvp-node:{}", prepared.tag) - ); - assert!(prepared.built); - assert!(!prepared.pushed); - assert!(runner.commands.iter().any(|(_, _, label, image_ref)| { - label == "build mvp node image" - && image_ref.as_deref() == Some(prepared.image_ref.as_str()) - })); - assert!(progress.events.iter().any(|event| matches!( - &event.kind, - NodeImageProgressEventKind::ImageReference { role, image_ref } - if role == "resolved" && image_ref == &prepared.image_ref - ))); - assert!(progress.events.iter().any(|event| matches!( - &event.kind, - NodeImageProgressEventKind::CommandStdout { line } - if line == "build mvp node image stdout" - ))); - } - - #[test] - fn node_image_content_hash_tracks_node_payload_not_unrelated_files() { - let root = workspace_root().expect("workspace root resolves"); - let scratch = root - .join("target/node-image-hash-test") - .join(std::process::id().to_string()); - fs::create_dir_all(&scratch).expect("scratch dir is writable"); - let node_bin = scratch.join("mvp-worker-node"); - fs::write(&node_bin, b"worker binary v1").expect("node bin fixture is writable"); - - let initial = - node_image_content_hash(&root, &node_bin, "base-v1").expect("initial hash succeeds"); - fs::write(scratch.join("unrelated.txt"), b"not part of the image") - .expect("unrelated fixture is writable"); - let after_unrelated = - node_image_content_hash(&root, &node_bin, "base-v1").expect("unrelated hash succeeds"); - fs::write(&node_bin, b"worker binary v2").expect("node bin fixture update is writable"); - let after_node_bin = - node_image_content_hash(&root, &node_bin, "base-v1").expect("node bin hash succeeds"); - let after_base = - node_image_content_hash(&root, &node_bin, "base-v2").expect("base hash succeeds"); - - let _ = fs::remove_dir_all(&scratch); - - assert_eq!(initial, after_unrelated); - assert_ne!(initial, after_node_bin); - assert_ne!(after_node_bin, after_base); - } - - #[test] - fn image_name_splits_tag_after_last_slash() { - let image = ImageName::parse("localhost:5000/team/mvp-node:trial").unwrap(); - - assert_eq!(image.repository, "localhost:5000/team/mvp-node"); - assert_eq!(image.requested_tag.as_deref(), Some("trial")); - assert_eq!( - image.ref_for_tag("git-abcdef"), - "localhost:5000/team/mvp-node:git-abcdef" - ); - } - - #[test] - fn image_name_keeps_registry_port_without_tag() { - let image = ImageName::parse("localhost:5000/team/mvp-node").unwrap(); - - assert_eq!(image.repository, "localhost:5000/team/mvp-node"); - assert_eq!(image.requested_tag, None); - } - - #[test] - fn alias_tags_include_requested_and_extra_without_version_duplicate() { - let image = ImageName::parse("ghcr.io/team/mvp-node:latest").unwrap(); - - let aliases = alias_tags(&image, Some("smoke"), "dirty-1234").unwrap(); - - assert_eq!( - aliases.into_iter().collect::>(), - vec!["latest".to_owned(), "smoke".to_owned()] - ); - } - - #[test] - fn docker_build_context_path_makes_worker_binary_relative_to_workspace() { - let root = Path::new("/workspace/swactor"); - - assert_eq!( - docker_build_context_path( - root, - Path::new("/workspace/swactor/target/debug/mvp-worker-node") - ) - .expect("absolute workspace path is valid"), - "target/debug/mvp-worker-node" - ); - assert_eq!( - docker_build_context_path(root, Path::new("target/debug/mvp-worker-node")) - .expect("relative workspace path is valid"), - "target/debug/mvp-worker-node" - ); - assert!( - docker_build_context_path(root, Path::new("/tmp/mvp-worker-node")).is_err(), - "Docker COPY inputs must stay inside the build context" - ); - } -} diff --git a/crates/mvp-system/src/chat/runtime.rs b/crates/mvp-system/src/chat/runtime.rs index 6787f04..f77463f 100644 --- a/crates/mvp-system/src/chat/runtime.rs +++ b/crates/mvp-system/src/chat/runtime.rs @@ -2511,22 +2511,6 @@ mod tests { } } - #[test] - fn observability_server_launch_contract_enables_orchestrator_dashboard() { - let build_args = mvp_orchestrator_build_args(); - assert!( - build_args - .windows(2) - .any(|pair| pair[0] == "--features" && pair[1] == "dashboard"), - "{build_args:?}" - ); - - let config = base_config(provider_kind::process()); - let args = config.orchestrator_cli_args("resolved-image"); - assert!(args.iter().any(|arg| arg == "--dashboard"), "{args:?}"); - assert!(!args.iter().any(|arg| arg == "--no-dashboard"), "{args:?}"); - } - fn valid_vastai() -> ResolvedVastAiConfig { ResolvedVastAiConfig { api_key: "secret".to_owned(), @@ -2583,48 +2567,6 @@ mod tests { .collect() } - #[test] - fn benchmark_observability_chat_progress_records_include_run_id_and_stamp() { - let temp = TempDir::new("chat-progress-archive"); - let archive_path = temp.path().join("frames.ndjson"); - let mut progress = ChatDatastream::new(77, Some(archive_path.clone())) - .expect("chat datastream constructs"); - - progress.emit( - CHAT_RUNTIME_CHANNEL, - "unit_phase", - "ready", - serde_json::json!({"ok": true}), - ); - progress.archive_pending().expect("archive pending frames"); - - let archive = fs::read_to_string(&archive_path).expect("read archive"); - let line = archive.lines().next().expect("archive line"); - let outer: serde_json::Value = serde_json::from_str(line).expect("outer archive JSON"); - let inner_text = outer - .get("payload") - .and_then(|payload| payload.get("value")) - .and_then(serde_json::Value::as_str) - .expect("inner event text"); - let inner: serde_json::Value = serde_json::from_str(inner_text).expect("inner event JSON"); - - assert_eq!( - inner.get("type").and_then(serde_json::Value::as_str), - Some("ChatProgress") - ); - assert_eq!( - inner.get("run_id").and_then(serde_json::Value::as_u64), - Some(77) - ); - assert_eq!( - inner - .get("benchmark") - .and_then(|benchmark| benchmark.get("schema")) - .and_then(serde_json::Value::as_u64), - Some(1) - ); - } - #[test] fn parsed_args_accepts_public_flags() { let parsed = ParsedArgs::parse(strings(&[ @@ -2662,85 +2604,6 @@ mod tests { assert_eq!(alias.pipeline_stages, Some(4)); } - #[test] - fn config_gpu_flag_selects_in_process_gpu_run() { - let temp = TempDir::new("gpu-flag-config"); - let cache = temp.path().join(REPO_MODEL_CACHE_DIR); - fs::create_dir_all(&cache).expect("create model cache"); - let cached_path = cache.join("default.gguf"); - fs::write(&cached_path, b"cached model").expect("write cached model"); - with_process_state(&[], Some(temp.path()), || { - let config = Config::from_args(strings(&["--gpu", "--skip-rebuild"])) - .expect("gpu config resolves"); - assert!(config.gpu_run); - assert_eq!(config.orchestrator_launch_mode(), "in_process_actor"); - assert_eq!( - config - .cached_model - .as_ref() - .map(|model| model.host_path.clone()), - Some(cached_path.canonicalize().expect("canonical cached model")) - ); - }); - } - - #[test] - fn benchmark_observability_parsed_args_accepts_run_id_and_forwards_to_orchestrator() { - let parsed = ParsedArgs::parse(strings(&["--run-id", "123"])).expect("run id parses"); - assert_eq!(parsed.run_id, Some(123)); - - let temp = TempDir::new("run-id-config"); - with_process_state(&[], Some(temp.path()), || { - let config = - Config::from_args(strings(&["--run-id", "123"])).expect("config resolves run id"); - assert_eq!(config.run_id, 123); - let args = config.orchestrator_cli_args("resolved-image"); - let run_id_arg = args - .windows(2) - .find(|pair| pair[0] == "--run-id") - .map(|pair| pair[1].as_str()); - assert_eq!(run_id_arg, Some("123"), "{args:?}"); - }); - } - - #[test] - fn benchmark_observability_parsed_args_rejects_zero_run_id() { - let error = - ParsedArgs::parse(strings(&["--run-id", "0"])).expect_err("zero run id should fail"); - assert_eq!(error, "--run-id must be greater than 0"); - } - - #[test] - fn parsed_args_accepts_cached_model_path() { - let parsed = ParsedArgs::parse(strings(&["--cached-model=/tmp/model.gguf"])) - .expect("cached model path parses"); - - assert_eq!( - parsed.cached_model, - Some(CachedModelSource::Path(PathBuf::from("/tmp/model.gguf"))) - ); - } - - #[test] - fn parsed_args_accepts_cached_model_equals_path_with_dash_prefix() { - let parsed = ParsedArgs::parse(strings(&["--cached-model=-model.gguf"])) - .expect("cached model path parses"); - - assert_eq!( - parsed.cached_model, - Some(CachedModelSource::Path(PathBuf::from("-model.gguf"))) - ); - } - - #[test] - fn parsed_args_accepts_dump_logs_equals_path_with_dash_prefix() { - let parsed = ParsedArgs::parse(strings(&["--dump-logs=-logs.ndjson"])) - .expect("dump log path parses"); - - assert!(parsed.dump_logs); - assert_eq!(parsed.dump_log_path, Some(PathBuf::from("-logs.ndjson"))); - } - #[test] fn parsed_args_rejects_conflicts_and_pruned_inputs() { for args in [ @@ -2939,202 +2802,6 @@ kind = "mock" }); } - #[test] - fn vastai_config_requires_secret_relay_bootstrap_and_remote_image() { - let missing_secret = TempDir::new("vastai-missing-secret"); - let missing_secret_config = write_config( - &missing_secret, - "chat.toml", - r#" -[provider] -kind = "vastai" - -[image] -node = "docker.io/acme/node:latest" - -[vastai] -relay_url = "https://relay.example" -bootstrap_command = "boot" -"#, - ); - with_process_state(&[], Some(missing_secret.path()), || { - let config_arg = missing_secret_config.to_string_lossy().into_owned(); - assert!(Config::from_args(strings(&["--config", config_arg.as_str()])).is_err()); - }); - - let missing_relay = TempDir::new("vastai-missing-relay"); - let missing_relay_config = write_config( - &missing_relay, - "chat.toml", - r#" -[provider] -kind = "vastai" - -[image] -node = "docker.io/acme/node:latest" - -[vastai] -bootstrap_command = "boot" -"#, - ); - with_process_state( - &[("VAST_API_KEY", Some("secret"))], - Some(missing_relay.path()), - || { - let config_arg = missing_relay_config.to_string_lossy().into_owned(); - assert!(Config::from_args(strings(&["--config", config_arg.as_str()])).is_err()); - }, - ); - - let local_image = TempDir::new("vastai-local-image"); - let local_image_config = write_config( - &local_image, - "chat.toml", - r#" -[provider] -kind = "vastai" - -[image] -node = "local-node:latest" - -[vastai] -relay_url = "https://relay.example" -bootstrap_command = "boot" -"#, - ); - with_process_state( - &[("VAST_API_KEY", Some("secret"))], - Some(local_image.path()), - || { - let config_arg = local_image_config.to_string_lossy().into_owned(); - assert!(Config::from_args(strings(&["--config", config_arg.as_str()])).is_err()); - }, - ); - - let valid = TempDir::new("vastai-valid"); - let valid_config = write_config( - &valid, - "chat.toml", - r#" -[provider] -kind = "vastai" - -[image] -node = "docker.io/acme/node:latest" - -[vastai] -relay_url = "https://relay.example" -bootstrap_command = "boot" -blacklist_hosts = [155385, 546483] -"#, - ); - with_process_state( - &[("VAST_API_KEY", Some("secret"))], - Some(valid.path()), - || { - let config_arg = valid_config.to_string_lossy().into_owned(); - let config = Config::from_args(strings(&["--config", config_arg.as_str()])) - .expect("valid Vast.ai config resolves"); - let vastai = config.vastai.as_ref().expect("resolved Vast.ai config"); - assert_eq!(vastai.api_key, "secret"); - assert_eq!(vastai.relay_url, "https://relay.example"); - assert_eq!(vastai.bootstrap_command, "boot"); - assert_eq!(vastai.image, "docker.io/acme/node:latest"); - assert_eq!(vastai.blacklist_hosts, vec![155385, 546483]); - let args = config.orchestrator_cli_args("docker.io/acme/node:latest"); - assert!( - !args - .iter() - .any(|arg| arg == "--vastai-api-key" || arg == "secret"), - "Vast.ai API key must not be exposed in orchestrator argv: {args:?}" - ); - assert!( - args.windows(2) - .any(|pair| pair == ["--vastai-bootstrap-command", "boot"]), - "non-secret Vast.ai config should still be forwarded" - ); - assert!( - args.windows(2) - .any(|pair| pair == ["--vastai-blacklist-host", "155385"]) - && args - .windows(2) - .any(|pair| pair == ["--vastai-blacklist-host", "546483"]), - "Vast.ai host blacklist must be forwarded to orchestrator argv: {args:?}" - ); - }, - ); - } - - #[test] - fn relay_only_endpoint_mask_requires_and_forwards_relay_url() { - let missing = TempDir::new("relay-mask-missing-url"); - let missing_config = write_config( - &missing, - "chat.toml", - r#" -[provider] -kind = "docker" - -[image] -node = "docker.io/acme/node:latest" -"#, - ); - with_process_state(&[], Some(missing.path()), || { - let config_arg = missing_config.to_string_lossy().into_owned(); - let error = match Config::from_args(strings(&[ - "--config", - config_arg.as_str(), - "--endpoint-addr-mask", - "relay-only", - ])) { - Ok(_) => panic!("relay-only mask without relay URL should fail"), - Err(error) => error, - }; - assert!( - error.contains("requires [relay].url, --relay-url, or [vastai].relay_url"), - "{error}" - ); - }); - - let fallback = TempDir::new("relay-mask-vastai-fallback"); - let fallback_config = write_config( - &fallback, - "chat.toml", - r#" -[provider] -kind = "docker" - -[image] -node = "docker.io/acme/node:latest" - -[relay] -endpoint_addr_mask = "relay-only" - -[vastai] -relay_url = "https://relay.example" -"#, - ); - with_process_state(&[], Some(fallback.path()), || { - let config_arg = fallback_config.to_string_lossy().into_owned(); - let config = Config::from_args(strings(&["--config", config_arg.as_str()])) - .expect("relay-only mask uses Vast.ai relay fallback"); - - assert_eq!(config.provider, provider_kind::docker()); - assert_eq!(config.relay_mode.as_deref(), Some("default")); - assert_eq!(config.relay_url.as_deref(), Some("https://relay.example")); - assert_eq!(config.endpoint_addr_mask, EndpointAddrMask::RelayOnly); - let args = config.orchestrator_cli_args("docker.io/acme/node:latest"); - assert!( - args.windows(2) - .any(|pair| pair == ["--relay-url", "https://relay.example"]) - ); - assert!( - args.windows(2) - .any(|pair| pair == ["--endpoint-addr-mask", "relay-only"]) - ); - }); - } - struct MockApproval { terminal: bool, answer: Result, @@ -3150,191 +2817,6 @@ relay_url = "https://relay.example" } } - #[test] - fn parse_approval_accepts_only_yes_variants() { - for value in ["y", "Y", " yes \n", "YeS"] { - assert!(parse_approval(value), "{value:?} should approve"); - } - for value in ["", "n", "no", "yep", " yes please"] { - assert!(!parse_approval(value), "{value:?} should decline"); - } - } - - #[test] - fn vastai_approval_is_used_only_when_required() { - let process = base_config(provider_kind::process()); - let mut approval = MockApproval { - terminal: false, - answer: Err("should not ask".to_owned()), - }; - confirm_vastai_if_needed_with_approval(&process, &mut approval) - .expect("non-Vast.ai skips approval"); - - let mut yes_config = base_config(provider_kind::vastai()); - yes_config.vastai = Some(valid_vastai()); - yes_config.vastai_yes = true; - let mut approval = MockApproval { - terminal: false, - answer: Err("should not ask".to_owned()), - }; - confirm_vastai_if_needed_with_approval(&yes_config, &mut approval) - .expect("--yes skips approval prompt"); - - let mut non_terminal = base_config(provider_kind::vastai()); - non_terminal.vastai = Some(valid_vastai()); - let mut approval = MockApproval { - terminal: false, - answer: Err("should not ask".to_owned()), - }; - assert!(confirm_vastai_if_needed_with_approval(&non_terminal, &mut approval).is_err()); - - let mut accepted = base_config(provider_kind::vastai()); - accepted.vastai = Some(valid_vastai()); - let mut approval = MockApproval { - terminal: true, - answer: Ok(true), - }; - confirm_vastai_if_needed_with_approval(&accepted, &mut approval) - .expect("interactive approval accepts"); - - let mut declined = base_config(provider_kind::vastai()); - declined.vastai = Some(valid_vastai()); - let mut approval = MockApproval { - terminal: true, - answer: Ok(false), - }; - assert!(confirm_vastai_if_needed_with_approval(&declined, &mut approval).is_err()); - } - - #[test] - fn cached_model_discovery_selects_first_sorted_gguf_file() { - let temp = TempDir::new("cached-model-selects"); - let cache_dir = temp.path().join(".model-cache"); - fs::create_dir_all(&cache_dir).expect("create cache dir"); - fs::write(cache_dir.join("z.gguf"), b"z").expect("write z model"); - fs::write(cache_dir.join("a.gguf"), b"a").expect("write a model"); - fs::write(cache_dir.join("ignored.txt"), b"ignored").expect("write ignored file"); - fs::create_dir(cache_dir.join("0.gguf")).expect("create ignored directory"); - - with_process_state(&[], Some(temp.path()), || { - let cached = CachedModelConfig::discover().expect("cached model discovered"); - assert_eq!(cached.host_path.file_name(), Some(OsStr::new("a.gguf"))); - }); - } - - #[test] - fn cached_model_discovery_errors_when_no_usable_model_exists() { - let missing = TempDir::new("cached-model-missing"); - with_process_state(&[], Some(missing.path()), || { - assert!(CachedModelConfig::discover().is_err()); - }); - - let empty = TempDir::new("cached-model-empty"); - let cache_dir = empty.path().join(".model-cache"); - fs::create_dir_all(&cache_dir).expect("create cache dir"); - fs::write(cache_dir.join("ignored.txt"), b"ignored").expect("write ignored file"); - fs::create_dir(cache_dir.join("not-a-file.gguf")).expect("create ignored directory"); - with_process_state(&[], Some(empty.path()), || { - assert!(CachedModelConfig::discover().is_err()); - }); - } - - #[test] - fn cached_model_path_resolves_regular_gguf_file() { - let temp = TempDir::new("cached-model-path"); - let model = temp.path().join("chosen.gguf"); - fs::write(&model, b"model").expect("write chosen model"); - let model_arg = model.to_string_lossy().into_owned(); - - with_process_state(&[], Some(temp.path()), || { - let parsed = ParsedArgs::parse(strings(&[&format!("--cached-model={model_arg}")])) - .expect("cached model path parses"); - assert_eq!( - parsed.cached_model, - Some(CachedModelSource::Path(PathBuf::from(model_arg.as_str()))) - ); - - let config = Config::from_args(strings(&[&format!("--cached-model={model_arg}")])) - .expect("cached model path resolves"); - assert_eq!( - config.cached_model.unwrap().host_path.file_name(), - Some(OsStr::new("chosen.gguf")) - ); - - let upper_model = temp.path().join("upper.GGUF"); - fs::write(&upper_model, b"model").expect("write uppercase model"); - let upper = CachedModelConfig::from_path(upper_model) - .expect("uppercase cached model extension resolves"); - assert_eq!(upper.host_path.file_name(), Some(OsStr::new("upper.GGUF"))); - }); - } - - #[test] - fn vastai_cached_pipeline_model_selects_matching_remote_gguf() { - let temp = TempDir::new("vastai-cached-pipeline-model"); - let cached_path = temp.path().join(DEFAULT_PIPELINE_CACHED_MODEL_FILE); - fs::write(&cached_path, b"cached model").expect("write cached model"); - let config_path = write_config( - &temp, - "chat.toml", - r#" -[provider] -kind = "vastai" - -[image] -node = "docker.io/acme/node:latest" - -[model] -id = "qwen2.5-7b-instruct-q4-k-m" -gguf_repo = "bartowski/Qwen2.5-7B-Instruct-GGUF" -gguf_file = "Qwen2.5-7B-Instruct-Q4_K_M.gguf" -max_context = 512 - -[vastai] -relay_url = "https://relay.example" -bootstrap_command = "boot" -"#, - ); - - let cached_arg = format!("--cached-model={}", cached_path.display()); - with_process_state(&[("VAST_API_KEY", Some("secret"))], None, || { - let config_arg = config_path.to_string_lossy().into_owned(); - let config = Config::from_args(strings(&[ - "--config", - config_arg.as_str(), - cached_arg.as_str(), - "--yes", - ])) - .expect("VastAI cached pipeline model resolves"); - let args = config.orchestrator_cli_args("docker.io/acme/node:prepared"); - - assert_eq!( - config - .cached_model - .as_ref() - .and_then(|model| model.host_path.file_name()), - Some(OsStr::new(DEFAULT_PIPELINE_CACHED_MODEL_FILE)) - ); - assert!( - args.windows(2) - .any(|pair| pair == ["--model-id", DEFAULT_PIPELINE_CACHED_MODEL_ID]) - ); - assert!( - args.windows(2) - .any(|pair| pair == ["--gguf-repo", DEFAULT_PIPELINE_CACHED_MODEL_REPO]) - ); - assert!( - args.windows(2) - .any(|pair| pair == ["--gguf-file", DEFAULT_PIPELINE_CACHED_MODEL_FILE]) - ); - let expected_context = DEFAULT_PIPELINE_CACHED_MODEL_MAX_CONTEXT.to_string(); - assert!( - args.windows(2) - .any(|pair| pair == ["--max-context", expected_context.as_str()]) - ); - }); - } - fn panic_prepare_node_image(_: NodeImageRequest) -> Result { panic!("image preparer must not be called when --skip-rebuild is set") } @@ -3445,278 +2927,6 @@ bootstrap_command = "boot" Err("build mvp node image failed with exit status: 42".to_owned()) } - #[test] - fn skip_rebuild_requires_existing_artifacts_and_skips_image_preparation() { - let temp = TempDir::new("skip-rebuild"); - let orch_bin = temp.path().join("mvp-orchestrator"); - let worker_bin = temp.path().join("mvp-worker-node"); - let mut config = base_config(provider_kind::docker()); - config.skip_rebuild = true; - config.orch_bin = orch_bin.clone(); - config.worker_bin = worker_bin.clone(); - config.node_image = "docker.io/acme/node:latest".to_owned(); - - assert!(prepare_runtime_with(&config, panic_prepare_node_image).is_err()); - - fs::write(&orch_bin, b"orch").expect("write orchestrator artifact"); - assert!(prepare_runtime_with(&config, panic_prepare_node_image).is_err()); - - fs::write(&worker_bin, b"worker").expect("write worker artifact"); - let image_ref = prepare_runtime_with(&config, panic_prepare_node_image) - .expect("skip rebuild uses existing artifacts"); - assert_eq!(image_ref, "docker.io/acme/node:latest"); - } - - #[test] - fn prepare_runtime_progress_records_local_prep_details() { - let temp = TempDir::new("prep-progress"); - let archive_path = temp.path().join("frames.ndjson"); - let orch_bin = temp.path().join("mvp-orchestrator"); - let worker_bin = temp.path().join("mvp-worker-node"); - fs::write(&orch_bin, b"orch").expect("write orchestrator artifact"); - fs::write(&worker_bin, b"worker").expect("write worker artifact"); - let mut config = base_config(provider_kind::docker()); - config.skip_rebuild = true; - config.orch_bin = orch_bin; - config.worker_bin = worker_bin; - config.node_image = "docker.io/acme/node:latest".to_owned(); - let mut progress = - ChatDatastream::new(91, Some(archive_path.clone())).expect("datastream constructs"); - - prepare_runtime_with_progress( - &config, - panic_prepare_node_image_with_progress, - Some(&mut progress), - ) - .expect("skip rebuild uses existing artifacts"); - progress.archive_pending().expect("archive prep frames"); - - let events = fs::read_to_string(&archive_path).expect("read archive"); - let inner_events = events - .lines() - .filter_map(|line| { - let outer: serde_json::Value = serde_json::from_str(line).ok()?; - if outer.get("channel").and_then(serde_json::Value::as_str) - != Some(CHAT_RUNTIME_CHANNEL) - { - return None; - } - let inner = outer - .get("payload")? - .get("value")? - .as_str() - .and_then(|text| serde_json::from_str::(text).ok())?; - Some(inner) - }) - .collect::>(); - let ensure_ready = inner_events - .iter() - .find(|event| { - event.get("phase").and_then(serde_json::Value::as_str) == Some("ensure_orch_binary") - && event.get("status").and_then(serde_json::Value::as_str) == Some("ready") - }) - .expect("ensure_orch_binary ready event"); - assert_eq!( - ensure_ready - .pointer("/detail/command_label") - .and_then(serde_json::Value::as_str), - Some("ensure_orch_binary") - ); - assert!( - ensure_ready - .pointer("/detail/elapsed_ms") - .and_then(serde_json::Value::as_u64) - .is_some() - ); - let image_skip = inner_events - .iter() - .find(|event| { - event.get("phase").and_then(serde_json::Value::as_str) == Some("prepare_node_image") - && event.get("status").and_then(serde_json::Value::as_str) == Some("skipped") - }) - .expect("prepare_node_image skipped event"); - assert_eq!( - image_skip - .pointer("/detail/reason") - .and_then(serde_json::Value::as_str), - Some("skip_rebuild") - ); - } - - #[test] - fn prepare_runtime_streams_node_image_command_progress() { - let temp = TempDir::new("node-image-command-progress"); - let archive_path = temp.path().join("frames.ndjson"); - let mut config = base_config(provider_kind::docker()); - config.skip_rebuild = false; - config.gpu_run = true; - config.node_image = "docker.io/acme/node:latest".to_owned(); - let mut progress = - ChatDatastream::new(92, Some(archive_path.clone())).expect("datastream constructs"); - - let image_ref = prepare_runtime_with_progress( - &config, - fake_prepare_node_image_with_progress, - Some(&mut progress), - ) - .expect("fake image preparation succeeds"); - progress.archive_pending().expect("archive prep frames"); - assert_eq!(image_ref, "docker.io/acme/node:prepared"); - - let events = runtime_events(&archive_path); - assert!(events.iter().any(|event| { - event.get("phase").and_then(serde_json::Value::as_str) == Some("prepare_node_image") - && event.get("status").and_then(serde_json::Value::as_str) == Some("image_ref") - && event - .pointer("/detail/image_ref") - .and_then(serde_json::Value::as_str) - == Some("docker.io/acme/node:prepared") - })); - assert!(events.iter().any(|event| { - event.get("phase").and_then(serde_json::Value::as_str) == Some("node_image_command") - && event.get("status").and_then(serde_json::Value::as_str) == Some("started") - && event - .pointer("/detail/command_label") - .and_then(serde_json::Value::as_str) - == Some("build mvp node image") - && event - .pointer("/detail/program") - .and_then(serde_json::Value::as_str) - == Some("fake-docker") - })); - assert!(events.iter().any(|event| { - event.get("status").and_then(serde_json::Value::as_str) == Some("stdout") - && event - .pointer("/detail/line") - .and_then(serde_json::Value::as_str) - == Some("building layer") - })); - assert!(events.iter().any(|event| { - event.get("status").and_then(serde_json::Value::as_str) == Some("stderr") - && event - .pointer("/detail/line") - .and_then(serde_json::Value::as_str) - == Some("pushing metadata") - })); - assert!(events.iter().any(|event| { - event.get("status").and_then(serde_json::Value::as_str) == Some("exited") - && event - .pointer("/detail/command_status") - .and_then(serde_json::Value::as_str) - == Some("exit status: 0") - && event.pointer("/detail/duration_ms").is_some() - })); - } - - #[test] - fn prepare_runtime_command_failure_preserves_label_and_status() { - let temp = TempDir::new("node-image-command-failure"); - let archive_path = temp.path().join("frames.ndjson"); - let mut config = base_config(provider_kind::docker()); - config.skip_rebuild = false; - config.gpu_run = true; - let mut progress = - ChatDatastream::new(93, Some(archive_path.clone())).expect("datastream constructs"); - - let error = prepare_runtime_with_progress( - &config, - failing_prepare_node_image_with_progress, - Some(&mut progress), - ) - .expect_err("fake image preparation failure propagates"); - progress.archive_pending().expect("archive prep frames"); - assert!(error.contains("build mvp node image"), "{error}"); - - let events = runtime_events(&archive_path); - let failure = events - .iter() - .find(|event| { - event.get("phase").and_then(serde_json::Value::as_str) == Some("node_image_command") - && event.get("status").and_then(serde_json::Value::as_str) == Some("failed") - }) - .expect("failed command progress event"); - assert_eq!( - failure - .pointer("/detail/command_label") - .and_then(serde_json::Value::as_str), - Some("build mvp node image") - ); - assert_eq!( - failure - .pointer("/detail/command_status") - .and_then(serde_json::Value::as_str), - Some("exit status: 42") - ); - } - - #[test] - fn vastai_skip_rebuild_uses_remote_image_without_worker_artifact() { - let temp = TempDir::new("vastai-skip-rebuild"); - let orch_bin = temp.path().join("mvp-orchestrator"); - let worker_bin = temp.path().join("mvp-worker-node"); - let mut config = base_config(provider_kind::vastai()); - config.skip_rebuild = true; - config.orch_bin = orch_bin.clone(); - config.worker_bin = worker_bin; - config.node_image = "docker.io/acme/node:latest".to_owned(); - - assert!(prepare_runtime_with(&config, panic_prepare_node_image).is_err()); - - fs::write(&orch_bin, b"orch").expect("write orchestrator artifact"); - let image_ref = prepare_runtime_with(&config, panic_prepare_node_image) - .expect("VastAI skip rebuild reuses remote image"); - assert_eq!(image_ref, "docker.io/acme/node:latest"); - } - - #[test] - fn artifact_roots_use_current_directory() { - let temp = TempDir::new("artifact-root"); - - with_process_state(&[], Some(temp.path()), || { - let path = default_orch_bin().expect("default orchestrator path resolves"); - assert!(path.starts_with(temp.path()), "{path:?}"); - assert!(path.ends_with("target/debug/mvp-orchestrator"), "{path:?}"); - }); - } - - #[cfg(target_os = "linux")] - #[test] - fn orch_child_shutdown_sends_sigterm_to_process_group() { - let temp = TempDir::new("orch-shutdown"); - let flag_path = temp.path().join("term.flag"); - let mut command = Command::new("sh"); - command - .args([ - "-c", - "trap 'echo term > \"$1\"; exit 0' TERM; while true; do sleep 1; done", - "sh", - ]) - .arg(&flag_path) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - unsafe { - command.pre_exec(|| { - if libc::setpgid(0, 0) == 0 { - Ok(()) - } else { - Err(std::io::Error::last_os_error()) - } - }); - } - let child = command.spawn().expect("spawn signal test child"); - thread::sleep(Duration::from_millis(100)); - let mut orch = OrchChild { - child, - cleaned: false, - shutdown_grace: Duration::from_millis(ORCH_SHUTDOWN_GRACE_MS), - }; - - orch.shutdown(); - - assert!(flag_path.exists(), "SIGTERM trap should write flag"); - } - #[test] fn prompt_loop_exits_cleanly_and_ignores_empty_prompts() { let mut rpc_writer = Vec::new(); diff --git a/crates/mvp-system/src/lib.rs b/crates/mvp-system/src/lib.rs index 46c18fe..f35d962 100644 --- a/crates/mvp-system/src/lib.rs +++ b/crates/mvp-system/src/lib.rs @@ -62,13 +62,3 @@ mod transport; #[cfg(test)] mod tests; - -#[cfg(test)] -#[path = "tests/shard_fetch_guarantees.rs"] -mod shard_fetch_guarantees; -#[cfg(test)] -#[path = "tests/shard_weight_lifecycle_guarantees.rs"] -mod shard_weight_lifecycle_guarantees; -#[cfg(test)] -#[path = "tests/weight_shards_guarantees.rs"] -mod weight_shards_guarantees; diff --git a/crates/mvp-system/src/node/actor.rs b/crates/mvp-system/src/node/actor.rs index 5e02dfd..caffb64 100644 --- a/crates/mvp-system/src/node/actor.rs +++ b/crates/mvp-system/src/node/actor.rs @@ -755,97 +755,3 @@ pub fn register_codecs(registry: &mut CodecRegistry) { registry.register::(JsonCodec::::default()); registry.register::(JsonCodec::::default()); } - -#[cfg(test)] -mod tests { - use super::*; - use iroh::SecretKey; - use swactor::config::RuntimeConfig; - use swactor::runtime::Runtime; - - #[test] - fn node_agent_runtime_loaded_reports_orchestrator() { - let runtime = Runtime::new(RuntimeConfig::default()); - let orchestrator_inbox = runtime - .new_inbox::() - .expect("orchestrator inbox"); - let orchestrator = *orchestrator_inbox.addr(); - let node_actor = ActorAddress::new_random(); - let datastream_publisher = ActorAddress::new_random(); - let endpoint = EndpointAddr::new(SecretKey::from_bytes(&[9; 32]).public()); - let actor = runtime - .spawn(NodeAgentActor::new(stage::NodeId(11), orchestrator, None)) - .expect("spawn node agent"); - - runtime - .send_to( - actor, - NodeAgentMsg::RuntimeLoaded { - run_id: 7, - node_id: 11, - stage_index: 3, - endpoint: endpoint.clone(), - node_actor, - datastream_publisher, - readiness_id: 99, - }, - ) - .expect("send runtime loaded"); - runtime.tick(); - - assert_eq!( - orchestrator_inbox.try_recv(), - Some(OrchestratorMsg::ObserveNodeRuntimeReady { - run_id: 7, - node_id: 11, - stage_index: 3, - endpoint, - node_actor, - datastream_publisher, - readiness_id: 99, - }) - ); - } - - #[test] - fn node_agent_runtime_ready_ack_reports_worker_loop() { - let runtime = Runtime::new(RuntimeConfig::default()); - let orchestrator_inbox = runtime - .new_inbox::() - .expect("orchestrator inbox"); - let reports = runtime - .new_inbox::() - .expect("node report inbox"); - let actor = runtime - .spawn(NodeAgentActor::new( - stage::NodeId(11), - *orchestrator_inbox.addr(), - Some(*reports.addr()), - )) - .expect("spawn node agent"); - - runtime - .send_to( - actor, - NodeAgentMsg::RuntimeReadyAck { - run_id: 7, - node_id: 11, - stage_index: 3, - readiness_id: 99, - }, - ) - .expect("send runtime ready ack"); - runtime.tick(); - - assert_eq!( - reports.try_recv(), - Some(NodeAgentReport::RuntimeReadyAck { - run_id: 7, - node_id: 11, - stage_index: 3, - readiness_id: 99, - }) - ); - assert_eq!(reports.try_recv(), None); - } -} diff --git a/crates/mvp-system/src/node/worker_node_runtime.rs b/crates/mvp-system/src/node/worker_node_runtime.rs index e6db707..807ea7d 100644 --- a/crates/mvp-system/src/node/worker_node_runtime.rs +++ b/crates/mvp-system/src/node/worker_node_runtime.rs @@ -4815,525 +4815,3 @@ fn tokenizer_from_env() -> TokenizerSource { .map(TokenizerSource::LocalPath) .unwrap_or(TokenizerSource::EmbeddedGguf) } - -#[cfg(test)] -mod tests { - use super::*; - use crate::orchestration::actor::OrchestratorMsg; - use distribution::swim::actor::MembershipChanged; - use distribution::swim::node::{SwimObservation, SwimObserver}; - - fn endpoint(seed: u8) -> EndpointAddr { - EndpointAddr::new(iroh::SecretKey::from_bytes(&[seed; 32]).public()) - } - - fn test_config(coordinator_endpoint: Option) -> DeploymentConfig { - DeploymentConfig { - run_id: 7, - logical_node_id: 11, - stage_index: 3, - coordinator_endpoint, - orchestrator_actor: Some(ActorAddress::new_random()), - datastream_frame_log: None, - debug_join_socket: None, - relay_mode: iroh::RelayMode::Disabled, - endpoint_addr_mask: EndpointAddrMask::Full, - worker_script: DEFAULT_WORKER_SCRIPT.to_owned(), - device: DEFAULT_DEVICE.to_owned(), - model_id: DEFAULT_MODEL_ID.to_owned(), - gguf_source: GgufSource::LocalPath("/tmp/model.gguf".to_owned()), - tokenizer: TokenizerSource::EmbeddedGguf, - self_test_prompt: None, - self_test_layer_end: 16, - self_test_max_tokens: 1, - arena_bytes: DEFAULT_ARENA_BYTES, - arena_alignment: DEFAULT_ARENA_ALIGNMENT, - } - } - - fn test_stack() -> DistributionRuntimeStack { - DistributionRuntimeStack::new(DistNodeId([1; 32]), DistributedNodeConfig::default()) - } - - #[test] - fn benchmark_observability_node_event_payload_includes_stamp() { - let config = test_config(None); - let payload = node_event_payload(&config, "phase", "ready", json!({"ok": true})); - - assert_eq!(payload.get("run_id").and_then(Value::as_u64), Some(7)); - assert_eq!(payload.get("node_id").and_then(Value::as_u64), Some(11)); - assert_eq!(payload.get("stage_index").and_then(Value::as_u64), Some(3)); - assert_eq!( - payload - .get("benchmark") - .and_then(|benchmark| benchmark.get("schema")) - .and_then(Value::as_u64), - Some(1) - ); - let producer_instance_id = payload - .get("producer_instance_id") - .and_then(Value::as_str) - .expect("producer instance id"); - assert!(producer_instance_id.starts_with("mvp-worker-node:7:node-11:pid-")); - assert_eq!( - payload - .get("benchmark") - .and_then(|benchmark| benchmark.get("producer_instance_id")) - .and_then(Value::as_str), - Some(producer_instance_id) - ); - } - - #[test] - fn worker_swim_telemetry_emits_probe_records_to_datastream() { - let stack = test_stack(); - let peer = DistNodeId([9; 32]); - stack.swim_telemetry.observe(SwimObservation::ProbeSent { - target: peer, - sequence: 41, - kind: "direct", - }); - stack - .swim_telemetry - .observe(SwimObservation::ProbeTimedOut { - target: peer, - sequence: 41, - kind: "direct", - budget_ticks: 15_000, - }); - let mut datastream = NodeDatastream::new(&test_config(None)); - - emit_swim_telemetry(&mut datastream, &stack, "unit_phase"); - - let frames = datastream.endpoint.mux().drain(); - let probe_records = frames - .iter() - .filter(|frame| { - datastream - .by_id - .get(&frame.channel) - .is_some_and(|channel| channel == SwimProbeEvent::CHANNEL) - }) - .map(|frame| SwimProbeEvent::decode(&frame.payload).expect("probe record decodes")) - .collect::>(); - - assert_eq!(probe_records.len(), 2); - assert_eq!(probe_records[0].event, "sent"); - assert_eq!(probe_records[0].sequence, 41); - assert_eq!(probe_records[0].local_phase, "unit_phase"); - assert_eq!(probe_records[0].probe_timeout_ms, 15_000); - assert_eq!(probe_records[1].event, "timed_out"); - assert_eq!(probe_records[1].budget_ms, Some(15_000)); - assert_eq!(probe_records[1].consecutive_timeouts, 1); - } - - #[test] - fn debug_join_client_serializes_endpoint_from_stdin() { - let secret = iroh::SecretKey::from_bytes(&[7; 32]); - let endpoint = EndpointAddr::new(secret.public()).with_relay_url( - "http://relay.example.com" - .parse::() - .unwrap(), - ); - - let line = debug_join_request_line(endpoint).expect("serialize debug join request"); - let request: DebugJoinRequestWire = - serde_json::from_str(&line).expect("deserialize debug join request"); - - match request { - DebugJoinRequestWire::JoinEndpoint { endpoint } => { - assert_eq!( - endpoint.relay_urls().next().map(ToString::to_string), - Some("http://relay.example.com/".to_owned()) - ); - } - } - } - - #[test] - fn debug_join_listener_queues_join_endpoint() { - let root = std::env::temp_dir().join(format!( - "mvp-worker-debug-join-test-{}-{}", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .expect("system time after epoch") - .as_nanos() - )); - std::fs::create_dir(&root).expect("create debug join test temp dir"); - let socket_path = root.join("debug-join.sock"); - let runtime = tokio::runtime::Runtime::new().expect("create tokio runtime"); - let mut commands = spawn_debug_join_listener(runtime.handle().clone(), socket_path.clone()) - .expect("spawn debug join listener"); - - let secret = iroh::SecretKey::from_bytes(&[8; 32]); - let endpoint = EndpointAddr::new(secret.public()).with_relay_url( - "http://relay.example.com" - .parse::() - .unwrap(), - ); - let request = DebugJoinRequestWire::JoinEndpoint { endpoint }; - let mut request_line = - serde_json::to_string(&request).expect("serialize debug join request"); - request_line.push('\n'); - - runtime.block_on(async { - let mut stream = tokio::net::UnixStream::connect(&socket_path) - .await - .expect("connect to debug join listener"); - stream - .write_all(request_line.as_bytes()) - .await - .expect("write debug join request"); - stream.flush().await.expect("flush debug join request"); - - let DebugJoinCommand::JoinEndpoint { endpoint, reply } = - commands.recv().await.expect("receive debug join command"); - assert_eq!( - endpoint.relay_urls().next().map(ToString::to_string), - Some("http://relay.example.com/".to_owned()) - ); - let peer_node_id = endpoint.id.to_string(); - assert!( - reply - .send(DebugJoinResponseWire::JoinQueued { - peer_node_id, - has_relay: true, - direct_addr_count: 0, - }) - .is_ok() - ); - - let mut reader = tokio::io::BufReader::new(stream); - let mut response_line = String::new(); - reader - .read_line(&mut response_line) - .await - .expect("read debug join response"); - let response: DebugJoinResponseWire = - serde_json::from_str(&response_line).expect("deserialize debug join response"); - match response { - DebugJoinResponseWire::JoinQueued { has_relay, .. } => { - assert!(has_relay); - } - DebugJoinResponseWire::JoinRejected { error, detail } => { - panic!("debug join was rejected: {error}: {detail}"); - } - } - }); - - drop(commands); - let _ = std::fs::remove_file(&socket_path); - std::fs::remove_dir(&root).expect("remove debug join test temp dir"); - } - - #[test] - fn runtime_ready_retry_waits_for_swim() { - let stack = test_stack(); - let node_actor = ActorAddress::new_random(); - let mut pending = PendingRuntimeReady::new( - &test_config(None), - endpoint(3), - node_actor, - ActorAddress::new_random(), - ); - pending.coordinator = Some(DistNodeId([2; 32])); - - assert_eq!(pending.maybe_send(&stack, node_actor), Ok(false)); - assert_eq!(pending.attempts, 0); - } - - #[test] - fn runtime_ready_retry_stops_after_matching_ack() { - let stack = test_stack(); - let node_actor = ActorAddress::new_random(); - let mut pending = PendingRuntimeReady::new( - &test_config(None), - endpoint(4), - node_actor, - ActorAddress::new_random(), - ); - - assert!(pending.observe_ack( - pending.run_id, - pending.node_id, - pending.stage_index, - pending.readiness_id, - )); - assert!(pending.acked); - assert_eq!(pending.maybe_send(&stack, node_actor), Ok(false)); - } - - #[test] - fn runtime_ready_retry_backoff_caps() { - let stack = test_stack(); - let orchestrator_inbox = stack - .runtime - .new_inbox::() - .expect("orchestrator inbox"); - let node_actor = stack - .runtime - .spawn(NodeAgentActor::new( - stage::NodeId(11), - *orchestrator_inbox.addr(), - None, - )) - .expect("spawn node agent"); - let coordinator = DistNodeId([2; 32]); - stack - .runtime - .send_to( - stack.actors.membership_fanout, - MembershipChanged { - node_id: coordinator, - state: MemberState::Alive, - incarnation: 1, - }, - ) - .expect("send membership change"); - stack.pump_runtime_once(); - - let mut pending = PendingRuntimeReady::new( - &test_config(None), - endpoint(5), - node_actor, - ActorAddress::new_random(), - ); - pending.coordinator = Some(coordinator); - for expected_attempts in 1..=4 { - pending.next_attempt_at = Instant::now(); - assert!( - pending - .maybe_send(&stack, node_actor) - .expect("runtime ready send") - ); - assert_eq!(pending.attempts, expected_attempts); - assert!(pending.backoff <= RUNTIME_READY_RETRY_MAX); - } - } - - fn fast_helper_wait_config() -> HelperCommandWaitConfig { - HelperCommandWaitConfig { - poll_interval: Duration::from_millis(5), - telemetry_interval: Duration::from_millis(10), - } - } - - fn drain_json_frames( - datastream: &NodeDatastream, - subscription: &DatastreamSubscription, - ) -> Vec<(String, Value)> { - datastream.endpoint.tick(); - subscription - .drain_available() - .into_iter() - .filter_map(|event| { - let DatastreamEvent::Frame(frame) = event else { - return None; - }; - let channel = datastream.by_id.get(&frame.channel.channel)?.clone(); - let value = serde_json::from_slice(&frame.payload).ok()?; - Some((channel, value)) - }) - .collect() - } - - #[test] - fn helper_wait_pumps_and_emits_busy_telemetry_during_quiet_stdout() { - let config = test_config(None); - let mut datastream = NodeDatastream::new(&config); - let channel = datastream.channel_by_name("mvp.worker.weights"); - let subscription = datastream.endpoint.subscribe_all("helper-wait-test"); - let (tx, rx) = mpsc::channel(); - thread::spawn(move || { - thread::sleep(Duration::from_millis(35)); - tx.send(HelperStdoutEvent::Line( - json!({"type":"WeightsLoaded","model_id":"fake"}).to_string() + "\n", - )) - .expect("send fake helper event"); - }); - let mut pump_count = 0_u32; - - let event = wait_for_helper_event( - &rx, - None, - "WeightsLoaded", - "LoadWeights", - &config, - &mut datastream, - channel, - "mvp.worker.weights", - fast_helper_wait_config(), - &mut || { - pump_count = pump_count.saturating_add(1); - }, - ) - .expect("quiet helper eventually returns expected event"); - - assert_eq!( - event.get("type").and_then(Value::as_str), - Some("WeightsLoaded") - ); - assert!(pump_count >= 2, "pump_count={pump_count}"); - let frames = drain_json_frames(&datastream, &subscription); - assert!(frames.iter().any(|(channel, value)| { - channel == "mvp.worker.weights" - && value.get("type").and_then(Value::as_str) == Some("WeightsLoaded") - })); - assert!(frames.iter().any(|(channel, value)| { - channel == NODE_WORKER_CHANNEL - && value.get("phase").and_then(Value::as_str) == Some("worker_command_wait") - && value.get("status").and_then(Value::as_str) == Some("waiting") - && value - .get("detail") - .and_then(|detail| detail.get("state")) - .and_then(Value::as_str) - == Some("busy_waiting_for_helper_stdout") - })); - } - - #[test] - fn helper_wait_errors_on_worker_fatal_event() { - let config = test_config(None); - let mut datastream = NodeDatastream::new(&config); - let channel = datastream.channel_by_name("mvp.worker.weights"); - let (tx, rx) = mpsc::channel(); - tx.send(HelperStdoutEvent::Line( - json!({"type":"WorkerFatal","error":"boom"}).to_string() + "\n", - )) - .expect("send fatal event"); - let mut pump_count = 0_u32; - - let error = wait_for_helper_event( - &rx, - None, - "WeightsLoaded", - "LoadWeights", - &config, - &mut datastream, - channel, - "mvp.worker.weights", - fast_helper_wait_config(), - &mut || { - pump_count = pump_count.saturating_add(1); - }, - ) - .expect_err("worker fatal must fail command wait"); - - assert!(error.contains("worker fatal")); - assert_eq!(pump_count, 1); - } - - #[test] - fn helper_wait_errors_on_closed_stdout_event() { - let config = test_config(None); - let mut datastream = NodeDatastream::new(&config); - let channel = datastream.channel_by_name("mvp.worker.weights"); - let (tx, rx) = mpsc::channel(); - tx.send(HelperStdoutEvent::Closed) - .expect("send closed stdout event"); - let mut pump_count = 0_u32; - - let error = wait_for_helper_event( - &rx, - None, - "WeightsLoaded", - "LoadWeights", - &config, - &mut datastream, - channel, - "mvp.worker.weights", - fast_helper_wait_config(), - &mut || { - pump_count = pump_count.saturating_add(1); - }, - ) - .expect_err("closed stdout must fail command wait"); - - assert!(error.contains("stdout closed")); - assert_eq!(pump_count, 0); - } - - #[test] - fn sampler_health_start_and_no_sample_records_are_json_events() { - let config = test_config(None); - let mut datastream = NodeDatastream::new(&config); - let health_channel = datastream.channel_by_name(NODE_SAMPLER_CHANNEL); - let subscription = datastream.endpoint.subscribe_all("sampler-health-test"); - let context = SamplerHealthContext::from_config(&config); - - for (sampler, sample_channel) in [ - ("gpu", datastream::hardware::gpu::HOST_GPU_CHANNEL), - ("cpu", datastream::hardware::cpu::HOST_CPU_CHANNEL), - ("net", datastream::hardware::net::HOST_NET_CHANNEL), - ] { - submit_sampler_started( - &datastream.producer, - health_channel, - context, - sampler, - sample_channel, - Duration::from_secs(1), - ); - } - submit_sampler_sample_health( - &datastream.producer, - health_channel, - context, - "gpu", - datastream::hardware::gpu::HOST_GPU_CHANNEL, - 0, - Some("nvidia-smi unavailable"), - ); - - let frames = drain_json_frames(&datastream, &subscription); - let sampler_events = frames - .iter() - .filter(|(channel, _)| channel == NODE_SAMPLER_CHANNEL) - .map(|(_, value)| value) - .collect::>(); - assert_eq!(sampler_events.len(), 7); - assert!(sampler_events.iter().all(|value| { - value - .get("producer_instance_id") - .and_then(Value::as_str) - .is_some_and(|id| id.starts_with("mvp-worker-node:7:node-11:pid-")) - })); - assert!(sampler_events.iter().all(|value| { - value - .get("benchmark") - .and_then(|benchmark| benchmark.get("producer_instance_id")) - == value.get("producer_instance_id") - })); - for sampler in ["gpu", "cpu", "net"] { - assert!(sampler_events.iter().any(|value| { - value.get("type").and_then(Value::as_str) == Some("SamplerHealth") - && value.get("schema").and_then(Value::as_str) - == Some("mvp.node.sampler.health.v1") - && value.get("run_id").and_then(Value::as_u64) == Some(7) - && value.get("node_id").and_then(Value::as_u64) == Some(11) - && value.get("stage_index").and_then(Value::as_u64) == Some(3) - && value.get("sampler").and_then(Value::as_str) == Some(sampler) - && value.get("status").and_then(Value::as_str) == Some("started") - })); - assert!(sampler_events.iter().any(|value| { - value.get("sampler").and_then(Value::as_str) == Some(sampler) - && value.get("status").and_then(Value::as_str) == Some("waiting") - && value - .get("detail") - .and_then(|detail| detail.get("state")) - .and_then(Value::as_str) - == Some("no_sample_yet") - })); - } - assert!(sampler_events.iter().any(|value| { - value.get("sampler").and_then(Value::as_str) == Some("gpu") - && value.get("status").and_then(Value::as_str) == Some("failed") - && value - .get("detail") - .and_then(|detail| detail.get("error")) - .and_then(Value::as_str) - == Some("nvidia-smi unavailable") - })); - } -} diff --git a/crates/mvp-system/src/orchestration/actor.rs b/crates/mvp-system/src/orchestration/actor.rs index 77b3c0a..b995019 100644 --- a/crates/mvp-system/src/orchestration/actor.rs +++ b/crates/mvp-system/src/orchestration/actor.rs @@ -489,63 +489,3 @@ impl From<&core::TokenObjectPayload> for TokenObjectPayloadWire { } } } - -#[cfg(test)] -mod tests { - use super::*; - use iroh::SecretKey; - use swactor::config::RuntimeConfig; - use swactor::runtime::Runtime; - - #[test] - fn orchestrator_actor_reports_node_runtime_ready() { - let runtime = Runtime::new(RuntimeConfig::default()); - let reports = runtime - .new_inbox::() - .expect("orchestrator report inbox"); - let report_to = *reports.addr(); - let actor = runtime - .spawn(OrchestratorActor::new( - core::RunConfig { - run_id: core::RunId(7), - max_tokens: 1, - prompt: Vec::new(), - }, - Some(report_to), - )) - .expect("spawn orchestrator actor"); - let endpoint = EndpointAddr::new(SecretKey::from_bytes(&[8; 32]).public()); - let node_actor = ActorAddress::new_random(); - let datastream_publisher = ActorAddress::new_random(); - - runtime - .send_to( - actor, - OrchestratorMsg::ObserveNodeRuntimeReady { - run_id: 7, - node_id: 11, - stage_index: 3, - endpoint: endpoint.clone(), - node_actor, - datastream_publisher, - readiness_id: 99, - }, - ) - .expect("send runtime ready"); - runtime.tick(); - - assert_eq!( - reports.try_recv(), - Some(OrchestratorReport::NodeRuntimeReady { - run_id: 7, - node_id: 11, - stage_index: 3, - endpoint, - node_actor, - datastream_publisher, - readiness_id: 99, - }) - ); - assert_eq!(reports.try_recv(), None); - } -} diff --git a/crates/mvp-system/src/orchestration/app.rs b/crates/mvp-system/src/orchestration/app.rs index 991bcbe..2cc2b94 100644 --- a/crates/mvp-system/src/orchestration/app.rs +++ b/crates/mvp-system/src/orchestration/app.rs @@ -6077,3218 +6077,3 @@ where .parse::() .map_err(|e| format!("invalid {name}={value:?}: {e}")) } - -#[cfg(test)] -mod tests { - use super::*; - use distribution::swim::node::{SwimObservation, SwimObserver}; - use std::{ffi::OsString, path::PathBuf}; - - static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); - - const ENV_KEYS: &[&str] = &[ - CACHED_MODEL_HOST_ENV, - "HOME", - "HF_TOKEN", - "MVP_CPU_LINE_PROFILE", - "MVP_CPU_LINE_PROFILE_INTERVAL_MS", - "CUDA_DEVICE_SCHEDULE", - "MVP_DASHBOARD", - "MVP_DOCKER_GPUS", - "MVP_GGUF_FILE", - "MVP_GGUF_LOCAL_PATH", - "MVP_GGUF_REPO", - "MVP_GGUF_REVISION", - "MVP_IROH_RELAY_MODE", - MVP_IROH_RELAY_URL_ENV, - "MVP_LAYER_END_EXCLUSIVE", - "MVP_LOGICAL_NODE_ID", - "MVP_MODEL_CACHE_DIR", - "MVP_MAX_CONTEXT", - "MVP_MODEL_ID", - "MVP_NODE_IMAGE", - "MVP_NODE_PROVIDER", - "MVP_PROVIDER", - "MVP_PROMPT_MAX_TOKENS", - "MVP_PROMPT_RPC_BIND", - "MVP_RUN_ID", - "MVP_RUNTIME_CONFIG", - "MVP_PIPELINE_STAGES", - "MVP_STAGE_INDEX", - "MVP_TOKEN_PROGRESS_EVERY", - "MVP_TINYGRAD_WORKER", - "MVP_TOKENIZER_LOCAL_PATH", - "MVP_VASTAI_API_KEY", - "MVP_VASTAI_BOOTSTRAP_COMMAND", - "MVP_VASTAI_CONFIRM_LEASE", - "MVP_VASTAI_DISK_GB", - "MVP_VASTAI_GPU_NAME", - "MVP_VASTAI_MIN_DOWN_MBPS", - "MVP_VASTAI_MIN_GPU_RAM_MB", - "MVP_VASTAI_MIN_RELIABILITY", - "MVP_VASTAI_MIN_UP_MBPS", - "MVP_VASTAI_ONSTART", - "MVP_VASTAI_POLL_INTERVAL_SECS", - "MVP_VASTAI_REQUIRE_VERIFIED", - "MVP_VASTAI_SSH_USER", - "MVP_VASTAI_SSH_IDENTITY", - "VASTAI_API_KEY", - SWACTOR_IROH_RELAY_URL_ENV, - MVP_DOCKER_CONTAINER_PREFIX_ENV, - MVP_WORKER_BIN_ENV, - ]; - - struct RestoreEnv { - saved: Vec<(&'static str, Option)>, - } - - impl Drop for RestoreEnv { - fn drop(&mut self) { - for (key, value) in &self.saved { - match value { - Some(value) => unsafe { std::env::set_var(key, value) }, - None => unsafe { std::env::remove_var(key) }, - } - } - } - } - - fn with_clean_env(settings: &[(&'static str, &'static str)], test: impl FnOnce() -> T) -> T { - let settings = settings - .iter() - .map(|(key, value)| (*key, OsString::from(value))) - .collect::>(); - with_clean_env_os(&settings, test) - } - - fn with_clean_env_os(settings: &[(&'static str, OsString)], test: impl FnOnce() -> T) -> T { - let _lock = ENV_LOCK - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - let saved = ENV_KEYS - .iter() - .map(|&key| (key, std::env::var_os(key))) - .collect::>(); - for key in ENV_KEYS { - unsafe { std::env::remove_var(key) }; - } - for (key, value) in settings { - assert!( - ENV_KEYS.contains(key), - "test env key {key} must be restored" - ); - unsafe { std::env::set_var(key, value) }; - } - let _restore = RestoreEnv { saved }; - test() - } - - fn selected_provider(settings: &[(&'static str, &'static str)]) -> ProviderKind { - with_clean_env(settings, || { - Config::from_layers_with_path_and_args(None, std::iter::empty::()) - .expect("config parses") - .provider - }) - } - - fn canonical_relay_url(raw: &str) -> String { - raw.parse::() - .expect("fixture relay URL parses") - .to_string() - } - - fn node_spec_env(settings: &[(&'static str, &'static str)]) -> Vec<(String, String)> { - with_clean_env(settings, || { - let config = Config::from_layers_with_path_and_args(None, std::iter::empty::()) - .expect("config parses"); - let coordinator = EndpointAddr::new(iroh::SecretKey::from_bytes(&[9; 32]).public()); - let orchestrator_actor = ActorAddress([12; 32]); - config - .node_spec(coordinator, orchestrator_actor) - .expect("node spec builds") - .env - }) - } - - fn env_value<'a>(env: &'a [(String, String)], key: &str) -> Option<&'a str> { - env.iter() - .find(|(env_key, _)| env_key == key) - .map(|(_, value)| value.as_str()) - } - - fn cached_model_config_with_args(model: &TempModelFile, args: &[&str]) -> Config { - with_clean_env_os( - &[ - ("MVP_RUNTIME_CONFIG", OsString::from("local")), - ("MVP_NODE_PROVIDER", OsString::from("docker")), - (CACHED_MODEL_HOST_ENV, model.raw_path.as_os_str().to_owned()), - ], - || { - Config::from_layers_with_path_and_args( - None, - args.iter().copied().map(str::to_owned), - ) - .expect("cached-model config parses") - }, - ) - } - - fn pipeline_config_with_cached_model(model: &TempModelFile, pipeline_stages: u32) -> Config { - let stages_arg = pipeline_stages.to_string(); - cached_model_config_with_args( - model, - &[ - "--pipeline-stages", - stages_arg.as_str(), - "--max-context", - "512", - ], - ) - } - - fn expected_layer_ranges(num_layers: u32, stage_count: u32) -> Vec<(u32, u32)> { - (0..stage_count) - .map(|stage_index| { - let start = (u64::from(num_layers) * u64::from(stage_index) - / u64::from(stage_count)) as u32; - let end = (u64::from(num_layers) * u64::from(stage_index + 1) - / u64::from(stage_count)) as u32; - (start, end) - }) - .collect() - } - #[test] - fn pipeline_weight_load_scheduler_keeps_all_unloaded_stages_pending() { - let model = TempModelFile::with_metadata( - "seven-stage-scheduler.gguf", - TestGgufMetadata { - num_layers: 30, - ..TestGgufMetadata::default() - }, - ); - let config = pipeline_config_with_cached_model(&model, 7); - let plan = config.build_run_plan().expect("seven-stage plan builds"); - let mut loaded = BTreeSet::new(); - - assert_eq!( - pending_pipeline_weight_load_stages(&plan, &loaded) - .iter() - .map(|stage| stage.stage_index) - .collect::>(), - vec![0, 1, 2, 3, 4, 5, 6] - ); - - loaded.insert(0); - loaded.insert(3); - assert_eq!( - pending_pipeline_weight_load_stages(&plan, &loaded) - .iter() - .map(|stage| stage.stage_index) - .collect::>(), - vec![1, 2, 4, 5, 6] - ); - - for stage_index in 0..7 { - loaded.insert(stage_index); - } - assert!( - pending_pipeline_weight_load_stages(&plan, &loaded).is_empty(), - "all stages loaded should leave no pending load" - ); - } - - fn assert_plan_matches_metadata( - config: &Config, - plan: &run_plan::RunPlan, - metadata: TestGgufMetadata, - stage_count: u32, - requested_context: u64, - ) { - assert_eq!(plan.run_id, run_plan::RunId(config.run_id)); - assert_eq!(plan.model.model_id, config.model_id); - assert_eq!(plan.model.gguf_source, config.gguf_source); - assert_eq!(plan.model.num_layers, metadata.num_layers); - assert_eq!(plan.model.hidden_dim, metadata.hidden_dim as u32); - assert_eq!( - plan.model.max_seq_len, - requested_context.min(metadata.context_length) as u32 - ); - assert_eq!(plan.model.eos_token_id, metadata.eos_token_id); - assert_eq!(plan.stages.len(), stage_count as usize); - assert_eq!(plan.edges.len(), stage_count as usize + 1); - assert_eq!(plan.max_tokens, config.default_max_tokens); - - let mut stages = plan.stages.clone(); - stages.sort_by_key(|stage| stage.stage_index); - let expected_ranges = expected_layer_ranges(metadata.num_layers, stage_count); - for (stage, expected_range) in stages.iter().zip(expected_ranges) { - assert_eq!(stage.stage_count, stage_count); - assert_eq!( - stage.node_id, - run_plan::NodeId(config.node_id + 1 + u64::from(stage.stage_index)) - ); - assert_eq!( - (stage.layer_start, stage.layer_end_exclusive), - expected_range - ); - assert!( - stage.layer_start < stage.layer_end_exclusive, - "stage {} must have a non-empty layer range", - stage.stage_index - ); - } - } - - fn token_object_spec(max_extent: u64, alignment: u32) -> run_plan::ObjectSpec { - run_plan::ObjectSpec { - kind: run_plan::ObjectKind::Token, - max_extent, - dtype_family: run_plan::DTypeFamily::BFloat, - dtype_width_bytes: 4, - shape: run_plan::ShapeRule::TokenIds, - layout: run_plan::LayoutRule::Contiguous, - alignment, - sequence_policy: run_plan::SequencePolicy::Ordered, - } - } - - fn decode_token_record_payload( - bytes: &[u8], - spec: run_plan::ObjectSpec, - ) -> (u64, Vec, bool) { - let read = ingress::read_object_record(bytes, ingress_object_spec_from_plan(spec), false) - .expect("token record should be valid"); - let ingress::ObjectRecordRead::Complete(record) = read else { - panic!("token record should be complete"); - }; - let payload = record.payload(bytes).expect("token payload is present"); - let tokens = payload - .chunks_exact(4) - .map(|chunk| u32::from_le_bytes(chunk.try_into().unwrap())) - .collect::>(); - assert!(payload.chunks_exact(4).remainder().is_empty()); - (record.sequence, tokens, record.flags.end_of_sequence) - } - - fn decode_token_record_with_flags( - bytes: &[u8], - spec: run_plan::ObjectSpec, - ) -> (u64, Vec, bool, bool) { - let read = ingress::read_object_record(bytes, ingress_object_spec_from_plan(spec), false) - .expect("token record should be valid"); - let ingress::ObjectRecordRead::Complete(record) = read else { - panic!("token record should be complete"); - }; - let payload = record.payload(bytes).expect("token payload is present"); - let tokens = payload - .chunks_exact(4) - .map(|chunk| u32::from_le_bytes(chunk.try_into().unwrap())) - .collect::>(); - assert!(payload.chunks_exact(4).remainder().is_empty()); - ( - record.sequence, - tokens, - record.flags.end_of_sequence, - record.flags.begin_sequence, - ) - } - - struct PipelineRuntimeTestFixture { - actor_runtime: Arc, - tokenizer_events: swactor::runtime::Inbox, - encode_requests: swactor::runtime::Inbox, - decode_requests: swactor::runtime::Inbox, - runtime: PipelinePromptRuntime, - token_in_rx: tokio_mpsc::UnboundedReceiver>, - } - - fn pipeline_runtime_fixture() -> PipelineRuntimeTestFixture { - let spec = token_object_spec(64, 64); - let (token_in_tx, token_in_rx) = tokio_mpsc::unbounded_channel(); - let (recv_tx, recv_rx) = mpsc::channel(); - pipeline_runtime_fixture_from_specs( - 11, - 12, - spec, - spec, - token_in_tx, - token_in_rx, - recv_tx, - recv_rx, - ) - } - - fn pipeline_runtime_fixture_from_plan(stage_count: u32) -> PipelineRuntimeTestFixture { - let model = TempModelFile::new(&format!("prompt-plan-{stage_count}.gguf")); - let config = pipeline_config_with_cached_model(&model, stage_count); - let plan = config.build_run_plan().expect("prompt runtime plan builds"); - let token_in = plan - .edges - .iter() - .find(|edge| edge.kind == run_plan::EdgeKind::TokenIn) - .expect("token-in edge"); - let token_out = plan - .edges - .iter() - .find(|edge| edge.kind == run_plan::EdgeKind::TokenOut) - .expect("token-out edge"); - let (token_in_tx, token_in_rx) = tokio_mpsc::unbounded_channel(); - let (recv_tx, recv_rx) = mpsc::channel(); - pipeline_runtime_fixture_from_specs( - token_in.edge_id.0, - token_out.edge_id.0, - token_in.object_spec, - token_out.object_spec, - token_in_tx, - token_in_rx, - recv_tx, - recv_rx, - ) - } - - #[allow(clippy::too_many_arguments)] - fn pipeline_runtime_fixture_from_specs( - token_in_edge_id: u64, - token_out_edge_id: u64, - token_spec: run_plan::ObjectSpec, - token_out_spec: run_plan::ObjectSpec, - token_in_tx: tokio_mpsc::UnboundedSender>, - token_in_rx: tokio_mpsc::UnboundedReceiver>, - recv_tx: mpsc::Sender>, - recv_rx: mpsc::Receiver>, - ) -> PipelineRuntimeTestFixture { - let actor_runtime = Arc::new(swactor::runtime::Runtime::new( - swactor::config::RuntimeConfig::default(), - )); - let encode_requests = actor_runtime - .new_inbox::() - .expect("encode request inbox"); - let decode_requests = actor_runtime - .new_inbox::() - .expect("decode request inbox"); - let tokenizer_events = actor_runtime - .new_inbox::() - .expect("tokenizer event inbox"); - PipelineRuntimeTestFixture { - runtime: PipelinePromptRuntime { - token_in_edge_id, - token_out_edge_id, - token_spec, - token_out_spec, - token_in_sender: PipelineSendHandle::Channel(token_in_tx), - recv_rx, - recv_tx, - recv_buffer: Vec::new(), - tokenizer_encode_actor: *encode_requests.addr(), - tokenizer_decode_actor: *decode_requests.addr(), - tokenizer_reply_to: *tokenizer_events.addr(), - pending_encode: None, - pending_decode: None, - next_sequence: 0, - generated_tokens: Vec::new(), - final_text: String::new(), - active: None, - started_at: None, - last_progress_at: None, - next_wait_log_at: None, - }, - actor_runtime, - tokenizer_events, - encode_requests, - decode_requests, - token_in_rx, - } - } - - fn start_fixture_prompt( - fixture: &mut PipelineRuntimeTestFixture, - request: SubmitPrompt, - events: mpsc::Sender, - datastream: &mut OrchDatastream, - run_id: u64, - node_id: u64, - ) { - fixture - .runtime - .start_prompt( - request, - events, - &fixture.actor_runtime, - None, - datastream, - run_id, - node_id, - ) - .expect("pipeline prompt starts"); - } - - fn drain_fixture_tokenizer( - fixture: &mut PipelineRuntimeTestFixture, - datastream: &mut OrchDatastream, - run_id: u64, - node_id: u64, - ) { - fixture - .runtime - .drain_tokenizer_events( - &fixture.actor_runtime, - &fixture.tokenizer_events, - None, - datastream, - run_id, - node_id, - ) - .expect("tokenizer events drain"); - } - - fn send_encoded_tokens( - fixture: &PipelineRuntimeTestFixture, - request_id: u64, - tokens: Vec, - ) { - fixture - .actor_runtime - .send_to( - *fixture.tokenizer_events.addr(), - TokenizerEvent::PromptEncoded { request_id, tokens }, - ) - .expect("send encoded tokens"); - } - - fn send_decoded_text(fixture: &PipelineRuntimeTestFixture, request_id: u64, text: &str) { - fixture - .actor_runtime - .send_to( - *fixture.tokenizer_events.addr(), - TokenizerEvent::TokensDecoded { - request_id, - text: text.to_owned(), - }, - ) - .expect("send decoded text"); - } - - fn assert_encode_request(fixture: &PipelineRuntimeTestFixture, request_id: u64, prompt: &str) { - match fixture.encode_requests.try_recv() { - Some(NodeAgentMsg::EncodePrompt { - request_id: actual_request_id, - prompt: actual_prompt, - reply_to, - }) => { - assert_eq!(actual_request_id, request_id); - assert_eq!(actual_prompt, prompt); - assert_eq!(reply_to, *fixture.tokenizer_events.addr()); - } - other => panic!("expected EncodePrompt request, got {other:?}"), - } - } - - fn assert_decode_request(fixture: &PipelineRuntimeTestFixture, request_id: u64, token: u32) { - match fixture.decode_requests.try_recv() { - Some(NodeAgentMsg::DecodeTokens { - request_id: actual_request_id, - tokens, - reply_to, - }) => { - assert_eq!(actual_request_id, request_id); - assert_eq!(tokens, vec![token]); - assert_eq!(reply_to, *fixture.tokenizer_events.addr()); - } - other => panic!("expected DecodeTokens request, got {other:?}"), - } - } - - #[test] - fn pipeline_prompt_token_record_round_trips_one_token_with_eos_and_plan_ring_alignment() { - let spec = token_object_spec(64, 64); - let bytes = encode_token_record(spec, 12, 7, &[513], true).expect("token record encodes"); - assert_eq!(&bytes[0..4], b"MO01"); - - let mut partial = bytes[..run_plan::MO01_HEADER_BYTES as usize + 2].to_vec(); - assert!( - take_pipeline_token_record(&mut partial, spec) - .expect("partial token record is not malformed") - .is_none() - ); - assert_eq!(partial.len(), run_plan::MO01_HEADER_BYTES as usize + 2); - - let mut buffer = bytes; - let record = take_pipeline_token_record(&mut buffer, spec) - .expect("one-u32 token-out payload should parse despite ring alignment") - .expect("complete token-out record is available"); - - assert_eq!(record.object_id, 9007); - assert_eq!(record.sequence, 7); - assert_eq!(record.token_id, 513); - assert!(record.eos); - assert!(buffer.is_empty()); - } - - #[test] - fn plan_derived_mo01_specs_accept_token_and_activation_record_extents() { - let metadata = TestGgufMetadata { - num_layers: 7, - hidden_dim: 13, - context_length: 64, - eos_token_id: 11, - }; - let model = TempModelFile::with_metadata("plan-mo01.gguf", metadata); - let config = pipeline_config_with_cached_model(&model, 3); - let plan = config - .build_run_plan() - .expect("plan-derived MO01 plan builds"); - let token_in = plan - .edges - .iter() - .find(|edge| edge.kind == run_plan::EdgeKind::TokenIn) - .expect("token-in edge"); - let activation = plan - .edges - .iter() - .find(|edge| edge.kind == run_plan::EdgeKind::Activation) - .expect("activation edge"); - - let token_record = encode_token_record( - token_in.object_spec, - token_in.edge_id.0, - 0, - &[65, 195, 169], - false, - ) - .expect("plan token record encodes"); - assert_eq!( - decode_token_record_payload(&token_record, token_in.object_spec), - (0, vec![65, 195, 169], false) - ); - - let activation_payload = vec![ - 0_u8; - usize::try_from(metadata.hidden_dim * 2) - .expect("activation payload size fits usize") - ]; - let activation_record = ingress::ObjectRecordBuilder::new(ingress_object_spec_from_plan( - activation.object_spec, - )) - .object_id(ingress::ObjectId(9100)) - .sequence(0) - .payload(activation_payload) - .encode(); - let read = ingress::read_object_record( - &activation_record, - ingress_object_spec_from_plan(activation.object_spec), - false, - ) - .expect("plan activation record parses"); - let ingress::ObjectRecordRead::Complete(record) = read else { - panic!("activation record should be complete"); - }; - assert_eq!(record.payload(&activation_record).unwrap().len(), 26); - } - - #[test] - fn pipeline_prompt_token_out_parser_rejects_payloads_that_are_not_one_u32() { - let spec = token_object_spec(64, 64); - let mut buffer = - encode_token_record(spec, 12, 0, &[65, 66], false).expect("multi-token record encodes"); - - let error = take_pipeline_token_record(&mut buffer, spec) - .expect_err("token-out records must carry exactly one generated token"); - - assert!( - error.contains("token-out payload must be exactly one u32, got 8"), - "unexpected error: {error}" - ); - } - - #[test] - fn pipeline_prompt_runtime_uses_tokenizer_events_for_encode_decode_and_continuations() { - let mut fixture = pipeline_runtime_fixture(); - let (event_tx, event_rx) = mpsc::channel(); - let mut datastream = OrchDatastream::new(91, None).expect("datastream opens"); - start_fixture_prompt( - &mut fixture, - SubmitPrompt { - request_id: 42, - prompt_text: "Hi".to_owned(), - max_tokens: 2, - }, - event_tx, - &mut datastream, - 91, - 3, - ); - assert_encode_request(&fixture, 42, "Hi"); - send_encoded_tokens(&fixture, 42, vec![1001, 1002]); - drain_fixture_tokenizer(&mut fixture, &mut datastream, 91, 3); - - let initial = fixture - .token_in_rx - .try_recv() - .expect("tokenizer tokens are sent to token-in"); - assert_eq!( - decode_token_record_payload(&initial, fixture.runtime.token_spec), - (0, vec![1001, 1002], false) - ); - assert!(fixture.token_in_rx.try_recv().is_err()); - - fixture - .runtime - .recv_tx - .send( - encode_token_record( - fixture.runtime.token_out_spec, - fixture.runtime.token_out_edge_id, - 0, - &[79], - false, - ) - .expect("first token-out record encodes"), - ) - .expect("token-out bytes enqueue"); - fixture - .runtime - .drain_tokens(&fixture.actor_runtime, None, &mut datastream, 91, 3) - .expect("first token drains"); - assert_decode_request(&fixture, 42, 79); - assert!(event_rx.try_recv().is_err()); - - send_decoded_text(&fixture, 42, "O"); - drain_fixture_tokenizer(&mut fixture, &mut datastream, 91, 3); - assert_eq!( - event_rx.try_recv().expect("first delta event"), - PromptEvent::TextDelta { - request_id: 42, - text: "O".to_owned(), - } - ); - let continuation = fixture - .token_in_rx - .try_recv() - .expect("non-terminal token is fed back to token-in"); - assert_eq!( - decode_token_record_payload(&continuation, fixture.runtime.token_spec), - (1, vec![79], false) - ); - - fixture - .runtime - .recv_tx - .send( - encode_token_record( - fixture.runtime.token_out_spec, - fixture.runtime.token_out_edge_id, - 1, - &[75], - false, - ) - .expect("second token-out record encodes"), - ) - .expect("token-out bytes enqueue"); - fixture - .runtime - .drain_tokens(&fixture.actor_runtime, None, &mut datastream, 91, 3) - .expect("second token drains"); - assert_decode_request(&fixture, 42, 75); - - send_decoded_text(&fixture, 42, "K"); - drain_fixture_tokenizer(&mut fixture, &mut datastream, 91, 3); - assert_eq!( - event_rx.try_recv().expect("second delta event"), - PromptEvent::TextDelta { - request_id: 42, - text: "K".to_owned(), - } - ); - match event_rx.try_recv().expect("done event at max tokens") { - PromptEvent::Done { - request_id, - final_text, - tokens_generated, - .. - } => { - assert_eq!(request_id, 42); - assert_eq!(final_text, "OK"); - assert_eq!(tokens_generated, 2); - } - event => panic!("expected Done at max tokens, got {event:?}"), - } - assert!(event_rx.try_recv().is_err()); - assert!(fixture.token_in_rx.try_recv().is_err()); - assert!(!fixture.runtime.is_active()); - } - - #[test] - fn pipeline_prompt_runtime_rejects_overlapping_prompt_without_dropping_active() { - let mut fixture = pipeline_runtime_fixture(); - let (first_event_tx, first_event_rx) = mpsc::channel(); - let (second_event_tx, second_event_rx) = mpsc::channel(); - let mut datastream = OrchDatastream::new(95, None).expect("datastream opens"); - start_fixture_prompt( - &mut fixture, - SubmitPrompt { - request_id: 501, - prompt_text: "first".to_owned(), - max_tokens: 1, - }, - first_event_tx, - &mut datastream, - 95, - 3, - ); - start_fixture_prompt( - &mut fixture, - SubmitPrompt { - request_id: 502, - prompt_text: "second".to_owned(), - max_tokens: 1, - }, - second_event_tx, - &mut datastream, - 95, - 3, - ); - - assert_encode_request(&fixture, 501, "first"); - assert!(fixture.encode_requests.try_recv().is_none()); - match second_event_rx - .try_recv() - .expect("overlapping prompt receives terminal fault") - { - PromptEvent::Fault { request_id, error } => { - assert_eq!(request_id, 502); - assert!(error.contains("pipeline prompt runtime is busy")); - } - event => panic!("expected busy fault, got {event:?}"), - } - assert!(first_event_rx.try_recv().is_err()); - assert!(fixture.runtime.is_active()); - } - - #[test] - fn pipeline_prompt_runtime_keeps_idle_prompt_active_while_reporting_wait_progress() { - let mut fixture = pipeline_runtime_fixture(); - let (event_tx, event_rx) = mpsc::channel(); - let mut datastream = OrchDatastream::new(96, None).expect("datastream opens"); - start_fixture_prompt( - &mut fixture, - SubmitPrompt { - request_id: 601, - prompt_text: "stalls".to_owned(), - max_tokens: 1, - }, - event_tx, - &mut datastream, - 96, - 3, - ); - fixture.runtime.last_progress_at = Some(Instant::now() - Duration::from_secs(3_600)); - fixture.runtime.next_wait_log_at = Some(Instant::now() - Duration::from_millis(1)); - - fixture - .runtime - .emit_wait_progress(None, &mut datastream, 96, 3); - - assert!( - event_rx.try_recv().is_err(), - "idle prompt should not receive a terminal timeout fault" - ); - assert!(fixture.runtime.is_active()); - } - - #[test] - fn pipeline_prompt_runtime_marks_each_prompt_start_without_resetting_stream_sequence() { - let mut fixture = pipeline_runtime_fixture(); - let (first_event_tx, first_event_rx) = mpsc::channel(); - let mut datastream = OrchDatastream::new(94, None).expect("datastream opens"); - start_fixture_prompt( - &mut fixture, - SubmitPrompt { - request_id: 101, - prompt_text: "first".to_owned(), - max_tokens: 1, - }, - first_event_tx, - &mut datastream, - 94, - 3, - ); - assert_encode_request(&fixture, 101, "first"); - send_encoded_tokens(&fixture, 101, vec![10, 11]); - drain_fixture_tokenizer(&mut fixture, &mut datastream, 94, 3); - let first_initial = fixture - .token_in_rx - .try_recv() - .expect("first prompt token-in record is emitted"); - assert_eq!( - decode_token_record_with_flags(&first_initial, fixture.runtime.token_spec), - (0, vec![10, 11], false, true) - ); - fixture - .runtime - .recv_tx - .send( - encode_token_record( - fixture.runtime.token_out_spec, - fixture.runtime.token_out_edge_id, - 0, - &[21], - false, - ) - .expect("first token-out record encodes"), - ) - .expect("first token-out bytes enqueue"); - fixture - .runtime - .drain_tokens(&fixture.actor_runtime, None, &mut datastream, 94, 3) - .expect("first token drains"); - assert_decode_request(&fixture, 101, 21); - send_decoded_text(&fixture, 101, "A"); - drain_fixture_tokenizer(&mut fixture, &mut datastream, 94, 3); - assert!(matches!( - first_event_rx.try_recv().expect("first delta"), - PromptEvent::TextDelta { - request_id: 101, - .. - } - )); - assert!(matches!( - first_event_rx.try_recv().expect("first done"), - PromptEvent::Done { - request_id: 101, - .. - } - )); - assert!(!fixture.runtime.is_active()); - - let (second_event_tx, _second_event_rx) = mpsc::channel(); - start_fixture_prompt( - &mut fixture, - SubmitPrompt { - request_id: 102, - prompt_text: "second".to_owned(), - max_tokens: 1, - }, - second_event_tx, - &mut datastream, - 94, - 3, - ); - assert_encode_request(&fixture, 102, "second"); - send_encoded_tokens(&fixture, 102, vec![20, 22, 24]); - drain_fixture_tokenizer(&mut fixture, &mut datastream, 94, 3); - let second_initial = fixture - .token_in_rx - .try_recv() - .expect("second prompt token-in record is emitted"); - assert_eq!( - decode_token_record_with_flags(&second_initial, fixture.runtime.token_spec), - (1, vec![20, 22, 24], false, true) - ); - } - - #[test] - fn prompt_runtime_uses_plan_derived_token_edges_for_cached_n_one() { - let mut fixture = pipeline_runtime_fixture_from_plan(1); - let (event_tx, event_rx) = mpsc::channel(); - let mut datastream = OrchDatastream::new(93, None).expect("datastream opens"); - start_fixture_prompt( - &mut fixture, - SubmitPrompt { - request_id: 88, - prompt_text: "Aé".to_owned(), - max_tokens: 1, - }, - event_tx, - &mut datastream, - 93, - 3, - ); - assert_encode_request(&fixture, 88, "Aé"); - send_encoded_tokens(&fixture, 88, vec![321, 654]); - drain_fixture_tokenizer(&mut fixture, &mut datastream, 93, 3); - - let initial = fixture - .token_in_rx - .try_recv() - .expect("plan-derived token-in record is emitted"); - assert_eq!( - decode_token_record_payload(&initial, fixture.runtime.token_spec), - (0, vec![321, 654], false) - ); - assert_eq!(fixture.runtime.token_in_edge_id, 1); - assert_eq!(fixture.runtime.token_out_edge_id, 2); - - fixture - .runtime - .recv_tx - .send( - encode_token_record( - fixture.runtime.token_out_spec, - fixture.runtime.token_out_edge_id, - 0, - &[33], - false, - ) - .expect("plan-derived token-out record encodes"), - ) - .expect("token-out bytes enqueue"); - fixture - .runtime - .drain_tokens(&fixture.actor_runtime, None, &mut datastream, 93, 3) - .expect("plan-derived token drains"); - assert_decode_request(&fixture, 88, 33); - send_decoded_text(&fixture, 88, "!"); - drain_fixture_tokenizer(&mut fixture, &mut datastream, 93, 3); - - assert_eq!( - event_rx.try_recv().expect("delta event"), - PromptEvent::TextDelta { - request_id: 88, - text: "!".to_owned(), - } - ); - match event_rx.try_recv().expect("done event at max tokens") { - PromptEvent::Done { - request_id, - final_text, - tokens_generated, - .. - } => { - assert_eq!(request_id, 88); - assert_eq!(final_text, "!"); - assert_eq!(tokens_generated, 1); - } - event => panic!("expected Done at max tokens, got {event:?}"), - } - assert!(fixture.token_in_rx.try_recv().is_err()); - } - - #[test] - fn pipeline_prompt_runtime_finishes_on_eos_without_feedback_token() { - let mut fixture = pipeline_runtime_fixture(); - let (event_tx, event_rx) = mpsc::channel(); - let mut datastream = OrchDatastream::new(92, None).expect("datastream opens"); - start_fixture_prompt( - &mut fixture, - SubmitPrompt { - request_id: 77, - prompt_text: "go".to_owned(), - max_tokens: 8, - }, - event_tx, - &mut datastream, - 92, - 4, - ); - assert_encode_request(&fixture, 77, "go"); - send_encoded_tokens(&fixture, 77, vec![700]); - drain_fixture_tokenizer(&mut fixture, &mut datastream, 92, 4); - fixture - .token_in_rx - .try_recv() - .expect("initial prompt token-in record is sent"); - - fixture - .runtime - .recv_tx - .send( - encode_token_record( - fixture.runtime.token_out_spec, - fixture.runtime.token_out_edge_id, - 0, - &[33], - true, - ) - .expect("eos token-out record encodes"), - ) - .expect("token-out bytes enqueue"); - fixture - .runtime - .drain_tokens(&fixture.actor_runtime, None, &mut datastream, 92, 4) - .expect("eos token drains"); - assert_decode_request(&fixture, 77, 33); - send_decoded_text(&fixture, 77, "!"); - drain_fixture_tokenizer(&mut fixture, &mut datastream, 92, 4); - - assert_eq!( - event_rx.try_recv().expect("delta event before eos done"), - PromptEvent::TextDelta { - request_id: 77, - text: "!".to_owned(), - } - ); - match event_rx.try_recv().expect("done event on eos") { - PromptEvent::Done { - request_id, - final_text, - tokens_generated, - .. - } => { - assert_eq!(request_id, 77); - assert_eq!(final_text, "!"); - assert_eq!(tokens_generated, 1); - } - event => panic!("expected Done on eos, got {event:?}"), - } - assert!(event_rx.try_recv().is_err()); - assert!(fixture.token_in_rx.try_recv().is_err()); - assert!(!fixture.runtime.is_active()); - } - - #[test] - fn planned_prompt_runtime_uses_pipeline_edges_for_single_or_multi_stage_cached_models() { - assert_eq!( - prompt_runtime_mode(None), - PromptRuntimeMode::DirectInferPrompt - ); - - for stage_count in [1_u32, 3] { - let model = TempModelFile::new(&format!("prompt-runtime-{stage_count}.gguf")); - let config = pipeline_config_with_cached_model(&model, stage_count); - let plan = config - .build_run_plan() - .expect("cached-model planned prompt runtime plan builds"); - - assert_eq!( - prompt_runtime_mode(Some(&plan)), - PromptRuntimeMode::PipelineTokenEdges, - "cached-model N={stage_count} must use the planned token-edge runtime" - ); - } - } - - #[test] - fn docker_container_prefix_defaults_and_trims_env_override() { - with_clean_env(&[], || { - assert_eq!(docker_container_prefix(), DEFAULT_DOCKER_CONTAINER_PREFIX); - }); - with_clean_env( - &[(MVP_DOCKER_CONTAINER_PREFIX_ENV, " custom-prefix ")], - || { - assert_eq!(docker_container_prefix(), "custom-prefix"); - }, - ); - } - - #[test] - fn expand_home_path_expands_leading_home_segment() { - with_clean_env(&[("HOME", "/tmp/mvp-vastai-home")], || { - assert_eq!( - expand_home_path("~/keys/deploy").expect("home path expands"), - PathBuf::from("/tmp/mvp-vastai-home/keys/deploy") - ); - assert_eq!( - expand_home_path("/tmp/not-~/expanded").expect("literal path stays literal"), - PathBuf::from("/tmp/not-~/expanded") - ); - }); - } - - #[test] - fn account_ssh_keys_output_contains_public_key_matches_exact_key_material() { - let public_key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITestKeyBody vastai"; - - assert!(account_ssh_keys_output_contains_public_key( - public_key, public_key - )); - assert!(account_ssh_keys_output_contains_public_key( - r#"{"keys":[{"public_key":"AAAAC3NzaC1lZDI1NTE5AAAAITestKeyBody"}]}"#, - public_key - )); - assert!(!account_ssh_keys_output_contains_public_key( - r#"{"keys":[{"public_key":"AAAAC3NzaC1lZDI1NTE5AAAADifferent"}]}"#, - public_key - )); - } - - #[test] - fn vastai_config_reads_ssh_identity_without_runtime_preparation() { - let config = with_clean_env( - &[ - ("MVP_RUNTIME_CONFIG", "deploy"), - ("MVP_NODE_PROVIDER", "vastai"), - ("MVP_VASTAI_API_KEY", "vast-key"), - ("MVP_VASTAI_BOOTSTRAP_COMMAND", "/usr/local/bin/mvp-node"), - ("MVP_VASTAI_SSH_IDENTITY", "/tmp/mvp-vastai-key"), - ], - || { - Config::from_layers_with_path_and_args(None, std::iter::empty::()) - .expect("vastai config parses without ssh-keygen or vastai CLI") - }, - ); - let vastai = config.vastai.expect("vastai config is present"); - assert_eq!( - vastai.ssh_identity.as_deref(), - Some(Path::new("/tmp/mvp-vastai-key")) - ); - assert_eq!(vastai.ssh_public_key, None); - assert_eq!(vastai.ssh_public_fingerprint, None); - } - - #[derive(Clone, Copy)] - struct TestGgufMetadata { - num_layers: u32, - hidden_dim: u64, - context_length: u64, - eos_token_id: u32, - } - - impl Default for TestGgufMetadata { - fn default() -> Self { - Self { - num_layers: 7, - hidden_dim: 13, - context_length: 64, - eos_token_id: 11, - } - } - } - - struct TempModelFile { - root: PathBuf, - raw_path: PathBuf, - canonical_path: PathBuf, - metadata: TestGgufMetadata, - } - - impl TempModelFile { - fn new(file_name: &str) -> Self { - Self::with_metadata(file_name, TestGgufMetadata::default()) - } - - fn with_metadata(file_name: &str, metadata: TestGgufMetadata) -> Self { - let root = std::env::temp_dir().join(format!( - "mvp-cached-model-test-{}-{}", - std::process::id(), - std::thread::current().name().unwrap_or("unnamed") - )); - let _ = std::fs::remove_dir_all(&root); - std::fs::create_dir_all(root.join("nested")).expect("create temp model dir"); - let canonical_path = root.join(file_name); - std::fs::write(&canonical_path, minimal_gguf(metadata)) - .expect("write temp GGUF metadata file"); - let raw_path = root.join("nested").join("..").join(file_name); - Self { - root, - raw_path, - canonical_path: canonical_path - .canonicalize() - .expect("canonicalize temp model file"), - metadata, - } - } - } - - fn minimal_gguf(metadata: TestGgufMetadata) -> Vec { - let mut bytes = Vec::new(); - bytes.extend_from_slice(b"GGUF"); - bytes.extend_from_slice(&3_u32.to_le_bytes()); - bytes.extend_from_slice(&0_u64.to_le_bytes()); - bytes.extend_from_slice(&6_u64.to_le_bytes()); - push_string_kv(&mut bytes, "general.architecture", "llama"); - push_string_kv(&mut bytes, "general.name", "fixture"); - push_u32_kv(&mut bytes, "llama.block_count", metadata.num_layers); - push_u32_kv( - &mut bytes, - "llama.embedding_length", - metadata - .hidden_dim - .try_into() - .expect("test hidden dimension fits u32"), - ); - push_u32_kv( - &mut bytes, - "llama.context_length", - metadata - .context_length - .try_into() - .expect("test context length fits u32"), - ); - push_u32_kv( - &mut bytes, - "tokenizer.ggml.eos_token_id", - metadata.eos_token_id, - ); - bytes - } - - fn push_string_kv(bytes: &mut Vec, key: &str, value: &str) { - push_gguf_string(bytes, key); - bytes.extend_from_slice(&8_u32.to_le_bytes()); - push_gguf_string(bytes, value); - } - - fn push_u32_kv(bytes: &mut Vec, key: &str, value: u32) { - push_gguf_string(bytes, key); - bytes.extend_from_slice(&4_u32.to_le_bytes()); - bytes.extend_from_slice(&value.to_le_bytes()); - } - - fn push_gguf_string(bytes: &mut Vec, value: &str) { - bytes.extend_from_slice(&(value.len() as u64).to_le_bytes()); - bytes.extend_from_slice(value.as_bytes()); - } - - impl Drop for TempModelFile { - fn drop(&mut self) { - let _ = std::fs::remove_dir_all(&self.root); - } - } - - struct TempTomlFile { - path: PathBuf, - } - - impl TempTomlFile { - fn new(file_name: &str, contents: &str) -> Self { - let path = std::env::temp_dir().join(format!( - "mvp-orchestrator-config-test-{}-{}-{file_name}", - std::process::id(), - std::thread::current().name().unwrap_or("unnamed") - )); - let _ = std::fs::remove_file(&path); - std::fs::write(&path, contents).expect("write temp TOML config"); - Self { path } - } - } - - impl Drop for TempTomlFile { - fn drop(&mut self) { - let _ = std::fs::remove_file(&self.path); - } - } - - #[test] - fn frame_archive_writes_jsonl_records_for_text_and_binary_payloads() { - static NEXT_TEMP_FILE: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); - - let suffix = NEXT_TEMP_FILE.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let path = std::env::temp_dir().join(format!( - "mvp-frame-archive-test-{}-{suffix}.jsonl", - std::process::id() - )); - let _ = std::fs::remove_file(&path); - - let stream = StreamId::new("test-node", Lifetime(42)); - let mut archive = FrameArchive::open(&path).expect("frame archive opens"); - archive - .record( - "orchestrator", - &stream, - "stdout", - &Frame::new( - ChannelId(1), - datastream::Position(7), - b"hello \xce\xbb".to_vec(), - ), - ) - .expect("text frame archives"); - archive - .record( - "orchestrator", - &stream, - "stderr", - &Frame::new( - ChannelId(2), - datastream::Position(8), - vec![0xff, 0x00, b'A'], - ), - ) - .expect("binary frame archives"); - drop(archive); - - let contents = std::fs::read_to_string(&path).expect("read frame archive jsonl"); - let records = contents - .lines() - .map(|line| serde_json::from_str::(line).expect("archive line is json")) - .collect::>(); - let _ = std::fs::remove_file(&path); - - assert_eq!(records.len(), 2); - assert_eq!(records[0]["arrival_seq"], json!(0)); - assert!( - records[0]["arrival_unix_ms"] - .as_u64() - .is_some_and(|value| value > 0) - ); - assert_eq!(records[0]["source"], json!("orchestrator")); - assert_eq!(records[0]["stream"], json!("test-node#42")); - assert_eq!(records[0]["channel"], json!("stdout")); - assert_eq!(records[0]["channel_id"], json!(1)); - assert_eq!(records[0]["position"], json!(7)); - assert_eq!( - records[0]["payload"], - json!({"encoding":"utf8","value":"hello λ"}) - ); - - assert_eq!(records[1]["arrival_seq"], json!(1)); - assert!( - records[1]["arrival_unix_ms"] - .as_u64() - .is_some_and(|value| value > 0) - ); - assert_eq!(records[1]["source"], json!("orchestrator")); - assert_eq!(records[1]["stream"], json!("test-node#42")); - assert_eq!(records[1]["channel"], json!("stderr")); - assert_eq!(records[1]["channel_id"], json!(2)); - assert_eq!(records[1]["position"], json!(8)); - assert_eq!( - records[1]["payload"], - json!({"encoding":"bytes","value":[255,0,65]}) - ); - } - - #[test] - fn emit_swim_probe_events_archives_probe_lifecycle_once_with_config() { - static NEXT_TEMP_FILE: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); - - let suffix = NEXT_TEMP_FILE.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let path = std::env::temp_dir().join(format!( - "mvp-swim-probe-archive-test-{}-{suffix}.jsonl", - std::process::id() - )); - let _ = std::fs::remove_file(&path); - let stack = - DistributionRuntimeStack::new(DistNodeId([7; 32]), DistributedNodeConfig::default()); - let peer = DistNodeId([8; 32]); - stack.swim_telemetry.observe(SwimObservation::ProbeSent { - target: peer, - sequence: 99, - kind: "direct", - }); - stack - .swim_telemetry - .observe(SwimObservation::ProbeTimedOut { - target: peer, - sequence: 99, - kind: "direct", - budget_ticks: 15_000, - }); - let mut datastream = OrchDatastream::new(77, Some(&path)).expect("datastream opens"); - - emit_swim_probe_events(&mut datastream, None, &stack, "weights_loaded_wait"); - emit_swim_probe_events(&mut datastream, None, &stack, "weights_loaded_wait"); - drop(datastream); - - let contents = std::fs::read_to_string(&path).expect("read frame archive jsonl"); - let records = contents - .lines() - .map(|line| serde_json::from_str::(line).expect("archive line is json")) - .collect::>(); - let _ = std::fs::remove_file(&path); - let probe_payloads = records - .iter() - .filter(|record| record["channel"] == json!(SwimProbeEvent::CHANNEL)) - .map(|record| { - let value = record["payload"]["value"] - .as_str() - .expect("probe payload archived as utf8 json"); - serde_json::from_str::(value).expect("probe payload parses") - }) - .collect::>(); - - assert_eq!(probe_payloads.len(), 2); - assert_eq!(probe_payloads[0]["event"], json!("sent")); - assert_eq!(probe_payloads[0]["sequence"], json!(99)); - assert_eq!( - probe_payloads[0]["local_phase"], - json!("weights_loaded_wait") - ); - assert_eq!(probe_payloads[0]["probe_timeout_ms"], json!(15_000)); - assert_eq!(probe_payloads[1]["event"], json!("timed_out")); - assert_eq!(probe_payloads[1]["budget_ms"], json!(15_000)); - assert_eq!(probe_payloads[1]["budget_ticks"], json!(15_000)); - assert_eq!(probe_payloads[1]["consecutive_timeouts"], json!(1)); - } - - #[test] - fn orchestrator_stdio_drain_archives_stdout_and_stderr_as_provision_logs() { - static NEXT_TEMP_FILE: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); - - let suffix = NEXT_TEMP_FILE.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let path = std::env::temp_dir().join(format!( - "mvp-orch-stdio-test-{}-{suffix}.jsonl", - std::process::id() - )); - let _ = std::fs::remove_file(&path); - let (tx, rx) = mpsc::channel(); - tx.send(OrchStdioLine { - stream: ProvisionLogStream::Stdout, - line: "offer pool selected".to_owned(), - }) - .expect("send stdout line"); - tx.send(OrchStdioLine { - stream: ProvisionLogStream::Stderr, - line: "lease chain detail".to_owned(), - }) - .expect("send stderr line"); - - let mut datastream = OrchDatastream::new(77, Some(&path)).expect("datastream opens"); - drain_orch_stdio_capture(Some(&rx), &mut datastream, None, 77, 9); - drop(datastream); - - let contents = std::fs::read_to_string(&path).expect("read frame archive jsonl"); - let records = contents - .lines() - .map(|line| serde_json::from_str::(line).expect("archive line is json")) - .collect::>(); - let _ = std::fs::remove_file(&path); - - let stdout_record = records - .iter() - .rev() - .find(|record| record["channel"] == "mvp.provisioning.logs.node.9.stdout") - .expect("stdout provisioning log frame archived"); - let stderr_record = records - .iter() - .rev() - .find(|record| record["channel"] == "mvp.provisioning.logs.node.9.stderr") - .expect("stderr provisioning log frame archived"); - assert_eq!(stdout_record["source"], "orchestrator"); - assert_eq!(stderr_record["source"], "orchestrator"); - - let stdout_payload = serde_json::from_str::( - stdout_record["payload"]["value"] - .as_str() - .expect("stdout payload is archived as text"), - ) - .expect("stdout payload is log record json"); - let stderr_payload = serde_json::from_str::( - stderr_record["payload"]["value"] - .as_str() - .expect("stderr payload is archived as text"), - ) - .expect("stderr payload is log record json"); - assert_eq!(stdout_payload["line"]["run_id"], 77); - assert_eq!(stdout_payload["line"]["node_id"], 9); - assert_eq!(stdout_payload["line"]["stream"], "Stdout"); - assert_eq!(stdout_payload["line"]["line"], "offer pool selected"); - assert_eq!(stderr_payload["line"]["stream"], "Stderr"); - assert_eq!(stderr_payload["line"]["line"], "lease chain detail"); - } - - #[test] - fn config_layers_defaults_toml_env_then_cli() { - let toml = TempTomlFile::new( - "layering.toml", - r#" -[runtime] -profile = "deploy" -run_id = 41 -node_id = 9 -stage_index = 3 -layer_end_exclusive = 24 - -[provider] -kind = "vastai" - -[image] -node = "docker.io/example/from-image-node:toml" - -[vastai] -image = "docker.io/example/from-vastai-image:toml" -api_key = "toml-key" -bootstrap_command = "/toml/bootstrap" -disk_gb = 60 -gpu_name = "RTX 4090" - -[prompt] -rpc_addr = "127.0.0.1:19999" -max_tokens = 17 -dashboard = true - -[model] -id = "toml-model" -gguf_repo = "toml/repo" -gguf_file = "toml.gguf" -gguf_revision = "toml-rev" -max_context = 384 - -[relay] -mode = "disabled" -"#, - ); - - let config = with_clean_env( - &[ - ("MVP_NODE_PROVIDER", "docker"), - ("MVP_NODE_IMAGE", "docker.io/example/from-env:latest"), - ("MVP_PROMPT_MAX_TOKENS", "23"), - ("MVP_MODEL_ID", "env-model"), - ("MVP_IROH_RELAY_MODE", "default"), - ], - || { - Config::from_layers_with_path_and_args( - Some(&toml.path), - [ - "--image", - "docker.io/example/from-cli:latest", - "--max-tokens", - "31", - "--model-id", - "cli-model", - "--max-context", - "768", - ] - .into_iter() - .map(str::to_owned), - ) - .expect("layered config parses") - }, - ); - - assert_eq!(config.provider, provider_kind::docker()); - assert_eq!(config.image, "docker.io/example/from-cli:latest"); - assert_eq!(config.default_max_tokens, 31); - assert_eq!(config.model_id, "cli-model"); - assert_eq!(config.run_id, 41); - assert_eq!(config.node_id, 9); - assert_eq!(config.stage_index, 3); - assert_eq!(config.layer_end_exclusive, Some(24)); - assert!(config.dashboard); - assert_eq!(config.max_context, Some(768)); - assert!(matches!(config.relay.mode, iroh::RelayMode::Default)); - assert!(config.vastai.is_none()); - } - - #[test] - fn pipeline_stages_layers_toml_env_then_cli_aliases() { - let toml = TempTomlFile::new( - "pipeline-stages.toml", - r#" -[runtime] -pipeline_stages = 2 - -[provider] -kind = "docker" -"#, - ); - - for (case, settings, args, expected) in [ - ("toml", vec![], vec![], 2), - ( - "env-over-toml", - vec![("MVP_PIPELINE_STAGES", "3")], - vec![], - 3, - ), - ( - "long-cli-over-env", - vec![("MVP_PIPELINE_STAGES", "3")], - vec!["--pipeline-stages", "4"], - 4, - ), - ( - "short-cli-over-env", - vec![("MVP_PIPELINE_STAGES", "3")], - vec!["-N", "5"], - 5, - ), - ] { - let config = with_clean_env(&settings, || { - Config::from_layers_with_path_and_args( - Some(&toml.path), - args.iter().copied().map(str::to_owned), - ) - .unwrap_or_else(|error| { - panic!("{case} pipeline stage config should parse: {error}") - }) - }); - - assert_eq!(config.pipeline_stages, expected, "{case}"); - } - } - - #[test] - fn pipeline_stages_rejects_zero_missing_and_non_numeric_values() { - for (case, settings, args, expected) in [ - ( - "env-zero", - vec![("MVP_PIPELINE_STAGES", "0")], - vec![], - "--pipeline-stages must be greater than 0", - ), - ( - "env-non-numeric", - vec![("MVP_PIPELINE_STAGES", "many")], - vec![], - "invalid MVP_PIPELINE_STAGES=\"many\"", - ), - ( - "long-cli-zero", - vec![], - vec!["--pipeline-stages", "0"], - "--pipeline-stages must be greater than 0", - ), - ( - "long-cli-non-numeric", - vec![], - vec!["--pipeline-stages", "many"], - "invalid --pipeline-stages=\"many\"", - ), - ( - "short-cli-missing", - vec![], - vec!["-N"], - "missing value after -N", - ), - ] { - let error = - with_clean_env(&settings, || { - match Config::from_layers_with_path_and_args( - None, - args.iter().copied().map(str::to_owned), - ) { - Ok(_) => panic!("invalid pipeline stages setting must fail"), - Err(error) => error, - } - }); - - assert!( - error.contains(expected), - "{case} error {error:?} should contain {expected:?}" - ); - } - } - - #[test] - fn vastai_eight_stage_plan_uses_remote_gguf_and_no_mounts() { - let config = with_clean_env(&[], || { - Config::from_layers_with_path_and_args( - None, - [ - "--provider", - "vastai", - "--pipeline-stages", - "8", - "--model-id", - "smollm2-135m-instruct-q4", - "--gguf-repo", - "QuantFactory/SmolLM2-135M-Instruct-GGUF", - "--gguf-file", - DEFAULT_PIPELINE_CACHED_MODEL_FILE, - "--max-context", - "256", - "--relay-mode", - "default", - "--vastai-bootstrap-command", - "boot", - "--vastai-blacklist-host", - "155385", - ] - .into_iter() - .map(str::to_owned), - ) - .expect("VastAI eight-stage pipeline config parses") - }); - let plan = config - .build_run_plan() - .expect("VastAI eight-stage run plan uses local metadata only"); - let coordinator = EndpointAddr::new(iroh::SecretKey::from_bytes(&[41; 32]).public()); - let orchestrator_actor = ActorAddress([42; 32]); - - let specs = stage_node_specs(&config, Some(&plan), coordinator, orchestrator_actor) - .expect("VastAI pipeline stage node specs build"); - - assert_eq!(config.provider, provider_kind::vastai()); - assert!( - config - .vastai - .as_ref() - .expect("VastAI runtime config") - .provisioning - .selection - .blacklist_hosts - .contains(&155385), - "VastAI CLI blacklist must reach provisioning policy" - ); - assert!( - config.cached_model.is_none(), - "VastAI must not mount host caches" - ); - assert_eq!(specs.len(), 8); - for (expected_stage_index, spec) in specs.iter().enumerate() { - let expected_stage_index = - u32::try_from(expected_stage_index).expect("fixture stage index fits u32"); - let expected_node_id = config.node_id + 1 + u64::from(expected_stage_index); - assert_eq!(spec.node_id, expected_node_id); - assert_eq!(spec.stage_index, Some(expected_stage_index)); - assert_eq!(env_value(&spec.env, "MVP_NODE_PROVIDER"), Some("vastai")); - assert_eq!(env_value(&spec.env, "MVP_PIPELINE_STAGES"), Some("8")); - assert_eq!( - env_value(&spec.env, "MVP_GGUF_REPO"), - Some("QuantFactory/SmolLM2-135M-Instruct-GGUF") - ); - assert_eq!( - env_value(&spec.env, "MVP_GGUF_FILE"), - Some(DEFAULT_PIPELINE_CACHED_MODEL_FILE) - ); - assert_eq!(env_value(&spec.env, "MVP_GGUF_LOCAL_PATH"), None); - assert_eq!(env_value(&spec.env, "MVP_MAX_CONTEXT"), Some("256")); - assert_eq!(spec.args, vec!["boot".to_owned()]); - assert!(spec.mounts.is_empty(), "VastAI stage specs must not mount"); - } - } - - #[test] - fn pipeline_cached_model_resolution_uses_toml_smollm2_gguf_file_with_cli_stage_count() { - let toml = TempTomlFile::new( - "pipeline-smollm2-model.toml", - r#" -[model] -gguf_repo = "QuantFactory/SmolLM2-135M-Instruct-GGUF" -gguf_file = "SmolLM2-135M-Instruct.Q4_0.gguf" -"#, - ); - - let config = with_clean_env(&[], || { - Config::from_layers_with_path_and_args( - Some(&toml.path), - ["--pipeline-stages", "3"].into_iter().map(str::to_owned), - ) - .expect("TOML SmolLM2 pipeline config parses") - }); - let expected_cached_host_path = default_pipeline_cached_model_path() - .canonicalize() - .expect("default cached SmolLM2 GGUF is present for cache resolution tests"); - let cached_model = config - .cached_model - .as_ref() - .expect("matching TOML SmolLM2 pipeline source should resolve the default cache"); - - assert_eq!(config.pipeline_stages, 3); - assert_eq!(config.provider, provider_kind::process()); - assert_eq!(cached_model.host_path, expected_cached_host_path); - assert_eq!( - cached_model.container_path, - "/models/cached/SmolLM2-135M-Instruct.Q4_0.gguf" - ); - assert_eq!( - config.gguf_source, - GgufSource::LocalPath(expected_cached_host_path.to_string_lossy().to_string()) - ); - } - - #[test] - fn pipeline_cached_model_resolution_does_not_use_default_cache_for_other_toml_gguf_file() { - let toml = TempTomlFile::new( - "pipeline-other-model.toml", - r#" -[model] -gguf_repo = "QuantFactory/SmolLM2-135M-Instruct-GGUF" -gguf_file = "SmolLM2-135M-Instruct.Q8_0.gguf" -"#, - ); - - let config = with_clean_env(&[], || { - Config::from_layers_with_path_and_args( - Some(&toml.path), - ["--pipeline-stages", "3"].into_iter().map(str::to_owned), - ) - .expect("non-default TOML GGUF pipeline config parses") - }); - - assert_eq!(config.pipeline_stages, 3); - assert!(config.cached_model.is_none()); - assert_eq!( - config.gguf_source, - GgufSource::HuggingFaceGguf { - repo: "QuantFactory/SmolLM2-135M-Instruct-GGUF".to_owned(), - file: "SmolLM2-135M-Instruct.Q8_0.gguf".to_owned(), - revision: None, - } - ); - - let error = config - .build_run_plan() - .expect_err("non-default remote TOML GGUF should still require an explicit cache"); - - assert!( - error.contains( - "QuantFactory/SmolLM2-135M-Instruct-GGUF/SmolLM2-135M-Instruct.Q8_0.gguf" - ), - "unexpected error: {error}" - ); - assert!( - error.contains("--cached-model-host-path"), - "unexpected error: {error}" - ); - } - - #[test] - fn cached_model_plan_phase_is_metadata_driven_for_each_pipeline_width() { - let metadata = TestGgufMetadata { - num_layers: 7, - hidden_dim: 13, - context_length: 64, - eos_token_id: 11, - }; - let model = TempModelFile::with_metadata("metadata-driven.gguf", metadata); - - for stage_count in [1_u32, 3, metadata.num_layers] { - let config = pipeline_config_with_cached_model(&model, stage_count); - assert!(config.uses_planned_execution()); - let plan = config - .build_run_plan() - .expect("metadata-derived cached-model plan builds"); - assert_plan_matches_metadata(&config, &plan, metadata, stage_count, 512); - } - } - - #[test] - fn cached_model_implicit_single_stage_matches_explicit_n_one_plan() { - let model = TempModelFile::new("implicit-n-one.gguf"); - let implicit = cached_model_config_with_args(&model, &["--max-context", "32"]); - let explicit = cached_model_config_with_args( - &model, - &["--pipeline-stages", "1", "--max-context", "32"], - ); - - let implicit_plan = implicit - .build_run_plan() - .expect("implicit cached-model N=1 plan builds"); - let explicit_plan = explicit - .build_run_plan() - .expect("explicit cached-model N=1 plan builds"); - - assert!(implicit.uses_planned_execution()); - assert!(explicit.uses_planned_execution()); - assert_plan_matches_metadata(&implicit, &implicit_plan, model.metadata, 1, 32); - assert_plan_matches_metadata(&explicit, &explicit_plan, model.metadata, 1, 32); - assert_eq!(implicit_plan.model, explicit_plan.model); - assert_eq!(implicit_plan.edges, explicit_plan.edges); - assert_eq!(implicit_plan.stages, explicit_plan.stages); - } - - #[test] - fn cached_model_plan_rejects_pipeline_width_above_model_layers() { - let metadata = TestGgufMetadata { - num_layers: 2, - ..TestGgufMetadata::default() - }; - let model = TempModelFile::with_metadata("too-many-stages.gguf", metadata); - let config = pipeline_config_with_cached_model(&model, metadata.num_layers + 1); - - let error = config - .build_run_plan() - .expect_err("pipeline width above layer count must reject before provisioning"); - - assert!( - error.contains("--pipeline-stages=3 exceeds GGUF layer count 2"), - "unexpected error: {error}" - ); - } - - #[test] - fn pipeline_planning_rejects_remote_gguf_source_without_local_cache() { - let config = with_clean_env(&[], || { - Config::from_layers_with_path_and_args( - None, - [ - "--pipeline-stages", - "3", - "--gguf-repo", - "example/remote", - "--gguf-file", - "remote.gguf", - ] - .into_iter() - .map(str::to_owned), - ) - .expect("non-default remote pipeline config parses") - }); - - let error = config - .build_run_plan() - .expect_err("remote GGUF source cannot be inspected before provisioning"); - - assert!( - error.contains("locally inspectable GGUF before provisioning"), - "unexpected error: {error}" - ); - assert!( - error.contains("example/remote/remote.gguf"), - "unexpected error: {error}" - ); - assert!( - error.contains("--cached-model-host-path"), - "unexpected error: {error}" - ); - } - - #[test] - fn toml_vastai_image_overrides_image_node_for_vastai_provider() { - let toml = TempTomlFile::new( - "vastai-image.toml", - r#" -[runtime] -profile = "deploy" - -[provider] -kind = "vastai" - -[image] -node = "docker.io/example/generic:toml" - -[vastai] -image = "docker.io/example/vastai:toml" -api_key = "k" -bootstrap_command = "/run" -"#, - ); - - let config = with_clean_env(&[], || { - Config::from_layers_with_path_and_args(Some(&toml.path), std::iter::empty::()) - .expect("VastAI TOML config parses") - }); - - assert_eq!(config.provider, provider_kind::vastai()); - assert_eq!(config.image, "docker.io/example/vastai:toml"); - } - - #[test] - fn missing_toml_uses_hardcoded_defaults() { - let missing_path = std::env::temp_dir().join(format!( - "mvp-orchestrator-missing-config-{}-{}.toml", - std::process::id(), - std::thread::current().name().unwrap_or("unnamed") - )); - let _ = std::fs::remove_file(&missing_path); - - let config = with_clean_env(&[], || { - Config::from_layers_with_path_and_args( - Some(&missing_path), - std::iter::empty::(), - ) - .expect("missing optional TOML config uses defaults") - }); - - assert_eq!(config.provider, provider_kind::process()); - assert_eq!(config.image, DEFAULT_IMAGE); - assert_eq!(config.model_id, DEFAULT_MODEL_ID); - assert_eq!(config.default_max_tokens, DEFAULT_MAX_TOKENS); - assert!(!config.dashboard); - assert_eq!(config.max_context, None); - } - - #[test] - fn node_spec_propagates_max_context_when_configured() { - let config = with_clean_env(&[], || { - Config::from_layers_with_path_and_args( - None, - ["--max-context", "256"].into_iter().map(str::to_owned), - ) - .expect("CLI max context config parses") - }); - let coordinator = EndpointAddr::new(iroh::SecretKey::from_bytes(&[3; 32]).public()); - let orchestrator_actor = ActorAddress([18; 32]); - let spec = config - .node_spec(coordinator, orchestrator_actor) - .expect("node spec builds"); - - assert_eq!(env_value(&spec.env, "MVP_MAX_CONTEXT"), Some("256")); - } - - #[test] - fn runtime_profile_selects_provider_and_node_provider_takes_precedence() { - assert_eq!( - selected_provider(&[("MVP_RUNTIME_CONFIG", "local")]), - provider_kind::process() - ); - assert_eq!( - selected_provider(&[("MVP_RUNTIME_CONFIG", "deploy")]), - provider_kind::vastai() - ); - assert_eq!( - selected_provider(&[ - ("MVP_RUNTIME_CONFIG", "deploy"), - ("MVP_NODE_PROVIDER", "docker"), - ]), - provider_kind::docker() - ); - } - - #[test] - fn relay_mode_env_uses_default_relay_and_accepts_disabled() { - with_clean_env(&[], || { - assert!(matches!( - relay_mode_from_env().expect("unset relay mode parses"), - iroh::RelayMode::Default - )); - }); - with_clean_env(&[("MVP_IROH_RELAY_MODE", "disabled")], || { - assert!(matches!( - relay_mode_from_env().expect("disabled relay mode parses"), - iroh::RelayMode::Disabled - )); - }); - } - - #[test] - fn node_spec_env_propagates_relay_url_only_for_custom_relay_config() { - const RELAY_URL: &str = "https://relay-node-spec.example.com"; - - let custom_env = node_spec_env(&[(MVP_IROH_RELAY_URL_ENV, RELAY_URL)]); - let expected_url = canonical_relay_url(RELAY_URL); - assert_eq!( - env_value(&custom_env, MVP_IROH_RELAY_URL_ENV), - Some(expected_url.as_str()) - ); - - let disabled_env = node_spec_env(&[ - ("MVP_IROH_RELAY_MODE", "disabled"), - (MVP_IROH_RELAY_URL_ENV, RELAY_URL), - ]); - assert_eq!(env_value(&disabled_env, MVP_IROH_RELAY_URL_ENV), None); - } - - #[test] - fn node_spec_preserves_relay_transport_in_coordinator_endpoint() { - with_clean_env(&[], || { - let config = Config::from_layers_with_path_and_args(None, std::iter::empty::()) - .expect("config parses"); - let secret = iroh::SecretKey::from_bytes(&[10; 32]); - let coordinator = EndpointAddr::new(secret.public()).with_relay_url( - "http://relay.example.com" - .parse::() - .unwrap(), - ); - let orchestrator_actor = ActorAddress([20; 32]); - - let spec = config - .node_spec(coordinator, orchestrator_actor) - .expect("node spec builds"); - let coordinator_endpoint_json = env_value(&spec.env, "MVP_COORDINATOR_ENDPOINT") - .expect("coordinator endpoint env is present"); - let coordinator_endpoint = - serde_json::from_str::(coordinator_endpoint_json) - .expect("coordinator endpoint env deserializes"); - - assert_eq!( - coordinator_endpoint - .relay_urls() - .next() - .map(|url| url.to_string()), - Some("http://relay.example.com/".to_owned()) - ); - }); - } - - #[test] - fn docker_config_construction_ignores_malformed_vastai_environment() { - let config = with_clean_env( - &[ - ("MVP_RUNTIME_CONFIG", "local"), - ("MVP_NODE_PROVIDER", "docker"), - ("MVP_VASTAI_CONFIRM_LEASE", "definitely-not-a-bool"), - ("MVP_VASTAI_DISK_GB", "not-a-u32"), - ("MVP_VASTAI_MIN_DOWN_MBPS", "not-a-float"), - ], - || { - Config::from_layers_with_path_and_args(None, std::iter::empty::()) - .expect("docker config ignores VastAI-only env") - }, - ); - - assert_eq!(config.provider, provider_kind::docker()); - assert!(config.vastai.is_none()); - } - - #[test] - fn docker_cached_model_builds_local_gguf_env_and_planned_file_mount_from_canonical_host_path() { - let model = TempModelFile::new("weights-q4.gguf"); - let config = with_clean_env_os( - &[ - ("MVP_RUNTIME_CONFIG", OsString::from("local")), - ("MVP_NODE_PROVIDER", OsString::from("docker")), - (CACHED_MODEL_HOST_ENV, model.raw_path.as_os_str().to_owned()), - ], - || { - Config::from_layers_with_path_and_args(None, std::iter::empty::()) - .expect("docker cached model config parses") - }, - ); - let coordinator = EndpointAddr::new(iroh::SecretKey::from_bytes(&[7; 32]).public()); - let orchestrator_actor = ActorAddress([14; 32]); - let spec = config - .node_spec(coordinator, orchestrator_actor) - .expect("cached model node spec builds"); - - assert_eq!( - env_value(&spec.env, "MVP_GGUF_LOCAL_PATH"), - Some("/models/cached/weights-q4.gguf") - ); - assert_eq!(env_value(&spec.env, "MVP_GGUF_REPO"), None); - assert_eq!(env_value(&spec.env, "MVP_GGUF_FILE"), None); - assert_eq!( - spec.mounts, - vec![ProviderMount { - host_path: model.canonical_path.to_string_lossy().to_string(), - container_path: "/models/cached/weights-q4.gguf".to_owned(), - readonly: true, - }] - ); - } - - #[test] - fn process_cached_model_builds_host_gguf_env_without_mounts() { - let model = TempModelFile::new("process-weights-q4.gguf"); - let config = with_clean_env_os( - &[ - ("MVP_RUNTIME_CONFIG", OsString::from("local")), - (CACHED_MODEL_HOST_ENV, model.raw_path.as_os_str().to_owned()), - ], - || { - Config::from_layers_with_path_and_args(None, std::iter::empty::()) - .expect("process cached model config parses") - }, - ); - let coordinator = EndpointAddr::new(iroh::SecretKey::from_bytes(&[8; 32]).public()); - let orchestrator_actor = ActorAddress([15; 32]); - let spec = config - .node_spec(coordinator, orchestrator_actor) - .expect("process cached model node spec builds"); - let host_path = model.canonical_path.to_string_lossy().to_string(); - - assert_eq!(config.provider, provider_kind::process()); - assert_eq!(env_value(&spec.env, "MVP_NODE_PROVIDER"), Some("process")); - assert_eq!( - env_value(&spec.env, "MVP_GGUF_LOCAL_PATH"), - Some(host_path.as_str()) - ); - assert_eq!(env_value(&spec.env, "MVP_DOCKER_GPUS"), None); - assert!(spec.mounts.is_empty(), "process workers use host paths"); - } - - #[test] - fn vectorized_local_docker_stage_node_specs_follow_three_stage_plan() { - let model = TempModelFile::new("pipeline-node-spec.gguf"); - let config = pipeline_config_with_cached_model(&model, 3); - let plan = config - .build_run_plan() - .expect("stage node spec plan builds"); - let coordinator = EndpointAddr::new(iroh::SecretKey::from_bytes(&[21; 32]).public()); - let orchestrator_actor = ActorAddress([23; 32]); - let coordinator_env = - serde_json::to_string(&coordinator).expect("coordinator endpoint serializes"); - let orchestrator_actor_env = - serde_json::to_string(&orchestrator_actor).expect("orchestrator actor serializes"); - let expected_run_id = config.run_id.to_string(); - let expected_mount = ProviderMount { - host_path: model.canonical_path.to_string_lossy().to_string(), - container_path: "/models/cached/pipeline-node-spec.gguf".to_owned(), - readonly: true, - }; - - let specs = stage_node_specs(&config, Some(&plan), coordinator, orchestrator_actor) - .expect("pipeline stage node specs build"); - - assert_eq!( - specs.len(), - 3, - "pipeline_stages=3 should provision three Docker workers, not one" - ); - assert!( - specs.iter().all(|spec| spec.node_id != config.node_id), - "coordinator node {} must not be counted as a worker: {specs:?}", - config.node_id - ); - - for (expected_stage_index, spec) in specs.iter().enumerate() { - let expected_stage_index = - u32::try_from(expected_stage_index).expect("fixture stage index fits u32"); - let expected_node_id = config.node_id + 1 + u64::from(expected_stage_index); - let expected_node_id_env = expected_node_id.to_string(); - let expected_stage_index_env = expected_stage_index.to_string(); - - assert_eq!(spec.run_id, config.run_id); - assert_eq!(spec.node_id, expected_node_id); - assert_eq!(spec.stage_index, Some(expected_stage_index)); - assert_eq!( - env_value(&spec.env, "MVP_RUN_ID"), - Some(expected_run_id.as_str()) - ); - assert_eq!( - env_value(&spec.env, "MVP_LOGICAL_NODE_ID"), - Some(expected_node_id_env.as_str()) - ); - assert_eq!( - env_value(&spec.env, "MVP_STAGE_INDEX"), - Some(expected_stage_index_env.as_str()) - ); - assert_eq!(env_value(&spec.env, "MVP_PIPELINE_STAGES"), Some("3")); - assert_eq!(env_value(&spec.env, "MVP_NODE_PROVIDER"), Some("docker")); - assert_eq!( - env_value(&spec.env, "MVP_MODEL_ID"), - Some(config.model_id.as_str()) - ); - assert_eq!( - env_value(&spec.env, "MVP_GGUF_LOCAL_PATH"), - Some("/models/cached/pipeline-node-spec.gguf") - ); - assert_eq!(env_value(&spec.env, "MVP_MAX_CONTEXT"), Some("512")); - assert_eq!( - env_value(&spec.env, "MVP_COORDINATOR_ENDPOINT"), - Some(coordinator_env.as_str()) - ); - assert_eq!( - env_value(&spec.env, "MVP_ORCHESTRATOR_ACTOR"), - Some(orchestrator_actor_env.as_str()) - ); - assert_eq!( - spec.mounts, - vec![expected_mount.clone()], - "pipeline cached GGUF mount should be read-only for stage {expected_stage_index}" - ); - } - } - - #[test] - fn vectorized_local_process_stage_node_specs_follow_three_stage_plan() { - let model = TempModelFile::new("process-pipeline-node-spec.gguf"); - let config = with_clean_env_os( - &[ - ("MVP_RUNTIME_CONFIG", OsString::from("local")), - (CACHED_MODEL_HOST_ENV, model.raw_path.as_os_str().to_owned()), - ], - || { - Config::from_layers_with_path_and_args( - None, - ["--pipeline-stages", "3", "--max-context", "512"] - .into_iter() - .map(str::to_owned), - ) - .expect("process pipeline config parses") - }, - ); - let plan = config - .build_run_plan() - .expect("process stage node spec plan builds"); - let coordinator = EndpointAddr::new(iroh::SecretKey::from_bytes(&[22; 32]).public()); - let orchestrator_actor = ActorAddress([24; 32]); - let host_path = model.canonical_path.to_string_lossy().to_string(); - - let specs = stage_node_specs(&config, Some(&plan), coordinator, orchestrator_actor) - .expect("process pipeline stage node specs build"); - - assert_eq!(config.provider, provider_kind::process()); - assert_eq!(specs.len(), 3); - for (expected_stage_index, spec) in specs.iter().enumerate() { - let expected_stage_index = - u32::try_from(expected_stage_index).expect("fixture stage index fits u32"); - assert_eq!(spec.stage_index, Some(expected_stage_index)); - assert_eq!(env_value(&spec.env, "MVP_NODE_PROVIDER"), Some("process")); - assert_eq!(env_value(&spec.env, "MVP_PIPELINE_STAGES"), Some("3")); - assert_eq!( - env_value(&spec.env, "MVP_GGUF_LOCAL_PATH"), - Some(host_path.as_str()) - ); - assert_eq!(env_value(&spec.env, "MVP_MAX_CONTEXT"), Some("512")); - assert!(spec.mounts.is_empty(), "process stage specs must not mount"); - } - } - - #[test] - fn planned_single_stage_cached_model_node_spec_follows_generated_plan() { - let model = TempModelFile::new("single-stage-node-spec.gguf"); - let config = pipeline_config_with_cached_model(&model, 1); - let plan = config - .build_run_plan() - .expect("single-stage cached plan builds"); - let coordinator = EndpointAddr::new(iroh::SecretKey::from_bytes(&[31; 32]).public()); - let orchestrator_actor = ActorAddress([33; 32]); - - let specs = stage_node_specs(&config, Some(&plan), coordinator, orchestrator_actor) - .expect("single planned stage node spec builds"); - - assert_eq!(specs.len(), 1); - let spec = &specs[0]; - let stage = &plan.stages[0]; - assert_eq!(stage.stage_index, 0); - assert_eq!(stage.layer_start, 0); - assert_eq!(stage.layer_end_exclusive, model.metadata.num_layers); - assert_eq!(spec.node_id, stage.node_id.0); - assert_ne!( - spec.node_id, config.node_id, - "planned cached N=1 still provisions a worker separate from the coordinator" - ); - assert_eq!(spec.stage_index, Some(0)); - assert_eq!(env_value(&spec.env, "MVP_PIPELINE_STAGES"), Some("1")); - assert_eq!( - env_value(&spec.env, "MVP_GGUF_LOCAL_PATH"), - Some("/models/cached/single-stage-node-spec.gguf") - ); - assert_eq!( - spec.mounts, - vec![ProviderMount { - host_path: model.canonical_path.to_string_lossy().to_string(), - container_path: "/models/cached/single-stage-node-spec.gguf".to_owned(), - readonly: true, - }] - ); - } - - #[test] - fn vectorized_local_docker_stage_provision_detail_preserves_plan_edges_and_layers() { - let model = TempModelFile::new("pipeline-stage-detail.gguf"); - let config = pipeline_config_with_cached_model(&model, 3); - let plan = config.build_run_plan().expect("stage detail plan builds"); - - let detail = stage_provision_detail(&config, Some(&plan)); - - assert_eq!( - detail, - json!({ - "run_id": config.run_id, - "stage_count": 3, - "stages": plan.stages.iter().map(|stage| { - json!({ - "node_id": stage.node_id.0, - "stage_index": stage.stage_index, - "layer_range": { - "start": stage.layer_start, - "end_exclusive": stage.layer_end_exclusive - }, - "inbound_edge_id": stage.inbound_edge.0, - "outbound_edge_id": stage.outbound_edge.0, - }) - }).collect::>(), - "model_id": config.model_id, - }) - ); - } - - #[test] - fn stage_provision_wire_preserves_plan_edges_for_single_and_multi_stage_cached_models() { - for stage_count in [1_u32, 3] { - let model = TempModelFile::new(&format!("wire-{stage_count}.gguf")); - let config = pipeline_config_with_cached_model(&model, stage_count); - let plan = config.build_run_plan().expect("wire test plan builds"); - let coordinator = EndpointAddr::new(iroh::SecretKey::from_bytes(&[41; 32]).public()); - let readies = plan - .stages - .iter() - .map(|stage| { - let byte = u8::try_from(stage.stage_index + 42).expect("fixture byte fits"); - let endpoint = - EndpointAddr::new(iroh::SecretKey::from_bytes(&[byte; 32]).public()); - ( - stage.node_id.0, - RuntimeReady { - endpoint: endpoint.clone(), - node_actor: ActorAddress([byte; 32]), - datastream_publisher: ActorAddress([byte.wrapping_add(80); 32]), - stage_index: stage.stage_index, - readiness_id: u64::from(stage.stage_index) + 100, - swim_node_id: DistNodeId(*endpoint.id.as_bytes()), - }, - ) - }) - .collect::>(); - - let stage_shard_plans = BTreeMap::new(); - - for stage in &plan.stages { - let wire = stage_provision_wire_from_plan( - &plan, - stage.stage_index, - &readies, - &coordinator, - &stage_shard_plans, - ) - .expect("stage provision wire builds from plan"); - let inbound = wire.inbound_edge.as_ref().expect("planned inbound edge"); - let outbound = wire.outbound_edge.as_ref().expect("planned outbound edge"); - - assert_eq!(wire.stage_count, stage_count); - assert_eq!(wire.node_id, stage.node_id.0); - assert_eq!(wire.layer_start, stage.layer_start); - assert_eq!(wire.layer_end_exclusive, stage.layer_end_exclusive); - assert_eq!(inbound.edge_id, stage.inbound_edge.0); - assert_eq!(outbound.edge_id, stage.outbound_edge.0); - assert_eq!( - inbound.kind, - if stage.stage_index == 0 { - StageEdgeKindWire::TokenIn - } else { - StageEdgeKindWire::Activation - } - ); - assert_eq!( - outbound.kind, - if stage.stage_index + 1 == stage_count { - StageEdgeKindWire::TokenOut - } else { - StageEdgeKindWire::Activation - } - ); - assert_eq!(wire.model_id, config.model_id); - assert_eq!(wire.gguf_source, config.gguf_source); - assert_eq!(wire.tokenizer, config.tokenizer); - } - } - } - - #[test] - fn vastai_cached_model_host_path_is_planning_only_and_keeps_remote_worker_source() { - let model = TempModelFile::with_metadata( - "Qwen2.5-7B-Instruct-Q4_K_M.gguf", - TestGgufMetadata { - num_layers: 28, - hidden_dim: 3584, - context_length: 32_768, - eos_token_id: 151_645, - }, - ); - - let config = with_clean_env_os( - &[ - ("MVP_RUNTIME_CONFIG", OsString::from("deploy")), - (CACHED_MODEL_HOST_ENV, model.raw_path.as_os_str().to_owned()), - ("MVP_VASTAI_API_KEY", OsString::from("test-key")), - ], - || { - Config::from_layers_with_path_and_args( - None, - [ - "--provider", - "vastai", - "--pipeline-stages", - "4", - "--model-id", - "qwen2.5-7b-instruct-q4-k-m", - "--gguf-repo", - "bartowski/Qwen2.5-7B-Instruct-GGUF", - "--gguf-file", - "Qwen2.5-7B-Instruct-Q4_K_M.gguf", - "--max-context", - "512", - "--vastai-bootstrap-command", - "boot", - ] - .into_iter() - .map(str::to_owned), - ) - .expect("VastAI planning-cache config parses") - }, - ); - - assert_eq!(config.provider, provider_kind::vastai()); - assert_eq!( - config - .cached_model - .as_ref() - .expect("VastAI planning cache retained") - .host_path, - model.canonical_path - ); - assert_eq!( - config.gguf_source, - GgufSource::HuggingFaceGguf { - repo: "bartowski/Qwen2.5-7B-Instruct-GGUF".to_owned(), - file: "Qwen2.5-7B-Instruct-Q4_K_M.gguf".to_owned(), - revision: None, - } - ); - - let plan = config - .build_run_plan() - .expect("VastAI planning cache supplies local GGUF metadata"); - assert_eq!(plan.model.num_layers, 28); - assert_eq!(plan.stages.len(), 4); - - let coordinator = EndpointAddr::new(iroh::SecretKey::from_bytes(&[55; 32]).public()); - let orchestrator_actor = ActorAddress([56; 32]); - let specs = stage_node_specs(&config, Some(&plan), coordinator, orchestrator_actor) - .expect("VastAI stage specs build with planning cache"); - assert_eq!(specs.len(), 4); - for spec in specs { - assert!(spec.mounts.is_empty(), "VastAI stage specs must not mount"); - assert_eq!( - env_value(&spec.env, "MVP_GGUF_REPO"), - Some("bartowski/Qwen2.5-7B-Instruct-GGUF") - ); - assert_eq!( - env_value(&spec.env, "MVP_GGUF_FILE"), - Some("Qwen2.5-7B-Instruct-Q4_K_M.gguf") - ); - assert_eq!(env_value(&spec.env, "MVP_GGUF_LOCAL_PATH"), None); - assert_eq!(env_value(&spec.env, "MVP_PIPELINE_STAGES"), Some("4")); - } - } - - #[test] - fn load_liveness_tracks_worker_progress_and_missing_gpu_samples() { - let mut progress = BTreeMap::new(); - let frame = CollectedDatastreamFrame { - stream: StreamId::new(NodeId::new("2"), Lifetime(17)), - channel_name: "mvp.worker.weights".to_owned(), - frame: Frame::new( - ChannelId(1), - datastream::Position(1), - serde_json::to_vec(&json!({ - "type": "GgufDownloadProgress", - "bytes_done": 4_218_u64, - "bytes_total": 4_683_u64, - })) - .expect("serialize progress frame"), - ), - }; - - update_load_progress_from_frame(&mut progress, &frame, Instant::now()); - let detail = stage_load_liveness_detail( - progress.get(&2), - 0, - 2, - Some(MemberState::Dead), - Some(DistNodeId([3; 32])), - None, - false, - "heartbeat_missed", - ); - - assert_eq!( - detail - .pointer("/load_progress/phase") - .and_then(Value::as_str), - Some("prefetching_model") - ); - assert_eq!( - detail - .pointer("/load_progress/bytes_done") - .and_then(Value::as_u64), - Some(4_218) - ); - assert_eq!( - detail - .pointer("/load_progress/bytes_total") - .and_then(Value::as_u64), - Some(4_683) - ); - assert_eq!( - detail.pointer("/classification").and_then(Value::as_str), - Some("heartbeat_missed") - ); - assert_eq!( - detail.pointer("/host_gpu_missing").and_then(Value::as_bool), - Some(true) - ); - - let gpu_frame = CollectedDatastreamFrame { - stream: StreamId::new(NodeId::new("2"), Lifetime(17)), - channel_name: "host.gpu".to_owned(), - frame: Frame::new(ChannelId(2), datastream::Position(2), b"{}".to_vec()), - }; - update_load_progress_from_frame(&mut progress, &gpu_frame, Instant::now()); - let detail = stage_load_liveness_detail( - progress.get(&2), - 0, - 2, - Some(MemberState::Alive), - Some(DistNodeId([3; 32])), - None, - true, - "waiting", - ); - assert_eq!( - detail.pointer("/host_gpu_missing").and_then(Value::as_bool), - Some(false) - ); - } - - #[test] - fn stage_shard_events_update_load_progress_and_liveness_phase() { - let mut progress = BTreeMap::new(); - let now = Instant::now(); - let frame = CollectedDatastreamFrame { - stream: StreamId::new(NodeId::new("7"), Lifetime(17)), - channel_name: "mvp.worker.weights".to_owned(), - frame: Frame::new( - ChannelId(1), - datastream::Position(1), - serde_json::to_vec(&json!({ - "type":"NodeEvent", - "phase":"stage_shard_fetch", - "status":"event", - "run_id":17, - "node_id":7, - "stage_index":3, - "detail":{ - "event":{ - "type":"StageShardRangeFetchReady", - "stage_index":3, - "range_index":1, - "range_count":4, - "tensor_index":2, - "tensor_count":9, - "bytes_done":384_u64, - "bytes_total":1024_u64 - } - } - })) - .expect("serialize stage shard progress frame"), - ), - }; - - update_load_progress_from_frame(&mut progress, &frame, now); - let entry = progress.get(&7).expect("stage shard progress tracked"); - assert_eq!(entry.stage_index, Some(3)); - assert_eq!(entry.phase.as_deref(), Some("fetching_stage_shard")); - assert_eq!( - entry.last_worker_event.as_deref(), - Some("StageShardRangeFetchReady") - ); - assert_eq!(entry.bytes_done, Some(384)); - assert_eq!(entry.bytes_total, Some(1024)); - assert_eq!(entry.last_progress, Some(now)); - - let ready_frame = CollectedDatastreamFrame { - stream: StreamId::new(NodeId::new("7"), Lifetime(17)), - channel_name: "mvp.worker.weights".to_owned(), - frame: Frame::new( - ChannelId(1), - datastream::Position(2), - serde_json::to_vec(&json!({ - "type":"NodeEvent", - "phase":"stage_shard_fetch", - "status":"event", - "run_id":17, - "node_id":7, - "stage_index":3, - "detail":{ - "event":{ - "type":"StageShardReady", - "stage_index":3, - "bytes_done":1024_u64, - "bytes_total":1024_u64 - } - } - })) - .expect("serialize stage shard ready frame"), - ), - }; - update_load_progress_from_frame(&mut progress, &ready_frame, Instant::now()); - let detail = stage_load_liveness_detail( - progress.get(&7), - 3, - 7, - Some(MemberState::Dead), - Some(DistNodeId([4; 32])), - None, - false, - "heartbeat_missed", - ); - assert_eq!( - detail - .pointer("/load_progress/phase") - .and_then(Value::as_str), - Some("stage_shard_ready") - ); - assert_eq!( - detail - .pointer("/load_progress/last_worker_event") - .and_then(Value::as_str), - Some("StageShardReady") - ); - assert!( - detail - .pointer("/load_progress/last_progress_age_ms") - .and_then(Value::as_u64) - .is_some() - ); - - let cache_frame = CollectedDatastreamFrame { - stream: StreamId::new(NodeId::new("8"), Lifetime(17)), - channel_name: "mvp.worker.weights".to_owned(), - frame: Frame::new( - ChannelId(1), - datastream::Position(3), - serde_json::to_vec(&json!({ - "type":"NodeEvent", - "phase":"stage_shard_fetch", - "status":"event", - "run_id":17, - "node_id":8, - "stage_index":4, - "detail":{"event":{"type":"StageShardCacheReady","stage_index":4,"cache_hit":true}} - })) - .expect("serialize stage shard cache frame"), - ), - }; - update_load_progress_from_frame(&mut progress, &cache_frame, Instant::now()); - assert_eq!( - progress.get(&8).and_then(|entry| entry.phase.as_deref()), - Some("stage_shard_cache_ready") - ); - } - - #[test] - fn stage_shard_plan_summary_events_include_fetch_facts() { - static NEXT_TEMP_FILE: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); - let suffix = NEXT_TEMP_FILE.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let path = std::env::temp_dir().join(format!( - "mvp-stage-shard-plan-summary-{}-{suffix}.jsonl", - std::process::id() - )); - let _ = std::fs::remove_file(&path); - let plan0 = test_stage_shard_plan(0, 2_000, 100, vec![(800, 400), (1_600, 200)]); - let plan1 = test_stage_shard_plan(1, 2_000, 120, vec![(1_000, 300)]); - let plans = BTreeMap::from([(0, plan0.clone()), (1, plan1.clone())]); - - let mut datastream = OrchDatastream::new(41, Some(&path)).expect("datastream opens"); - emit_stage_shard_plan_summaries(None, &mut datastream, 41, 9, &plans); - drop(datastream); - - let contents = std::fs::read_to_string(&path).expect("read summary archive"); - let _ = std::fs::remove_file(&path); - let summaries = contents - .lines() - .map(|line| serde_json::from_str::(line).expect("archive line is json")) - .filter(|record| { - record.get("channel").and_then(Value::as_str) == Some(MVP_ORCH_BOOTSTRAP) - }) - .map(|record| { - let payload = record - .pointer("/payload/value") - .and_then(Value::as_str) - .expect("bootstrap payload is text"); - serde_json::from_str::(payload).expect("bootstrap payload is json") - }) - .filter(|event| event.get("phase").and_then(Value::as_str) == Some("stage_shard_plan")) - .collect::>(); - assert_eq!(summaries.len(), 2); - - let first = summaries - .iter() - .find(|event| event.pointer("/detail/stage_index").and_then(Value::as_u64) == Some(0)) - .expect("stage 0 summary emitted"); - assert_eq!( - first - .pointer("/detail/planned_fetch_bytes") - .and_then(Value::as_u64), - Some(plan0.planned_fetch_bytes()) - ); - assert_eq!( - first - .pointer("/detail/source_total_bytes") - .and_then(Value::as_u64), - Some(plan0.source_total_bytes) - ); - assert_eq!( - first - .pointer("/detail/tensor_count") - .and_then(Value::as_u64), - Some(plan0.tensors.len() as u64) - ); - assert_eq!( - first.pointer("/detail/range_count").and_then(Value::as_u64), - Some(plan0.planned_range_count() as u64) - ); - assert!( - first - .pointer("/detail/planned_fraction") - .and_then(Value::as_f64) - .expect("planned fraction is numeric") - < 1.0 - ); - assert!( - plan0.planned_fetch_bytes() < plan0.source_total_bytes, - "nontrivial stage shard fetches less than the source GGUF" - ); - } - - #[test] - fn stage_provision_dispatch_suppresses_active_progress_and_recovers_when_stale() { - let now = Instant::now(); - assert_eq!( - stage_provision_dispatch(None, 0, None, now), - StageProvisionDispatch::Send("initial") - ); - assert_eq!( - stage_provision_dispatch(None, 1, Some(now - Duration::from_secs(15)), now), - StageProvisionDispatch::Send("no_progress_after_send") - ); - - let active = StageLoadProgress { - node_id: 2, - stage_index: Some(0), - phase: Some("fetching_stage_shard".to_owned()), - last_progress: Some(now - Duration::from_secs(5)), - ..StageLoadProgress::default() - }; - assert_eq!( - stage_provision_dispatch(Some(&active), 1, Some(now - Duration::from_secs(15)), now), - StageProvisionDispatch::Suppress("active_progress") - ); - - let stale = StageLoadProgress { - last_progress: Some(now - STAGE_PROVISION_ACTIVE_RESEND_AFTER - Duration::from_secs(1)), - ..active.clone() - }; - assert_eq!( - stage_provision_dispatch(Some(&stale), 1, Some(now - Duration::from_secs(15)), now), - StageProvisionDispatch::Suppress("recent_stale_progress_resend") - ); - assert_eq!( - stage_provision_dispatch( - Some(&stale), - 1, - Some(now - STAGE_PROVISION_ACTIVE_RESEND_AFTER - Duration::from_secs(1)), - now, - ), - StageProvisionDispatch::Send("stale_progress") - ); - - let failed = StageLoadProgress { - phase: Some("failed".to_owned()), - failure_reason: Some("cache write failed".to_owned()), - ..active - }; - assert_eq!( - stage_provision_dispatch(Some(&failed), 1, Some(now), now), - StageProvisionDispatch::Suppress("worker_load_failed") - ); - } - - fn test_stage_shard_plan( - stage_index: u32, - source_total_bytes: u64, - metadata_end: u64, - ranges: Vec<(u64, u64)>, - ) -> StageShardPlan { - StageShardPlan { - source: GgufSource::HuggingFaceGguf { - repo: "org/repo".to_owned(), - file: "model.gguf".to_owned(), - revision: None, - }, - stage_index, - stage_count: 2, - layer_start: stage_index * 2, - layer_end_exclusive: stage_index * 2 + 2, - metadata_count: 1, - metadata_end, - data_start: 512, - alignment: 32, - source_total_bytes, - tensors: vec![crate::gguf_shard::StageShardTensor { - name: format!("blk.{stage_index}.attn_q.weight"), - dims: vec![2, 2], - ggml_type: 0, - source_offset: 0, - byte_len: 16, - }], - merged_tensor_ranges: ranges - .into_iter() - .map(|(start, len)| crate::gguf_shard::ByteRange { start, len }) - .collect(), - cache_key: format!("test-stage-{stage_index}"), - } - } - - #[derive(Default)] - struct FakeProvisionState { - stopped: Vec, - completed: Vec, - } - - #[derive(Default)] - struct FakeProvisionPlugin { - stopped: Vec, - shared: Option>>, - } - - impl ProvisionPlugin for FakeProvisionPlugin { - fn start_node( - &mut self, - _spec: NodeProvisionSpec, - _sink: PluginSink, - ) -> Result { - unreachable!("guard tests construct handles directly") - } - - fn complete_bootstrap( - &mut self, - handle: &crate::provisioning::PluginNodeHandle, - ) -> Result<(), String> { - if let Some(shared) = &self.shared { - shared - .lock() - .expect("fake provision state") - .completed - .push(handle.id); - } - Ok(()) - } - - fn stop_node( - &mut self, - handle: &crate::provisioning::PluginNodeHandle, - ) -> Result<(), String> { - self.stopped.push(handle.id); - if let Some(shared) = &self.shared { - shared - .lock() - .expect("fake provision state") - .stopped - .push(handle.id); - } - Ok(()) - } - } - - fn provider_start_spec(node_id: u64) -> NodeProvisionSpec { - NodeProvisionSpec { - run_id: 41, - node_id, - stage_index: Some(u32::try_from(node_id).unwrap_or(u32::MAX)), - image: "registry.example/mvp-worker:latest".to_owned(), - env: Vec::new(), - args: Vec::new(), - mounts: Vec::new(), - } - } - - #[test] - fn provider_start_outcome_keeps_later_successes_after_earlier_failure() { - let outcome = collect_provider_start_outcome(vec![ - ( - provider_start_spec(2), - Err("synthetic provider start failed".to_owned()), - ), - ( - provider_start_spec(3), - Ok(crate::provisioning::PluginNodeHandle { - id: 22, - provider_process_id: None, - }), - ), - ]); - - assert_eq!( - outcome.first_error.as_deref(), - Some("synthetic provider start failed") - ); - assert_eq!( - outcome - .successful_handles - .iter() - .map(|handle| handle.id) - .collect::>(), - vec![22], - "cleanup must include successful starts even when an earlier stage failed" - ); - assert_eq!(outcome.results.len(), 2); - } - - #[test] - fn provisioned_cluster_guard_completes_bootstrap_without_dropping_stop_handles() { - let state = Arc::new(std::sync::Mutex::new(FakeProvisionState::default())); - let plugin = FakeProvisionPlugin { - shared: Some(Arc::clone(&state)), - ..FakeProvisionPlugin::default() - }; - let mut guard = ProvisionedClusterGuard::new( - Box::new(plugin), - vec![ - crate::provisioning::PluginNodeHandle { - id: 7, - provider_process_id: None, - }, - crate::provisioning::PluginNodeHandle { - id: 8, - provider_process_id: None, - }, - ], - ); - - guard - .complete_bootstrap() - .expect("runtime-ready bootstrap completion succeeds"); - assert_eq!( - state - .lock() - .expect("fake provision state") - .completed - .clone(), - vec![7, 8] - ); - assert!( - state - .lock() - .expect("fake provision state") - .stopped - .is_empty() - ); - - guard - .stop() - .expect("node stop still succeeds after completion"); - assert_eq!( - state.lock().expect("fake provision state").stopped.clone(), - vec![8, 7] - ); - } - - #[test] - fn provisioned_node_guard_stops_node_on_drop() { - let mut plugin = FakeProvisionPlugin::default(); - { - let _guard = ProvisionedNodeGuard::new( - &mut plugin, - crate::provisioning::PluginNodeHandle { - id: 7, - provider_process_id: Some(99), - }, - ); - } - assert_eq!(plugin.stopped, vec![7]); - } - - #[test] - fn provisioned_node_guard_explicit_stop_runs_once() { - let mut plugin = FakeProvisionPlugin::default(); - { - let mut guard = ProvisionedNodeGuard::new( - &mut plugin, - crate::provisioning::PluginNodeHandle { - id: 8, - provider_process_id: None, - }, - ); - guard.stop().expect("first stop succeeds"); - guard.stop().expect("second stop is a no-op"); - } - assert_eq!(plugin.stopped, vec![8]); - } - - #[test] - fn stop_requested_observes_shutdown_signal_only() { - let (_tx, rx) = mpsc::channel(); - assert!(!stop_requested(&rx)); - - let (tx, rx) = mpsc::channel(); - tx.send(()).expect("send shutdown"); - assert!(stop_requested(&rx)); - assert!(!stop_requested(&rx)); - } - - fn endpoint(seed: u8) -> EndpointAddr { - EndpointAddr::new(iroh::SecretKey::from_bytes(&[seed; 32]).public()) - } - - fn insert_route(stack: &DistributionRuntimeStack, actor: ActorAddress, owner: DistNodeId) { - let mut route_view = match stack.route_view.write() { - Ok(route_view) => route_view, - Err(poisoned) => poisoned.into_inner(), - }; - route_view.insert(actor, owner); - } - - fn mark_alive(stack: &DistributionRuntimeStack, node_id: DistNodeId) { - stack - .runtime - .send_to( - stack.actors.membership_fanout, - distribution::swim::actor::MembershipChanged { - node_id, - state: MemberState::Alive, - incarnation: 1, - }, - ) - .expect("send membership change"); - stack.pump_runtime_once(); - } - - #[test] - fn runtime_ready_barrier_waits_for_specific_swim_and_route() { - let remote = DistNodeId([2; 32]); - let node_actor = ActorAddress::new_random(); - let datastream_publisher = ActorAddress::new_random(); - let ready = RuntimeReady { - endpoint: endpoint(2), - node_actor, - datastream_publisher, - stage_index: 3, - readiness_id: 99, - swim_node_id: remote, - }; - - let stack = - DistributionRuntimeStack::new(DistNodeId([1; 32]), DistributedNodeConfig::default()); - assert!(!runtime_ready_barrier_met(&stack, &ready)); - - let stack = - DistributionRuntimeStack::new(DistNodeId([1; 32]), DistributedNodeConfig::default()); - mark_alive(&stack, remote); - assert!(!runtime_ready_barrier_met(&stack, &ready)); - - let stack = - DistributionRuntimeStack::new(DistNodeId([1; 32]), DistributedNodeConfig::default()); - insert_route(&stack, node_actor, remote); - assert!(!runtime_ready_barrier_met(&stack, &ready)); - - let stack = - DistributionRuntimeStack::new(DistNodeId([1; 32]), DistributedNodeConfig::default()); - mark_alive(&stack, remote); - insert_route(&stack, node_actor, remote); - assert!(runtime_ready_barrier_met(&stack, &ready)); - } - - #[test] - fn enqueue_runtime_ready_ack_reports_to_node_agent() { - use crate::node_actor::{NodeAgentActor, NodeAgentReport}; - use crate::orchestration::actor::OrchestratorMsg; - use crate::staging as stage; - - let stack = - DistributionRuntimeStack::new(DistNodeId([1; 32]), DistributedNodeConfig::default()); - let orchestrator_inbox = stack - .runtime - .new_inbox::() - .expect("orchestrator inbox"); - let reports = stack - .runtime - .new_inbox::() - .expect("node report inbox"); - let node_actor = stack - .runtime - .spawn(NodeAgentActor::new( - stage::NodeId(11), - *orchestrator_inbox.addr(), - Some(*reports.addr()), - )) - .expect("spawn node agent"); - let ready = RuntimeReady { - endpoint: endpoint(9), - node_actor, - datastream_publisher: ActorAddress::new_random(), - stage_index: 3, - readiness_id: 99, - swim_node_id: DistNodeId([2; 32]), - }; - - enqueue_runtime_ready_ack(&stack, &ready, 7, 11).expect("enqueue runtime ready ack"); - stack.pump_runtime_once(); - - assert_eq!( - reports.try_recv(), - Some(NodeAgentReport::RuntimeReadyAck { - run_id: 7, - node_id: 11, - stage_index: ready.stage_index, - readiness_id: 99, - }) - ); - - assert_eq!( - orchestrator_inbox.try_recv(), - Some(OrchestratorMsg::ObserveNodeRuntimeReadyAck { - run_id: 7, - node_id: 11, - stage_index: ready.stage_index, - readiness_id: 99, - }) - ); - } -} diff --git a/crates/mvp-system/src/orchestration/config.rs b/crates/mvp-system/src/orchestration/config.rs index 858921c..3622ee7 100644 --- a/crates/mvp-system/src/orchestration/config.rs +++ b/crates/mvp-system/src/orchestration/config.rs @@ -192,287 +192,3 @@ impl TomlConfigOverlay { toml::from_str(text) } } - -#[cfg(test)] -mod tests { - use super::*; - use super::{ResolvedVastAiConfig, looks_remote_image}; - use std::sync::atomic::{AtomicU64, Ordering}; - - static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0); - - fn valid_resolved_vastai_config() -> ResolvedVastAiConfig { - ResolvedVastAiConfig { - api_key: "vast-key".to_owned(), - relay_url: "https://relay.example.com".to_owned(), - image: "ghcr.io/swactor/mvp-node:latest".to_owned(), - bootstrap_command: "/usr/local/bin/mvp-node".to_owned(), - disk_gb: Some(80), - gpu_name: Some("RTX 4090".to_owned()), - min_gpu_ram_mb: Some(16_000), - min_down_mbps: Some(100.0), - min_up_mbps: Some(25.0), - max_dph_total: Some(0.10), - min_reliability: Some(0.98), - require_verified: Some(true), - blacklist_hosts: vec![155385], - onstart: None, - ssh_identity: Some("~/.ssh/swactor_vastai_ed25519".to_owned()), - } - } - - #[test] - fn config_toml_parses_explicit_chat_and_vastai_fields() { - let config = TomlConfigOverlay::from_str( - r#" - -[runtime] -profile = "deploy" -run_id = 42 -node_id = 7 -stage_index = 2 -layer_end_exclusive = 24 -pipeline_stages = 3 - -[provider] -kind = "vastai" - -[image] -node = "ghcr.io/swactor/mvp-node:latest" -tag = "trial" -build = false -push = true -force_refresh = true - -[relay] -mode = "disabled" -url = "https://relay.example.com" - -[prompt] -rpc_addr = "127.0.0.1:19000" -max_tokens = 128 -dashboard = false - - -[model] -id = "smollm2-135m-instruct-q4" -gguf_local_path = "/models/local.gguf" -gguf_repo = "QuantFactory/SmolLM2-135M-Instruct-GGUF" -gguf_file = "SmolLM2-135M-Instruct.Q4_K_M.gguf" -gguf_revision = "main" -tokenizer_local_path = "/models/tokenizer.json" -max_context = 512 - -[docker] -gpus = "all" -cached_model_host_path = "/cache/model.gguf" - -[observability] -datastream_frame_log = "/tmp/frames.jsonl" - -[vastai] -api_key = "vast-key" -image = "registry.example.com/team/mvp-node:latest" -bootstrap_command = "/opt/mvp/node --join" -disk_gb = 80 -ssh_user = "ubuntu" -confirm_lease = true -gpu_name = "RTX 4090" -min_gpu_ram_mb = 24000 -min_down_mbps = 250.5 -min_up_mbps = 50.25 -max_dph_total = 0.10 -min_reliability = 0.99 -require_verified = true -blacklist_hosts = [155385, 59017] -onstart = "echo preparing" -ssh_identity = "~/.ssh/swactor_vastai_ed25519" -poll_interval_secs = 30 -"#, - ) - .expect("explicit config TOML parses"); - - assert_eq!(config.runtime.profile.as_deref(), Some("deploy")); - assert_eq!(config.runtime.run_id, Some(42)); - assert_eq!(config.runtime.node_id, Some(7)); - assert_eq!(config.runtime.stage_index, Some(2)); - assert_eq!(config.runtime.layer_end_exclusive, Some(24)); - assert_eq!(config.runtime.pipeline_stages, Some(3)); - assert_eq!(config.provider.kind.as_deref(), Some("vastai")); - assert_eq!( - config.image.node.as_deref(), - Some("ghcr.io/swactor/mvp-node:latest") - ); - assert_eq!(config.image.tag.as_deref(), Some("trial")); - assert_eq!(config.image.build, Some(false)); - assert_eq!(config.image.push, Some(true)); - assert_eq!(config.image.force_refresh, Some(true)); - assert_eq!(config.relay.mode.as_deref(), Some("disabled")); - assert_eq!( - config.relay.url.as_deref(), - Some("https://relay.example.com") - ); - assert_eq!(config.prompt.rpc_addr.as_deref(), Some("127.0.0.1:19000")); - assert_eq!(config.prompt.max_tokens, Some(128)); - assert_eq!(config.prompt.dashboard, Some(false)); - assert_eq!(config.model.id.as_deref(), Some("smollm2-135m-instruct-q4")); - assert_eq!( - config.model.gguf_local_path.as_deref(), - Some("/models/local.gguf") - ); - assert_eq!( - config.model.gguf_repo.as_deref(), - Some("QuantFactory/SmolLM2-135M-Instruct-GGUF") - ); - assert_eq!( - config.model.gguf_file.as_deref(), - Some("SmolLM2-135M-Instruct.Q4_K_M.gguf") - ); - assert_eq!(config.model.gguf_revision.as_deref(), Some("main")); - assert_eq!( - config.model.tokenizer_local_path.as_deref(), - Some("/models/tokenizer.json") - ); - assert_eq!(config.model.max_context, Some(512)); - assert_eq!(config.docker.gpus.as_deref(), Some("all")); - assert_eq!( - config.docker.cached_model_host_path.as_deref(), - Some("/cache/model.gguf") - ); - assert_eq!( - config.observability.datastream_frame_log.as_deref(), - Some("/tmp/frames.jsonl") - ); - assert_eq!(config.vastai.api_key.as_deref(), Some("vast-key")); - assert_eq!( - config.vastai.image.as_deref(), - Some("registry.example.com/team/mvp-node:latest") - ); - assert_eq!( - config.vastai.bootstrap_command.as_deref(), - Some("/opt/mvp/node --join") - ); - assert_eq!(config.vastai.disk_gb, Some(80)); - assert_eq!(config.vastai.ssh_user.as_deref(), Some("ubuntu")); - assert_eq!(config.vastai.confirm_lease, Some(true)); - assert_eq!(config.vastai.gpu_name.as_deref(), Some("RTX 4090")); - assert_eq!(config.vastai.min_gpu_ram_mb, Some(24_000)); - assert_eq!(config.vastai.min_down_mbps, Some(250.5)); - assert_eq!(config.vastai.min_up_mbps, Some(50.25)); - assert_eq!(config.vastai.max_dph_total, Some(0.10)); - assert_eq!(config.vastai.min_reliability, Some(0.99)); - assert_eq!(config.vastai.require_verified, Some(true)); - assert_eq!(config.vastai.blacklist_hosts, vec![155385, 59017]); - assert_eq!(config.vastai.poll_interval_secs, Some(30)); - assert_eq!(config.vastai.onstart.as_deref(), Some("echo preparing")); - assert_eq!( - config.vastai.ssh_identity.as_deref(), - Some("~/.ssh/swactor_vastai_ed25519") - ); - } - - #[test] - fn config_toml_rejects_non_numeric_pipeline_stages() { - let error = TomlConfigOverlay::from_str( - r#" -[runtime] -pipeline_stages = "many" -"#, - ) - .expect_err("non-numeric pipeline_stages must not parse"); - - assert!( - error.to_string().contains("pipeline_stages"), - "error should identify pipeline_stages: {error}" - ); - } - - #[test] - fn explicit_config_path_error_includes_the_requested_path() { - let counter = TEMP_COUNTER.fetch_add(1, Ordering::Relaxed); - let path = std::env::temp_dir().join(format!( - "mvp-system-missing-config-{}-{counter}.toml", - std::process::id() - )); - let _ = fs::remove_file(&path); - - let error = TomlConfigOverlay::load_required(&path) - .expect_err("missing explicit config path errors"); - - assert!( - error.contains(&path.display().to_string()), - "error {error:?} must include requested path {}", - path.display() - ); - } - - #[test] - fn looks_remote_image_accepts_registry_refs_and_rejects_local_refs() { - for image in [ - "ghcr.io/swactor/mvp-node:latest", - "registry.example.com:5000/team/mvp-node@sha256:abcdef", - "localhost:5000/team/mvp-node:latest", - ] { - assert!(looks_remote_image(image), "{image:?} should be remote"); - } - - for image in [ - "swactor-mvp-node:latest", - "team/mvp-node:latest", - "mvp-node@sha256:abcdef", - ] { - assert!(!looks_remote_image(image), "{image:?} should be local"); - } - } - - #[test] - fn resolved_vastai_config_validate_rejects_missing_required_fields() { - let cases = [ - ( - "VAST_API_KEY", - ResolvedVastAiConfig { - api_key: " ".to_owned(), - ..valid_resolved_vastai_config() - }, - ), - ( - "relay.url", - ResolvedVastAiConfig { - relay_url: "\t".to_owned(), - ..valid_resolved_vastai_config() - }, - ), - ( - "vastai.image", - ResolvedVastAiConfig { - image: String::new(), - ..valid_resolved_vastai_config() - }, - ), - ( - "vastai.bootstrap_command", - ResolvedVastAiConfig { - bootstrap_command: "\n".to_owned(), - ..valid_resolved_vastai_config() - }, - ), - ( - "vastai.ssh_identity", - ResolvedVastAiConfig { - ssh_identity: Some(" ".to_owned()), - ..valid_resolved_vastai_config() - }, - ), - ]; - - for (label, config) in cases { - let error = config - .validate() - .expect_err("missing field must be rejected"); - assert!( - error.contains(label), - "error {error:?} must identify missing {label}" - ); - } - } -} diff --git a/crates/mvp-system/src/orchestration/distribution_stack.rs b/crates/mvp-system/src/orchestration/distribution_stack.rs index e93e05e..982ddf8 100644 --- a/crates/mvp-system/src/orchestration/distribution_stack.rs +++ b/crates/mvp-system/src/orchestration/distribution_stack.rs @@ -260,41 +260,3 @@ impl ActorInterface for MembershipFanout { let _ = ctx.send(self.directory, DirectoryIn::Membership(change)); } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn distribution_stack_reports_member_state_and_route_owner() { - let stack = DistributionRuntimeStack::new( - distribution::types::NodeId([1; 32]), - DistributedNodeConfig::default(), - ); - let remote = distribution::types::NodeId([2; 32]); - - stack - .runtime - .send_to( - stack.actors.membership_fanout, - distribution::swim::actor::MembershipChanged { - node_id: remote, - state: MemberState::Alive, - incarnation: 1, - }, - ) - .expect("send membership change"); - stack.pump_runtime_once(); - assert_eq!(stack.member_state(remote), Some(MemberState::Alive)); - - let actor = ActorAddress::new_random(); - { - let mut route_view = match stack.route_view.write() { - Ok(route_view) => route_view, - Err(poisoned) => poisoned.into_inner(), - }; - route_view.insert(actor, remote); - } - assert_eq!(stack.route_owner(actor), Some(remote)); - } -} diff --git a/crates/mvp-system/src/orchestration/provider_adapters/vastai/mod.rs b/crates/mvp-system/src/orchestration/provider_adapters/vastai/mod.rs index c9f9975..03dd0e6 100644 --- a/crates/mvp-system/src/orchestration/provider_adapters/vastai/mod.rs +++ b/crates/mvp-system/src/orchestration/provider_adapters/vastai/mod.rs @@ -1743,368 +1743,3 @@ where result } } - -#[cfg(test)] -mod tests { - use super::*; - - struct NoopLeaseClient; - - impl VastAiLeaseClient for NoopLeaseClient { - fn provision_one( - &mut self, - _request: ProvisionRequest, - ) -> Result { - panic!("build_request tests must not provision a real Vast.ai lease") - } - - fn ssh_endpoint( - &mut self, - _contract_id: u64, - _label: &str, - _lifecycle: &LifecyclePolicy, - _ssh_user: &str, - ) -> Result { - panic!("build_request tests must not query a real Vast.ai endpoint") - } - - fn destroy_contract(&mut self, _contract_id: u64) -> Result<(), String> { - panic!("build_request tests must not destroy a real Vast.ai lease") - } - } - - struct NoopBootstrapLauncher; - - impl VastAiBootstrapLauncher for NoopBootstrapLauncher { - type Handle = (); - - fn start_bootstrap( - &mut self, - _spec: NodeProvisionSpec, - _endpoint: VastAiSshEndpoint, - _sink: PluginSink, - _producer: Option, - _lifecycle: LifecyclePolicy, - ) -> Result { - panic!("build_request tests must not start SSH bootstrap") - } - - fn stop_bootstrap(&mut self, _handle: &mut Self::Handle, _reason: BootstrapStopReason) { - panic!("build_request tests must not stop SSH bootstrap") - } - } - - fn node_spec_with_bootstrap_args() -> NodeProvisionSpec { - NodeProvisionSpec { - run_id: 9, - node_id: 11, - stage_index: Some(2), - image: "registry.example.com/mvp-worker:latest".to_owned(), - env: vec![("EXISTING".to_owned(), "1".to_owned())], - args: vec!["python".to_owned(), "worker.py".to_owned()], - mounts: Vec::new(), - } - } - - fn plugin_with_onstart( - onstart: Option, - ) -> VastAiProvisioningPlugin { - let config = VastAiProvisioningConfig { - onstart, - ..VastAiProvisioningConfig::default() - }; - VastAiProvisioningPlugin::new(NoopLeaseClient, NoopBootstrapLauncher, config) - } - - #[derive(Clone, Default)] - struct ObservationSink { - observations: Arc>>, - } - - impl crate::provisioning::PluginObservationSink for ObservationSink { - fn observe(&self, observation: PluginObservation) { - self.observations.lock().push(observation); - } - } - - #[derive(Clone)] - struct RecordingLeaseClient { - destroyed_contracts: Arc>>, - } - - impl VastAiLeaseClient for RecordingLeaseClient { - fn provision_one( - &mut self, - _request: ProvisionRequest, - ) -> Result { - Ok(ProvisionedInstance { - index: 0, - contract_id: 42, - offer_id: 7, - host_id: Some(99), - gpu_name: "RTX 4060".to_owned(), - gpu_ram: Some(8_192.0), - dph_total: 0.064, - }) - } - - fn ssh_endpoint( - &mut self, - _contract_id: u64, - _label: &str, - _lifecycle: &LifecyclePolicy, - ssh_user: &str, - ) -> Result { - Ok(VastAiSshEndpoint { - host: "ssh5.vast.ai".to_owned(), - port: 22_017, - user: ssh_user.to_owned(), - }) - } - - fn destroy_contract(&mut self, contract_id: u64) -> Result<(), String> { - self.destroyed_contracts.lock().push(contract_id); - Ok(()) - } - } - - #[derive(Clone)] - struct RecordingBootstrapLauncher { - stop_reasons: Arc>>, - } - - impl VastAiBootstrapLauncher for RecordingBootstrapLauncher { - type Handle = u64; - - fn start_bootstrap( - &mut self, - spec: NodeProvisionSpec, - _endpoint: VastAiSshEndpoint, - _sink: PluginSink, - _producer: Option, - _lifecycle: LifecyclePolicy, - ) -> Result { - Ok(spec.node_id) - } - - fn stop_bootstrap(&mut self, _handle: &mut Self::Handle, reason: BootstrapStopReason) { - self.stop_reasons.lock().push(reason); - } - } - - #[test] - fn vastai_provisioning_build_request_keeps_bootstrap_args_out_of_onstart() { - let plugin = plugin_with_onstart(None); - let request = - plugin.build_request(&node_spec_with_bootstrap_args(), "test-label".to_owned()); - - assert_eq!(request.onstart, None); - assert_eq!(request.label.as_deref(), Some("test-label")); - assert_eq!(request.env.get("EXISTING").map(String::as_str), Some("1")); - } - - #[test] - fn vastai_provisioning_build_request_uses_explicit_onstart() { - let plugin = plugin_with_onstart(Some("echo explicit setup".to_owned())); - let request = - plugin.build_request(&node_spec_with_bootstrap_args(), "test-label".to_owned()); - - assert_eq!(request.onstart.as_deref(), Some("echo explicit setup")); - } - - #[test] - fn vastai_complete_bootstrap_keeps_optional_log_tail_until_node_stop() { - let destroyed_contracts = Arc::new(Mutex::new(Vec::new())); - let stop_reasons = Arc::new(Mutex::new(Vec::new())); - let sink = PluginSink::new(Arc::new(ObservationSink::default())); - let mut plugin = VastAiProvisioningPlugin::new( - RecordingLeaseClient { - destroyed_contracts: destroyed_contracts.clone(), - }, - RecordingBootstrapLauncher { - stop_reasons: stop_reasons.clone(), - }, - VastAiProvisioningConfig::default(), - ); - - let handle = plugin - .start_node(node_spec_with_bootstrap_args(), sink) - .expect("VastAI node starts"); - - plugin - .complete_bootstrap(&handle) - .expect("runtime-ready bootstrap completion succeeds"); - assert!( - stop_reasons.lock().is_empty(), - "runtime-ready keeps the SSH log tail alive for post-bootstrap diagnostics" - ); - - plugin.stop_node(&handle).expect("VastAI node stops"); - assert_eq!(*stop_reasons.lock(), vec![BootstrapStopReason::NodeStop]); - assert_eq!(*destroyed_contracts.lock(), vec![42]); - } - - #[test] - fn vastai_provisioning_next_ssh_backoff_doubles_until_thirty_second_cap() { - for (current, expected) in [ - (Duration::from_secs(1), Duration::from_secs(2)), - (Duration::from_secs(15), Duration::from_secs(30)), - (Duration::from_secs(20), Duration::from_secs(30)), - (Duration::from_secs(30), Duration::from_secs(30)), - ] { - assert_eq!( - next_ssh_backoff(current), - expected, - "backoff from {current:?}" - ); - } - } - - #[test] - fn ssh_bootstrap_observation_classifies_auth_and_transport_failures() { - for (line, expected) in [ - ("Permission denied (publickey).", Some("auth_denied")), - ( - "ssh: connect to host ssh5.vast.ai port 22017: Connection refused", - Some("refused"), - ), - ("ssh: connect timed out", Some("timeout")), - ("debug1: permanently_set_uid", None), - ] { - assert_eq!(classify_ssh_observation(line), expected, "{line}"); - } - } - - #[test] - fn ssh_bootstrap_args_include_verbose_flag_and_identity_when_configured() { - let endpoint = VastAiSshEndpoint { - host: "ssh5.vast.ai".to_owned(), - port: 22017, - user: "ubuntu".to_owned(), - }; - - let args = ssh_bootstrap_args(&endpoint, "python worker.py", Some(Path::new("/tmp/key"))); - - assert_eq!( - args, - vec![ - "-v", - "-p", - "22017", - "-o", - "BatchMode=yes", - "-o", - "StrictHostKeyChecking=accept-new", - "-i", - "/tmp/key", - "-o", - "IdentitiesOnly=yes", - "ubuntu@ssh5.vast.ai", - "python worker.py", - ] - ); - assert_eq!(args.iter().filter(|arg| arg.as_str() == "-v").count(), 1); - } - - #[test] - fn ssh_bootstrap_args_keep_identity_absent_when_not_configured() { - let endpoint = VastAiSshEndpoint { - host: "ssh5.vast.ai".to_owned(), - port: 22017, - user: "ubuntu".to_owned(), - }; - - let args = ssh_bootstrap_args(&endpoint, "python worker.py", None); - - assert_eq!( - args, - vec![ - "-v", - "-p", - "22017", - "-o", - "BatchMode=yes", - "-o", - "StrictHostKeyChecking=accept-new", - "ubuntu@ssh5.vast.ai", - "python worker.py", - ] - ); - assert!(!args.iter().any(|arg| arg == "-i")); - assert!(!args.iter().any(|arg| arg == "IdentitiesOnly=yes")); - } - - #[test] - fn provider_terminal_start_error_ignores_package_names_while_loading() { - let package_log = "#7 1.745 libevent-core-2.1-7t64 liberror-perl libglib2.0-data"; - - assert_eq!( - provider_terminal_start_error(46132050, "loading", "running", Some(package_log)), - None - ); - } - - #[test] - fn provider_terminal_start_error_reports_standalone_failure_words() { - assert_eq!( - provider_terminal_start_error( - 46132050, - "loading", - "running", - Some("ERROR: build failed") - ), - Some("instance 46132050 error: ERROR: build failed".to_owned()) - ); - } - - #[test] - fn ssh_bootstrap_actor_stop_kills_child() { - let runtime = Arc::new(swactor::runtime::Runtime::new( - swactor::config::RuntimeConfig::default(), - )); - let child = std::process::Command::new("sleep") - .arg("30") - .spawn() - .expect("spawn sleep child"); - let pid = child.id(); - let bridge = BootstrapDatastreamBridge::new( - node_spec_with_bootstrap_args(), - PluginSink::new(Arc::new(ObservationSink::default())), - None, - ); - let actor = runtime - .spawn(SshBootstrapActor { - bridge, - endpoint: VastAiSshEndpoint { - host: "127.0.0.1".to_owned(), - port: 22, - user: "ubuntu".to_owned(), - }, - ssh_identity: None, - sender: runtime.create_sender(), - child: Some(child), - stdout_reader: None, - stderr_reader: None, - stdout_closed: true, - stderr_closed: true, - pending_status: None, - pending_wait_error: None, - attempt: 1, - backoff: Duration::from_secs(1), - observation_class: None, - stopped: false, - start_on_boot: false, - }) - .expect("spawn ssh bootstrap actor"); - - runtime - .send_to(actor, SshBootstrapMsg::Stop) - .expect("send stop"); - runtime.tick(); - #[cfg(target_os = "linux")] - assert!( - !std::path::Path::new(&format!("/proc/{pid}")).exists(), - "child process should be reaped" - ); - } -} diff --git a/crates/mvp-system/src/orchestration/provisioning.rs b/crates/mvp-system/src/orchestration/provisioning.rs index bace936..aab405a 100644 --- a/crates/mvp-system/src/orchestration/provisioning.rs +++ b/crates/mvp-system/src/orchestration/provisioning.rs @@ -583,293 +583,3 @@ fn spawn_stderr_reader( ) { BootstrapDatastreamBridge::new(spec, sink, None).spawn_stderr_reader(stderr); } - -#[cfg(test)] -mod tests { - use super::*; - use std::sync::Mutex; - use std::time::{Duration, Instant}; - - #[cfg(unix)] - use std::os::unix::fs::PermissionsExt; - - #[derive(Default)] - struct RecordingSink { - observations: Mutex>, - } - - impl RecordingSink { - fn observations(&self) -> Vec { - self.observations - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .clone() - } - } - - impl PluginObservationSink for RecordingSink { - fn observe(&self, observation: PluginObservation) { - self.observations - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .push(observation); - } - } - - fn recording_sink() -> (Arc, PluginSink) { - let recorder = Arc::new(RecordingSink::default()); - (Arc::clone(&recorder), PluginSink::new(recorder)) - } - - fn wait_for_observation( - recorder: &RecordingSink, - predicate: impl Fn(&PluginObservation) -> bool, - ) -> PluginObservation { - let deadline = Instant::now() + Duration::from_secs(5); - loop { - if let Some(observation) = recorder - .observations() - .into_iter() - .find(|observation| predicate(observation)) - { - return observation; - } - assert!( - Instant::now() < deadline, - "timed out waiting for observation; observed: {:?}", - recorder.observations() - ); - thread::sleep(Duration::from_millis(20)); - } - } - - fn process_spec(node_id: u64, args: Vec) -> NodeProvisionSpec { - NodeProvisionSpec { - run_id: 17, - node_id, - stage_index: Some(2), - image: "unused-for-local-process".to_owned(), - env: vec![("MVP_RUN_ID".to_owned(), "17".to_owned())], - args, - mounts: Vec::new(), - } - } - - #[cfg(unix)] - struct TempScript { - root: PathBuf, - path: PathBuf, - } - - #[cfg(unix)] - impl TempScript { - fn new(name: &str, content: &str) -> Self { - let root = std::env::temp_dir().join(format!( - "mvp-local-process-plugin-{name}-{}-{}", - std::process::id(), - thread::current().name().unwrap_or("unnamed") - )); - let _ = fs::remove_dir_all(&root); - fs::create_dir_all(&root).expect("create temp script directory"); - let path = root.join("helper.sh"); - fs::write(&path, content).expect("write local process helper"); - let mut permissions = fs::metadata(&path) - .expect("stat local process helper") - .permissions(); - permissions.set_mode(0o755); - fs::set_permissions(&path, permissions).expect("chmod local process helper"); - Self { root, path } - } - } - - #[cfg(unix)] - impl Drop for TempScript { - fn drop(&mut self) { - let _ = fs::remove_dir_all(&self.root); - } - } - - #[cfg(unix)] - fn process_alive(pid: u32) -> bool { - unsafe { - if libc::kill(pid as i32, 0) == 0 { - true - } else { - std::io::Error::last_os_error().raw_os_error() != Some(libc::ESRCH) - } - } - } - - fn spec_with_mounts(mounts: Vec) -> NodeProvisionSpec { - NodeProvisionSpec { - run_id: 17, - node_id: 23, - stage_index: Some(2), - image: "swactor-mvp-node:test".to_owned(), - env: vec![("MVP_RUN_ID".to_owned(), "17".to_owned())], - args: vec!["--serve".to_owned()], - mounts, - } - } - - #[test] - fn node_provision_spec_serde_preserves_readonly_mounts() { - let spec = spec_with_mounts(vec![ProviderMount { - host_path: "/cache/models/model.gguf".to_owned(), - container_path: "/models/cached/model.gguf".to_owned(), - readonly: true, - }]); - - let json = serde_json::to_string(&spec).expect("serialize mounted node spec"); - let decoded: NodeProvisionSpec = - serde_json::from_str(&json).expect("deserialize mounted node spec"); - - assert_eq!(decoded, spec); - } - - #[test] - fn node_provision_spec_serde_omits_empty_mounts_and_accepts_missing_mounts() { - let spec = spec_with_mounts(Vec::new()); - - let json = serde_json::to_value(&spec).expect("serialize unmounted node spec"); - assert_eq!(json.get("mounts"), None); - - let decoded: NodeProvisionSpec = serde_json::from_value(serde_json::json!({ - "run_id": 17, - "node_id": 23, - "stage_index": 2, - "image": "swactor-mvp-node:test", - "env": [["MVP_RUN_ID", "17"]], - "args": ["--serve"] - })) - .expect("deserialize node spec written before mounts existed"); - - assert!(decoded.mounts.is_empty()); - } - - #[test] - fn docker_mount_arg_uses_bind_src_dst_and_readonly_flag() { - let mount = ProviderMount { - host_path: "/cache/models/model.gguf".to_owned(), - container_path: "/models/cached/model.gguf".to_owned(), - readonly: true, - }; - - assert_eq!( - docker_mount_arg(&mount), - "type=bind,src=/cache/models/model.gguf,dst=/models/cached/model.gguf,readonly" - ); - - let writable_mount = ProviderMount { - readonly: false, - ..mount - }; - assert_eq!( - docker_mount_arg(&writable_mount), - "type=bind,src=/cache/models/model.gguf,dst=/models/cached/model.gguf" - ); - } - - #[cfg(unix)] - #[test] - fn local_process_plugin_observes_stdio_frames_and_graceful_shutdown() { - let helper = TempScript::new( - "graceful", - r#"#!/bin/sh -printf '%s\n' '{"mvp_stdio_event":1,"kind":"datastream_frame","channel":"mvp.node.bootstrap","payload":{"type":"TestFrame","status":"ready"}}' -printf '%s\n' 'local-process-stderr' >&2 -while IFS= read -r line; do - if [ "$line" = shutdown ]; then - exit 0 - fi -done -exit 0 -"#, - ); - let mut plugin = LocalProcessPlugin::new("/bin/sh"); - let (recorder, sink) = recording_sink(); - let spec = process_spec(23, vec![helper.path.to_string_lossy().to_string()]); - - let handle = plugin - .start_node(spec, sink) - .expect("start local process node"); - - assert!(handle.provider_process_id.is_some()); - wait_for_observation(&recorder, |observation| { - matches!( - observation, - PluginObservation::DatastreamFrame { - channel, - payload, - .. - } if channel == "mvp.node.bootstrap" - && payload.contains("\"TestFrame\"") - && payload.contains("\"ready\"") - ) - }); - wait_for_observation(&recorder, |observation| { - matches!( - observation, - PluginObservation::StderrLine { line, .. } if line == "local-process-stderr" - ) - }); - - plugin.stop_node(&handle).expect("stop local process node"); - - wait_for_observation(&recorder, |observation| { - matches!( - observation, - PluginObservation::Exited { - node_id: 23, - status: Some(0), - .. - } - ) - }); - } - - #[cfg(unix)] - #[test] - fn local_process_plugin_kills_and_reaps_unresponsive_child() { - let helper = TempScript::new( - "unresponsive", - r#"#!/bin/sh -while :; do - sleep 1 -done -"#, - ); - let mut plugin = LocalProcessPlugin::new("/bin/sh"); - let (recorder, sink) = recording_sink(); - let spec = process_spec(24, vec![helper.path.to_string_lossy().to_string()]); - let handle = plugin - .start_node(spec, sink) - .expect("start unresponsive local process node"); - let pid = handle - .provider_process_id - .expect("local process handle exposes child pid"); - - plugin - .stop_node(&handle) - .expect("stop unresponsive local process node"); - - wait_for_observation(&recorder, |observation| { - matches!( - observation, - PluginObservation::Exited { - node_id: 24, - status, - .. - } if *status != Some(0) - ) - }); - let deadline = Instant::now() + Duration::from_secs(1); - while process_alive(pid) && Instant::now() < deadline { - thread::sleep(Duration::from_millis(20)); - } - assert!( - !process_alive(pid), - "local process child {pid} is still live" - ); - } -} diff --git a/crates/mvp-system/src/prompt/rpc.rs b/crates/mvp-system/src/prompt/rpc.rs index 6cfeba6..96768dd 100644 --- a/crates/mvp-system/src/prompt/rpc.rs +++ b/crates/mvp-system/src/prompt/rpc.rs @@ -98,47 +98,3 @@ pub fn read_submit_prompt(reader: &mut impl BufRead) -> Result(reader: &mut R) -> Result { .map_err(|e| format!("read i64: {e}"))?; Ok(i64::from_le_bytes(bytes)) } - -#[cfg(test)] -mod tests { - use super::*; - use std::io::Cursor; - - #[test] - fn reads_planning_metadata_from_minimal_gguf_header() { - let mut bytes = Vec::new(); - bytes.extend_from_slice(b"GGUF"); - bytes.extend_from_slice(&3_u32.to_le_bytes()); - bytes.extend_from_slice(&0_u64.to_le_bytes()); - bytes.extend_from_slice(&6_u64.to_le_bytes()); - push_string_kv(&mut bytes, "general.architecture", "llama"); - push_string_kv(&mut bytes, "general.name", "fixture"); - push_u32_kv(&mut bytes, "llama.block_count", 30); - push_u32_kv(&mut bytes, "llama.embedding_length", 576); - push_u32_kv(&mut bytes, "llama.context_length", 8192); - push_u32_kv(&mut bytes, "tokenizer.ggml.eos_token_id", 2); - - let metadata = read_gguf_planning_metadata_from_reader(Cursor::new(bytes)).unwrap(); - - assert_eq!(metadata.version, 3); - assert_eq!(metadata.architecture, "llama"); - assert_eq!(metadata.name.as_deref(), Some("fixture")); - assert_eq!(metadata.num_layers, 30); - assert_eq!(metadata.hidden_dim, 576); - assert_eq!(metadata.context_length, 8192); - assert_eq!(metadata.eos_token_id, 2); - } - - #[test] - fn model_facts_clamp_effective_context_to_gguf_context() { - let metadata = GgufPlanningMetadata { - version: 3, - architecture: "llama".to_owned(), - name: None, - num_layers: 30, - hidden_dim: 576, - context_length: 256, - eos_token_id: 2, - }; - - let facts = metadata - .to_model_facts( - "fixture", - GgufSource::LocalPath("/models/fixture.gguf".to_owned()), - TokenizerSource::EmbeddedGguf, - Some(512), - ) - .unwrap(); - - assert_eq!(facts.max_seq_len, 256); - assert_eq!(facts.dtype_family, DTypeFamily::BFloat); - assert_eq!(facts.dtype_width_bytes, 2); - } - - #[test] - fn model_facts_use_explicit_context_when_it_fits_inside_gguf_context() { - let metadata = GgufPlanningMetadata { - version: 3, - architecture: "llama".to_owned(), - name: Some("fixture".to_owned()), - num_layers: 30, - hidden_dim: 576, - context_length: 8192, - eos_token_id: 2, - }; - - let facts = metadata - .to_model_facts( - "fixture-model", - GgufSource::LocalPath("/models/fixture.gguf".to_owned()), - TokenizerSource::EmbeddedGguf, - Some(384), - ) - .unwrap(); - - assert_eq!(facts.model_id, "fixture-model"); - assert_eq!( - facts.gguf_source, - GgufSource::LocalPath("/models/fixture.gguf".to_owned()) - ); - assert_eq!(facts.num_layers, 30); - assert_eq!(facts.hidden_dim, 576); - assert_eq!(facts.dtype_family, DTypeFamily::BFloat); - assert_eq!(facts.dtype_width_bytes, 2); - assert_eq!(facts.max_seq_len, 384); - assert_eq!(facts.eos_token_id, 2); - assert_eq!(facts.tokenizer, TokenizerSource::EmbeddedGguf); - } - - #[test] - fn reader_rejects_missing_required_planning_metadata() { - for (missing_key, expected_error) in [ - ("general.architecture", "missing general.architecture"), - ( - "llama.block_count", - "missing layer count key llama.block_count", - ), - ( - "llama.embedding_length", - "missing hidden dimension key llama.embedding_length", - ), - ( - "llama.context_length", - "missing context length key llama.context_length", - ), - ( - "tokenizer.ggml.eos_token_id", - "missing EOS token id key tokenizer.ggml.eos_token_id", - ), - ] { - let error = read_gguf_planning_metadata_from_reader(Cursor::new(minimal_gguf_without( - missing_key, - ))) - .expect_err("metadata with a missing planning field must reject"); - assert!( - error.contains(expected_error), - "missing {missing_key} error {error:?} should contain {expected_error:?}" - ); - } - } - - fn minimal_gguf_without(missing_key: &str) -> Vec { - let fields = [ - "general.architecture", - "general.name", - "llama.block_count", - "llama.embedding_length", - "llama.context_length", - "tokenizer.ggml.eos_token_id", - ]; - let mut bytes = Vec::new(); - bytes.extend_from_slice(b"GGUF"); - bytes.extend_from_slice(&3_u32.to_le_bytes()); - bytes.extend_from_slice(&0_u64.to_le_bytes()); - bytes.extend_from_slice( - &(fields.iter().filter(|field| **field != missing_key).count() as u64).to_le_bytes(), - ); - if missing_key != "general.architecture" { - push_string_kv(&mut bytes, "general.architecture", "llama"); - } - if missing_key != "general.name" { - push_string_kv(&mut bytes, "general.name", "fixture"); - } - if missing_key != "llama.block_count" { - push_u32_kv(&mut bytes, "llama.block_count", 30); - } - if missing_key != "llama.embedding_length" { - push_u32_kv(&mut bytes, "llama.embedding_length", 576); - } - if missing_key != "llama.context_length" { - push_u32_kv(&mut bytes, "llama.context_length", 8192); - } - if missing_key != "tokenizer.ggml.eos_token_id" { - push_u32_kv(&mut bytes, "tokenizer.ggml.eos_token_id", 2); - } - bytes - } - - fn push_string_kv(bytes: &mut Vec, key: &str, value: &str) { - push_string(bytes, key); - bytes.extend_from_slice(&8_u32.to_le_bytes()); - push_string(bytes, value); - } - - fn push_u32_kv(bytes: &mut Vec, key: &str, value: u32) { - push_string(bytes, key); - bytes.extend_from_slice(&4_u32.to_le_bytes()); - bytes.extend_from_slice(&value.to_le_bytes()); - } - - fn push_string(bytes: &mut Vec, value: &str) { - bytes.extend_from_slice(&(value.len() as u64).to_le_bytes()); - bytes.extend_from_slice(value.as_bytes()); - } -} diff --git a/crates/mvp-system/src/staging/gguf_shard.rs b/crates/mvp-system/src/staging/gguf_shard.rs index 6bd8601..028b482 100644 --- a/crates/mvp-system/src/staging/gguf_shard.rs +++ b/crates/mvp-system/src/staging/gguf_shard.rs @@ -991,389 +991,3 @@ fn read_i64(reader: &mut R) -> Result { .map_err(|e| format!("read i64: {e}"))?; Ok(i64::from_le_bytes(bytes)) } - -#[cfg(test)] -mod tests { - use super::*; - use parking_lot::Mutex; - use std::io::{Read, Write}; - use std::net::{TcpListener, TcpStream}; - use std::sync::Arc; - use std::thread; - - #[test] - fn stage_plans_select_only_assigned_layer_tensors_and_boundaries() { - let fixture = SyntheticGguf::new(8); - let source = GgufSource::HuggingFaceGguf { - repo: "org/repo".to_owned(), - file: "model file.gguf".to_owned(), - revision: Some("abc123".to_owned()), - }; - - let stage0 = plan_stage_shard(&fixture.path, source.clone(), 0, 4, 0, 2).unwrap(); - let stage1 = plan_stage_shard(&fixture.path, source.clone(), 1, 4, 2, 4).unwrap(); - let stage3 = plan_stage_shard(&fixture.path, source.clone(), 3, 4, 6, 8).unwrap(); - - assert_eq!( - names(&stage0), - vec![ - "token_embd.weight", - "blk.0.attn_q.weight", - "blk.0.ffn_up.weight", - "blk.1.attn_q.weight", - "blk.1.ffn_up.weight", - ] - ); - assert_eq!( - names(&stage1), - vec![ - "blk.2.attn_q.weight", - "blk.2.ffn_up.weight", - "blk.3.attn_q.weight", - "blk.3.ffn_up.weight", - ] - ); - assert_eq!( - names(&stage3), - vec![ - "blk.6.attn_q.weight", - "blk.6.ffn_up.weight", - "blk.7.attn_q.weight", - "blk.7.ffn_up.weight", - "output_norm.weight", - "output.weight", - ] - ); - assert_eq!( - stage0.source_url().unwrap(), - "https://huggingface.co/org/repo/resolve/abc123/model%20file.gguf" - ); - assert!( - stage1 - .merged_tensor_ranges - .iter() - .all(|range| range.start >= stage1.data_start) - ); - assert!( - stage1 - .merged_tensor_ranges - .iter() - .map(|range| range.len) - .sum::() - < fixture.bytes_len - ); - } - - #[test] - fn tied_output_final_stage_includes_token_embedding_when_output_weight_is_missing() { - let fixture = SyntheticGguf::without_output_weight(4); - let source = GgufSource::HuggingFaceGguf { - repo: "org/repo".to_owned(), - file: "model.gguf".to_owned(), - revision: None, - }; - - let final_stage = plan_stage_shard(&fixture.path, source, 1, 2, 2, 4).unwrap(); - - assert!(names(&final_stage).contains(&"token_embd.weight")); - assert!(names(&final_stage).contains(&"output_norm.weight")); - assert!(!names(&final_stage).contains(&"output.weight")); - } - - #[test] - fn materialized_stage_shard_fetches_only_http_ranges_and_loads_as_gguf() { - let fixture = SyntheticGguf::new(4); - let bytes = std::fs::read(&fixture.path).unwrap(); - let server = RangeServer::start(bytes); - let source = GgufSource::HuggingFaceGguf { - repo: "org/repo".to_owned(), - file: "model.gguf".to_owned(), - revision: None, - }; - let plan = plan_stage_shard(&fixture.path, source, 1, 2, 2, 4).unwrap(); - let output_path = fixture.path.with_file_name("stage-1.gguf"); - let mut events = Vec::new(); - - materialize_stage_shard_from_url(&plan, &server.url, &output_path, |event| { - events.push(event); - }) - .unwrap(); - - let materialized = read_gguf_directory(&output_path).unwrap(); - assert_eq!(materialized.tensors.len(), plan.tensors.len()); - assert_eq!( - materialized - .tensors - .iter() - .map(|tensor| tensor.name.as_str()) - .collect::>(), - names(&plan) - ); - let ready = events.iter().find(|event| { - event.get("type").and_then(serde_json::Value::as_str) == Some("StageShardReady") - }); - assert_eq!( - ready - .and_then(|event| event.get("bytes_done")) - .and_then(serde_json::Value::as_u64), - Some(plan.planned_fetch_bytes()) - ); - assert_eq!( - ready - .and_then(|event| event.get("bytes_total")) - .and_then(serde_json::Value::as_u64), - Some(plan.planned_fetch_bytes()) - ); - let ranges = server.ranges(); - let expected_ranges = std::iter::once((0, plan.metadata_end.saturating_sub(1))) - .chain(plan.merged_tensor_ranges.iter().map(|range| { - ( - range.start, - range - .end_exclusive() - .expect("planned range end") - .saturating_sub(1), - ) - })) - .collect::>(); - assert_eq!(ranges, expected_ranges); - assert!( - ranges.len() < plan.tensors.len() + 1, - "coalesced tensor ranges should replace one HTTP request per tensor" - ); - let range_ready = events - .iter() - .filter(|event| { - event.get("type").and_then(serde_json::Value::as_str) - == Some("StageShardRangeFetchReady") - }) - .collect::>(); - assert_eq!(range_ready.len(), plan.planned_range_count()); - assert_eq!( - range_ready - .last() - .and_then(|event| event.get("bytes_done")) - .and_then(serde_json::Value::as_u64), - Some(plan.planned_fetch_bytes()) - ); - assert_eq!( - range_ready - .last() - .and_then(|event| event.get("range_index")) - .and_then(serde_json::Value::as_u64), - Some((plan.planned_range_count() - 1) as u64) - ); - let total_requested = ranges - .iter() - .map(|(start, end)| end.saturating_sub(*start) + 1) - .sum::(); - assert_eq!(total_requested, plan.planned_fetch_bytes()); - assert!(total_requested < fixture.bytes_len); - validate_stage_shard_cache(&output_path, &plan).unwrap(); - } - - #[test] - fn corrupted_stage_shard_cache_is_rejected_before_reuse() { - let fixture = SyntheticGguf::new(4); - let source = GgufSource::HuggingFaceGguf { - repo: "org/repo".to_owned(), - file: "model.gguf".to_owned(), - revision: None, - }; - let plan = plan_stage_shard(&fixture.path, source, 1, 2, 2, 4).unwrap(); - let output_path = fixture.path.with_file_name(plan.cache_file_name()); - - std::fs::write(&output_path, b"not a gguf").unwrap(); - - let error = validate_stage_shard_cache(&output_path, &plan).unwrap_err(); - assert!(error.contains("invalid cached stage shard")); - } - - fn names(plan: &StageShardPlan) -> Vec<&str> { - plan.tensors - .iter() - .map(|tensor| tensor.name.as_str()) - .collect() - } - - struct RangeServer { - url: String, - ranges: Arc>>, - } - - impl RangeServer { - fn start(bytes: Vec) -> Self { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let url = format!("http://{}", listener.local_addr().unwrap()); - let bytes = Arc::new(bytes); - let ranges = Arc::new(Mutex::new(Vec::new())); - let server_ranges = Arc::clone(&ranges); - thread::spawn(move || { - for stream in listener.incoming().flatten() { - handle_range_request(stream, Arc::clone(&bytes), Arc::clone(&server_ranges)); - } - }); - Self { url, ranges } - } - - fn ranges(&self) -> Vec<(u64, u64)> { - self.ranges.lock().clone() - } - } - - fn handle_range_request( - mut stream: TcpStream, - bytes: Arc>, - ranges: Arc>>, - ) { - let mut request = Vec::new(); - let mut buffer = [0_u8; 1024]; - while !request.windows(4).any(|window| window == b"\r\n\r\n") { - let Ok(read) = stream.read(&mut buffer) else { - return; - }; - if read == 0 { - return; - } - request.extend_from_slice(&buffer[..read]); - } - let request = String::from_utf8_lossy(&request); - let Some(range_header) = request - .lines() - .find(|line| line.to_ascii_lowercase().starts_with("range: bytes=")) - else { - let body = b"missing range"; - let _ = write!( - stream, - "HTTP/1.1 400 Bad Request\r\nContent-Length: {}\r\n\r\n", - body.len() - ); - let _ = stream.write_all(body); - return; - }; - let range = range_header - .split_once("bytes=") - .map(|(_, range)| range.trim()) - .unwrap(); - let (start, end) = range.split_once('-').unwrap(); - let start = start.parse::().unwrap(); - let end = end.parse::().unwrap(); - let content = &bytes[start as usize..=end as usize]; - ranges.lock().push((start, end)); - let _ = write!( - stream, - "HTTP/1.1 206 Partial Content\r\nContent-Length: {}\r\nContent-Range: bytes {}-{}/{}\r\nAccept-Ranges: bytes\r\n\r\n", - content.len(), - start, - end, - bytes.len() - ); - let _ = stream.write_all(content); - } - - struct SyntheticGguf { - path: std::path::PathBuf, - bytes_len: u64, - _dir: std::path::PathBuf, - } - - impl SyntheticGguf { - fn new(layers: u32) -> Self { - Self::build(layers, true) - } - - fn without_output_weight(layers: u32) -> Self { - Self::build(layers, false) - } - - fn build(layers: u32, output_weight: bool) -> Self { - let dir = std::env::temp_dir().join(format!( - "mvp-gguf-shard-test-{}-{:?}", - std::process::id(), - std::thread::current().id() - )); - let _ = std::fs::remove_dir_all(&dir); - std::fs::create_dir_all(&dir).unwrap(); - let path = dir.join(if output_weight { - "model.gguf" - } else { - "tied.gguf" - }); - let bytes = synthetic_gguf(layers, output_weight); - std::fs::File::create(&path) - .unwrap() - .write_all(&bytes) - .unwrap(); - Self { - path, - bytes_len: bytes.len() as u64, - _dir: dir, - } - } - } - - fn synthetic_gguf(layers: u32, output_weight: bool) -> Vec { - let mut tensors = vec!["token_embd.weight".to_owned()]; - for layer in 0..layers { - tensors.push(format!("blk.{layer}.attn_q.weight")); - tensors.push(format!("blk.{layer}.ffn_up.weight")); - } - tensors.push("output_norm.weight".to_owned()); - if output_weight { - tensors.push("output.weight".to_owned()); - } - - let mut metadata = Vec::new(); - push_string_kv(&mut metadata, "general.architecture", "llama"); - push_u32_kv(&mut metadata, "general.alignment", 32); - push_u32_kv(&mut metadata, "llama.block_count", layers); - push_u32_kv(&mut metadata, "llama.embedding_length", 8); - push_u32_kv(&mut metadata, "llama.context_length", 16); - push_u32_kv(&mut metadata, "tokenizer.ggml.eos_token_id", 2); - - let mut tensor_infos = Vec::new(); - let mut data = Vec::new(); - let mut offset = 0_u64; - for (index, name) in tensors.iter().enumerate() { - push_gguf_string(&mut tensor_infos, name); - tensor_infos.extend_from_slice(&2_u32.to_le_bytes()); - tensor_infos.extend_from_slice(&2_u64.to_le_bytes()); - tensor_infos.extend_from_slice(&2_u64.to_le_bytes()); - tensor_infos.extend_from_slice(&0_u32.to_le_bytes()); - tensor_infos.extend_from_slice(&offset.to_le_bytes()); - let len = 16 + index as u64; - data.extend(std::iter::repeat_n(index as u8, len as usize)); - offset += len; - } - - let mut bytes = Vec::new(); - bytes.extend_from_slice(b"GGUF"); - bytes.extend_from_slice(&3_u32.to_le_bytes()); - bytes.extend_from_slice(&(tensors.len() as u64).to_le_bytes()); - bytes.extend_from_slice(&6_u64.to_le_bytes()); - bytes.extend_from_slice(&metadata); - bytes.extend_from_slice(&tensor_infos); - while bytes.len() % 32 != 0 { - bytes.push(0); - } - bytes.extend_from_slice(&data); - bytes - } - - fn push_string_kv(bytes: &mut Vec, key: &str, value: &str) { - push_gguf_string(bytes, key); - bytes.extend_from_slice(&8_u32.to_le_bytes()); - push_gguf_string(bytes, value); - } - - fn push_u32_kv(bytes: &mut Vec, key: &str, value: u32) { - push_gguf_string(bytes, key); - bytes.extend_from_slice(&4_u32.to_le_bytes()); - bytes.extend_from_slice(&value.to_le_bytes()); - } - - fn push_gguf_string(bytes: &mut Vec, value: &str) { - bytes.extend_from_slice(&(value.len() as u64).to_le_bytes()); - bytes.extend_from_slice(value.as_bytes()); - } -} diff --git a/crates/mvp-system/src/tests/bootstrap_datastream_guarantees.rs b/crates/mvp-system/src/tests/bootstrap_datastream_guarantees.rs deleted file mode 100644 index 207e986..0000000 --- a/crates/mvp-system/src/tests/bootstrap_datastream_guarantees.rs +++ /dev/null @@ -1,120 +0,0 @@ -use std::sync::Arc; - -use crate::provisioning::{ - NodeProvisionSpec, PluginObservation, PluginObservationSink, PluginSink, ProvisionLogStream, -}; -use datastream::{DatastreamEndpoint, Record}; -use iroh::{EndpointAddr, SecretKey}; -use mvp_system::observability::provisioning_logs::{BootstrapDatastreamBridge, node_stream_id}; -use mvp_system::observability::telemetry::MvpProvisionLogRecord; -use parking_lot::Mutex; -use serde_json::json; - -#[derive(Default)] -struct RecordingSink { - observations: Mutex>, -} - -impl RecordingSink { - fn observations(&self) -> Vec { - self.observations.lock().clone() - } -} - -impl PluginObservationSink for RecordingSink { - fn observe(&self, observation: PluginObservation) { - self.observations.lock().push(observation); - } -} - -fn recording_sink() -> (Arc, PluginSink) { - let recording = Arc::new(RecordingSink::default()); - (recording.clone(), PluginSink::new(recording)) -} - -fn spec() -> NodeProvisionSpec { - NodeProvisionSpec { - run_id: 7, - node_id: 42, - stage_index: Some(3), - image: "worker:latest".to_owned(), - env: Vec::new(), - args: Vec::new(), - mounts: Vec::new(), - } -} - -#[test] -fn bootstrap_bridge_writes_node_stream_and_forwards_plugin_observations() { - let endpoint = DatastreamEndpoint::new(node_stream_id(7, 42)); - let subscription = endpoint.subscribe_all("test"); - let (recording, sink) = recording_sink(); - let bridge = BootstrapDatastreamBridge::new(spec(), sink, Some(endpoint.producer())); - - bridge.observe_stdout_line("ssh stdout diagnostic"); - bridge.observe_stderr_line("debug1: ssh stderr diagnostic"); - endpoint.tick(); - - let observations = recording.observations(); - assert_eq!( - observations, - vec![ - PluginObservation::StdoutLine { - run_id: 7, - node_id: 42, - line: "ssh stdout diagnostic".to_owned(), - }, - PluginObservation::StderrLine { - run_id: 7, - node_id: 42, - line: "debug1: ssh stderr diagnostic".to_owned(), - }, - ] - ); - - let deliveries = subscription.drain_available(); - let logs = deliveries - .iter() - .filter_map(|event| match event { - datastream::DatastreamEvent::Frame(delivery) - if delivery.channel.stream == node_stream_id(7, 42) => - { - MvpProvisionLogRecord::decode(&delivery.payload).ok() - } - _ => None, - }) - .collect::>(); - assert_eq!(logs.len(), 2, "{deliveries:?}"); - - let stdout = &logs[0]; - let stderr = &logs[1]; - assert_eq!(stdout.line.stream, ProvisionLogStream::Stdout); - assert_eq!(stdout.line.line, "ssh stdout diagnostic"); - assert_eq!(stderr.line.stream, ProvisionLogStream::Stderr); - assert_eq!(stderr.line.line, "debug1: ssh stderr diagnostic"); -} - -#[test] -fn stdout_ready_json_is_log_only() { - let (recording, sink) = recording_sink(); - let bridge = BootstrapDatastreamBridge::new(spec(), sink, None); - let line = serde_json::to_string(&json!({ - "type": "ready", - "endpoint": EndpointAddr::new(SecretKey::from_bytes(&[7; 32]).public()), - "node_actor": "ignored-by-stdout-bridge", - "logical_node_id": 42, - "stage_index": 3, - })) - .unwrap(); - - bridge.observe_stdout_line(line.clone()); - - assert_eq!( - recording.observations(), - vec![PluginObservation::StdoutLine { - run_id: 7, - node_id: 42, - line, - }] - ); -} diff --git a/crates/mvp-system/src/tests/engine_builder_guarantees.rs b/crates/mvp-system/src/tests/engine_builder_guarantees.rs deleted file mode 100644 index 3da7d73..0000000 --- a/crates/mvp-system/src/tests/engine_builder_guarantees.rs +++ /dev/null @@ -1,200 +0,0 @@ -//! Black-box contract tests for the pool-based engine builder. -//! -//! The builder contract is topology/lifecycle only: it acquires a neutral node -//! pool, launches the same node image, waits for readiness/convergence, lets a -//! planner assign roles, and stays agnostic to workload input semantics. - -use mvp_system::orchestration::engine_builder as engine; -use mvp_system::orchestration::engine_builder::WorkloadAdapter; - -fn model() -> engine::ModelSpec { - engine::ModelSpec::mvp_tiny_open_llm_fixture() -} - -fn image() -> engine::NodeImageSpec { - engine::NodeImageSpec::new("mvp-node:cuda").worker_runtime( - engine::WorkerRuntimeSpec::TinygradCuda { - worker_script: "/opt/mvp/mvp_tinygrad_worker.py".to_owned(), - device_env: "CUDA".to_owned(), - }, - ) -} - -fn full_pool() -> engine::StaticPoolProvider { - engine::StaticPoolProvider::new(vec![ - engine::NodeLease::new( - "coordinator", - engine::NodeId(900), - [engine::NodeCapability::Coordinator], - ) - .resources(engine::ResourceFacts::cpu_only(2, 2 << 30)), - engine::NodeLease::new( - "worker-0", - engine::NodeId(11), - [engine::NodeCapability::Worker], - ) - .resources(engine::ResourceFacts::cuda(1, 8 << 30, 4, 8 << 30)), - engine::NodeLease::new( - "worker-1", - engine::NodeId(12), - [engine::NodeCapability::Worker], - ) - .resources(engine::ResourceFacts::cuda(1, 8 << 30, 4, 8 << 30)), - ]) -} - -#[test] -fn builder_launches_pool_converges_and_assigns_planned_roles() { - let cluster = engine::ClusterBuilder::new("cluster-a", model()) - .run_id(77) - .image(image()) - .pool_provider(full_pool()) - .launcher(engine::StaticNodeLauncher) - .planner(engine::FixedLinearPipelinePlanner::new(2)) - .launch() - .expect("launch cluster"); - - let summaries = cluster.node_summaries(); - let coordinator = summaries - .iter() - .find(|node| node.node_id == engine::NodeId(900)) - .expect("coordinator summary"); - assert_eq!(coordinator.roles, vec![engine::RoleKind::Coordinator]); - let worker0 = summaries - .iter() - .find(|node| node.node_id == engine::NodeId(11)) - .expect("worker 0 summary"); - assert_eq!( - worker0.roles, - vec![engine::RoleKind::StageWorker { stage_index: 0 }] - ); - let worker1 = summaries - .iter() - .find(|node| node.node_id == engine::NodeId(12)) - .expect("worker 1 summary"); - assert_eq!( - worker1.roles, - vec![engine::RoleKind::StageWorker { stage_index: 1 }] - ); - - let plan = cluster.role_plan(); - assert_eq!(plan.run_plan.stages.len(), 2); - assert_eq!(plan.run_plan.edges.len(), 3); - assert_eq!(plan.run_plan.stages[0].layer_start, 0); - assert_eq!(plan.run_plan.stages[0].layer_end_exclusive, 2); - assert_eq!(plan.run_plan.stages[1].layer_start, 2); - assert_eq!(plan.run_plan.stages[1].layer_end_exclusive, 4); - assert!( - cluster - .events() - .contains(&engine::EngineEvent::EngineReady { - cluster_id: "cluster-a".to_owned() - }) - ); - - let shutdown_events = cluster.shutdown().expect("shutdown cluster"); - assert!( - shutdown_events.contains(&engine::EngineEvent::ShutdownComplete { - cluster_id: "cluster-a".to_owned() - }) - ); -} - -#[test] -fn planner_rejects_when_pool_cannot_supply_stage_workers() { - let short_pool = engine::StaticPoolProvider::new(vec![ - engine::NodeLease::new( - "coordinator", - engine::NodeId(900), - [engine::NodeCapability::Coordinator], - ), - engine::NodeLease::new( - "worker-0", - engine::NodeId(11), - [engine::NodeCapability::Worker], - ), - engine::NodeLease::new( - "observer", - engine::NodeId(901), - [engine::NodeCapability::Coordinator], - ), - ]); - - let result = engine::ClusterBuilder::new("cluster-short", model()) - .run_id(77) - .image(image()) - .pool_provider(short_pool) - .launcher(engine::StaticNodeLauncher) - .planner(engine::FixedLinearPipelinePlanner::new(2)) - .launch(); - - match result { - Err(engine::EngineBuildError::Planning(engine::PlanningError::InsufficientWorkers { - required, - available, - })) => { - assert_eq!(required, 2); - assert_eq!(available, 1); - } - Ok(_) => panic!("expected insufficient worker planning error, got launched cluster"), - Err(other) => panic!("unexpected error: {other:?}"), - } -} - -struct ProbeWorkload; - -impl engine::WorkloadAdapter for ProbeWorkload { - type Input = Vec<&'static str>; - type Output = usize; - type Error = std::convert::Infallible; - - fn submit( - &self, - _cluster: &mut engine::ClusterHandle, - input: Self::Input, - ) -> Result { - Ok(input.len()) - } -} - -#[test] -fn workload_input_semantics_live_outside_the_cluster_builder() { - let mut cluster = engine::ClusterBuilder::new("cluster-opaque", model()) - .run_id(77) - .image(image()) - .pool_provider(full_pool()) - .launcher(engine::StaticNodeLauncher) - .planner(engine::FixedLinearPipelinePlanner::new(2)) - .launch() - .expect("launch cluster"); - - let observed = ProbeWorkload - .submit(&mut cluster, vec!["not", "tokens"]) - .expect("submit probe workload"); - assert_eq!(observed, 2); - - cluster.shutdown().expect("shutdown cluster"); -} - -#[test] -fn runtime_node_builds_the_reusable_iroh_swactor_stack() { - let mut node = engine::RuntimeNode::start_default().expect("start runtime node"); - - let orchestrator = node - .spawn_orchestrator_actor( - crate::run_fsm::RunConfig { - run_id: crate::run_fsm::RunId(77), - max_tokens: 1, - prompt: vec![1, 2, 3], - }, - None, - ) - .expect("spawn orchestrator actor"); - let worker = node - .spawn_node_agent_actor(mvp_system::staging::NodeId(11), orchestrator, None) - .expect("spawn node agent actor"); - - node.wait_for_routes(&[orchestrator, worker]) - .expect("local actor routes"); - assert_eq!(node.node_id().0, *node.endpoint_addr().id.as_bytes()); -} diff --git a/crates/mvp-system/src/tests/local_mock_pipeline_integration.rs b/crates/mvp-system/src/tests/local_e2e_guarantees.rs similarity index 99% rename from crates/mvp-system/src/tests/local_mock_pipeline_integration.rs rename to crates/mvp-system/src/tests/local_e2e_guarantees.rs index f5d8fce..727b9de 100644 --- a/crates/mvp-system/src/tests/local_mock_pipeline_integration.rs +++ b/crates/mvp-system/src/tests/local_e2e_guarantees.rs @@ -1,4 +1,4 @@ -//! Local mock integration coverage for the MVP system lifecycle. +//! Local end-to-end behavior guarantee for the MVP system lifecycle. //! //! This test composes the crate's MVP contract harnesses through one in-process //! mock environment. It deliberately avoids Docker, real SWIM, real iroh, GGUF, diff --git a/crates/mvp-system/src/tests/mod.rs b/crates/mvp-system/src/tests/mod.rs index 31d55d2..33dff01 100644 --- a/crates/mvp-system/src/tests/mod.rs +++ b/crates/mvp-system/src/tests/mod.rs @@ -1,14 +1,8 @@ -mod bootstrap_datastream_guarantees; -mod engine_builder_guarantees; +mod local_e2e_guarantees; mod local_mock; -mod local_mock_pipeline_integration; -mod observability_surface_guarantees; -mod orchestrator_run_fsm_guarantees; -mod relay_provisioning_guarantees; -mod run_plan_guarantees; -mod shared_ring_helper_abi_guarantees; -mod stage_controller_guarantees; -mod telemetry_guarantees; -mod tx_rx_edge_actor_guarantees; -mod weight_lifecycle_guarantees; -mod worker_edge_adapter_guarantees; +mod node_guarantees; +mod observability_guarantees; +mod orchestration_guarantees; +mod prompt_guarantees; +mod staging_guarantees; +mod transport_guarantees; diff --git a/crates/mvp-system/src/tests/node_guarantees.rs b/crates/mvp-system/src/tests/node_guarantees.rs new file mode 100644 index 0000000..07d0470 --- /dev/null +++ b/crates/mvp-system/src/tests/node_guarantees.rs @@ -0,0 +1,95 @@ +//! Behavior guarantees for the `node` module. + +use crate::node_actor::{NodeAgentActor, NodeAgentMsg, NodeAgentReport}; +use crate::orchestration::actor::OrchestratorMsg; +use iroh::{EndpointAddr, SecretKey}; +use mvp_system::staging as stage; +use swactor::actor::ActorAddress; +use swactor::config::RuntimeConfig; +use swactor::runtime::Runtime; + +#[test] +fn node_agent_runtime_loaded_reports_orchestrator() { + let runtime = Runtime::new(RuntimeConfig::default()); + let orchestrator_inbox = runtime + .new_inbox::() + .expect("orchestrator inbox"); + let orchestrator = *orchestrator_inbox.addr(); + let node_actor = ActorAddress::new_random(); + let datastream_publisher = ActorAddress::new_random(); + let endpoint = EndpointAddr::new(SecretKey::from_bytes(&[9; 32]).public()); + let actor = runtime + .spawn(NodeAgentActor::new(stage::NodeId(11), orchestrator, None)) + .expect("spawn node agent"); + + runtime + .send_to( + actor, + NodeAgentMsg::RuntimeLoaded { + run_id: 7, + node_id: 11, + stage_index: 3, + endpoint: endpoint.clone(), + node_actor, + datastream_publisher, + readiness_id: 99, + }, + ) + .expect("send runtime loaded"); + runtime.tick(); + + assert_eq!( + orchestrator_inbox.try_recv(), + Some(OrchestratorMsg::ObserveNodeRuntimeReady { + run_id: 7, + node_id: 11, + stage_index: 3, + endpoint, + node_actor, + datastream_publisher, + readiness_id: 99, + }) + ); +} + +#[test] +fn node_agent_runtime_ready_ack_reports_worker_loop() { + let runtime = Runtime::new(RuntimeConfig::default()); + let orchestrator_inbox = runtime + .new_inbox::() + .expect("orchestrator inbox"); + let reports = runtime + .new_inbox::() + .expect("node report inbox"); + let actor = runtime + .spawn(NodeAgentActor::new( + stage::NodeId(11), + *orchestrator_inbox.addr(), + Some(*reports.addr()), + )) + .expect("spawn node agent"); + + runtime + .send_to( + actor, + NodeAgentMsg::RuntimeReadyAck { + run_id: 7, + node_id: 11, + stage_index: 3, + readiness_id: 99, + }, + ) + .expect("send runtime ready ack"); + runtime.tick(); + + assert_eq!( + reports.try_recv(), + Some(NodeAgentReport::RuntimeReadyAck { + run_id: 7, + node_id: 11, + stage_index: 3, + readiness_id: 99, + }) + ); + assert_eq!(reports.try_recv(), None); +} diff --git a/crates/mvp-system/src/tests/observability_surface_guarantees.rs b/crates/mvp-system/src/tests/observability_guarantees.rs similarity index 92% rename from crates/mvp-system/src/tests/observability_surface_guarantees.rs rename to crates/mvp-system/src/tests/observability_guarantees.rs index d86527e..992b466 100644 --- a/crates/mvp-system/src/tests/observability_surface_guarantees.rs +++ b/crates/mvp-system/src/tests/observability_guarantees.rs @@ -1,3 +1,5 @@ +//! Behavior guarantees for the `observability` module. + //! Black-box contract tests for the MVP observability surface. //! //! These tests intentionally know only the public event stream surface: @@ -347,31 +349,3 @@ fn event_ordering_reflects_component_contracts_and_one_terminal_outcome() { .count(); assert_eq!(terminal_count, 1); } - -// This proves observability tests are independent of transport, storage, and -// batching policy by asserting the same event facts after batching is changed. -#[test] -fn event_contract_survives_transport_storage_and_batching_policy() { - // Build the same logical events under two batching policies. - let unbatched = - obs::EventSubscriberHarness::collect(successful_run_trace(), obs::Batching::None); - let batched = - obs::EventSubscriberHarness::collect(successful_run_trace(), obs::Batching::Fixed(8)); - - // Flattened public event facts must match as an ordered stream. - let unbatched_kinds = unbatched - .flattened_events() - .iter() - .map(|event| event.kind()) - .collect::>(); - let batched_kinds = batched - .flattened_events() - .iter() - .map(|event| event.kind()) - .collect::>(); - assert_eq!(batched_kinds, unbatched_kinds); - - // Neither subscriber depends on transport or storage implementation names. - assert!(!unbatched.used_transport_specific_assertions()); - assert!(!batched.used_storage_specific_assertions()); -} diff --git a/crates/mvp-system/src/tests/orchestration_guarantees.rs b/crates/mvp-system/src/tests/orchestration_guarantees.rs new file mode 100644 index 0000000..39d9d00 --- /dev/null +++ b/crates/mvp-system/src/tests/orchestration_guarantees.rs @@ -0,0 +1,1074 @@ +//! Behavior guarantees for the `orchestration` module. + +mod run_plan { + //! Black-box contract tests for MVP RunPlan formation. + //! + //! These tests intentionally know only the public planning surface: + //! + //! - `plan_run(input) -> Result` + //! - `derive_stage_provision(&plan, stage_index) -> Result` + //! + //! They assert the guarantees in `specs/BEHAVIOR_GUARANTEES.md`. + //! The planner implementation, placement heuristic, helper APIs, internal graph + //! representation, and allocation strategy are not observable here. + + use crate::run_plan as plan; + + // Local aliases keep the test prose readable while the file imports only the + // public planning module. The aliases do not grant access to planner internals. + type DTypeFamily = plan::DTypeFamily; + type EdgeEndpoint = plan::EdgeEndpoint; + type EdgeId = plan::EdgeId; + type EdgeKind = plan::EdgeKind; + type EdgePlan = plan::EdgePlan; + type GgufSource = plan::GgufSource; + type HostPinning = plan::HostPinning; + type InboundEdgeProvision = plan::InboundEdgeProvision; + type ModelFacts = plan::ModelFacts; + use plan::NodeId; + type LayoutRule = plan::LayoutRule; + type ObjectKind = plan::ObjectKind; + type OutboundEdgeProvision = plan::OutboundEdgeProvision; + type PromptSource = plan::PromptSource; + type PlacementInput = plan::PlacementInput; + type PlanRejectionKind = plan::PlanRejectionKind; + type PlannerInput = plan::PlannerInput; + type RingSpec = plan::RingSpec; + type RingDirection = plan::RingDirection; + type RunPlan = plan::RunPlan; + type RuntimeConfig = plan::RuntimeConfig; + type SamplingPolicy = plan::SamplingPolicy; + type SequencePolicy = plan::SequencePolicy; + type ShapeRule = plan::ShapeRule; + type StagePlacement = plan::StagePlacement; + type TokenOutputPolicy = plan::TokenOutputPolicy; + type TokenizerSource = plan::TokenizerSource; + type WakeCoalescing = plan::WakeCoalescing; + + // Keep test node ids small and readable. The concrete identity mechanism is + // outside this contract; these ids exist only so assertions can name topology + // facts without depending on any address or discovery machinery. + fn node(id: u64) -> NodeId { + NodeId(id) + } + + // The candidate pool is deliberately larger than some test placements. That + // lets the tests distinguish "known to the orchestrator" from "assigned to a + // stage", which is one of the planner authority boundaries. + fn valid_nodes() -> Vec { + vec![node(10), node(11), node(12), node(13)] + } + + // Fixed linear placement is the smallest placement input that still exercises + // the contract. It supplies stage-to-node intent, while the planner remains + // responsible for validating it and minting the full RunPlan topology. + fn linear_placement(stage_count: u32) -> PlacementInput { + PlacementInput::FixedLinear( + (0..stage_count) + .map(|stage_index| StagePlacement { + stage_index, + node_id: node(10 + u64::from(stage_index)), + }) + .collect(), + ) + } + + // This is the canonical valid fixture for RunPlan guarantees. Each test tweaks + // only the fact it is trying to prove, so a failure points at the violated + // contract instead of at accidental fixture drift. + fn valid_input(stage_count: u32, num_layers: u32) -> PlannerInput { + PlannerInput { + run_id: 7.into(), + orchestrator_node_id: node(99), + model: ModelFacts { + model_id: "test-gguf".into(), + gguf_source: GgufSource::LocalPath("/models/test-gguf.gguf".into()), + num_layers, + hidden_dim: 4096, + dtype_family: DTypeFamily::BFloat, + dtype_width_bytes: 2, + max_seq_len: 2048, + eos_token_id: 2, + tokenizer: TokenizerSource::LocalPath("/tokenizers/test-gguf.json".into()), + }, + runtime: RuntimeConfig { + max_tokens: 4, + prompt: PromptSource::Inline("hello from planner input".into()), + sampling: SamplingPolicy { + temperature_millis: 125, + top_k: 7, + }, + token_output_policy: TokenOutputPolicy::EmitAll, + }, + candidate_pool: valid_nodes(), + stage_count, + placement: linear_placement(stage_count), + activation_ring: RingSpec::test_default_activation(), + token_ring: RingSpec::test_default_token(), + } + } + + // Tests frequently need to compare a provisioned edge id back to the canonical + // edge record in the RunPlan. This helper makes that lookup explicit without + // giving tests access to any planner-private index. + fn plan_edges_by_id(plan: &RunPlan) -> std::collections::BTreeMap { + plan.edges + .iter() + .map(|edge| (edge.edge_id, edge)) + .collect::>() + } + + // Edge endpoints can be orchestrator or stage endpoints. Tests use this helper + // when they care only about stage adjacency and want orchestrator endpoints to + // remain visibly outside the stage index space. + fn edge_stage_index(endpoint: &EdgeEndpoint) -> Option { + match endpoint { + EdgeEndpoint::Orchestrator { .. } => None, + EdgeEndpoint::Stage { stage_index, .. } => Some(*stage_index), + } + } + + // Provisioning sends concrete node ids across the data-flow boundary. This + // helper extracts the observable node id from either endpoint shape so tests + // can compare projection output to plan topology. + fn edge_node_id(endpoint: &EdgeEndpoint) -> NodeId { + match endpoint { + EdgeEndpoint::Orchestrator { node_id } => *node_id, + EdgeEndpoint::Stage { node_id, .. } => *node_id, + } + } + + // This proves RunPlan formation is a total public boundary for valid input: + // the caller observes one complete plan, not hidden follow-up topology work or + // a partially initialized result. + #[test] + fn valid_input_emits_one_complete_plan() { + // Build one ordinary valid planning request. + let input = valid_input(3, 36); + + // Planning valid input must produce a usable plan, not a deferred partial. + let plan = plan::plan_run(input).expect("valid input must emit a plan"); + + // The plan-level identifiers and counts must be complete immediately. + assert_eq!(plan.run_id, 7.into()); + assert_eq!(plan.stages.len(), 3); + assert_eq!(plan.edges.len(), 4); + assert_eq!(plan.max_tokens, 4); + assert_eq!(plan.model.model_id, "test-gguf"); + assert_eq!( + plan.model.gguf_source, + GgufSource::LocalPath("/models/test-gguf.gguf".into()) + ); + assert_eq!(plan.model.num_layers, 36); + assert_eq!(plan.model.hidden_dim, 4096); + assert_eq!(plan.model.dtype_family, DTypeFamily::BFloat); + assert_eq!(plan.model.dtype_width_bytes, 2); + assert_eq!(plan.model.max_seq_len, 2048); + assert_eq!(plan.model.eos_token_id, 2); + assert_eq!( + plan.model.tokenizer, + TokenizerSource::LocalPath("/tokenizers/test-gguf.json".into()) + ); + assert_eq!( + plan.runtime.sampling, + SamplingPolicy { + temperature_millis: 125, + top_k: 7, + } + ); + assert_eq!( + plan.runtime.prompt, + PromptSource::Inline("hello from planner input".into()) + ); + assert_eq!(plan.runtime.token_output_policy, TokenOutputPolicy::EmitAll); + + // Every stage must be bound to this run and know the run's stage count. + for stage in &plan.stages { + assert_eq!(stage.run_id, plan.run_id); + assert_eq!(stage.stage_count, 3); + assert_eq!(stage.gguf_source, plan.model.gguf_source); + } + + // Every edge must also be bound to this run; no edge can be a loose fact. + for edge in &plan.edges { + assert_eq!(edge.run_id, plan.run_id); + } + } + // This proves layer assignment is a contiguous, non-overlapping partition of + // the intended GGUF block range, with one non-empty range per stage. + #[test] + fn stage_ranges_partition_the_model_layers() { + // Exercise several deterministic sizes so the check covers one-stage and + // multi-stage partitioning without relying on random generation. + for (stage_count, num_layers) in [(1, 12), (2, 24), (3, 36), (4, 40)] { + // Produce the plan from public inputs. + let plan = plan::plan_run(valid_input(stage_count, num_layers)).unwrap(); + + // Read only the public stage assignments and sort by stage index. + let mut ranges = plan + .stages + .iter() + .map(|stage| { + ( + stage.stage_index, + stage.layer_start, + stage.layer_end_exclusive, + ) + }) + .collect::>(); + ranges.sort_by_key(|(stage_index, _, _)| *stage_index); + + // Walk the sorted ranges as a proof of contiguity. The next start must + // equal the previous end, and every range must consume at least one + // layer inside the model range. + let mut expected_start = 0; + for (_, start, end) in ranges { + assert_eq!(start, expected_start, "range gap or overlap"); + assert!(end > start, "stage range must be non-empty"); + assert!(end <= num_layers, "stage range exceeds model layer range"); + expected_start = end; + } + + // The final end must cover the whole intended block range. + assert_eq!(expected_start, num_layers); + } + } + // This proves every edge has one public producer and one public consumer, and + // that the returned edge graph is exactly the MVP linear pipeline. + #[test] + fn edge_graph_is_exactly_the_linear_pipeline() { + // Use four stages so the activation chain has multiple interior edges. + let stage_count = 4; + let plan = plan::plan_run(valid_input(stage_count, 40)).unwrap(); + + // Token-in must be unique and must enter stage 0 from the orchestrator. + let token_in = plan + .edges + .iter() + .filter(|edge| edge.kind == EdgeKind::TokenIn) + .collect::>(); + assert_eq!(token_in.len(), 1); + assert!(matches!( + token_in[0].producer, + EdgeEndpoint::Orchestrator { node_id } if node_id == node(99) + )); + assert_eq!( + token_in[0].consumer, + EdgeEndpoint::Stage { + node_id: node(10), + stage_index: 0, + } + ); + + // Activation edges must be the only stage-to-stage edges, one per adjacent + // stage pair. + let activation_edges = plan + .edges + .iter() + .filter(|edge| edge.kind == EdgeKind::Activation) + .collect::>(); + assert_eq!(activation_edges.len(), (stage_count - 1) as usize); + + // Each activation edge produced by stage i must be consumed by stage i+1. + for stage_index in 0..stage_count - 1 { + let edge = activation_edges + .iter() + .find(|edge| edge_stage_index(&edge.producer) == Some(stage_index)) + .expect("activation edge produced by stage"); + + assert_eq!( + edge.consumer, + EdgeEndpoint::Stage { + node_id: node(11 + u64::from(stage_index)), + stage_index: stage_index + 1, + } + ); + assert_ne!(edge.producer, edge.consumer, "self-edge is forbidden"); + } + + // Token-out must be unique and must leave the final stage for the + // orchestrator. + let token_out = plan + .edges + .iter() + .filter(|edge| edge.kind == EdgeKind::TokenOut) + .collect::>(); + assert_eq!(token_out.len(), 1); + assert_eq!( + edge_stage_index(&token_out[0].producer), + Some(stage_count - 1) + ); + assert!(matches!( + token_out[0].consumer, + EdgeEndpoint::Orchestrator { node_id } if node_id == node(99) + )); + } + + // This proves edge ids are run-unique and that stage plans refer only to edge + // ids present in the returned RunPlan, so stages receive assigned ids rather + // than deriving data-flow identity themselves. + #[test] + fn edge_ids_are_unique_and_stage_references_resolve_to_plan_edges() { + // Produce a plan with enough edges to make duplicate ids observable. + let plan = plan::plan_run(valid_input(4, 40)).unwrap(); + + // Insert every public edge id into a set; a duplicate shrinks the set. + let edge_ids = plan + .edges + .iter() + .map(|edge| edge.edge_id) + .collect::>(); + + assert_eq!(edge_ids.len(), plan.edges.len(), "duplicate edge id"); + + // Stage plans may reference only ids that the RunPlan itself assigned. + for stage in &plan.stages { + assert!( + edge_ids.contains(&stage.inbound_edge), + "stage inbound edge id must come from RunPlan edges" + ); + assert!( + edge_ids.contains(&stage.outbound_edge), + "stage outbound edge id must come from RunPlan edges" + ); + } + } + + // This proves deriving ProvisionStage is deterministic and stage-local: + // repeated projection returns the same value, and the projected layer range is + // exactly the range assigned to that stage in the RunPlan. + #[test] + fn provision_stage_projection_is_deterministic_and_stage_local() { + // Start from one committed plan; projection is a pure public view of it. + let plan = plan::plan_run(valid_input(3, 36)).unwrap(); + + // Check every stage projection, not only one representative stage. + for stage_index in 0..3 { + // Derive twice to prove projection does not depend on hidden mutable + // state or call order. + let first = plan::derive_stage_provision(&plan, stage_index).unwrap(); + let second = plan::derive_stage_provision(&plan, stage_index).unwrap(); + + // Find the corresponding public stage assignment in the plan. + let stage = plan + .stages + .iter() + .find(|stage| stage.stage_index == stage_index) + .unwrap(); + + // The projected message must be stable and expose only that stage's + // assigned run position and layer range. + assert_eq!(first, second); + assert_eq!(first.stage_index, stage_index); + assert_eq!(first.stage_count, 3); + assert_eq!(first.layer_start, stage.layer_start); + assert_eq!(first.layer_end_exclusive, stage.layer_end_exclusive); + assert_eq!(first.gguf_source, stage.gguf_source); + assert_eq!(first.tokenizer, plan.model.tokenizer); + assert_eq!(first.model.model_id, plan.model.model_id); + assert_eq!(first.model.hidden_dim, plan.model.hidden_dim); + assert_eq!(first.model.dtype_family, plan.model.dtype_family); + assert_eq!(first.model.dtype_width_bytes, plan.model.dtype_width_bytes); + assert_eq!(first.model.max_seq_len, plan.model.max_seq_len); + assert_eq!(first.runtime.role_id, plan::RoleId(u64::from(stage_index))); + assert_eq!(first.runtime.input_port, plan::PortId("input".into())); + assert_eq!(first.runtime.output_port, plan::PortId("output".into())); + let expected_sampling = if stage_index + 1 == stage.stage_count { + Some(plan.runtime.sampling) + } else { + None + }; + assert_eq!(first.runtime.sampling, expected_sampling); + } + } + + // This proves each stage receives exactly one inbound and one outbound edge + // provision, and that the provisioned ids are the ids assigned to that stage by + // the RunPlan. + #[test] + fn provision_stage_contains_exactly_the_assigned_inbound_and_outbound_edges() { + // Build a valid plan and test projection for every stage in it. + let plan = plan::plan_run(valid_input(3, 36)).unwrap(); + + for stage in &plan.stages { + // Derive the public provisioning message for this stage. + let provision = plan::derive_stage_provision(&plan, stage.stage_index).unwrap(); + + // The message exposes exactly the inbound and outbound ids assigned in + // that stage's StagePlan. + assert_eq!(provision.inbound.edge_id, stage.inbound_edge); + assert_eq!(provision.outbound.edge_id, stage.outbound_edge); + } + } + // This proves typed rejection is the public behavior for invalid authority and + // topology inputs. No invalid case is allowed to emit a partial plan. + #[test] + fn invalid_authority_and_topology_inputs_reject_without_plan() { + // Each case changes one authority/topology fact from the valid fixture and + // names the typed rejection the planner must expose. + let cases = [ + ( + invalid_unknown_node(), + PlanRejectionKind::UnknownNode, + "unknown node id", + ), + ( + invalid_duplicate_stage_assignment(), + PlanRejectionKind::DuplicateStageAssignment, + "duplicate stage assignment", + ), + ( + invalid_missing_stage_assignment(), + PlanRejectionKind::MissingStage, + "missing stage", + ), + ( + invalid_zero_stage_count(), + PlanRejectionKind::InvalidStageCount, + "invalid stage count", + ), + ( + invalid_model_stage_layout(), + PlanRejectionKind::ModelStageLayoutMismatch, + "model/stage layout mismatch", + ), + ]; + + // Invalid inputs must not produce a partial plan. The observable result is + // a typed rejection kind. + for (input, expected, label) in cases { + let err = plan::plan_run(input).expect_err(label); + assert_eq!(err.kind(), expected, "{label}"); + } + } + // Unknown-node rejection needs the placement to name a node outside the + // orchestrator's candidate pool. The rest of the input stays valid so the + // expected rejection is isolated to authority over node identity. + fn invalid_unknown_node() -> PlannerInput { + let mut input = valid_input(3, 36); + input.placement = PlacementInput::FixedLinear(vec![ + StagePlacement { + stage_index: 0, + node_id: node(10), + }, + StagePlacement { + stage_index: 1, + node_id: node(404), + }, + StagePlacement { + stage_index: 2, + node_id: node(12), + }, + ]); + input + } + + // Duplicate-stage rejection is observable when two placement entries claim the + // same stage index. This checks that the planner does not silently pick one and + // continue with ambiguous authority. + fn invalid_duplicate_stage_assignment() -> PlannerInput { + let mut input = valid_input(3, 36); + input.placement = PlacementInput::FixedLinear(vec![ + StagePlacement { + stage_index: 0, + node_id: node(10), + }, + StagePlacement { + stage_index: 1, + node_id: node(11), + }, + StagePlacement { + stage_index: 1, + node_id: node(12), + }, + ]); + input + } + + // Missing-stage rejection is observable when placement skips an index inside + // `0..stage_count`. This checks that the planner does not invent hidden stage + // ownership to patch an incomplete placement. + fn invalid_missing_stage_assignment() -> PlannerInput { + let mut input = valid_input(3, 36); + input.placement = PlacementInput::FixedLinear(vec![ + StagePlacement { + stage_index: 0, + node_id: node(10), + }, + StagePlacement { + stage_index: 2, + node_id: node(12), + }, + ]); + input + } + + // Zero stages cannot form the MVP pipeline. This fixture isolates the invalid + // stage-count path without adding any other contradictory facts. + fn invalid_zero_stage_count() -> PlannerInput { + valid_input(0, 36) + } + + // The current contract requires one non-empty layer range per stage. Fewer + // layers than stages forces an empty range unless explicitly allowed, so this + // fixture should reject at the model/stage-layout boundary. + fn invalid_model_stage_layout() -> PlannerInput { + let mut input = valid_input(4, 3); + input.placement = linear_placement(4); + input + } + + // Zero sequence length makes activation capacity zero. The planner must reject + // before creating edges whose object specs cannot carry an activation. + fn invalid_zero_activation_extent() -> PlannerInput { + let mut input = valid_input(3, 36); + input.model.max_seq_len = 0; + input + } + + // Dtype width participates directly in activation extent and object layout. + // A zero width is not a valid dtype fact and must reject before planning. + fn invalid_dtype_width() -> PlannerInput { + let mut input = valid_input(3, 36); + input.model.dtype_width_bytes = 0; + input + } + + // Hidden dimension participates directly in activation shape. A zero hidden + // dimension represents an unsupported shape/layout fact for the MVP contract. + fn invalid_unsupported_shape_or_layout() -> PlannerInput { + let mut input = valid_input(3, 36); + input.model.hidden_dim = 0; + input + } + + // Ring alignment must be a usable alignment contract for shared memory and + // device copy boundaries. A non-power-of-two alignment makes the ring spec + // invalid before any edge can be provisioned. + fn invalid_ring_alignment() -> PlannerInput { + let mut input = valid_input(3, 36); + input.activation_ring.alignment = 3; + input + } +} + +mod run_fsm { + //! Black-box contract tests for the MVP orchestrator run FSM. + //! + //! These tests intentionally know only the public orchestrator surface: + //! + //! - pool, plan, stage, endpoint, token, fault, and stop events in + //! - commands, lifecycle events, and terminal outcome out + //! + //! They assert the guarantees in + //! `specs/BEHAVIOR_GUARANTEES.md`. + + use crate::run_fsm as fsm; + + // A three-stage plan proves multi-stage provisioning and readiness without + // making tests depend on any placement heuristic. The plan is already valid; + // these tests are about how the orchestrator consumes it. + fn committed_plan() -> fsm::RunPlan { + fsm::RunPlan::test_linear( + fsm::RunId(7), + vec![ + fsm::StageRef { + stage_index: 0, + node_id: fsm::NodeId(10), + }, + fsm::StageRef { + stage_index: 1, + node_id: fsm::NodeId(11), + }, + fsm::StageRef { + stage_index: 2, + node_id: fsm::NodeId(12), + }, + ], + ) + } + + // The harness is the black-box public boundary for the run FSM. It accepts + // observable events and records emitted commands/events; tests never inspect an + // internal FSM enum or private readiness counter. + fn new_run() -> fsm::OrchestratorHarness { + fsm::OrchestratorHarness::new(fsm::RunConfig { + run_id: fsm::RunId(7), + max_tokens: 4, + prompt: vec![101, 102, 103], + }) + } + + // Stage readiness events are generated from the committed plan so the tests + // prove readiness by stage identity instead of relying on command ordering. + fn stage_ready_events(plan: &fsm::RunPlan) -> Vec { + plan.stages + .iter() + .map(|stage| fsm::RunEvent::StageReady { + run_id: plan.run_id, + stage_index: stage.stage_index, + }) + .collect() + } + + // Transcript positions turn ordering claims into proofs over observable output. + // If an event is missing, the test fails at the boundary where users and other + // components would also lose the guarantee. + fn position_of(events: &[fsm::LifecycleEvent], needle: &fsm::LifecycleEvent) -> usize { + events + .iter() + .position(|event| event == needle) + .expect("expected lifecycle event missing") + } + + // This proves planning and provisioning are gated by PoolReady, and that the + // orchestrator provisions exactly the committed stages and local token endpoints + // from a valid RunPlan. + #[test] + fn planning_and_provisioning_start_only_after_pool_ready() { + // Start the run and give it a valid plan, but no PoolReady event. + let plan = committed_plan(); + let mut harness = new_run(); + harness.observe(fsm::RunEvent::PlanAvailable(plan.clone())); + + // Without PoolReady, provisioning must not begin. + assert!( + !harness + .commands() + .iter() + .any(|command| { matches!(command, fsm::RunCommand::ProvisionStage { .. }) }) + ); + + // Once PoolReady is observed, the committed plan may be provisioned. + harness.observe(fsm::RunEvent::PoolReady { + nodes: plan.stage_nodes(), + }); + + // Every planned stage gets exactly one provision command. + let provisioned = harness + .commands() + .iter() + .filter_map(|command| match command { + fsm::RunCommand::ProvisionStage { provision } => Some(provision.stage_index), + _ => None, + }) + .collect::>(); + let expected = plan + .stages + .iter() + .map(|stage| stage.stage_index) + .collect::>(); + assert_eq!(provisioned, expected); + + // Provisioning must not mention nodes outside the committed plan. + let plan_nodes = plan + .stage_nodes() + .into_iter() + .collect::>(); + for command in harness.commands() { + if let fsm::RunCommand::ProvisionStage { provision } = command { + assert!(plan_nodes.contains(&provision.node_id)); + } + } + + // Token endpoints are created locally from the same committed plan. + assert!(harness.commands().iter().any(|command| { + matches!(command, fsm::RunCommand::CreateTokenInEndpoint { run_id } if *run_id == fsm::RunId(7)) + })); + assert!(harness.commands().iter().any(|command| { + matches!(command, fsm::RunCommand::CreateTokenOutEndpoint { run_id } if *run_id == fsm::RunId(7)) + })); + } + + // This proves prompt injection is blocked until every planned stage and both + // local token endpoints are ready. Duplicate readiness must not count as a + // missing stage, and foreign readiness must fault or reject. + #[test] + fn readiness_barrier_controls_prompt_injection() { + // Provision a valid plan after PoolReady. + let plan = committed_plan(); + let mut harness = new_run(); + harness.observe(fsm::RunEvent::PoolReady { + nodes: plan.stage_nodes(), + }); + harness.observe(fsm::RunEvent::PlanAvailable(plan.clone())); + + // A duplicate StageReady for stage 0 cannot satisfy stage 1 or 2. + harness.observe(fsm::RunEvent::StageReady { + run_id: fsm::RunId(7), + stage_index: 0, + }); + harness.observe(fsm::RunEvent::StageReady { + run_id: fsm::RunId(7), + stage_index: 0, + }); + harness.observe(fsm::RunEvent::TokenInEndpointReady); + harness.observe(fsm::RunEvent::TokenOutEndpointReady); + assert!( + !harness + .commands() + .iter() + .any(|command| { matches!(command, fsm::RunCommand::InjectTokenObject { .. }) }) + ); + + // Complete the remaining stage readiness facts. + for event in stage_ready_events(&plan).into_iter().skip(1) { + harness.observe(event); + } + + // Prompt injection is the public start signal after the full barrier. + assert!(harness.commands().iter().any(|command| { + matches!( + command, + fsm::RunCommand::InjectTokenObject { + run_id: fsm::RunId(7), + object: fsm::TokenObjectInjection { + sequence: 0, + payload: fsm::TokenObjectPayload::Prompt { tokens }, + }, + } if tokens.as_slice() == [101, 102, 103] + ) + })); + + // Unknown stage readiness must not silently advance another run. + let mut invalid = new_run(); + invalid.observe(fsm::RunEvent::PoolReady { + nodes: plan.stage_nodes(), + }); + invalid.observe(fsm::RunEvent::PlanAvailable(plan)); + invalid.observe(fsm::RunEvent::StageReady { + run_id: fsm::RunId(7), + stage_index: 99, + }); + assert!(invalid.events().iter().any(|event| { + matches!(event, fsm::LifecycleEvent::RunFaulted { .. }) + || matches!(event, fsm::LifecycleEvent::RunRejected { .. }) + })); + } + + // This proves execution has one start signal and advances by the token feedback + // rule: inject sequence 0 first, then inject k + 1 only after consuming k. + #[test] + fn execution_injects_next_sequence_only_after_consuming_previous_token() { + // Drive a run through the complete readiness barrier. + let plan = committed_plan(); + let mut harness = new_run(); + harness.observe(fsm::RunEvent::PoolReady { + nodes: plan.stage_nodes(), + }); + harness.observe(fsm::RunEvent::PlanAvailable(plan.clone())); + harness.observe(fsm::RunEvent::TokenInEndpointReady); + harness.observe(fsm::RunEvent::TokenOutEndpointReady); + for event in stage_ready_events(&plan) { + harness.observe(event); + } + + // Sequence 0 must be injected first as a prompt token object. + let initial_objects = harness + .commands() + .iter() + .filter_map(|command| match command { + fsm::RunCommand::InjectTokenObject { object, .. } => Some(object), + _ => None, + }) + .collect::>(); + assert_eq!(initial_objects.len(), 1); + assert_eq!( + *initial_objects[0], + fsm::TokenObjectInjection { + sequence: 0, + payload: fsm::TokenObjectPayload::Prompt { + tokens: vec![101, 102, 103], + }, + } + ); + assert_eq!(harness.injected_sequences(), vec![0]); + + // Consuming token 0 permits injecting sequence 1. + harness.observe(fsm::RunEvent::TokenReceived { + sequence: 0, + token_id: 201, + eos: false, + }); + assert_eq!(harness.injected_sequences(), vec![0, 1]); + let decode_object = harness + .commands() + .iter() + .filter_map(|command| match command { + fsm::RunCommand::InjectTokenObject { object, .. } => Some(object), + _ => None, + }) + .last() + .expect("decode injection must be recorded"); + assert_eq!( + *decode_object, + fsm::TokenObjectInjection { + sequence: 1, + payload: fsm::TokenObjectPayload::Decode { + token_id: 201, + sampling: fsm::SamplingData { source_sequence: 0 }, + }, + } + ); + + // No additional injection may happen without consuming sequence 1. + harness.advance_time_ms(10); + assert_eq!(harness.injected_sequences(), vec![0, 1]); + + // EOS stops further injection after the consumed sequence. + harness.observe(fsm::RunEvent::TokenReceived { + sequence: 1, + token_id: 2, + eos: true, + }); + assert_eq!(harness.injected_sequences(), vec![0, 1]); + } + + // This proves every run-level fault source records one terminal fault, and the + // first failure reason is retained if later failures arrive. + #[test] + fn first_run_fault_reason_is_terminal_and_sticky() { + // Prepare an executing run so both setup and execution-time faults would be + // meaningful if observed. + let plan = committed_plan(); + let mut harness = new_run(); + harness.observe(fsm::RunEvent::PoolReady { + nodes: plan.stage_nodes(), + }); + harness.observe(fsm::RunEvent::PlanAvailable(plan.clone())); + harness.observe(fsm::RunEvent::TokenInEndpointReady); + harness.observe(fsm::RunEvent::TokenOutEndpointReady); + for event in stage_ready_events(&plan) { + harness.observe(event); + } + + // Inject the first failure source. + harness.observe(fsm::RunEvent::StageFault { + run_id: fsm::RunId(7), + stage_index: 1, + reason: fsm::StageFaultReason::WorkerCrashed, + }); + + // Inject later failures that must not replace the terminal reason. + harness.observe(fsm::RunEvent::EndpointFault { + run_id: fsm::RunId(7), + endpoint: fsm::EndpointKind::TokenOut, + }); + harness.observe(fsm::RunEvent::MembershipLost { + run_id: fsm::RunId(7), + node_id: fsm::NodeId(11), + }); + + // Exactly one terminal fault is recorded. + let faults = harness + .events() + .iter() + .filter_map(|event| match event { + fsm::LifecycleEvent::RunFaulted { reason, .. } => Some(reason), + _ => None, + }) + .collect::>(); + assert_eq!(faults.len(), 1); + assert_eq!( + *faults[0], + fsm::RunFaultReason::StageFault { + stage_index: 1, + reason: fsm::StageFaultReason::WorkerCrashed, + } + ); + } + + #[test] + fn membership_loss_faults_run() { + let plan = committed_plan(); + let mut membership_lost = new_run(); + membership_lost.observe(fsm::RunEvent::PoolReady { + nodes: plan.stage_nodes(), + }); + membership_lost.observe(fsm::RunEvent::PlanAvailable(plan.clone())); + membership_lost.observe(fsm::RunEvent::MembershipLost { + run_id: fsm::RunId(7), + node_id: fsm::NodeId(12), + }); + assert!(membership_lost.events().iter().any(|event| { + matches!( + event, + fsm::LifecycleEvent::RunFaulted { + reason: fsm::RunFaultReason::MembershipLost { + node_id: fsm::NodeId(12) + }, + .. + } + ) + })); + } + + // This proves terminal outcomes are mutually exclusive, reject new work, and + // always lead into teardown for success, fault, and operator stop. + #[test] + fn terminal_outcome_is_single_and_requires_teardown() { + // Complete a run by reaching EOS. + let plan = committed_plan(); + let mut harness = new_run(); + harness.observe(fsm::RunEvent::PoolReady { + nodes: plan.stage_nodes(), + }); + harness.observe(fsm::RunEvent::PlanAvailable(plan.clone())); + harness.observe(fsm::RunEvent::TokenInEndpointReady); + harness.observe(fsm::RunEvent::TokenOutEndpointReady); + for event in stage_ready_events(&plan) { + harness.observe(event); + } + harness.observe(fsm::RunEvent::TokenReceived { + sequence: 0, + token_id: 2, + eos: true, + }); + + // Completed and Faulted are mutually exclusive public outcomes. + let completed = harness + .events() + .iter() + .filter(|event| matches!(event, fsm::LifecycleEvent::RunCompleted { .. })) + .count(); + let faulted = harness + .events() + .iter() + .filter(|event| matches!(event, fsm::LifecycleEvent::RunFaulted { .. })) + .count(); + assert_eq!(completed, 1); + assert_eq!(faulted, 0); + + // New token work after terminal outcome begins must be rejected. + let before = harness.injected_sequences(); + harness.observe(fsm::RunEvent::TokenReceived { + sequence: 99, + token_id: 333, + eos: false, + }); + assert_eq!(harness.injected_sequences(), before); + + // Teardown commands must be emitted for every provisioned stage and local + // endpoint after the terminal outcome. + let stopped_stages = harness + .commands() + .iter() + .filter_map(|command| match command { + fsm::RunCommand::StopRun { stage_index, .. } => Some(*stage_index), + _ => None, + }) + .collect::>(); + let expected_stages = plan + .stages + .iter() + .map(|stage| stage.stage_index) + .collect::>(); + assert_eq!(stopped_stages, expected_stages); + assert!( + harness.commands().iter().any(|command| { + matches!(command, fsm::RunCommand::TearDownTokenEndpoints { .. }) + }) + ); + + let mut stopped = new_run(); + stopped.observe(fsm::RunEvent::PoolReady { + nodes: plan.stage_nodes(), + }); + stopped.observe(fsm::RunEvent::PlanAvailable(plan)); + stopped.observe(fsm::RunEvent::OperatorStop { + run_id: fsm::RunId(7), + }); + let stopped_count = stopped + .events() + .iter() + .filter(|event| matches!(event, fsm::LifecycleEvent::RunOperatorStopped { .. })) + .count(); + let stopped_faults = stopped + .events() + .iter() + .filter(|event| matches!(event, fsm::LifecycleEvent::RunFaulted { .. })) + .count(); + assert_eq!(stopped_count, 1); + assert_eq!(stopped_faults, 0); + assert!( + stopped.commands().iter().any(|command| { + matches!(command, fsm::RunCommand::TearDownTokenEndpoints { .. }) + }) + ); + } + + // This proves run_torn_down is emitted exactly once and only after teardown + // observes every planned stage stop and local endpoint stop. + #[test] + fn run_torn_down_is_emitted_once_after_teardown_terminal_state() { + // Fault a provisioned run so teardown is required. + let plan = committed_plan(); + let mut harness = new_run(); + harness.observe(fsm::RunEvent::PoolReady { + nodes: plan.stage_nodes(), + }); + harness.observe(fsm::RunEvent::PlanAvailable(plan.clone())); + harness.observe(fsm::RunEvent::StageFault { + run_id: fsm::RunId(7), + stage_index: 0, + reason: fsm::StageFaultReason::WorkerCrashed, + }); + + // StageStopped from only a prefix of stages is not enough to finish + // teardown. + harness.observe(fsm::RunEvent::StageStopped { + run_id: fsm::RunId(7), + stage_index: 0, + }); + assert!( + !harness + .events() + .iter() + .any(|event| { matches!(event, fsm::LifecycleEvent::RunTornDown { .. }) }) + ); + + // StageStopped for every stage still is not enough until local endpoints stop. + harness.observe(fsm::RunEvent::StageStopped { + run_id: fsm::RunId(7), + stage_index: 1, + }); + harness.observe(fsm::RunEvent::StageStopped { + run_id: fsm::RunId(7), + stage_index: 2, + }); + assert!( + !harness + .events() + .iter() + .any(|event| { matches!(event, fsm::LifecycleEvent::RunTornDown { .. }) }) + ); + harness.observe(fsm::RunEvent::TokenEndpointsStopped); + + // The final event may now appear, exactly once. + let torn_down_count = harness + .events() + .iter() + .filter(|event| matches!(event, fsm::LifecycleEvent::RunTornDown { .. })) + .count(); + assert_eq!(torn_down_count, 1); + + // Ordering is proven over the lifecycle transcript. + let fault_pos = position_of( + harness.events(), + &fsm::LifecycleEvent::RunFaulted { + run_id: fsm::RunId(7), + reason: fsm::RunFaultReason::StageFault { + stage_index: 0, + reason: fsm::StageFaultReason::WorkerCrashed, + }, + }, + ); + let torn_down_pos = position_of( + harness.events(), + &fsm::LifecycleEvent::RunTornDown { + run_id: fsm::RunId(7), + }, + ); + assert!(fault_pos < torn_down_pos); + } +} diff --git a/crates/mvp-system/src/tests/orchestrator_run_fsm_guarantees.rs b/crates/mvp-system/src/tests/orchestrator_run_fsm_guarantees.rs deleted file mode 100644 index e22f0bb..0000000 --- a/crates/mvp-system/src/tests/orchestrator_run_fsm_guarantees.rs +++ /dev/null @@ -1,520 +0,0 @@ -//! Black-box contract tests for the MVP orchestrator run FSM. -//! -//! These tests intentionally know only the public orchestrator surface: -//! -//! - pool, plan, stage, endpoint, token, fault, and stop events in -//! - commands, lifecycle events, and terminal outcome out -//! -//! They assert the guarantees in -//! `specs/mvp_system/orchestrator_run_fsm_contract.md`. - -use crate::run_fsm as fsm; - -// A three-stage plan proves multi-stage provisioning and readiness without -// making tests depend on any placement heuristic. The plan is already valid; -// these tests are about how the orchestrator consumes it. -fn committed_plan() -> fsm::RunPlan { - fsm::RunPlan::test_linear( - fsm::RunId(7), - vec![ - fsm::StageRef { - stage_index: 0, - node_id: fsm::NodeId(10), - }, - fsm::StageRef { - stage_index: 1, - node_id: fsm::NodeId(11), - }, - fsm::StageRef { - stage_index: 2, - node_id: fsm::NodeId(12), - }, - ], - ) -} - -// The harness is the black-box public boundary for the run FSM. It accepts -// observable events and records emitted commands/events; tests never inspect an -// internal FSM enum or private readiness counter. -fn new_run() -> fsm::OrchestratorHarness { - fsm::OrchestratorHarness::new(fsm::RunConfig { - run_id: fsm::RunId(7), - max_tokens: 4, - prompt: vec![101, 102, 103], - }) -} - -// Stage readiness events are generated from the committed plan so the tests -// prove readiness by stage identity instead of relying on command ordering. -fn stage_ready_events(plan: &fsm::RunPlan) -> Vec { - plan.stages - .iter() - .map(|stage| fsm::RunEvent::StageReady { - run_id: plan.run_id, - stage_index: stage.stage_index, - }) - .collect() -} - -// Transcript positions turn ordering claims into proofs over observable output. -// If an event is missing, the test fails at the boundary where users and other -// components would also lose the guarantee. -fn position_of(events: &[fsm::LifecycleEvent], needle: &fsm::LifecycleEvent) -> usize { - events - .iter() - .position(|event| event == needle) - .expect("expected lifecycle event missing") -} - -// This proves planning and provisioning are gated by PoolReady, and that the -// orchestrator provisions exactly the committed stages and local token endpoints -// from a valid RunPlan. -#[test] -fn planning_and_provisioning_start_only_after_pool_ready() { - // Start the run and give it a valid plan, but no PoolReady event. - let plan = committed_plan(); - let mut harness = new_run(); - harness.observe(fsm::RunEvent::PlanAvailable(plan.clone())); - - // Without PoolReady, provisioning must not begin. - assert!( - !harness - .commands() - .iter() - .any(|command| { matches!(command, fsm::RunCommand::ProvisionStage { .. }) }) - ); - - // Once PoolReady is observed, the committed plan may be provisioned. - harness.observe(fsm::RunEvent::PoolReady { - nodes: plan.stage_nodes(), - }); - - // Every planned stage gets exactly one provision command. - let provisioned = harness - .commands() - .iter() - .filter_map(|command| match command { - fsm::RunCommand::ProvisionStage { provision } => Some(provision.stage_index), - _ => None, - }) - .collect::>(); - let expected = plan - .stages - .iter() - .map(|stage| stage.stage_index) - .collect::>(); - assert_eq!(provisioned, expected); - - // Provisioning must not mention nodes outside the committed plan. - let plan_nodes = plan - .stage_nodes() - .into_iter() - .collect::>(); - for command in harness.commands() { - if let fsm::RunCommand::ProvisionStage { provision } = command { - assert!(plan_nodes.contains(&provision.node_id)); - } - } - - // Token endpoints are created locally from the same committed plan. - assert!(harness.commands().iter().any(|command| { - matches!(command, fsm::RunCommand::CreateTokenInEndpoint { run_id } if *run_id == fsm::RunId(7)) - })); - assert!(harness.commands().iter().any(|command| { - matches!(command, fsm::RunCommand::CreateTokenOutEndpoint { run_id } if *run_id == fsm::RunId(7)) - })); -} - -// This proves prompt injection is blocked until every planned stage and both -// local token endpoints are ready. Duplicate readiness must not count as a -// missing stage, and foreign readiness must fault or reject. -#[test] -fn readiness_barrier_controls_prompt_injection() { - // Provision a valid plan after PoolReady. - let plan = committed_plan(); - let mut harness = new_run(); - harness.observe(fsm::RunEvent::PoolReady { - nodes: plan.stage_nodes(), - }); - harness.observe(fsm::RunEvent::PlanAvailable(plan.clone())); - - // A duplicate StageReady for stage 0 cannot satisfy stage 1 or 2. - harness.observe(fsm::RunEvent::StageReady { - run_id: fsm::RunId(7), - stage_index: 0, - }); - harness.observe(fsm::RunEvent::StageReady { - run_id: fsm::RunId(7), - stage_index: 0, - }); - harness.observe(fsm::RunEvent::TokenInEndpointReady); - harness.observe(fsm::RunEvent::TokenOutEndpointReady); - assert!( - !harness - .commands() - .iter() - .any(|command| { matches!(command, fsm::RunCommand::InjectTokenObject { .. }) }) - ); - - // Complete the remaining stage readiness facts. - for event in stage_ready_events(&plan).into_iter().skip(1) { - harness.observe(event); - } - - // Prompt injection is the public start signal after the full barrier. - assert!(harness.commands().iter().any(|command| { - matches!( - command, - fsm::RunCommand::InjectTokenObject { - run_id: fsm::RunId(7), - object: fsm::TokenObjectInjection { - sequence: 0, - payload: fsm::TokenObjectPayload::Prompt { tokens }, - }, - } if tokens.as_slice() == [101, 102, 103] - ) - })); - - // Unknown stage readiness must not silently advance another run. - let mut invalid = new_run(); - invalid.observe(fsm::RunEvent::PoolReady { - nodes: plan.stage_nodes(), - }); - invalid.observe(fsm::RunEvent::PlanAvailable(plan)); - invalid.observe(fsm::RunEvent::StageReady { - run_id: fsm::RunId(7), - stage_index: 99, - }); - assert!(invalid.events().iter().any(|event| { - matches!(event, fsm::LifecycleEvent::RunFaulted { .. }) - || matches!(event, fsm::LifecycleEvent::RunRejected { .. }) - })); -} - -// This proves execution has one start signal and advances by the token feedback -// rule: inject sequence 0 first, then inject k + 1 only after consuming k. -#[test] -fn execution_injects_next_sequence_only_after_consuming_previous_token() { - // Drive a run through the complete readiness barrier. - let plan = committed_plan(); - let mut harness = new_run(); - harness.observe(fsm::RunEvent::PoolReady { - nodes: plan.stage_nodes(), - }); - harness.observe(fsm::RunEvent::PlanAvailable(plan.clone())); - harness.observe(fsm::RunEvent::TokenInEndpointReady); - harness.observe(fsm::RunEvent::TokenOutEndpointReady); - for event in stage_ready_events(&plan) { - harness.observe(event); - } - - // Sequence 0 must be injected first as a prompt token object. - let initial_objects = harness - .commands() - .iter() - .filter_map(|command| match command { - fsm::RunCommand::InjectTokenObject { object, .. } => Some(object), - _ => None, - }) - .collect::>(); - assert_eq!(initial_objects.len(), 1); - assert_eq!( - *initial_objects[0], - fsm::TokenObjectInjection { - sequence: 0, - payload: fsm::TokenObjectPayload::Prompt { - tokens: vec![101, 102, 103], - }, - } - ); - assert_eq!(harness.injected_sequences(), vec![0]); - - // Consuming token 0 permits injecting sequence 1. - harness.observe(fsm::RunEvent::TokenReceived { - sequence: 0, - token_id: 201, - eos: false, - }); - assert_eq!(harness.injected_sequences(), vec![0, 1]); - let decode_object = harness - .commands() - .iter() - .filter_map(|command| match command { - fsm::RunCommand::InjectTokenObject { object, .. } => Some(object), - _ => None, - }) - .last() - .expect("decode injection must be recorded"); - assert_eq!( - *decode_object, - fsm::TokenObjectInjection { - sequence: 1, - payload: fsm::TokenObjectPayload::Decode { - token_id: 201, - sampling: fsm::SamplingData { source_sequence: 0 }, - }, - } - ); - - // No additional injection may happen without consuming sequence 1. - harness.advance_time_ms(10); - assert_eq!(harness.injected_sequences(), vec![0, 1]); - - // EOS stops further injection after the consumed sequence. - harness.observe(fsm::RunEvent::TokenReceived { - sequence: 1, - token_id: 2, - eos: true, - }); - assert_eq!(harness.injected_sequences(), vec![0, 1]); -} - -// This proves every run-level fault source records one terminal fault, and the -// first failure reason is retained if later failures arrive. -#[test] -fn first_run_fault_reason_is_terminal_and_sticky() { - // Prepare an executing run so both setup and execution-time faults would be - // meaningful if observed. - let plan = committed_plan(); - let mut harness = new_run(); - harness.observe(fsm::RunEvent::PoolReady { - nodes: plan.stage_nodes(), - }); - harness.observe(fsm::RunEvent::PlanAvailable(plan.clone())); - harness.observe(fsm::RunEvent::TokenInEndpointReady); - harness.observe(fsm::RunEvent::TokenOutEndpointReady); - for event in stage_ready_events(&plan) { - harness.observe(event); - } - - // Inject the first failure source. - harness.observe(fsm::RunEvent::StageFault { - run_id: fsm::RunId(7), - stage_index: 1, - reason: fsm::StageFaultReason::WorkerCrashed, - }); - - // Inject later failures that must not replace the terminal reason. - harness.observe(fsm::RunEvent::EndpointFault { - run_id: fsm::RunId(7), - endpoint: fsm::EndpointKind::TokenOut, - }); - harness.observe(fsm::RunEvent::MembershipLost { - run_id: fsm::RunId(7), - node_id: fsm::NodeId(11), - }); - - // Exactly one terminal fault is recorded. - let faults = harness - .events() - .iter() - .filter_map(|event| match event { - fsm::LifecycleEvent::RunFaulted { reason, .. } => Some(reason), - _ => None, - }) - .collect::>(); - assert_eq!(faults.len(), 1); - assert_eq!( - *faults[0], - fsm::RunFaultReason::StageFault { - stage_index: 1, - reason: fsm::StageFaultReason::WorkerCrashed, - } - ); -} - -#[test] -fn membership_loss_faults_run() { - let plan = committed_plan(); - let mut membership_lost = new_run(); - membership_lost.observe(fsm::RunEvent::PoolReady { - nodes: plan.stage_nodes(), - }); - membership_lost.observe(fsm::RunEvent::PlanAvailable(plan.clone())); - membership_lost.observe(fsm::RunEvent::MembershipLost { - run_id: fsm::RunId(7), - node_id: fsm::NodeId(12), - }); - assert!(membership_lost.events().iter().any(|event| { - matches!( - event, - fsm::LifecycleEvent::RunFaulted { - reason: fsm::RunFaultReason::MembershipLost { - node_id: fsm::NodeId(12) - }, - .. - } - ) - })); -} - -// This proves terminal outcomes are mutually exclusive, reject new work, and -// always lead into teardown for success, fault, and operator stop. -#[test] -fn terminal_outcome_is_single_and_requires_teardown() { - // Complete a run by reaching EOS. - let plan = committed_plan(); - let mut harness = new_run(); - harness.observe(fsm::RunEvent::PoolReady { - nodes: plan.stage_nodes(), - }); - harness.observe(fsm::RunEvent::PlanAvailable(plan.clone())); - harness.observe(fsm::RunEvent::TokenInEndpointReady); - harness.observe(fsm::RunEvent::TokenOutEndpointReady); - for event in stage_ready_events(&plan) { - harness.observe(event); - } - harness.observe(fsm::RunEvent::TokenReceived { - sequence: 0, - token_id: 2, - eos: true, - }); - - // Completed and Faulted are mutually exclusive public outcomes. - let completed = harness - .events() - .iter() - .filter(|event| matches!(event, fsm::LifecycleEvent::RunCompleted { .. })) - .count(); - let faulted = harness - .events() - .iter() - .filter(|event| matches!(event, fsm::LifecycleEvent::RunFaulted { .. })) - .count(); - assert_eq!(completed, 1); - assert_eq!(faulted, 0); - - // New token work after terminal outcome begins must be rejected. - let before = harness.injected_sequences(); - harness.observe(fsm::RunEvent::TokenReceived { - sequence: 99, - token_id: 333, - eos: false, - }); - assert_eq!(harness.injected_sequences(), before); - - // Teardown commands must be emitted for every provisioned stage and local - // endpoint after the terminal outcome. - let stopped_stages = harness - .commands() - .iter() - .filter_map(|command| match command { - fsm::RunCommand::StopRun { stage_index, .. } => Some(*stage_index), - _ => None, - }) - .collect::>(); - let expected_stages = plan - .stages - .iter() - .map(|stage| stage.stage_index) - .collect::>(); - assert_eq!(stopped_stages, expected_stages); - assert!( - harness - .commands() - .iter() - .any(|command| { matches!(command, fsm::RunCommand::TearDownTokenEndpoints { .. }) }) - ); - - let mut stopped = new_run(); - stopped.observe(fsm::RunEvent::PoolReady { - nodes: plan.stage_nodes(), - }); - stopped.observe(fsm::RunEvent::PlanAvailable(plan)); - stopped.observe(fsm::RunEvent::OperatorStop { - run_id: fsm::RunId(7), - }); - let stopped_count = stopped - .events() - .iter() - .filter(|event| matches!(event, fsm::LifecycleEvent::RunOperatorStopped { .. })) - .count(); - let stopped_faults = stopped - .events() - .iter() - .filter(|event| matches!(event, fsm::LifecycleEvent::RunFaulted { .. })) - .count(); - assert_eq!(stopped_count, 1); - assert_eq!(stopped_faults, 0); - assert!( - stopped - .commands() - .iter() - .any(|command| { matches!(command, fsm::RunCommand::TearDownTokenEndpoints { .. }) }) - ); -} - -// This proves run_torn_down is emitted exactly once and only after teardown -// observes every planned stage stop and local endpoint stop. -#[test] -fn run_torn_down_is_emitted_once_after_teardown_terminal_state() { - // Fault a provisioned run so teardown is required. - let plan = committed_plan(); - let mut harness = new_run(); - harness.observe(fsm::RunEvent::PoolReady { - nodes: plan.stage_nodes(), - }); - harness.observe(fsm::RunEvent::PlanAvailable(plan.clone())); - harness.observe(fsm::RunEvent::StageFault { - run_id: fsm::RunId(7), - stage_index: 0, - reason: fsm::StageFaultReason::WorkerCrashed, - }); - - // StageStopped from only a prefix of stages is not enough to finish - // teardown. - harness.observe(fsm::RunEvent::StageStopped { - run_id: fsm::RunId(7), - stage_index: 0, - }); - assert!( - !harness - .events() - .iter() - .any(|event| { matches!(event, fsm::LifecycleEvent::RunTornDown { .. }) }) - ); - - // StageStopped for every stage still is not enough until local endpoints stop. - harness.observe(fsm::RunEvent::StageStopped { - run_id: fsm::RunId(7), - stage_index: 1, - }); - harness.observe(fsm::RunEvent::StageStopped { - run_id: fsm::RunId(7), - stage_index: 2, - }); - assert!( - !harness - .events() - .iter() - .any(|event| { matches!(event, fsm::LifecycleEvent::RunTornDown { .. }) }) - ); - harness.observe(fsm::RunEvent::TokenEndpointsStopped); - - // The final event may now appear, exactly once. - let torn_down_count = harness - .events() - .iter() - .filter(|event| matches!(event, fsm::LifecycleEvent::RunTornDown { .. })) - .count(); - assert_eq!(torn_down_count, 1); - - // Ordering is proven over the lifecycle transcript. - let fault_pos = position_of( - harness.events(), - &fsm::LifecycleEvent::RunFaulted { - run_id: fsm::RunId(7), - reason: fsm::RunFaultReason::StageFault { - stage_index: 0, - reason: fsm::StageFaultReason::WorkerCrashed, - }, - }, - ); - let torn_down_pos = position_of( - harness.events(), - &fsm::LifecycleEvent::RunTornDown { - run_id: fsm::RunId(7), - }, - ); - assert!(fault_pos < torn_down_pos); -} diff --git a/crates/mvp-system/src/tests/prompt_guarantees.rs b/crates/mvp-system/src/tests/prompt_guarantees.rs new file mode 100644 index 0000000..ce6ce1d --- /dev/null +++ b/crates/mvp-system/src/tests/prompt_guarantees.rs @@ -0,0 +1,42 @@ +//! Behavior guarantees for the `prompt` module. + +use mvp_system::prompt::rpc::{PromptEvent, SubmitPrompt}; + +#[test] +fn zero_request_limits_take_loop_defaults() { + let request = SubmitPrompt { + request_id: 7, + prompt_text: "hello".to_owned(), + max_tokens: 0, + } + .with_defaults(32); + + assert_eq!(request.max_tokens, 32); +} + +#[test] +fn event_terminal_state_is_explicit() { + assert!( + !PromptEvent::TextDelta { + request_id: 1, + text: "a".to_owned(), + } + .is_terminal() + ); + assert!( + PromptEvent::Done { + request_id: 1, + final_text: "a".to_owned(), + tokens_generated: 1, + elapsed_ms: 2, + } + .is_terminal() + ); + assert!( + PromptEvent::Fault { + request_id: 1, + error: "boom".to_owned(), + } + .is_terminal() + ); +} diff --git a/crates/mvp-system/src/tests/relay_provisioning_guarantees.rs b/crates/mvp-system/src/tests/relay_provisioning_guarantees.rs deleted file mode 100644 index 6352d62..0000000 --- a/crates/mvp-system/src/tests/relay_provisioning_guarantees.rs +++ /dev/null @@ -1,171 +0,0 @@ -use std::ffi::OsString; -use std::sync::Mutex; - -use iroh::{RelayMode, RelayUrl}; -use mvp_system::orchestration::provider_adapters::relay::{ - LocalShimRelayProvider, MVP_IROH_RELAY_MODE_ENV, MVP_IROH_RELAY_URL_ENV, RelayProvider, - RelayProviderKind, RelayProvisionRequest, RelayPurpose, SWACTOR_IROH_RELAY_URL_ENV, - StaticRelayProvider, relay_runtime_config_from_env, -}; - -static ENV_LOCK: Mutex<()> = Mutex::new(()); - -const RELAY_ENV_KEYS: &[&str] = &[ - MVP_IROH_RELAY_MODE_ENV, - MVP_IROH_RELAY_URL_ENV, - SWACTOR_IROH_RELAY_URL_ENV, -]; - -struct RestoreEnv { - saved: Vec<(&'static str, Option)>, -} - -impl Drop for RestoreEnv { - fn drop(&mut self) { - for (key, value) in &self.saved { - match value { - Some(value) => unsafe { std::env::set_var(key, value) }, - None => unsafe { std::env::remove_var(key) }, - } - } - } -} - -fn with_relay_env(settings: &[(&'static str, &'static str)], test: impl FnOnce() -> T) -> T { - let _lock = ENV_LOCK - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - let saved = RELAY_ENV_KEYS - .iter() - .map(|&key| (key, std::env::var_os(key))) - .collect::>(); - for key in RELAY_ENV_KEYS { - unsafe { std::env::remove_var(key) }; - } - for (key, value) in settings { - assert!( - RELAY_ENV_KEYS.contains(key), - "test env key {key} must be restored" - ); - unsafe { std::env::set_var(key, value) }; - } - let _restore = RestoreEnv { saved }; - test() -} - -fn provision_request(run_id: u64) -> RelayProvisionRequest { - RelayProvisionRequest { - run_id, - purpose: RelayPurpose::Combined, - } -} - -fn canonical_relay_url(raw: &str) -> String { - raw.parse::() - .expect("fixture relay URL parses") - .to_string() -} - -fn assert_custom_relay_mode(mode: RelayMode, expected_url: &str) { - let expected_url = expected_url - .parse::() - .expect("fixture relay URL parses"); - match mode { - RelayMode::Custom(relay_map) => { - assert_eq!(relay_map.len(), 1, "custom relay map must contain one URL"); - assert!( - relay_map.contains(&expected_url), - "custom relay map must contain {expected_url}" - ); - } - other => panic!("expected custom relay mode for {expected_url}, got {other:?}"), - } -} - -#[test] -fn local_shim_provisions_disabled_lease_without_endpoints() { - let mut provider = LocalShimRelayProvider; - - let lease = provider - .provision_relay(provision_request(77)) - .expect("local shim provisioning succeeds"); - - assert_eq!(lease.endpoints, Vec::new()); - assert!(matches!( - provider - .relay_mode(&lease) - .expect("local shim relay mode resolves"), - RelayMode::Disabled - )); -} - -#[test] -fn static_provider_provisions_one_endpoint_and_custom_relay_mode() { - const RELAY_URL: &str = "https://relay-static.example.com"; - let mut provider = - StaticRelayProvider::from_url_str(RELAY_URL).expect("static relay URL parses"); - let expected_url = canonical_relay_url(RELAY_URL); - - let lease = provider - .provision_relay(provision_request(88)) - .expect("static relay provisioning succeeds"); - - assert_eq!(lease.endpoints.len(), 1); - assert_eq!(lease.endpoints[0].url, expected_url); - assert_eq!(lease.endpoints[0].provider, RelayProviderKind::Static); - assert_custom_relay_mode( - provider - .relay_mode(&lease) - .expect("static relay mode resolves"), - &expected_url, - ); -} - -#[test] -fn default_relay_mode_uses_configured_mvp_or_swactor_relay_url() { - const MVP_URL: &str = "https://relay-mvp.example.com"; - const SWACTOR_URL: &str = "https://relay-swactor.example.com"; - - for (name, settings, expected_url) in [ - ( - "mvp relay URL", - [ - (MVP_IROH_RELAY_MODE_ENV, "default"), - (MVP_IROH_RELAY_URL_ENV, MVP_URL), - ], - MVP_URL, - ), - ( - "swactor relay URL fallback", - [ - (MVP_IROH_RELAY_MODE_ENV, "default"), - (SWACTOR_IROH_RELAY_URL_ENV, SWACTOR_URL), - ], - SWACTOR_URL, - ), - ] { - with_relay_env(&settings, || { - let config = relay_runtime_config_from_env(901).unwrap_or_else(|error| { - panic!("{name} should resolve custom relay config: {error}") - }); - let expected_url = canonical_relay_url(expected_url); - assert_eq!(config.url.as_deref(), Some(expected_url.as_str())); - assert_custom_relay_mode(config.mode, &expected_url); - }); - } -} - -#[test] -fn disabled_relay_mode_ignores_configured_url() { - with_relay_env( - &[ - (MVP_IROH_RELAY_MODE_ENV, "disabled"), - (MVP_IROH_RELAY_URL_ENV, "https://ignored-relay.example.com"), - ], - || { - let config = relay_runtime_config_from_env(902).expect("disabled relay mode resolves"); - assert!(matches!(config.mode, RelayMode::Disabled)); - assert_eq!(config.url, None); - }, - ); -} diff --git a/crates/mvp-system/src/tests/run_plan_guarantees.rs b/crates/mvp-system/src/tests/run_plan_guarantees.rs deleted file mode 100644 index 1fdcb34..0000000 --- a/crates/mvp-system/src/tests/run_plan_guarantees.rs +++ /dev/null @@ -1,821 +0,0 @@ -//! Black-box contract tests for MVP RunPlan formation. -//! -//! These tests intentionally know only the public planning surface: -//! -//! - `plan_run(input) -> Result` -//! - `derive_stage_provision(&plan, stage_index) -> Result` -//! -//! They assert the guarantees in `specs/mvp_system/run_plan_contract.md`. -//! The planner implementation, placement heuristic, helper APIs, internal graph -//! representation, and allocation strategy are not observable here. - -use crate::run_plan as plan; - -// Local aliases keep the test prose readable while the file imports only the -// public planning module. The aliases do not grant access to planner internals. -type DTypeFamily = plan::DTypeFamily; -type EdgeEndpoint = plan::EdgeEndpoint; -type EdgeId = plan::EdgeId; -type EdgeKind = plan::EdgeKind; -type EdgePlan = plan::EdgePlan; -type GgufSource = plan::GgufSource; -type HostPinning = plan::HostPinning; -type InboundEdgeProvision = plan::InboundEdgeProvision; -type ModelFacts = plan::ModelFacts; -use plan::NodeId; -type LayoutRule = plan::LayoutRule; -type ObjectKind = plan::ObjectKind; -type OutboundEdgeProvision = plan::OutboundEdgeProvision; -type PromptSource = plan::PromptSource; -type PlacementInput = plan::PlacementInput; -type PlanRejectionKind = plan::PlanRejectionKind; -type PlannerInput = plan::PlannerInput; -type RingSpec = plan::RingSpec; -type RingDirection = plan::RingDirection; -type RunPlan = plan::RunPlan; -type RuntimeConfig = plan::RuntimeConfig; -type SamplingPolicy = plan::SamplingPolicy; -type SequencePolicy = plan::SequencePolicy; -type ShapeRule = plan::ShapeRule; -type StagePlacement = plan::StagePlacement; -type TokenOutputPolicy = plan::TokenOutputPolicy; -type TokenizerSource = plan::TokenizerSource; -type WakeCoalescing = plan::WakeCoalescing; - -// Keep test node ids small and readable. The concrete identity mechanism is -// outside this contract; these ids exist only so assertions can name topology -// facts without depending on any address or discovery machinery. -fn node(id: u64) -> NodeId { - NodeId(id) -} - -// The candidate pool is deliberately larger than some test placements. That -// lets the tests distinguish "known to the orchestrator" from "assigned to a -// stage", which is one of the planner authority boundaries. -fn valid_nodes() -> Vec { - vec![node(10), node(11), node(12), node(13)] -} - -// Fixed linear placement is the smallest placement input that still exercises -// the contract. It supplies stage-to-node intent, while the planner remains -// responsible for validating it and minting the full RunPlan topology. -fn linear_placement(stage_count: u32) -> PlacementInput { - PlacementInput::FixedLinear( - (0..stage_count) - .map(|stage_index| StagePlacement { - stage_index, - node_id: node(10 + u64::from(stage_index)), - }) - .collect(), - ) -} - -// This is the canonical valid fixture for RunPlan guarantees. Each test tweaks -// only the fact it is trying to prove, so a failure points at the violated -// contract instead of at accidental fixture drift. -fn valid_input(stage_count: u32, num_layers: u32) -> PlannerInput { - PlannerInput { - run_id: 7.into(), - orchestrator_node_id: node(99), - model: ModelFacts { - model_id: "test-gguf".into(), - gguf_source: GgufSource::LocalPath("/models/test-gguf.gguf".into()), - num_layers, - hidden_dim: 4096, - dtype_family: DTypeFamily::BFloat, - dtype_width_bytes: 2, - max_seq_len: 2048, - eos_token_id: 2, - tokenizer: TokenizerSource::LocalPath("/tokenizers/test-gguf.json".into()), - }, - runtime: RuntimeConfig { - max_tokens: 4, - prompt: PromptSource::Inline("hello from planner input".into()), - sampling: SamplingPolicy { - temperature_millis: 125, - top_k: 7, - }, - token_output_policy: TokenOutputPolicy::EmitAll, - }, - candidate_pool: valid_nodes(), - stage_count, - placement: linear_placement(stage_count), - activation_ring: RingSpec::test_default_activation(), - token_ring: RingSpec::test_default_token(), - } -} - -// Tests frequently need to compare a provisioned edge id back to the canonical -// edge record in the RunPlan. This helper makes that lookup explicit without -// giving tests access to any planner-private index. -fn plan_edges_by_id(plan: &RunPlan) -> std::collections::BTreeMap { - plan.edges - .iter() - .map(|edge| (edge.edge_id, edge)) - .collect::>() -} - -// Edge endpoints can be orchestrator or stage endpoints. Tests use this helper -// when they care only about stage adjacency and want orchestrator endpoints to -// remain visibly outside the stage index space. -fn edge_stage_index(endpoint: &EdgeEndpoint) -> Option { - match endpoint { - EdgeEndpoint::Orchestrator { .. } => None, - EdgeEndpoint::Stage { stage_index, .. } => Some(*stage_index), - } -} - -// Provisioning sends concrete node ids across the data-flow boundary. This -// helper extracts the observable node id from either endpoint shape so tests -// can compare projection output to plan topology. -fn edge_node_id(endpoint: &EdgeEndpoint) -> NodeId { - match endpoint { - EdgeEndpoint::Orchestrator { node_id } => *node_id, - EdgeEndpoint::Stage { node_id, .. } => *node_id, - } -} - -// This proves RunPlan formation is a total public boundary for valid input: -// the caller observes one complete plan, not hidden follow-up topology work or -// a partially initialized result. -#[test] -fn valid_input_emits_one_complete_plan() { - // Build one ordinary valid planning request. - let input = valid_input(3, 36); - - // Planning valid input must produce a usable plan, not a deferred partial. - let plan = plan::plan_run(input).expect("valid input must emit a plan"); - - // The plan-level identifiers and counts must be complete immediately. - assert_eq!(plan.run_id, 7.into()); - assert_eq!(plan.stages.len(), 3); - assert_eq!(plan.edges.len(), 4); - assert_eq!(plan.max_tokens, 4); - assert_eq!(plan.model.model_id, "test-gguf"); - assert_eq!( - plan.model.gguf_source, - GgufSource::LocalPath("/models/test-gguf.gguf".into()) - ); - assert_eq!(plan.model.num_layers, 36); - assert_eq!(plan.model.hidden_dim, 4096); - assert_eq!(plan.model.dtype_family, DTypeFamily::BFloat); - assert_eq!(plan.model.dtype_width_bytes, 2); - assert_eq!(plan.model.max_seq_len, 2048); - assert_eq!(plan.model.eos_token_id, 2); - assert_eq!( - plan.model.tokenizer, - TokenizerSource::LocalPath("/tokenizers/test-gguf.json".into()) - ); - assert_eq!( - plan.runtime.sampling, - SamplingPolicy { - temperature_millis: 125, - top_k: 7, - } - ); - assert_eq!( - plan.runtime.prompt, - PromptSource::Inline("hello from planner input".into()) - ); - assert_eq!(plan.runtime.token_output_policy, TokenOutputPolicy::EmitAll); - - // Every stage must be bound to this run and know the run's stage count. - for stage in &plan.stages { - assert_eq!(stage.run_id, plan.run_id); - assert_eq!(stage.stage_count, 3); - assert_eq!(stage.gguf_source, plan.model.gguf_source); - } - - // Every edge must also be bound to this run; no edge can be a loose fact. - for edge in &plan.edges { - assert_eq!(edge.run_id, plan.run_id); - } -} - -// This proves planning does not mutate the candidate pool supplied by the -// caller. The only topology facts the caller can use after planning are the -// facts emitted in the RunPlan itself. -#[test] -fn planner_does_not_mutate_candidate_pool() { - // Keep a copy of the caller-owned pool before the planner sees it. - let input = valid_input(3, 36); - let original_pool = input.candidate_pool.clone(); - - // Run planning through the public API only. - let _ = plan::plan_run(input.clone()).expect("valid input must emit a plan"); - - // The input pool remains the caller's fact; topology facts must be in the - // returned plan, not back-written into the input. - assert_eq!(input.candidate_pool, original_pool); -} - -// This proves layer assignment is a contiguous, non-overlapping partition of -// the intended GGUF block range, with one non-empty range per stage. -#[test] -fn stage_ranges_partition_the_model_layers() { - // Exercise several deterministic sizes so the check covers one-stage and - // multi-stage partitioning without relying on random generation. - for (stage_count, num_layers) in [(1, 12), (2, 24), (3, 36), (4, 40)] { - // Produce the plan from public inputs. - let plan = plan::plan_run(valid_input(stage_count, num_layers)).unwrap(); - - // Read only the public stage assignments and sort by stage index. - let mut ranges = plan - .stages - .iter() - .map(|stage| { - ( - stage.stage_index, - stage.layer_start, - stage.layer_end_exclusive, - ) - }) - .collect::>(); - ranges.sort_by_key(|(stage_index, _, _)| *stage_index); - - // Walk the sorted ranges as a proof of contiguity. The next start must - // equal the previous end, and every range must consume at least one - // layer inside the model range. - let mut expected_start = 0; - for (_, start, end) in ranges { - assert_eq!(start, expected_start, "range gap or overlap"); - assert!(end > start, "stage range must be non-empty"); - assert!(end <= num_layers, "stage range exceeds model layer range"); - expected_start = end; - } - - // The final end must cover the whole intended block range. - assert_eq!(expected_start, num_layers); - } -} - -// This proves stage indices are exactly the dense range required by the -// contract. Missing, duplicate, or out-of-range stage indices are observable in -// the returned RunPlan and fail this check. -#[test] -fn stage_indices_are_exactly_zero_to_stage_count_minus_one() { - // Check several stage counts so the dense-index guarantee is not tied to - // the canonical three-stage fixture. - for stage_count in 1..=4 { - // Produce a valid plan and observe only its public stage indices. - let plan = plan::plan_run(valid_input(stage_count, stage_count * 8)).unwrap(); - let observed = plan - .stages - .iter() - .map(|stage| stage.stage_index) - .collect::>(); - - // Compare against the contract's exact dense index set. - let expected = (0..stage_count).collect::>(); - - assert_eq!(observed, expected); - } -} - -// This proves every edge has one public producer and one public consumer, and -// that the returned edge graph is exactly the MVP linear pipeline. -#[test] -fn edge_graph_is_exactly_the_linear_pipeline() { - // Use four stages so the activation chain has multiple interior edges. - let stage_count = 4; - let plan = plan::plan_run(valid_input(stage_count, 40)).unwrap(); - - // Token-in must be unique and must enter stage 0 from the orchestrator. - let token_in = plan - .edges - .iter() - .filter(|edge| edge.kind == EdgeKind::TokenIn) - .collect::>(); - assert_eq!(token_in.len(), 1); - assert!(matches!( - token_in[0].producer, - EdgeEndpoint::Orchestrator { node_id } if node_id == node(99) - )); - assert_eq!( - token_in[0].consumer, - EdgeEndpoint::Stage { - node_id: node(10), - stage_index: 0, - } - ); - - // Activation edges must be the only stage-to-stage edges, one per adjacent - // stage pair. - let activation_edges = plan - .edges - .iter() - .filter(|edge| edge.kind == EdgeKind::Activation) - .collect::>(); - assert_eq!(activation_edges.len(), (stage_count - 1) as usize); - - // Each activation edge produced by stage i must be consumed by stage i+1. - for stage_index in 0..stage_count - 1 { - let edge = activation_edges - .iter() - .find(|edge| edge_stage_index(&edge.producer) == Some(stage_index)) - .expect("activation edge produced by stage"); - - assert_eq!( - edge.consumer, - EdgeEndpoint::Stage { - node_id: node(11 + u64::from(stage_index)), - stage_index: stage_index + 1, - } - ); - assert_ne!(edge.producer, edge.consumer, "self-edge is forbidden"); - } - - // Token-out must be unique and must leave the final stage for the - // orchestrator. - let token_out = plan - .edges - .iter() - .filter(|edge| edge.kind == EdgeKind::TokenOut) - .collect::>(); - assert_eq!(token_out.len(), 1); - assert_eq!( - edge_stage_index(&token_out[0].producer), - Some(stage_count - 1) - ); - assert!(matches!( - token_out[0].consumer, - EdgeEndpoint::Orchestrator { node_id } if node_id == node(99) - )); -} - -// This proves edge ids are run-unique and that stage plans refer only to edge -// ids present in the returned RunPlan, so stages receive assigned ids rather -// than deriving data-flow identity themselves. -#[test] -fn edge_ids_are_unique_and_stage_references_resolve_to_plan_edges() { - // Produce a plan with enough edges to make duplicate ids observable. - let plan = plan::plan_run(valid_input(4, 40)).unwrap(); - - // Insert every public edge id into a set; a duplicate shrinks the set. - let edge_ids = plan - .edges - .iter() - .map(|edge| edge.edge_id) - .collect::>(); - - assert_eq!(edge_ids.len(), plan.edges.len(), "duplicate edge id"); - - // Stage plans may reference only ids that the RunPlan itself assigned. - for stage in &plan.stages { - assert!( - edge_ids.contains(&stage.inbound_edge), - "stage inbound edge id must come from RunPlan edges" - ); - assert!( - edge_ids.contains(&stage.outbound_edge), - "stage outbound edge id must come from RunPlan edges" - ); - } -} - -// This proves deriving ProvisionStage is deterministic and stage-local: -// repeated projection returns the same value, and the projected layer range is -// exactly the range assigned to that stage in the RunPlan. -#[test] -fn provision_stage_projection_is_deterministic_and_stage_local() { - // Start from one committed plan; projection is a pure public view of it. - let plan = plan::plan_run(valid_input(3, 36)).unwrap(); - - // Check every stage projection, not only one representative stage. - for stage_index in 0..3 { - // Derive twice to prove projection does not depend on hidden mutable - // state or call order. - let first = plan::derive_stage_provision(&plan, stage_index).unwrap(); - let second = plan::derive_stage_provision(&plan, stage_index).unwrap(); - - // Find the corresponding public stage assignment in the plan. - let stage = plan - .stages - .iter() - .find(|stage| stage.stage_index == stage_index) - .unwrap(); - - // The projected message must be stable and expose only that stage's - // assigned run position and layer range. - assert_eq!(first, second); - assert_eq!(first.stage_index, stage_index); - assert_eq!(first.stage_count, 3); - assert_eq!(first.layer_start, stage.layer_start); - assert_eq!(first.layer_end_exclusive, stage.layer_end_exclusive); - assert_eq!(first.gguf_source, stage.gguf_source); - assert_eq!(first.tokenizer, plan.model.tokenizer); - assert_eq!(first.model.model_id, plan.model.model_id); - assert_eq!(first.model.hidden_dim, plan.model.hidden_dim); - assert_eq!(first.model.dtype_family, plan.model.dtype_family); - assert_eq!(first.model.dtype_width_bytes, plan.model.dtype_width_bytes); - assert_eq!(first.model.max_seq_len, plan.model.max_seq_len); - assert_eq!(first.runtime.role_id, plan::RoleId(u64::from(stage_index))); - assert_eq!(first.runtime.input_port, plan::PortId("input".into())); - assert_eq!(first.runtime.output_port, plan::PortId("output".into())); - let expected_sampling = if stage_index + 1 == stage.stage_count { - Some(plan.runtime.sampling) - } else { - None - }; - assert_eq!(first.runtime.sampling, expected_sampling); - } -} - -// This proves each stage receives exactly one inbound and one outbound edge -// provision, and that the provisioned ids are the ids assigned to that stage by -// the RunPlan. -#[test] -fn provision_stage_contains_exactly_the_assigned_inbound_and_outbound_edges() { - // Build a valid plan and test projection for every stage in it. - let plan = plan::plan_run(valid_input(3, 36)).unwrap(); - - for stage in &plan.stages { - // Derive the public provisioning message for this stage. - let provision = plan::derive_stage_provision(&plan, stage.stage_index).unwrap(); - - // The message exposes exactly the inbound and outbound ids assigned in - // that stage's StagePlan. - assert_eq!(provision.inbound.edge_id, stage.inbound_edge); - assert_eq!(provision.outbound.edge_id, stage.outbound_edge); - } -} - -// This proves the data-flow addressing contract of provisioning. The outbound -// side carries the consumer node id; exhaustive struct destructuring also makes -// remote actor-address fields a compile-time contract violation. -#[test] -fn provision_stage_uses_node_id_addressing_not_remote_actor_addresses() { - // Build one plan and an edge lookup using only public edge records. - let plan = plan::plan_run(valid_input(3, 36)).unwrap(); - let edges = plan_edges_by_id(&plan); - - for stage in &plan.stages { - // Project the stage-local provisioning message. - let provision = plan::derive_stage_provision(&plan, stage.stage_index).unwrap(); - - // Destructure the inbound provision exhaustively. If a remote actor - // address becomes part of this public type, this test must be updated - // consciously instead of silently accepting it. - let InboundEdgeProvision { - edge_id: inbound_edge_id, - kind: _, - object_spec: _, - ring_spec: _, - } = provision.inbound.clone(); - - // Destructure the outbound provision exhaustively. The only remote - // routing fact it may expose is the consumer node id. - let OutboundEdgeProvision { - edge_id: outbound_edge_id, - kind: _, - consumer_node_id, - object_spec: _, - ring_spec: _, - } = provision.outbound.clone(); - - assert_eq!(inbound_edge_id, stage.inbound_edge); - assert_eq!(outbound_edge_id, stage.outbound_edge); - - // The provisioned consumer node id must match the consumer endpoint of - // the canonical RunPlan edge. - let outbound_edge = edges.get(&outbound_edge_id).unwrap(); - assert_eq!(consumer_node_id, edge_node_id(&outbound_edge.consumer)); - } -} - -// This proves every edge carries object and ring specs, activation capacity is -// derived from model facts, and edge kind selects the correct object kind. -#[test] -fn object_and_ring_specs_are_present_and_match_edge_kind() { - // Use the canonical model facts so the expected activation capacity is - // known directly from the public input. - let plan = plan::plan_run(valid_input(3, 36)).unwrap(); - let expected_activation_extent = 2048 * 4096 * 2; - - // Every edge must carry complete movement specs; no worker or transport may - // invent these later. - for edge in &plan.edges { - assert!(edge.object_spec.max_extent > 0); - assert!(edge.ring_spec.data_capacity > 0); - assert!(edge.object_spec.alignment > 0); - assert_eq!(edge.object_spec.layout, LayoutRule::Contiguous); - assert_eq!(edge.object_spec.sequence_policy, SequencePolicy::Ordered); - assert!(edge.ring_spec.alignment > 0); - assert_eq!(edge.ring_spec.direction, RingDirection::Egress); - assert_eq!(edge.ring_spec.host_pinning, HostPinning::Pageable); - assert_eq!(edge.ring_spec.wake_coalescing, WakeCoalescing::PendingBit); - - // Edge kind selects the object kind, and activation capacity is derived - // from model facts. - match edge.kind { - EdgeKind::Activation => { - assert_eq!(edge.object_spec.kind, ObjectKind::Activation); - assert_eq!(edge.object_spec.max_extent, expected_activation_extent); - assert_eq!(edge.object_spec.dtype_family, DTypeFamily::BFloat); - assert_eq!(edge.object_spec.dtype_width_bytes, 2); - assert_eq!( - edge.object_spec.shape, - ShapeRule::ActivationRows { - max_seq_len: 2048, - hidden_dim: 4096, - } - ); - } - EdgeKind::TokenIn | EdgeKind::TokenOut => { - assert_eq!(edge.object_spec.kind, ObjectKind::Token); - assert_eq!(edge.object_spec.dtype_width_bytes, 4); - assert_eq!(edge.object_spec.shape, ShapeRule::TokenIds); - } - } - } -} - -#[test] -fn token_object_alignment_is_token_width_not_ring_alignment() { - let mut input = valid_input(3, 36); - input.token_ring.alignment = 64; - let plan = plan::plan_run(input).unwrap(); - - for edge in plan - .edges - .iter() - .filter(|edge| matches!(edge.kind, EdgeKind::TokenIn | EdgeKind::TokenOut)) - { - assert_eq!(edge.object_spec.dtype_width_bytes, 4); - assert_eq!(edge.object_spec.alignment, 4); - assert_eq!(edge.ring_spec.alignment, 64); - } -} - -#[test] -fn activation_object_alignment_is_dtype_width_not_ring_alignment() { - let mut input = valid_input(3, 36); - input.model.hidden_dim = 13; - input.activation_ring.alignment = 64; - let plan = plan::plan_run(input).unwrap(); - - for edge in plan - .edges - .iter() - .filter(|edge| matches!(edge.kind, EdgeKind::Activation)) - { - assert_eq!(edge.object_spec.dtype_width_bytes, 2); - assert_eq!(edge.object_spec.alignment, 2); - assert_eq!(edge.ring_spec.alignment, 64); - } -} - -// This proves prompt-token objects are sized from the effective model context, -// not from a small fixed constant, and that token rings can carry one complete -// MO01 record for that object. -#[test] -fn token_edges_size_extent_from_context_and_expand_ring_capacity() { - let mut input = valid_input(3, 36); - input.model.max_seq_len = 512; - input.token_ring.data_capacity = 64; - - let plan = plan::plan_run(input).expect("valid input must emit a plan"); - let expected_token_extent = 512 * 4; - let expected_token_record_capacity = plan::MO01_HEADER_BYTES + expected_token_extent; - let token_edges = plan - .edges - .iter() - .filter(|edge| matches!(edge.kind, EdgeKind::TokenIn | EdgeKind::TokenOut)) - .collect::>(); - - assert_eq!(token_edges.len(), 2); - for edge in token_edges { - assert_eq!(edge.object_spec.kind, ObjectKind::Token); - assert_eq!(edge.object_spec.max_extent, expected_token_extent); - assert_eq!(edge.object_spec.dtype_width_bytes, 4); - assert_eq!(edge.object_spec.shape, ShapeRule::TokenIds); - assert!( - edge.ring_spec.data_capacity >= expected_token_record_capacity, - "token ring must carry MO01 header plus max token payload" - ); - assert_eq!(edge.ring_spec.data_capacity, expected_token_record_capacity); - } -} - -// This proves object and ring specs are projected consistently into every stage -// provision. A stage provision narrows the ring direction to the local role -// while preserving the edge's capacity, alignment, pinning, and wake policy. -#[test] -fn object_and_ring_specs_are_copied_consistently_into_stage_provisions() { - // Build a canonical plan and index its public edge records. - let plan = plan::plan_run(valid_input(3, 36)).unwrap(); - let edges = plan_edges_by_id(&plan); - - for stage in &plan.stages { - // Project a stage-local provisioning message. - let provision = plan::derive_stage_provision(&plan, stage.stage_index).unwrap(); - - // The inbound edge spec must preserve the plan edge facts and mark the - // local ring as ingress. - let inbound_edge = edges.get(&provision.inbound.edge_id).unwrap(); - let mut expected_inbound_ring = inbound_edge.ring_spec; - expected_inbound_ring.direction = RingDirection::Ingress; - assert_eq!(provision.inbound.kind, inbound_edge.kind); - assert_eq!(provision.inbound.object_spec, inbound_edge.object_spec); - assert_eq!(provision.inbound.ring_spec, expected_inbound_ring); - - // The outbound edge spec must preserve the plan edge facts and mark the - // local ring as egress. - let outbound_edge = edges.get(&provision.outbound.edge_id).unwrap(); - let mut expected_outbound_ring = outbound_edge.ring_spec; - expected_outbound_ring.direction = RingDirection::Egress; - assert_eq!(provision.outbound.kind, outbound_edge.kind); - assert_eq!(provision.outbound.object_spec, outbound_edge.object_spec); - assert_eq!(provision.outbound.ring_spec, expected_outbound_ring); - } -} - -// This proves typed rejection is the public behavior for invalid authority and -// topology inputs. No invalid case is allowed to emit a partial plan. -#[test] -fn invalid_authority_and_topology_inputs_reject_without_plan() { - // Each case changes one authority/topology fact from the valid fixture and - // names the typed rejection the planner must expose. - let cases = [ - ( - invalid_unknown_node(), - PlanRejectionKind::UnknownNode, - "unknown node id", - ), - ( - invalid_duplicate_stage_assignment(), - PlanRejectionKind::DuplicateStageAssignment, - "duplicate stage assignment", - ), - ( - invalid_missing_stage_assignment(), - PlanRejectionKind::MissingStage, - "missing stage", - ), - ( - invalid_zero_stage_count(), - PlanRejectionKind::InvalidStageCount, - "invalid stage count", - ), - ( - invalid_model_stage_layout(), - PlanRejectionKind::ModelStageLayoutMismatch, - "model/stage layout mismatch", - ), - ]; - - // Invalid inputs must not produce a partial plan. The observable result is - // a typed rejection kind. - for (input, expected, label) in cases { - let err = plan::plan_run(input).expect_err(label); - assert_eq!(err.kind(), expected, "{label}"); - } -} - -// This proves invalid object and ring spec facts reject before provisioning. -// The planner may choose the exact diagnostic payload, but the rejection kind -// must be typed and no RunPlan may be emitted. -#[test] -fn invalid_object_or_ring_specs_reject_without_plan() { - // Each case changes one object/ring fact from the valid fixture and names - // the typed rejection expected at the planning boundary. - let cases = [ - ( - invalid_zero_activation_extent(), - PlanRejectionKind::InvalidObjectSpec, - "zero activation extent", - ), - ( - invalid_dtype_width(), - PlanRejectionKind::InvalidObjectSpec, - "invalid dtype width", - ), - ( - invalid_unsupported_shape_or_layout(), - PlanRejectionKind::UnsupportedShapeOrLayout, - "unsupported shape/layout", - ), - ( - invalid_ring_alignment(), - PlanRejectionKind::InvalidRingSpec, - "invalid ring alignment", - ), - ]; - - // Rejection happens before provisioning: the only public output is the - // typed error, never a RunPlan with invalid specs. - for (input, expected, label) in cases { - let err = plan::plan_run(input).expect_err(label); - assert_eq!(err.kind(), expected, "{label}"); - } -} - -// Unknown-node rejection needs the placement to name a node outside the -// orchestrator's candidate pool. The rest of the input stays valid so the -// expected rejection is isolated to authority over node identity. -fn invalid_unknown_node() -> PlannerInput { - let mut input = valid_input(3, 36); - input.placement = PlacementInput::FixedLinear(vec![ - StagePlacement { - stage_index: 0, - node_id: node(10), - }, - StagePlacement { - stage_index: 1, - node_id: node(404), - }, - StagePlacement { - stage_index: 2, - node_id: node(12), - }, - ]); - input -} - -// Duplicate-stage rejection is observable when two placement entries claim the -// same stage index. This checks that the planner does not silently pick one and -// continue with ambiguous authority. -fn invalid_duplicate_stage_assignment() -> PlannerInput { - let mut input = valid_input(3, 36); - input.placement = PlacementInput::FixedLinear(vec![ - StagePlacement { - stage_index: 0, - node_id: node(10), - }, - StagePlacement { - stage_index: 1, - node_id: node(11), - }, - StagePlacement { - stage_index: 1, - node_id: node(12), - }, - ]); - input -} - -// Missing-stage rejection is observable when placement skips an index inside -// `0..stage_count`. This checks that the planner does not invent hidden stage -// ownership to patch an incomplete placement. -fn invalid_missing_stage_assignment() -> PlannerInput { - let mut input = valid_input(3, 36); - input.placement = PlacementInput::FixedLinear(vec![ - StagePlacement { - stage_index: 0, - node_id: node(10), - }, - StagePlacement { - stage_index: 2, - node_id: node(12), - }, - ]); - input -} - -// Zero stages cannot form the MVP pipeline. This fixture isolates the invalid -// stage-count path without adding any other contradictory facts. -fn invalid_zero_stage_count() -> PlannerInput { - valid_input(0, 36) -} - -// The current contract requires one non-empty layer range per stage. Fewer -// layers than stages forces an empty range unless explicitly allowed, so this -// fixture should reject at the model/stage-layout boundary. -fn invalid_model_stage_layout() -> PlannerInput { - let mut input = valid_input(4, 3); - input.placement = linear_placement(4); - input -} - -// Zero sequence length makes activation capacity zero. The planner must reject -// before creating edges whose object specs cannot carry an activation. -fn invalid_zero_activation_extent() -> PlannerInput { - let mut input = valid_input(3, 36); - input.model.max_seq_len = 0; - input -} - -// Dtype width participates directly in activation extent and object layout. -// A zero width is not a valid dtype fact and must reject before planning. -fn invalid_dtype_width() -> PlannerInput { - let mut input = valid_input(3, 36); - input.model.dtype_width_bytes = 0; - input -} - -// Hidden dimension participates directly in activation shape. A zero hidden -// dimension represents an unsupported shape/layout fact for the MVP contract. -fn invalid_unsupported_shape_or_layout() -> PlannerInput { - let mut input = valid_input(3, 36); - input.model.hidden_dim = 0; - input -} - -// Ring alignment must be a usable alignment contract for shared memory and -// device copy boundaries. A non-power-of-two alignment makes the ring spec -// invalid before any edge can be provisioned. -fn invalid_ring_alignment() -> PlannerInput { - let mut input = valid_input(3, 36); - input.activation_ring.alignment = 3; - input -} diff --git a/crates/mvp-system/src/tests/shard_fetch_guarantees.rs b/crates/mvp-system/src/tests/shard_fetch_guarantees.rs deleted file mode 100644 index af7962b..0000000 --- a/crates/mvp-system/src/tests/shard_fetch_guarantees.rs +++ /dev/null @@ -1,168 +0,0 @@ -use std::collections::BTreeMap; - -use mvp_system::staging::shard_fetch as fetch; -use mvp_system::staging::weight_shards as shards; - -fn assignment(stage_index: u32) -> shards::ShardAssignment { - let model_ref = - shards::ModelArtifactRef::parse("hf://org/repo@abcdef123456/model.gguf").unwrap(); - let split_scheme = shards::SplitScheme::GgufLayerContiguousV1; - let split_id = shards::SplitId::derive(&model_ref, split_scheme); - shards::ShardAssignment::new( - model_ref, - split_id, - split_scheme, - stage_index, - 8, - shards::LayerRange::new(stage_index * 4, stage_index * 4 + 4).unwrap(), - ) - .unwrap() -} - -fn fetched_for(assignment: &shards::ShardAssignment) -> fetch::FetchedShard { - fetch::FetchedShard::new( - format!("/cache/stage-{:05}.gguf", assignment.stage_index), - shards::ShardManifest::for_assignment( - assignment, - shards::ContentHash::literal(format!("sha256:stage-{}", assignment.stage_index)) - .unwrap(), - ), - ) -} - -#[derive(Default)] -struct MemoryCache { - entries: BTreeMap, - inserts: usize, -} - -impl fetch::ShardCache for MemoryCache { - fn get(&self, cache_key: &str) -> Option { - self.entries.get(cache_key).cloned() - } - - fn insert(&mut self, cache_key: String, shard: fetch::FetchedShard) { - self.inserts += 1; - self.entries.insert(cache_key, shard); - } -} - -struct RecordingFetcher { - calls: Vec, - result: Result, -} - -impl fetch::ShardFetcher for RecordingFetcher { - fn fetch( - &mut self, - request: &fetch::FetchShard, - ) -> Result { - self.calls.push(request.clone()); - self.result.clone() - } -} - -#[test] -fn shard_locator_derives_expected_uri() { - let assignment = assignment(3); - let location = fetch::ShardLocator::locate(&assignment); - - assert_eq!( - location.uri, - format!( - "hf://org/repo@abcdef123456/shards/{}/stage-00003.gguf", - assignment.split_id.as_str() - ) - ); - assert!(location.cache_key.contains(assignment.split_id.as_str())); - assert!(location.cache_key.ends_with(":00003")); -} - -#[test] -fn cache_key_depends_on_model_split_and_stage() { - let stage_three = assignment(3); - let stage_four = assignment(4); - - let different_model_ref = - shards::ModelArtifactRef::parse("hf://other/repo@abcdef123456/model.gguf").unwrap(); - let split_scheme = shards::SplitScheme::GgufLayerContiguousV1; - let different_model = shards::ShardAssignment::new( - different_model_ref.clone(), - shards::SplitId::derive(&different_model_ref, split_scheme), - split_scheme, - 3, - 8, - shards::LayerRange::new(12, 16).unwrap(), - ) - .unwrap(); - - let different_split = shards::ShardAssignment::new( - stage_three.model_ref.clone(), - shards::SplitId::literal("split-other").unwrap(), - stage_three.split_scheme, - 3, - 8, - stage_three.layer_range, - ) - .unwrap(); - - let keys = [ - fetch::ShardLocator::locate(&stage_three).cache_key, - fetch::ShardLocator::locate(&stage_four).cache_key, - fetch::ShardLocator::locate(&different_model).cache_key, - fetch::ShardLocator::locate(&different_split).cache_key, - ]; - - assert_eq!( - keys.iter().collect::>().len(), - keys.len() - ); -} - -#[test] -fn coordinator_downloads_and_caches_missing_shard() { - let assignment = assignment(2); - let expected = fetched_for(&assignment); - let mut cache = MemoryCache::default(); - let mut fetcher = RecordingFetcher { - calls: Vec::new(), - result: Ok(expected.clone()), - }; - - let outcome = - fetch::ShardFetchCoordinator::get_or_fetch(&assignment, &mut cache, &mut fetcher).unwrap(); - - assert_eq!(outcome.status, fetch::ShardFetchStatus::Downloaded); - assert_eq!(outcome.shard, expected); - assert_eq!(fetcher.calls.len(), 1); - assert_eq!(fetcher.calls[0].location, outcome.location); - assert_eq!(cache.inserts, 1); - assert_eq!( - cache.entries.get(&outcome.location.cache_key), - Some(&expected) - ); -} - -#[test] -fn coordinator_uses_cache_hit_without_fetching() { - let assignment = assignment(5); - let location = fetch::ShardLocator::locate(&assignment); - let cached = fetched_for(&assignment); - let mut cache = MemoryCache::default(); - cache - .entries - .insert(location.cache_key.clone(), cached.clone()); - let mut fetcher = RecordingFetcher { - calls: Vec::new(), - result: Err(fetch::FetchError::Unavailable), - }; - - let outcome = - fetch::ShardFetchCoordinator::get_or_fetch(&assignment, &mut cache, &mut fetcher).unwrap(); - - assert_eq!(outcome.status, fetch::ShardFetchStatus::CacheHit); - assert_eq!(outcome.location, location); - assert_eq!(outcome.shard, cached); - assert!(fetcher.calls.is_empty()); - assert_eq!(cache.inserts, 0); -} diff --git a/crates/mvp-system/src/tests/shard_weight_lifecycle_guarantees.rs b/crates/mvp-system/src/tests/shard_weight_lifecycle_guarantees.rs deleted file mode 100644 index 5a766bf..0000000 --- a/crates/mvp-system/src/tests/shard_weight_lifecycle_guarantees.rs +++ /dev/null @@ -1,308 +0,0 @@ -use std::collections::BTreeMap; - -use mvp_system::staging::shard_fetch as fetch; -use mvp_system::staging::shard_weight_lifecycle as lifecycle; -use mvp_system::staging::weight_shards as shards; - -fn assignment() -> shards::ShardAssignment { - let model_ref = - shards::ModelArtifactRef::parse("hf://org/repo@abcdef123456/model.gguf").unwrap(); - let split_scheme = shards::SplitScheme::GgufLayerContiguousV1; - let split_id = shards::SplitId::derive(&model_ref, split_scheme); - shards::ShardAssignment::new( - model_ref, - split_id, - split_scheme, - 3, - 8, - shards::LayerRange::new(12, 16).unwrap(), - ) - .unwrap() -} - -fn fetched_for(assignment: &shards::ShardAssignment) -> fetch::FetchedShard { - fetch::FetchedShard::new( - "/cache/stage-00003.gguf", - shards::ShardManifest::for_assignment( - assignment, - shards::ContentHash::literal("sha256:stage-3").unwrap(), - ), - ) -} - -fn event_kinds(events: &[lifecycle::ShardLifecycleEvent]) -> Vec<&'static str> { - events - .iter() - .map(|event| match event { - lifecycle::ShardLifecycleEvent::Assigned { .. } => "assigned", - lifecycle::ShardLifecycleEvent::Located { .. } => "located", - lifecycle::ShardLifecycleEvent::Fetching { .. } => "fetching", - lifecycle::ShardLifecycleEvent::CacheHit { .. } => "cache-hit", - lifecycle::ShardLifecycleEvent::Fetched { .. } => "fetched", - lifecycle::ShardLifecycleEvent::Validated { .. } => "validated", - lifecycle::ShardLifecycleEvent::Binding { .. } => "binding", - lifecycle::ShardLifecycleEvent::Ready { .. } => "ready", - lifecycle::ShardLifecycleEvent::Faulted { .. } => "faulted", - }) - .collect() -} - -#[derive(Default)] -struct MemoryCache { - entries: BTreeMap, - inserts: usize, -} - -impl fetch::ShardCache for MemoryCache { - fn get(&self, cache_key: &str) -> Option { - self.entries.get(cache_key).cloned() - } - - fn insert(&mut self, cache_key: String, shard: fetch::FetchedShard) { - self.inserts += 1; - self.entries.insert(cache_key, shard); - } -} - -struct RecordingFetcher { - calls: Vec, - result: Result, -} - -impl fetch::ShardFetcher for RecordingFetcher { - fn fetch( - &mut self, - request: &fetch::FetchShard, - ) -> Result { - self.calls.push(request.clone()); - self.result.clone() - } -} - -struct RecordingBinder { - calls: Vec, - result: Result<(), lifecycle::BindError>, -} - -impl lifecycle::WorkerShardBinder for RecordingBinder { - fn bind(&mut self, shard: &shards::ValidatedShard) -> Result<(), lifecycle::BindError> { - self.calls.push(shard.clone()); - self.result.clone() - } -} - -#[test] -fn lifecycle_happy_path_reaches_ready() { - let assignment = assignment(); - let fetched = fetched_for(&assignment); - let mut cache = MemoryCache::default(); - let mut fetcher = RecordingFetcher { - calls: Vec::new(), - result: Ok(fetched.clone()), - }; - let mut binder = RecordingBinder { - calls: Vec::new(), - result: Ok(()), - }; - let mut lifecycle = lifecycle::ShardWeightLifecycle::new(); - - lifecycle.load(assignment.clone(), &mut cache, &mut fetcher, &mut binder); - - assert_eq!(lifecycle.state(), lifecycle::ShardLifecycleState::Ready); - assert_eq!( - event_kinds(lifecycle.events()), - vec![ - "assigned", - "located", - "fetching", - "fetched", - "validated", - "binding", - "ready" - ] - ); - assert_eq!(fetcher.calls.len(), 1); - assert_eq!(cache.inserts, 1); - assert_eq!(binder.calls.len(), 1); - assert_eq!(binder.calls[0].assignment, assignment); - assert_eq!(binder.calls[0].local_path, fetched.local_path); -} - -#[test] -fn lifecycle_cache_hit_skips_fetch_but_still_validates_and_binds() { - let assignment = assignment(); - let fetched = fetched_for(&assignment); - let location = fetch::ShardLocator::locate(&assignment); - let mut cache = MemoryCache::default(); - cache.entries.insert(location.cache_key.clone(), fetched); - let mut fetcher = RecordingFetcher { - calls: Vec::new(), - result: Err(fetch::FetchError::Unavailable), - }; - let mut binder = RecordingBinder { - calls: Vec::new(), - result: Ok(()), - }; - let mut lifecycle = lifecycle::ShardWeightLifecycle::new(); - - lifecycle.load(assignment, &mut cache, &mut fetcher, &mut binder); - - assert_eq!(lifecycle.state(), lifecycle::ShardLifecycleState::Ready); - assert_eq!( - event_kinds(lifecycle.events()), - vec![ - "assigned", - "located", - "fetching", - "cache-hit", - "validated", - "binding", - "ready" - ] - ); - assert!(fetcher.calls.is_empty()); - assert_eq!(cache.inserts, 0); - assert_eq!(binder.calls.len(), 1); -} - -#[test] -fn lifecycle_fetch_failure_faults_without_validation_or_bind() { - let assignment = assignment(); - let mut cache = MemoryCache::default(); - let mut fetcher = RecordingFetcher { - calls: Vec::new(), - result: Err(fetch::FetchError::NotFound), - }; - let mut binder = RecordingBinder { - calls: Vec::new(), - result: Ok(()), - }; - let mut lifecycle = lifecycle::ShardWeightLifecycle::new(); - - lifecycle.load(assignment, &mut cache, &mut fetcher, &mut binder); - - assert_eq!(lifecycle.state(), lifecycle::ShardLifecycleState::Faulted); - assert_eq!(fetcher.calls.len(), 1); - assert!(binder.calls.is_empty()); - assert!(matches!( - lifecycle.events().last(), - Some(lifecycle::ShardLifecycleEvent::Faulted { - reason: lifecycle::ShardLifecycleFault::Fetch(fetch::FetchError::NotFound) - }) - )); -} - -#[test] -fn lifecycle_validation_failure_faults_without_bind() { - let assignment = assignment(); - let mut bad_manifest = shards::ShardManifest::for_assignment( - &assignment, - shards::ContentHash::literal("sha256:stage-3").unwrap(), - ); - bad_manifest.stage_index = 4; - let mut cache = MemoryCache::default(); - let mut fetcher = RecordingFetcher { - calls: Vec::new(), - result: Ok(fetch::FetchedShard::new( - "/cache/bad-stage.gguf", - bad_manifest, - )), - }; - let mut binder = RecordingBinder { - calls: Vec::new(), - result: Ok(()), - }; - let mut lifecycle = lifecycle::ShardWeightLifecycle::new(); - - lifecycle.load(assignment, &mut cache, &mut fetcher, &mut binder); - - assert_eq!(lifecycle.state(), lifecycle::ShardLifecycleState::Faulted); - assert!(binder.calls.is_empty()); - assert!(matches!( - lifecycle.events().last(), - Some(lifecycle::ShardLifecycleEvent::Faulted { - reason: lifecycle::ShardLifecycleFault::Validation( - shards::ShardValidationError::StageIndexMismatch - ) - }) - )); -} - -#[test] -fn lifecycle_bind_failure_faults_after_validation() { - let assignment = assignment(); - let fetched = fetched_for(&assignment); - let mut cache = MemoryCache::default(); - let mut fetcher = RecordingFetcher { - calls: Vec::new(), - result: Ok(fetched), - }; - let mut binder = RecordingBinder { - calls: Vec::new(), - result: Err(lifecycle::BindError::WorkerRejected), - }; - let mut lifecycle = lifecycle::ShardWeightLifecycle::new(); - - lifecycle.load(assignment, &mut cache, &mut fetcher, &mut binder); - - assert_eq!(lifecycle.state(), lifecycle::ShardLifecycleState::Faulted); - assert_eq!(binder.calls.len(), 1); - assert!(event_kinds(lifecycle.events()).contains(&"validated")); - assert!(matches!( - lifecycle.events().last(), - Some(lifecycle::ShardLifecycleEvent::Faulted { - reason: lifecycle::ShardLifecycleFault::Bind(lifecycle::BindError::WorkerRejected) - }) - )); -} - -#[test] -fn ready_terminal_state_is_idempotent() { - let assignment = assignment(); - let fetched = fetched_for(&assignment); - let mut cache = MemoryCache::default(); - let mut fetcher = RecordingFetcher { - calls: Vec::new(), - result: Ok(fetched.clone()), - }; - let mut binder = RecordingBinder { - calls: Vec::new(), - result: Ok(()), - }; - let mut lifecycle = lifecycle::ShardWeightLifecycle::new(); - - lifecycle.load(assignment.clone(), &mut cache, &mut fetcher, &mut binder); - let event_count = lifecycle.events().len(); - lifecycle.load(assignment, &mut cache, &mut fetcher, &mut binder); - - assert_eq!(lifecycle.state(), lifecycle::ShardLifecycleState::Ready); - assert_eq!(lifecycle.events().len(), event_count); - assert_eq!(fetcher.calls.len(), 1); - assert_eq!(binder.calls.len(), 1); -} - -#[test] -fn fault_terminal_state_is_idempotent() { - let assignment = assignment(); - let fetched = fetched_for(&assignment); - let mut cache = MemoryCache::default(); - let mut fetcher = RecordingFetcher { - calls: Vec::new(), - result: Err(fetch::FetchError::Unavailable), - }; - let mut binder = RecordingBinder { - calls: Vec::new(), - result: Ok(()), - }; - let mut lifecycle = lifecycle::ShardWeightLifecycle::new(); - - lifecycle.load(assignment.clone(), &mut cache, &mut fetcher, &mut binder); - let event_count = lifecycle.events().len(); - fetcher.result = Ok(fetched); - lifecycle.load(assignment, &mut cache, &mut fetcher, &mut binder); - - assert_eq!(lifecycle.state(), lifecycle::ShardLifecycleState::Faulted); - assert_eq!(lifecycle.events().len(), event_count); - assert_eq!(fetcher.calls.len(), 1); - assert!(binder.calls.is_empty()); -} diff --git a/crates/mvp-system/src/tests/shared_ring_helper_abi_guarantees.rs b/crates/mvp-system/src/tests/shared_ring_helper_abi_guarantees.rs deleted file mode 100644 index ea5c292..0000000 --- a/crates/mvp-system/src/tests/shared_ring_helper_abi_guarantees.rs +++ /dev/null @@ -1,235 +0,0 @@ -//! Black-box contract tests for the MVP shared ring helper ABI. -//! -//! These tests intentionally know only the public helper surface: -//! -//! - helper-created bounded SPSC rings -//! - reserve, write, publish, read, consume, and wake operations -//! - cursor snapshots and wake hints observed through the helper -//! -//! They assert the guarantees in -//! `specs/mvp_system/shared_ring_helper_abi_contract.md`. - -use data_plane::ring; - -// A small ring forces wraparound and full/empty transitions quickly. The helper -// still owns the actual shared-memory atomics and process-local address math. -fn new_ring() -> ring::RingHelperHarness { - ring::RingHelperHarness::create(ring::RingConfig { - node_id: ring::NodeId(10), - ring_id: ring::RingId(7000), - capacity: 8, - producer: ring::EndpointId("producer".into()), - consumer: ring::EndpointId("consumer".into()), - }) -} - -// This helper reads all currently committed bytes through the consumer API. -// It proves visibility through acquire reads instead of peeking at backing -// memory directly. -fn drain_committed(helper: &mut ring::RingHelperHarness) -> Vec { - let readable = helper.consumer_readable(); - let bytes = helper.consumer_read(readable); - helper.consumer_consume(readable); - bytes -} - -// Wake hints must remain hints. This helper checks the public wake enum without -// depending on scheduler internals. -fn assert_wake_is_payload_free(wake: &ring::WakeHint) { - match wake { - ring::WakeHint::RingReadable { ring_id } | ring::WakeHint::RingWritable { ring_id } => { - assert_eq!(*ring_id, ring::RingId(7000)); - } - } -} - -// This proves ring identity is bounded SPSC, has one producer and one consumer, -// uses unique RingId values, and does not allow stale wakes to alias a -// replacement ring. -#[test] -fn ring_identity_is_unique_bounded_spsc_and_stale_wakes_do_not_alias() { - // Create one ring and inspect its public identity. - let mut helper = new_ring(); - let identity = helper.identity(); - assert_eq!(identity.ring_id, ring::RingId(7000)); - assert_eq!(identity.capacity, 8); - assert_eq!(identity.producer_count, 1); - assert_eq!(identity.consumer_count, 1); - - // Retire it and create a replacement with the same numeric id but a new - // generation. Stale wake from the retired generation must not wake the new - // ring. - let stale_wake = helper.retire_and_capture_stale_wake(); - let mut replacement = ring::RingHelperHarness::create_replacement(identity.ring_id); - replacement.deliver_wake(stale_wake); - assert!(!replacement.wake_log().iter().any(|wake| wake.was_accepted)); -} - -// This proves commit and consume are monotonic logical byte positions, writes -// are invisible before commit, and physical wrap uses cursor modulo capacity. -#[test] -fn cursors_are_monotonic_and_wrap_by_modulo_capacity() { - // Reserve and write without publishing. - let mut helper = new_ring(); - let reservation = helper.producer_reserve(6).expect("space must exist"); - helper.producer_write(&reservation, b"abcdef"); - - // Producer-local write is not readable before commit. - assert_eq!(helper.consumer_readable(), 0); - - // Publishing makes the prefix readable and advances commit. - helper.producer_commit(reservation); - assert_eq!(helper.cursor_snapshot().commit, 6); - assert_eq!(drain_committed(&mut helper), b"abcdef"); - assert_eq!(helper.cursor_snapshot().consume, 6); - - // Wrap the physical index while logical cursors keep increasing. - let wrapped = helper - .producer_reserve(5) - .expect("space must exist after consume"); - helper.producer_write(&wrapped, b"ghijk"); - helper.producer_commit(wrapped); - assert_eq!(helper.cursor_snapshot().commit, 11); - assert_eq!( - helper.cursor_snapshot().commit % helper.identity().capacity, - 3 - ); - assert_eq!(drain_committed(&mut helper), b"ghijk"); - assert_eq!(helper.cursor_snapshot().consume, 11); -} - -// This proves the producer computes free space from acquired consume, never -// reserves beyond capacity, writes before publishing commit, and emits readable -// wake hints after publication. -#[test] -fn producer_respects_free_space_and_publishes_after_writing() { - // Reserve the full ring and publish it. - let mut helper = new_ring(); - let reservation = helper - .producer_reserve(8) - .expect("full ring reservation fits"); - helper.producer_write(&reservation, b"12345678"); - helper.producer_commit(reservation); - - // With no consumed bytes, producer cannot reserve additional space. - assert!(matches!( - helper.producer_reserve(1), - Err(ring::ReserveError::InsufficientSpace) - )); - - // The readable wake must be a payload-free hint. - let wake = helper - .wake_hints() - .iter() - .find(|wake| matches!(wake, ring::WakeHint::RingReadable { .. })) - .expect("readable wake must be emitted"); - assert_wake_is_payload_free(wake); - - // Consumer sees the written bytes, proving commit was not published before - // the payload became valid. - assert_eq!(drain_committed(&mut helper), b"12345678"); -} - -// This proves the consumer computes readable bytes from acquired commit, never -// reads beyond committed data, advances consume only after release, and emits -// writable wake hints after freeing space. -#[test] -fn consumer_reads_only_committed_bytes_and_releases_after_safe_consume() { - // Publish three committed bytes. - let mut helper = new_ring(); - let reservation = helper.producer_reserve(3).expect("space must exist"); - helper.producer_write(&reservation, b"abc"); - helper.producer_commit(reservation); - - // The consumer cannot read beyond the committed prefix. - assert_eq!(helper.consumer_readable(), 3); - assert!(matches!( - helper.consumer_try_read(4), - Err(ring::ReadError::BeyondCommittedBytes) - )); - - // Reading alone does not release bytes. - assert_eq!(helper.consumer_read(3), b"abc"); - assert_eq!(helper.cursor_snapshot().consume, 0); - - // Consuming releases space and emits a writable hint. - helper.consumer_consume(3); - assert_eq!(helper.cursor_snapshot().consume, 3); - let wake = helper - .wake_hints() - .iter() - .find(|wake| matches!(wake, ring::WakeHint::RingWritable { .. })) - .expect("writable wake must be emitted"); - assert_wake_is_payload_free(wake); -} - -// This proves wake hints carry no byte ranges, counts, pointers, or credits, -// and coalescing cannot hide the only readable or writable transition. -#[test] -fn wake_hints_are_edge_hints_without_hiding_transitions() { - // Create an empty ring and publish one byte, causing empty-to-readable. - let mut helper = new_ring(); - let reservation = helper.producer_reserve(1).expect("space must exist"); - helper.producer_write(&reservation, b"x"); - helper.producer_commit(reservation); - - // The readable transition must be discoverable even if duplicate wakes are - // coalesced. - helper.coalesce_duplicate_wakes(); - assert!( - helper - .scheduler_state() - .readable_rings - .contains(&ring::RingId(7000)) - ); - - // Fill then release space to cause full-to-writable. - let _ = drain_committed(&mut helper); - helper.coalesce_duplicate_wakes(); - assert!( - helper - .scheduler_state() - .writable_rings - .contains(&ring::RingId(7000)) - ); - - // Every wake remains a payload-free hint. - for wake in helper.wake_hints() { - assert_wake_is_payload_free(wake); - } -} - -// This proves Python-facing helper operations own shared atomics and wrap math, -// while returned pointers are process-local addresses derived from arena base -// plus arena offsets. -#[test] -fn helper_abi_owns_atomics_wrap_math_and_process_local_pointers() { - // Ask the helper for a process-local view of a layout. - let helper = new_ring(); - let view = helper.map_process_local_view(ring::ArenaBase(0x1000)); - - // The helper returns process-local addresses derived from offsets. - assert_eq!( - view.data_pointer, - ring::ProcessLocalPointer::from_base_plus_offset( - ring::ArenaBase(0x1000), - view.layout.data_offset - ) - ); - - // Python operations use helper calls for cursor and wrap behavior instead - // of implementing atomics directly. - for operation in helper.python_visible_operations() { - match operation { - ring::PythonOperation::ReserveViaHelper { .. } - | ring::PythonOperation::CommitViaHelper { .. } - | ring::PythonOperation::ReadableViaHelper { .. } - | ring::PythonOperation::ConsumeViaHelper { .. } - | ring::PythonOperation::MapPointerViaHelper { .. } => {} - ring::PythonOperation::DirectAtomicAccess { .. } - | ring::PythonOperation::DirectWrapArithmetic { .. } => { - panic!("Python operation bypassed helper ABI: {operation:?}") - } - } - } -} diff --git a/crates/mvp-system/src/tests/stage_controller_guarantees.rs b/crates/mvp-system/src/tests/stage_controller_guarantees.rs deleted file mode 100644 index 9e6c6c1..0000000 --- a/crates/mvp-system/src/tests/stage_controller_guarantees.rs +++ /dev/null @@ -1,545 +0,0 @@ -//! Black-box contract tests for MVP StageController behavior. -//! -//! These tests intentionally know only the public stage-controller surface: -//! -//! - `ProvisionStage`, worker, edge, object, stop, and fault events in -//! - worker commands, lifecycle events, and teardown events out -//! -//! They assert the guarantees in -//! `specs/mvp_system/stage_controller_contract.md`. - -use mvp_system::staging as stage; - -// This provision fixture represents a single middle stage. It has one inbound -// and one outbound edge so tests can prove the controller uses assigned edges -// without relying on endpoint internals. -fn valid_provision() -> stage::ProvisionStage { - stage::ProvisionStage { - run_id: stage::RunId(7), - authorized_orchestrator: stage::NodeId(99), - node_id: stage::NodeId(11), - stage_index: 1, - stage_count: 3, - layer_range: stage::LayerRange { - start: 12, - end_exclusive: 24, - }, - inbound: stage::EdgeProvision::inbound(stage::EdgeId(7001)), - outbound: stage::EdgeProvision::outbound(stage::EdgeId(7002)), - weight_source: stage::WeightSource::embedded_gguf("model", "model.gguf"), - shard_plan: None, - } -} - -// The harness exposes only public messages. Tests intentionally do not inspect -// private controller states such as "Preparing" or "Executing"; they infer -// controller behavior from emitted commands and lifecycle events. -fn new_controller() -> stage::StageControllerHarness { - stage::StageControllerHarness::new(stage::NodeId(11)) -} - -// Preparation readiness has four independent prerequisites. Listing them as -// public observations lets tests prove StageReady is a barrier across worker, -// weights, inbound edge, and outbound edge readiness. -fn preparation_ready_events() -> Vec { - vec![ - stage::StageEvent::WorkerReady, - stage::StageEvent::WeightsReady, - stage::StageEvent::InboundEdgeReady { - edge_id: stage::EdgeId(7001), - }, - stage::StageEvent::OutboundEdgeReady { - edge_id: stage::EdgeId(7002), - }, - ] -} - -// This helper provisions and readies a stage through public events. Tests that -// focus on execution use it to avoid duplicating setup while still going through -// the same observable path as production. -fn ready_stage() -> stage::StageControllerHarness { - let mut harness = new_controller(); - harness.observe(stage::StageEvent::ProvisionStage { - from: stage::NodeId(99), - provision: valid_provision(), - }); - for event in preparation_ready_events() { - harness.observe(event); - } - harness -} - -// This proves provisioning is authorized, validated before setup, and does not -// allow a stage to rewire its assigned inbound or outbound edge. -#[test] -fn provisioning_validates_authority_and_assigned_shape_before_setup() { - // Send a valid provision from the authorized orchestrator. - let mut harness = new_controller(); - harness.observe(stage::StageEvent::ProvisionStage { - from: stage::NodeId(99), - provision: valid_provision(), - }); - - // Setup commands should be derived from the provided assignment. - assert!(harness.commands().iter().any(|command| { - matches!( - command, - stage::StageCommand::EstablishInboundEdge { - edge_id: stage::EdgeId(7001), - .. - } - ) - })); - assert!(harness.commands().iter().any(|command| { - matches!( - command, - stage::StageCommand::EstablishOutboundEdge { - edge_id: stage::EdgeId(7002), - .. - } - ) - })); - - // The controller must not emit any command that replaces the provisioned - // edge ids with a locally chosen edge. - assert!( - !harness - .commands() - .iter() - .any(|command| { matches!(command, stage::StageCommand::RewireEdge { .. }) }) - ); - - // An unauthorized provision attempt must fault before setup can begin. - let mut unauthorized = new_controller(); - unauthorized.observe(stage::StageEvent::ProvisionStage { - from: stage::NodeId(123), - provision: valid_provision(), - }); - assert!(unauthorized.events().iter().any(|event| { - matches!( - event, - stage::StageLifecycleEvent::StageFault { - reason: stage::StageFaultReason::UnauthorizedProvision, - .. - } - ) - })); - assert!( - !unauthorized - .commands() - .iter() - .any(|command| { matches!(command, stage::StageCommand::ConfigureWorkerRole { .. }) }) - ); -} - -#[test] -fn duplicate_identical_provision_is_idempotent() { - let mut harness = new_controller(); - let provision = valid_provision(); - harness.observe(stage::StageEvent::ProvisionStage { - from: stage::NodeId(99), - provision: provision.clone(), - }); - let command_count = harness.commands().len(); - let event_count = harness.events().len(); - - harness.observe(stage::StageEvent::ProvisionStage { - from: stage::NodeId(99), - provision, - }); - - assert_eq!(harness.commands().len(), command_count); - assert_eq!(harness.events().len(), event_count); -} - -// This proves StageReady is emitted only after worker readiness, weight -// readiness, inbound edge readiness, and outbound edge readiness are all -// observed. -#[test] -fn stage_ready_waits_for_worker_weights_and_both_edges() { - // Provision the stage so preparation can begin. - let mut harness = new_controller(); - harness.observe(stage::StageEvent::ProvisionStage { - from: stage::NodeId(99), - provision: valid_provision(), - }); - - // Feed every readiness event except the final one and prove no prefix is - // enough for StageReady. - let mut events = preparation_ready_events(); - let final_event = events.pop().expect("fixture has final setup event"); - for event in events { - harness.observe(event); - assert!( - !harness - .events() - .iter() - .any(|event| { matches!(event, stage::StageLifecycleEvent::StageReady { .. }) }) - ); - } - - // The final prerequisite crosses the barrier. - harness.observe(final_event); - - // StageReady appears exactly once for the provisioned stage. - let ready_count = harness - .events() - .iter() - .filter(|event| { - matches!( - event, - stage::StageLifecycleEvent::StageReady { - run_id: stage::RunId(7), - stage_index: 1, - } - ) - }) - .count(); - assert_eq!(ready_count, 1); -} - -// This proves a ready stage admits work only from inbound ObjectLoaded, issues -// one ExecuteStep per accepted object, and binds output with the same sequence. -#[test] -fn accepted_inbound_object_creates_one_same_sequence_execute_step() { - // Bring the stage to ready state through public setup events. - let mut harness = ready_stage(); - - // Deliver the first inbound object, sequence 0. - harness.observe(stage::StageEvent::ObjectLoaded { - edge_id: stage::EdgeId(7001), - object_id: stage::ObjectId(9000), - sequence: 0, - handle: stage::DeviceHandle::new_current(42), - }); - - // Exactly one ExecuteStep command must result from that accepted object. - let execute_steps = harness - .commands() - .iter() - .filter_map(|command| match command { - stage::StageCommand::ExecuteStep(step) => Some(step), - _ => None, - }) - .collect::>(); - assert_eq!(execute_steps.len(), 1); - - // The output binding must preserve the input sequence. - assert_eq!(execute_steps[0].input.sequence, 0); - assert_eq!(execute_steps[0].outputs[0].sequence, 0); - - // A second object while the first step is active must not create another - // active ExecuteStep in the MVP. - harness.observe(stage::StageEvent::ObjectLoaded { - edge_id: stage::EdgeId(7001), - object_id: stage::ObjectId(9001), - sequence: 1, - handle: stage::DeviceHandle::new_current(43), - }); - let active_steps = harness - .commands() - .iter() - .filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_))) - .count(); - assert_eq!(active_steps, 1); -} - -// This proves the sequence contract: sequence 0 is accepted as prefill, decode -// sequences must strictly increase, and duplicate, skipped, or out-of-order -// inputs fault the stage. -#[test] -fn duplicate_skipped_and_out_of_order_sequences_fault() { - // Each invalid trace starts from a freshly readied stage. - let invalid_traces = vec![vec![0, 0], vec![0, 2], vec![0, 1, 0]]; - - for trace in invalid_traces { - // Accept the first object and complete its step when needed so the next - // object is admitted through the normal public path. - let mut harness = ready_stage(); - for (i, sequence) in trace.iter().enumerate() { - harness.observe(stage::StageEvent::ObjectLoaded { - edge_id: stage::EdgeId(7001), - object_id: stage::ObjectId(9000 + i as u64), - sequence: *sequence, - handle: stage::DeviceHandle::new_current(100 + i as u64), - }); - if i + 1 < trace.len() { - harness.observe(stage::StageEvent::StepCompleted { - step_id: stage::StepId(i as u64), - }); - } - } - - // The transcript must contain a sequence fault for the invalid trace. - assert!(harness.events().iter().any(|event| { - matches!( - event, - stage::StageLifecycleEvent::StageFault { - reason: stage::StageFaultReason::SequenceViolation, - .. - } - ) - })); - } -} - -// This proves compute completion is observed only after the worker reports -// StepCompleted, and completion returns the stage to ready-for-next-object. -#[test] -fn step_completed_releases_input_and_admits_next_object() { - // Start one accepted step. - let mut harness = ready_stage(); - harness.observe(stage::StageEvent::ObjectLoaded { - edge_id: stage::EdgeId(7001), - object_id: stage::ObjectId(9000), - sequence: 0, - handle: stage::DeviceHandle::new_current(42), - }); - - // Before worker completion, no compute-complete lifecycle event is allowed. - assert!(!harness.events().iter().any(|event| { - matches!( - event, - stage::StageLifecycleEvent::StepAccepted { sequence: 1, .. } - ) - })); - - // Worker StepCompleted is the public completion signal. - harness.observe(stage::StageEvent::StepCompleted { - step_id: stage::StepId(0), - }); - - // The controller releases per-step input according to policy. - assert!(harness.commands().iter().any(|command| { - matches!( - command, - stage::StageCommand::ReleaseInputHandle { - object_id: stage::ObjectId(9000), - .. - } - ) - })); - - // The next sequence is now admissible. - harness.observe(stage::StageEvent::ObjectLoaded { - edge_id: stage::EdgeId(7001), - object_id: stage::ObjectId(9001), - sequence: 1, - handle: stage::DeviceHandle::new_current(43), - }); - let execute_count = harness - .commands() - .iter() - .filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_))) - .count(); - assert_eq!(execute_count, 2); -} - -// This proves worker, object, output-edge, edge, and step failures fault the -// stage with stable public reasons, and after fault no new run work is accepted -// until StopRun. -#[test] -fn stage_failures_map_to_stable_fault_reasons_and_reject_new_work() { - let cases = vec![ - ( - stage::StageEvent::WorkerCrashed, - stage::StageFaultReason::WorkerCrashed, - ), - ( - stage::StageEvent::StepFailed { - step_id: stage::StepId(0), - }, - stage::StageFaultReason::StepFailed, - ), - ( - stage::StageEvent::ObjectFailed { - edge_id: stage::EdgeId(7001), - object_id: Some(stage::ObjectId(9000)), - }, - stage::StageFaultReason::ObjectFailed, - ), - ( - stage::StageEvent::OutputFault { - edge_id: stage::EdgeId(7002), - }, - stage::StageFaultReason::OutputFault, - ), - ( - stage::StageEvent::EdgeFault { - edge_id: stage::EdgeId(7002), - }, - stage::StageFaultReason::EdgeFault, - ), - ]; - - for (fault_event, expected_reason) in cases { - let mut harness = ready_stage(); - harness.observe(fault_event); - - assert!(harness.events().iter().any(|event| { - matches!( - event, - stage::StageLifecycleEvent::StageFault { - reason, - .. - } if *reason == expected_reason - ) - })); - - let before = harness - .commands() - .iter() - .filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_))) - .count(); - harness.observe(stage::StageEvent::ObjectLoaded { - edge_id: stage::EdgeId(7001), - object_id: stage::ObjectId(9999), - sequence: 0, - handle: stage::DeviceHandle::new_current(77), - }); - let after = harness - .commands() - .iter() - .filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_))) - .count(); - assert_eq!(after, before); - - harness.observe(stage::StageEvent::StopRun { - run_id: stage::RunId(7), - }); - assert!( - harness - .commands() - .iter() - .any(|command| { matches!(command, stage::StageCommand::StopLocalEdges { .. }) }) - ); - assert!( - !harness - .events() - .iter() - .any(|event| { matches!(event, stage::StageLifecycleEvent::StageStopped { .. }) }) - ); - - harness.observe(stage::StageEvent::LocalEdgesStopped { - run_id: stage::RunId(7), - }); - harness.observe(stage::StageEvent::WorkerRingsQuiesced { - run_id: stage::RunId(7), - }); - harness.observe(stage::StageEvent::DeviceObjectsReleased { - run_id: stage::RunId(7), - }); - harness.observe(stage::StageEvent::WorkerRoleReset { - run_id: stage::RunId(7), - }); - assert!( - harness - .events() - .iter() - .any(|event| { matches!(event, stage::StageLifecycleEvent::StageStopped { .. }) }) - ); - } -} - -// This proves StopRun starts teardown but StageStopped is held back until local -// edges are stopped, worker rings are quiesced, device objects are released, and -// the worker role reset completes. -#[test] -fn stop_run_waits_for_local_teardown_completion_before_stage_stopped() { - let mut harness = ready_stage(); - - harness.observe(stage::StageEvent::StopRun { - run_id: stage::RunId(7), - }); - assert!( - harness - .commands() - .iter() - .any(|command| { matches!(command, stage::StageCommand::StopLocalEdges { .. }) }) - ); - assert!( - !harness - .events() - .iter() - .any(|event| { matches!(event, stage::StageLifecycleEvent::StageStopped { .. }) }) - ); - assert!( - !harness.commands().iter().any(|command| { - matches!(command, stage::StageCommand::ReleaseRunDeviceObjects { .. }) - }) - ); - - let execute_before_stopping_work = harness - .commands() - .iter() - .filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_))) - .count(); - harness.observe(stage::StageEvent::ObjectLoaded { - edge_id: stage::EdgeId(7001), - object_id: stage::ObjectId(9999), - sequence: 0, - handle: stage::DeviceHandle::new_current(77), - }); - let execute_after_stopping_work = harness - .commands() - .iter() - .filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_))) - .count(); - assert_eq!(execute_after_stopping_work, execute_before_stopping_work); - - harness.observe(stage::StageEvent::LocalEdgesStopped { - run_id: stage::RunId(7), - }); - assert!( - !harness - .events() - .iter() - .any(|event| { matches!(event, stage::StageLifecycleEvent::StageStopped { .. }) }) - ); - assert!( - !harness.commands().iter().any(|command| { - matches!(command, stage::StageCommand::ReleaseRunDeviceObjects { .. }) - }) - ); - - harness.observe(stage::StageEvent::WorkerRingsQuiesced { - run_id: stage::RunId(7), - }); - assert!(harness.commands().iter().any(|command| { - matches!( - command, - stage::StageCommand::ReleaseRunDeviceObjects { - run_id: stage::RunId(7) - } - ) - })); - assert!( - !harness - .events() - .iter() - .any(|event| { matches!(event, stage::StageLifecycleEvent::StageStopped { .. }) }) - ); - - harness.observe(stage::StageEvent::DeviceObjectsReleased { - run_id: stage::RunId(7), - }); - assert!( - !harness - .events() - .iter() - .any(|event| { matches!(event, stage::StageLifecycleEvent::StageStopped { .. }) }) - ); - - harness.observe(stage::StageEvent::WorkerRoleReset { - run_id: stage::RunId(7), - }); - assert!(harness.events().iter().any(|event| { - matches!( - event, - stage::StageLifecycleEvent::StageStopped { - run_id: stage::RunId(7), - stage_index: 1, - } - ) - })); -} diff --git a/crates/mvp-system/src/tests/staging_guarantees.rs b/crates/mvp-system/src/tests/staging_guarantees.rs new file mode 100644 index 0000000..eac50d2 --- /dev/null +++ b/crates/mvp-system/src/tests/staging_guarantees.rs @@ -0,0 +1,747 @@ +//! Behavior guarantees for the `staging` module. + +mod stage_controller { + //! Black-box contract tests for MVP StageController behavior. + //! + //! These tests intentionally know only the public stage-controller surface: + //! + //! - `ProvisionStage`, worker, edge, object, stop, and fault events in + //! - worker commands, lifecycle events, and teardown events out + //! + //! They assert the guarantees in + //! `specs/BEHAVIOR_GUARANTEES.md`. + + use mvp_system::staging as stage; + + // This provision fixture represents a single middle stage. It has one inbound + // and one outbound edge so tests can prove the controller uses assigned edges + // without relying on endpoint internals. + fn valid_provision() -> stage::ProvisionStage { + stage::ProvisionStage { + run_id: stage::RunId(7), + authorized_orchestrator: stage::NodeId(99), + node_id: stage::NodeId(11), + stage_index: 1, + stage_count: 3, + layer_range: stage::LayerRange { + start: 12, + end_exclusive: 24, + }, + inbound: stage::EdgeProvision::inbound(stage::EdgeId(7001)), + outbound: stage::EdgeProvision::outbound(stage::EdgeId(7002)), + weight_source: stage::WeightSource::embedded_gguf("model", "model.gguf"), + shard_plan: None, + } + } + + // The harness exposes only public messages. Tests intentionally do not inspect + // private controller states such as "Preparing" or "Executing"; they infer + // controller behavior from emitted commands and lifecycle events. + fn new_controller() -> stage::StageControllerHarness { + stage::StageControllerHarness::new(stage::NodeId(11)) + } + + // Preparation readiness has four independent prerequisites. Listing them as + // public observations lets tests prove StageReady is a barrier across worker, + // weights, inbound edge, and outbound edge readiness. + fn preparation_ready_events() -> Vec { + vec![ + stage::StageEvent::WorkerReady, + stage::StageEvent::WeightsReady, + stage::StageEvent::InboundEdgeReady { + edge_id: stage::EdgeId(7001), + }, + stage::StageEvent::OutboundEdgeReady { + edge_id: stage::EdgeId(7002), + }, + ] + } + + // This helper provisions and readies a stage through public events. Tests that + // focus on execution use it to avoid duplicating setup while still going through + // the same observable path as production. + fn ready_stage() -> stage::StageControllerHarness { + let mut harness = new_controller(); + harness.observe(stage::StageEvent::ProvisionStage { + from: stage::NodeId(99), + provision: valid_provision(), + }); + for event in preparation_ready_events() { + harness.observe(event); + } + harness + } + + // This proves provisioning is authorized, validated before setup, and does not + // allow a stage to rewire its assigned inbound or outbound edge. + #[test] + fn provisioning_validates_authority_and_assigned_shape_before_setup() { + // Send a valid provision from the authorized orchestrator. + let mut harness = new_controller(); + harness.observe(stage::StageEvent::ProvisionStage { + from: stage::NodeId(99), + provision: valid_provision(), + }); + + // Setup commands should be derived from the provided assignment. + assert!(harness.commands().iter().any(|command| { + matches!( + command, + stage::StageCommand::EstablishInboundEdge { + edge_id: stage::EdgeId(7001), + .. + } + ) + })); + assert!(harness.commands().iter().any(|command| { + matches!( + command, + stage::StageCommand::EstablishOutboundEdge { + edge_id: stage::EdgeId(7002), + .. + } + ) + })); + + // The controller must not emit any command that replaces the provisioned + // edge ids with a locally chosen edge. + assert!( + !harness + .commands() + .iter() + .any(|command| { matches!(command, stage::StageCommand::RewireEdge { .. }) }) + ); + + // An unauthorized provision attempt must fault before setup can begin. + let mut unauthorized = new_controller(); + unauthorized.observe(stage::StageEvent::ProvisionStage { + from: stage::NodeId(123), + provision: valid_provision(), + }); + assert!(unauthorized.events().iter().any(|event| { + matches!( + event, + stage::StageLifecycleEvent::StageFault { + reason: stage::StageFaultReason::UnauthorizedProvision, + .. + } + ) + })); + assert!( + !unauthorized.commands().iter().any(|command| { + matches!(command, stage::StageCommand::ConfigureWorkerRole { .. }) + }) + ); + } + // This proves StageReady is emitted only after worker readiness, weight + // readiness, inbound edge readiness, and outbound edge readiness are all + // observed. + #[test] + fn stage_ready_waits_for_worker_weights_and_both_edges() { + // Provision the stage so preparation can begin. + let mut harness = new_controller(); + harness.observe(stage::StageEvent::ProvisionStage { + from: stage::NodeId(99), + provision: valid_provision(), + }); + + // Feed every readiness event except the final one and prove no prefix is + // enough for StageReady. + let mut events = preparation_ready_events(); + let final_event = events.pop().expect("fixture has final setup event"); + for event in events { + harness.observe(event); + assert!( + !harness.events().iter().any(|event| { + matches!(event, stage::StageLifecycleEvent::StageReady { .. }) + }) + ); + } + + // The final prerequisite crosses the barrier. + harness.observe(final_event); + + // StageReady appears exactly once for the provisioned stage. + let ready_count = harness + .events() + .iter() + .filter(|event| { + matches!( + event, + stage::StageLifecycleEvent::StageReady { + run_id: stage::RunId(7), + stage_index: 1, + } + ) + }) + .count(); + assert_eq!(ready_count, 1); + } + + // This proves a ready stage admits work only from inbound ObjectLoaded, issues + // one ExecuteStep per accepted object, and binds output with the same sequence. + #[test] + fn accepted_inbound_object_creates_one_same_sequence_execute_step() { + // Bring the stage to ready state through public setup events. + let mut harness = ready_stage(); + + // Deliver the first inbound object, sequence 0. + harness.observe(stage::StageEvent::ObjectLoaded { + edge_id: stage::EdgeId(7001), + object_id: stage::ObjectId(9000), + sequence: 0, + handle: stage::DeviceHandle::new_current(42), + }); + + // Exactly one ExecuteStep command must result from that accepted object. + let execute_steps = harness + .commands() + .iter() + .filter_map(|command| match command { + stage::StageCommand::ExecuteStep(step) => Some(step), + _ => None, + }) + .collect::>(); + assert_eq!(execute_steps.len(), 1); + + // The output binding must preserve the input sequence. + assert_eq!(execute_steps[0].input.sequence, 0); + assert_eq!(execute_steps[0].outputs[0].sequence, 0); + + // A second object while the first step is active must not create another + // active ExecuteStep in the MVP. + harness.observe(stage::StageEvent::ObjectLoaded { + edge_id: stage::EdgeId(7001), + object_id: stage::ObjectId(9001), + sequence: 1, + handle: stage::DeviceHandle::new_current(43), + }); + let active_steps = harness + .commands() + .iter() + .filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_))) + .count(); + assert_eq!(active_steps, 1); + } + + // This proves the sequence contract: sequence 0 is accepted as prefill, decode + // sequences must strictly increase, and duplicate, skipped, or out-of-order + // inputs fault the stage. + #[test] + fn duplicate_skipped_and_out_of_order_sequences_fault() { + // Each invalid trace starts from a freshly readied stage. + let invalid_traces = vec![vec![0, 0], vec![0, 2], vec![0, 1, 0]]; + + for trace in invalid_traces { + // Accept the first object and complete its step when needed so the next + // object is admitted through the normal public path. + let mut harness = ready_stage(); + for (i, sequence) in trace.iter().enumerate() { + harness.observe(stage::StageEvent::ObjectLoaded { + edge_id: stage::EdgeId(7001), + object_id: stage::ObjectId(9000 + i as u64), + sequence: *sequence, + handle: stage::DeviceHandle::new_current(100 + i as u64), + }); + if i + 1 < trace.len() { + harness.observe(stage::StageEvent::StepCompleted { + step_id: stage::StepId(i as u64), + }); + } + } + + // The transcript must contain a sequence fault for the invalid trace. + assert!(harness.events().iter().any(|event| { + matches!( + event, + stage::StageLifecycleEvent::StageFault { + reason: stage::StageFaultReason::SequenceViolation, + .. + } + ) + })); + } + } + + // This proves compute completion is observed only after the worker reports + // StepCompleted, and completion returns the stage to ready-for-next-object. + #[test] + fn step_completed_releases_input_and_admits_next_object() { + // Start one accepted step. + let mut harness = ready_stage(); + harness.observe(stage::StageEvent::ObjectLoaded { + edge_id: stage::EdgeId(7001), + object_id: stage::ObjectId(9000), + sequence: 0, + handle: stage::DeviceHandle::new_current(42), + }); + + // Before worker completion, no compute-complete lifecycle event is allowed. + assert!(!harness.events().iter().any(|event| { + matches!( + event, + stage::StageLifecycleEvent::StepAccepted { sequence: 1, .. } + ) + })); + + // Worker StepCompleted is the public completion signal. + harness.observe(stage::StageEvent::StepCompleted { + step_id: stage::StepId(0), + }); + + // The controller releases per-step input according to policy. + assert!(harness.commands().iter().any(|command| { + matches!( + command, + stage::StageCommand::ReleaseInputHandle { + object_id: stage::ObjectId(9000), + .. + } + ) + })); + + // The next sequence is now admissible. + harness.observe(stage::StageEvent::ObjectLoaded { + edge_id: stage::EdgeId(7001), + object_id: stage::ObjectId(9001), + sequence: 1, + handle: stage::DeviceHandle::new_current(43), + }); + let execute_count = harness + .commands() + .iter() + .filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_))) + .count(); + assert_eq!(execute_count, 2); + } + + // This proves worker, object, output-edge, edge, and step failures fault the + // stage with stable public reasons, and after fault no new run work is accepted + // until StopRun. + #[test] + fn stage_failures_map_to_stable_fault_reasons_and_reject_new_work() { + let cases = vec![ + ( + stage::StageEvent::WorkerCrashed, + stage::StageFaultReason::WorkerCrashed, + ), + ( + stage::StageEvent::StepFailed { + step_id: stage::StepId(0), + }, + stage::StageFaultReason::StepFailed, + ), + ( + stage::StageEvent::ObjectFailed { + edge_id: stage::EdgeId(7001), + object_id: Some(stage::ObjectId(9000)), + }, + stage::StageFaultReason::ObjectFailed, + ), + ( + stage::StageEvent::OutputFault { + edge_id: stage::EdgeId(7002), + }, + stage::StageFaultReason::OutputFault, + ), + ( + stage::StageEvent::EdgeFault { + edge_id: stage::EdgeId(7002), + }, + stage::StageFaultReason::EdgeFault, + ), + ]; + + for (fault_event, expected_reason) in cases { + let mut harness = ready_stage(); + harness.observe(fault_event); + + assert!(harness.events().iter().any(|event| { + matches!( + event, + stage::StageLifecycleEvent::StageFault { + reason, + .. + } if *reason == expected_reason + ) + })); + + let before = harness + .commands() + .iter() + .filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_))) + .count(); + harness.observe(stage::StageEvent::ObjectLoaded { + edge_id: stage::EdgeId(7001), + object_id: stage::ObjectId(9999), + sequence: 0, + handle: stage::DeviceHandle::new_current(77), + }); + let after = harness + .commands() + .iter() + .filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_))) + .count(); + assert_eq!(after, before); + + harness.observe(stage::StageEvent::StopRun { + run_id: stage::RunId(7), + }); + assert!( + harness.commands().iter().any(|command| { + matches!(command, stage::StageCommand::StopLocalEdges { .. }) + }) + ); + assert!( + !harness.events().iter().any(|event| { + matches!(event, stage::StageLifecycleEvent::StageStopped { .. }) + }) + ); + + harness.observe(stage::StageEvent::LocalEdgesStopped { + run_id: stage::RunId(7), + }); + harness.observe(stage::StageEvent::WorkerRingsQuiesced { + run_id: stage::RunId(7), + }); + harness.observe(stage::StageEvent::DeviceObjectsReleased { + run_id: stage::RunId(7), + }); + harness.observe(stage::StageEvent::WorkerRoleReset { + run_id: stage::RunId(7), + }); + assert!( + harness.events().iter().any(|event| { + matches!(event, stage::StageLifecycleEvent::StageStopped { .. }) + }) + ); + } + } + + // This proves StopRun starts teardown but StageStopped is held back until local + // edges are stopped, worker rings are quiesced, device objects are released, and + // the worker role reset completes. + #[test] + fn stop_run_waits_for_local_teardown_completion_before_stage_stopped() { + let mut harness = ready_stage(); + + harness.observe(stage::StageEvent::StopRun { + run_id: stage::RunId(7), + }); + assert!( + harness + .commands() + .iter() + .any(|command| { matches!(command, stage::StageCommand::StopLocalEdges { .. }) }) + ); + assert!( + !harness + .events() + .iter() + .any(|event| { matches!(event, stage::StageLifecycleEvent::StageStopped { .. }) }) + ); + assert!(!harness.commands().iter().any(|command| { + matches!(command, stage::StageCommand::ReleaseRunDeviceObjects { .. }) + })); + + let execute_before_stopping_work = harness + .commands() + .iter() + .filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_))) + .count(); + harness.observe(stage::StageEvent::ObjectLoaded { + edge_id: stage::EdgeId(7001), + object_id: stage::ObjectId(9999), + sequence: 0, + handle: stage::DeviceHandle::new_current(77), + }); + let execute_after_stopping_work = harness + .commands() + .iter() + .filter(|command| matches!(command, stage::StageCommand::ExecuteStep(_))) + .count(); + assert_eq!(execute_after_stopping_work, execute_before_stopping_work); + + harness.observe(stage::StageEvent::LocalEdgesStopped { + run_id: stage::RunId(7), + }); + assert!( + !harness + .events() + .iter() + .any(|event| { matches!(event, stage::StageLifecycleEvent::StageStopped { .. }) }) + ); + assert!(!harness.commands().iter().any(|command| { + matches!(command, stage::StageCommand::ReleaseRunDeviceObjects { .. }) + })); + + harness.observe(stage::StageEvent::WorkerRingsQuiesced { + run_id: stage::RunId(7), + }); + assert!(harness.commands().iter().any(|command| { + matches!( + command, + stage::StageCommand::ReleaseRunDeviceObjects { + run_id: stage::RunId(7) + } + ) + })); + assert!( + !harness + .events() + .iter() + .any(|event| { matches!(event, stage::StageLifecycleEvent::StageStopped { .. }) }) + ); + + harness.observe(stage::StageEvent::DeviceObjectsReleased { + run_id: stage::RunId(7), + }); + assert!( + !harness + .events() + .iter() + .any(|event| { matches!(event, stage::StageLifecycleEvent::StageStopped { .. }) }) + ); + + harness.observe(stage::StageEvent::WorkerRoleReset { + run_id: stage::RunId(7), + }); + assert!(harness.events().iter().any(|event| { + matches!( + event, + stage::StageLifecycleEvent::StageStopped { + run_id: stage::RunId(7), + stage_index: 1, + } + ) + })); + } +} + +mod weight_lifecycle { + //! Black-box contract tests for MVP stage-local weight lifecycle. + //! + //! These tests intentionally know only the public weight-work surface: + //! + //! - `ProvisionStage` assignment in + //! - artifact, parse, allocation, binding, and cache outcomes in + //! - `WeightsReady`, `StageReady`, and `StageFault` out + //! + //! They assert the guarantees in + //! `specs/BEHAVIOR_GUARANTEES.md`. + + use mvp_system::staging::weight_lifecycle as weights; + + // A valid assignment gives the stage exactly one layer range and one source. + // Tests vary only source or failure outcome so the assignment contract remains + // visible. + fn valid_assignment() -> weights::WeightAssignment { + weights::WeightAssignment { + run_id: weights::RunId(7), + stage_index: 1, + plan_layer_range: weights::LayerRange { + start: 12, + end_exclusive: 24, + }, + assigned_layer_range: weights::LayerRange { + start: 12, + end_exclusive: 24, + }, + source: weights::WeightSource::WholeGguf { + uri: "test://model.gguf".into(), + }, + } + } + + // Weight loading is intentionally opaque. The harness accepts public loader and + // worker outcomes and records only stage-visible events and commands. + fn new_weight_harness() -> weights::WeightLifecycleHarness { + weights::WeightLifecycleHarness::new(weights::NodeId(11)) + } + + // The success facts represent the observable prerequisites for WeightsReady: + // artifact bytes exist, the assigned layer range is valid, and the worker has + // loaded or bound that range. + fn successful_load_events() -> Vec { + vec![ + weights::WeightEvent::ArtifactAvailable { + bytes: weights::ArtifactBytes::Local, + }, + weights::WeightEvent::LayerRangeValidated, + weights::WeightEvent::WorkerRangeBound, + ] + } + + // Each failure case maps one public loader or worker failure to the stable + // stage fault reason expected at the control boundary. + fn failure_cases() -> Vec<(weights::WeightEvent, weights::StageFaultReason)> { + vec![ + ( + weights::WeightEvent::DownloadFailed, + weights::StageFaultReason::WeightDownloadFailed, + ), + ( + weights::WeightEvent::ParseFailed, + weights::StageFaultReason::WeightParseFailed, + ), + ( + weights::WeightEvent::DeviceAllocationFailed, + weights::StageFaultReason::DeviceAllocationFailed, + ), + ( + weights::WeightEvent::BindingFailed, + weights::StageFaultReason::WeightBindingFailed, + ), + ( + weights::WeightEvent::InvalidLayerRange, + weights::StageFaultReason::InvalidLayerRange, + ), + ] + } + + // This proves a stage receives weight source and exactly one assigned layer + // range from provisioning, validates it against the plan, and does not claim + // graph-visible ownership outside that range. + #[test] + fn assignment_is_stage_local_and_range_limited() { + // Start weight work from the provisioned assignment. + let mut harness = new_weight_harness(); + harness.observe(weights::WeightEvent::Provisioned(valid_assignment())); + + // The load command may use the physical source, but its graph-visible layer + // range must be the assigned range. + for command in harness.commands() { + if let weights::WeightCommand::LoadOrBindRange { range, .. } = command { + assert_eq!( + *range, + weights::LayerRange { + start: 12, + end_exclusive: 24, + } + ); + } + } + + // There must be no command claiming ownership of neighboring layers. + assert!(!harness.commands().iter().any(|command| { + matches!( + command, + weights::WeightCommand::AdvertiseLoadedLayerRange { + range, + .. + } if range.start < 12 || range.end_exclusive > 24 + ) + })); + } + + // This proves whole GGUF download, shard download, and cache use are physical + // mechanisms with the same public outcome: WeightsReady or StageFault. + #[test] + fn supported_physical_sources_have_same_visible_success_contract() { + // Exercise every supported source without asserting how bytes are obtained. + let sources = vec![ + weights::WeightSource::WholeGguf { + uri: "test://model.gguf".into(), + }, + weights::WeightSource::ShardSet { + uris: vec!["test://model.layers.12-24.gguf".into()], + }, + weights::WeightSource::CachedArtifact { + cache_key: "model:layers:12-24".into(), + }, + ]; + + for source in sources { + // Install the source in an otherwise valid assignment. + let mut assignment = valid_assignment(); + assignment.source = source; + let mut harness = new_weight_harness(); + harness.observe(weights::WeightEvent::Provisioned(assignment)); + + // Drive the same public success facts for every source. + for event in successful_load_events() { + harness.observe(event); + } + + // The system-visible success outcome is WeightsReady. + assert!(harness.events().iter().any(|event| { + matches!( + event, + weights::WeightLifecycleEvent::WeightsReady { + run_id: weights::RunId(7), + stage_index: 1, + } + ) + })); + } + } + + // This proves WeightsReady requires artifact availability, layer validation, + // and worker bind/load completion, and that WeightsReady precedes StageReady. + #[test] + fn weights_ready_requires_all_weight_facts_and_precedes_stage_ready() { + // Start from a valid assignment. + let mut harness = new_weight_harness(); + harness.observe(weights::WeightEvent::Provisioned(valid_assignment())); + + // Feed every success fact except the final one and prove no prefix is + // sufficient for WeightsReady. + let mut events = successful_load_events(); + let final_event = events.pop().expect("fixture has final weight event"); + for event in events { + harness.observe(event); + assert!(!harness.events().iter().any(|event| { + matches!(event, weights::WeightLifecycleEvent::WeightsReady { .. }) + })); + } + + // The final weight prerequisite emits WeightsReady. + harness.observe(final_event); + let weights_ready_pos = harness + .events() + .iter() + .position(|event| matches!(event, weights::WeightLifecycleEvent::WeightsReady { .. })) + .expect("WeightsReady must be emitted"); + + // StageReady may occur only after the StageController observes WeightsReady + // and the other local setup prerequisites. + harness.observe(weights::WeightEvent::OtherStagePrerequisitesReady); + let stage_ready_pos = harness + .events() + .iter() + .position(|event| matches!(event, weights::WeightLifecycleEvent::StageReady { .. })) + .expect("StageReady must be emitted after prerequisites"); + assert!(weights_ready_pos < stage_ready_pos); + } + + // This proves every weight failure source faults the stage and suppresses both + // WeightsReady and StageReady. + #[test] + fn weight_failures_emit_stage_fault_without_readiness() { + // Each failure source gets an isolated attempt. + for (failure_event, expected_reason) in failure_cases() { + let mut harness = new_weight_harness(); + harness.observe(weights::WeightEvent::Provisioned(valid_assignment())); + + // Deliver the public failure outcome from weight work. + harness.observe(failure_event); + + // The stable fault reason must be observable. + assert!(harness.events().iter().any(|event| { + matches!( + event, + weights::WeightLifecycleEvent::StageFault { + reason, + .. + } if *reason == expected_reason + ) + })); + + // Readiness cannot also be emitted after a faulted weight attempt. + assert!(!harness.events().iter().any(|event| { + matches!(event, weights::WeightLifecycleEvent::WeightsReady { .. }) + || matches!(event, weights::WeightLifecycleEvent::StageReady { .. }) + })); + } + } +} diff --git a/crates/mvp-system/src/tests/telemetry_guarantees.rs b/crates/mvp-system/src/tests/telemetry_guarantees.rs deleted file mode 100644 index ea85102..0000000 --- a/crates/mvp-system/src/tests/telemetry_guarantees.rs +++ /dev/null @@ -1,173 +0,0 @@ -use crate::provisioning::{ - ProvisionEvent, ProvisionEventKind, ProvisionLogLine, ProvisionLogStream, -}; -use datastream::{ChannelId, ChannelKind, Frame, Lifetime, Record, StreamId}; -use mvp_system::observability::frame_archive::FrameArchive; -use mvp_system::observability::lifecycle as obs; -use mvp_system::observability::telemetry::{ - self, MvpLifecycleRecord, MvpProvisionEventRecord, MvpProvisionLogRecord, -}; - -#[test] -fn mvp_lifecycle_record_round_trips_on_owned_datastream_channel() { - let record = MvpLifecycleRecord::new(obs::Event::StageScoped { - kind: obs::EventKind::StageFaulted, - run_id: obs::RunId(7), - stage_index: obs::StageIndex(2), - reason: Some(obs::FaultReason::WorkerCrashed), - component: obs::Component::StageController, - }); - - assert_eq!(MvpLifecycleRecord::CHANNEL, telemetry::MVP_LIFECYCLE); - assert_eq!(MvpLifecycleRecord::channel_name(), "mvp.lifecycle"); - assert_eq!(record.kind(), obs::EventKind::StageFaulted); - assert_eq!( - MvpLifecycleRecord::decode(&record.encode()).unwrap(), - record - ); -} - -#[test] -fn mvp_channel_registry_marks_lifecycle_payloads_as_typed() { - let registry = telemetry::channel_registry(); - - assert_eq!( - registry.classify_name(MvpLifecycleRecord::channel_name()), - ChannelKind::Typed - ); -} - -#[test] -fn provisioning_records_round_trip_on_owned_datastream_channels() { - let event = MvpProvisionEventRecord::new(ProvisionEvent { - run_id: 77, - node_id: 11, - kind: ProvisionEventKind::NodeLive, - provider: Some("docker".to_owned()), - message: None, - }); - let event_without_provider = MvpProvisionEventRecord::new(ProvisionEvent { - run_id: 77, - node_id: 12, - kind: ProvisionEventKind::ProvisionStart, - provider: None, - message: Some("queued".to_owned()), - }); - let log = MvpProvisionLogRecord::new(ProvisionLogLine { - run_id: 77, - node_id: 11, - stream: ProvisionLogStream::Stdout, - line: "{\"type\":\"ready\"}".to_owned(), - }); - - assert_eq!( - MvpProvisionEventRecord::CHANNEL, - telemetry::MVP_PROVISIONING_EVENTS - ); - assert_eq!( - MvpProvisionLogRecord::CHANNEL, - telemetry::MVP_PROVISIONING_LOGS - ); - assert_eq!( - MvpProvisionEventRecord::decode(&event.encode()).unwrap(), - event - ); - assert_eq!(MvpProvisionLogRecord::decode(&log.encode()).unwrap(), log); - assert_eq!( - MvpProvisionEventRecord::decode(&event_without_provider.encode()).unwrap(), - event_without_provider - ); - assert_eq!( - telemetry::mvp_provision_log_channel(11, ProvisionLogStream::Stdout).as_str(), - "mvp.provisioning.logs.node.11.stdout" - ); -} - -#[test] -fn mvp_channel_registry_marks_provisioning_payloads_as_typed() { - let registry = telemetry::channel_registry(); - - assert_eq!( - registry.classify_name(MvpProvisionEventRecord::channel_name()), - ChannelKind::Typed - ); - assert_eq!( - registry.classify_name(MvpProvisionLogRecord::channel_name()), - ChannelKind::Typed - ); -} - -#[test] -fn frame_archive_writes_jsonl_records_for_text_and_binary_payloads() { - static NEXT_TEMP_FILE: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); - - let suffix = NEXT_TEMP_FILE.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let path = std::env::temp_dir().join(format!( - "mvp-observability-frame-archive-test-{}-{suffix}.jsonl", - std::process::id() - )); - let _ = std::fs::remove_file(&path); - - let stream = StreamId::new("test-node", Lifetime(42)); - let mut archive = FrameArchive::open(&path).expect("frame archive opens"); - archive - .record( - "orchestrator", - &stream, - "stdout", - &Frame::new( - ChannelId(1), - datastream::Position(7), - b"hello \xce\xbb".to_vec(), - ), - ) - .expect("text frame archives"); - archive - .record( - "orchestrator", - &stream, - "stderr", - &Frame::new( - ChannelId(2), - datastream::Position(8), - vec![0xff, 0x00, b'A'], - ), - ) - .expect("binary frame archives"); - drop(archive); - - let contents = std::fs::read_to_string(&path).expect("read frame archive jsonl"); - let records = contents - .lines() - .map(|line| serde_json::from_str::(line).expect("archive line is json")) - .collect::>(); - let _ = std::fs::remove_file(&path); - - assert_eq!(records.len(), 2); - assert_eq!(records[0]["arrival_seq"], serde_json::json!(0)); - assert!( - records[0]["arrival_unix_ms"] - .as_u64() - .is_some_and(|value| value > 0) - ); - assert_eq!(records[0]["source"], serde_json::json!("orchestrator")); - assert_eq!(records[0]["stream"], serde_json::json!("test-node#42")); - assert_eq!(records[0]["channel"], serde_json::json!("stdout")); - assert_eq!(records[0]["channel_id"], serde_json::json!(1)); - assert_eq!(records[0]["position"], serde_json::json!(7)); - assert_eq!( - records[0]["payload"], - serde_json::json!({"encoding": "utf8", "value": "hello λ"}) - ); - - assert_eq!(records[1]["arrival_seq"], serde_json::json!(1)); - assert_eq!(records[1]["source"], serde_json::json!("orchestrator")); - assert_eq!(records[1]["stream"], serde_json::json!("test-node#42")); - assert_eq!(records[1]["channel"], serde_json::json!("stderr")); - assert_eq!(records[1]["channel_id"], serde_json::json!(2)); - assert_eq!(records[1]["position"], serde_json::json!(8)); - assert_eq!( - records[1]["payload"], - serde_json::json!({"encoding": "bytes", "value": [255, 0, 65]}) - ); -} diff --git a/crates/mvp-system/src/tests/transport_guarantees.rs b/crates/mvp-system/src/tests/transport_guarantees.rs new file mode 100644 index 0000000..e31e101 --- /dev/null +++ b/crates/mvp-system/src/tests/transport_guarantees.rs @@ -0,0 +1,38 @@ +//! Behavior guarantees for the `transport` module. + +use std::net::{Ipv4Addr, SocketAddr}; + +use mvp_system::transport::endpoint_advertisement::{EndpointAddrMask, advertised_endpoint}; + +#[test] +fn relay_only_mask_preserves_relay_urls_and_removes_direct_addresses() { + let relay = "http://relay.example.com" + .parse::() + .expect("relay URL parses"); + let endpoint = iroh::EndpointAddr::new(iroh::SecretKey::from_bytes(&[9; 32]).public()) + .with_relay_url(relay.clone()) + .with_ip_addr(SocketAddr::from((Ipv4Addr::LOCALHOST, 7777))); + + let masked = advertised_endpoint(endpoint, EndpointAddrMask::RelayOnly) + .expect("relay-only endpoint builds"); + + assert_eq!(masked.ip_addrs().count(), 0); + assert_eq!( + masked.relay_urls().next().map(ToString::to_string), + Some(relay.to_string()) + ); +} + +#[test] +fn relay_only_mask_rejects_endpoint_without_relay_url() { + let endpoint = iroh::EndpointAddr::new(iroh::SecretKey::from_bytes(&[7; 32]).public()) + .with_ip_addr(SocketAddr::from((Ipv4Addr::LOCALHOST, 7777))); + + let error = advertised_endpoint(endpoint, EndpointAddrMask::RelayOnly) + .expect_err("missing relay URL fails"); + + assert!( + error.contains("requires an endpoint relay URL"), + "unexpected error: {error}" + ); +} diff --git a/crates/mvp-system/src/tests/tx_rx_edge_actor_guarantees.rs b/crates/mvp-system/src/tests/tx_rx_edge_actor_guarantees.rs deleted file mode 100644 index 0c70b7e..0000000 --- a/crates/mvp-system/src/tests/tx_rx_edge_actor_guarantees.rs +++ /dev/null @@ -1,276 +0,0 @@ -//! Black-box contract tests for MVP Tx and Rx edge actors. -//! -//! These tests intentionally know only the public edge-actor surface: -//! -//! - lifecycle, object identity, stop, stream fault, and object fault events in -//! - role-facing lifecycle/object events out -//! -//! They assert the guarantees in -//! `specs/mvp_system/tx_rx_edge_actor_contract.md`. - -use data_plane::edge_actor; - -// The edge id fixture gives both actors a shared identity while keeping Tx and -// Rx lifecycle tests independent from driver and ring internals. -fn edge_id() -> edge_actor::EdgeId { - edge_actor::EdgeId(7001) -} - -#[test] -fn object_id_allocator_is_scoped_to_one_producer_edge() { - let mut stage0_output = edge_actor::ObjectIdAllocator::new(edge_actor::EdgeId(7001)); - let mut stage1_output = edge_actor::ObjectIdAllocator::new(edge_actor::EdgeId(7002)); - - assert_eq!( - stage0_output.alloc(), - edge_actor::ObjectKey::new(edge_actor::EdgeId(7001), edge_actor::ObjectId(1)) - ); - assert_eq!( - stage1_output.alloc(), - edge_actor::ObjectKey::new(edge_actor::EdgeId(7002), edge_actor::ObjectId(1)) - ); - assert_eq!( - stage0_output.alloc(), - edge_actor::ObjectKey::new(edge_actor::EdgeId(7001), edge_actor::ObjectId(2)) - ); -} - -// Tx starts in provisioning and represents the producer side of one edge. The -// harness records only actor messages, not bytes or flow-control details. -fn new_tx() -> edge_actor::TxActorHarness { - edge_actor::TxActorHarness::new(edge_actor::TxConfig { - edge_id: edge_id(), - role_port: edge_actor::PortId("out".into()), - }) -} - -// Rx starts in provisioning and represents the consumer side of one edge. It -// exposes complete object identities and opaque handles to the role layer. -fn new_rx() -> edge_actor::RxActorHarness { - edge_actor::RxActorHarness::new(edge_actor::RxConfig { - edge_id: edge_id(), - role_port: edge_actor::PortId("in".into()), - }) -} - -// This helper is a compile-time and runtime guard for payload isolation. If the -// public actor message enum grows a payload-bearing variant, this exhaustive -// match has to be updated and the test discussion becomes explicit. -fn assert_actor_message_is_payload_free(message: &edge_actor::ActorMessage) { - match message { - edge_actor::ActorMessage::Lifecycle { .. } - | edge_actor::ActorMessage::ObjectIdentity { .. } - | edge_actor::ActorMessage::OpaqueHandle { .. } - | edge_actor::ActorMessage::CoarseFault { .. } => {} - } -} - -// This proves Tx and Rx actors are tied to one edge id, receive lifecycle/object -// events only, and do not traffic payload bytes, pointers, ranges, credits, or -// free-space counts. -#[test] -fn edge_actor_messages_are_lifecycle_identity_and_handle_only() { - // Create one Tx and one Rx actor for the same edge id. - let mut tx = new_tx(); - let mut rx = new_rx(); - - // Drive typical lifecycle and object events. - tx.observe(edge_actor::TxEvent::EdgeReady { edge_id: edge_id() }); - tx.observe(edge_actor::TxEvent::ObjectProduced { - edge_id: edge_id(), - object_id: edge_actor::ObjectId(9000), - sequence: 0, - }); - rx.observe(edge_actor::RxEvent::EdgeReady { edge_id: edge_id() }); - rx.observe(edge_actor::RxEvent::ObjectLoaded { - edge_id: edge_id(), - object_id: edge_actor::ObjectId(9000), - sequence: 0, - handle: edge_actor::OpaqueHandle::new(42), - }); - - // Every emitted actor message must stay payload-free. - for message in tx.messages().iter().chain(rx.messages()) { - assert_actor_message_is_payload_free(message); - } - - // Both actors remain tied to exactly one edge id. - assert!( - tx.messages() - .iter() - .all(|message| message.edge_id() == edge_id()) - ); - assert!( - rx.messages() - .iter() - .all(|message| message.edge_id() == edge_id()) - ); -} - -// This proves Tx starts in provisioning, becomes ready only after EdgeReady, -// allows producing only after ready, reports produced object identity, and moves -// to faulted on stream or object faults. -#[test] -fn tx_lifecycle_gates_production_and_faults_on_stream_or_object_failure() { - // Before EdgeReady, production is rejected. - let mut tx = new_tx(); - tx.observe(edge_actor::TxEvent::ObjectProduced { - edge_id: edge_id(), - object_id: edge_actor::ObjectId(9000), - sequence: 0, - }); - assert!( - !tx.messages() - .iter() - .any(|message| { matches!(message, edge_actor::ActorMessage::ObjectIdentity { .. }) }) - ); - - // EdgeReady admits production. - tx.observe(edge_actor::TxEvent::EdgeReady { edge_id: edge_id() }); - tx.observe(edge_actor::TxEvent::ObjectProduced { - edge_id: edge_id(), - object_id: edge_actor::ObjectId(9000), - sequence: 0, - }); - assert!(tx.messages().iter().any(|message| { - matches!( - message, - edge_actor::ActorMessage::ObjectIdentity { - edge_id: edge_actor::EdgeId(7001), - object_id: edge_actor::ObjectId(9000), - sequence: 0, - .. - } - ) - })); - - // Stream fault moves Tx to faulted and suppresses later production. - tx.observe(edge_actor::TxEvent::StreamFault { edge_id: edge_id() }); - let produced_before = tx - .messages() - .iter() - .filter(|message| matches!(message, edge_actor::ActorMessage::ObjectIdentity { .. })) - .count(); - tx.observe(edge_actor::TxEvent::ObjectProduced { - edge_id: edge_id(), - object_id: edge_actor::ObjectId(9001), - sequence: 1, - }); - let produced_after = tx - .messages() - .iter() - .filter(|message| matches!(message, edge_actor::ActorMessage::ObjectIdentity { .. })) - .count(); - assert_eq!(produced_after, produced_before); -} - -// This proves Rx starts in provisioning, becomes ready only after EdgeReady, -// exposes loaded objects only after ObjectLoaded, and moves to faulted on stream -// or object faults. -#[test] -fn rx_lifecycle_gates_loaded_objects_and_faults_on_stream_or_object_failure() { - // Before EdgeReady, loaded objects are not exposed to the role layer. - let mut rx = new_rx(); - rx.observe(edge_actor::RxEvent::ObjectLoaded { - edge_id: edge_id(), - object_id: edge_actor::ObjectId(9000), - sequence: 0, - handle: edge_actor::OpaqueHandle::new(42), - }); - assert!( - !rx.messages() - .iter() - .any(|message| { matches!(message, edge_actor::ActorMessage::OpaqueHandle { .. }) }) - ); - - // EdgeReady admits ObjectLoaded exposure. - rx.observe(edge_actor::RxEvent::EdgeReady { edge_id: edge_id() }); - rx.observe(edge_actor::RxEvent::ObjectLoaded { - edge_id: edge_id(), - object_id: edge_actor::ObjectId(9000), - sequence: 0, - handle: edge_actor::OpaqueHandle::new(42), - }); - assert!(rx.messages().iter().any(|message| { - matches!( - message, - edge_actor::ActorMessage::OpaqueHandle { - edge_id: edge_actor::EdgeId(7001), - object_id: edge_actor::ObjectId(9000), - sequence: 0, - .. - } - ) - })); - - // Object failure faults Rx and suppresses later loaded objects. - rx.observe(edge_actor::RxEvent::ObjectFailed { edge_id: edge_id() }); - let loaded_before = rx - .messages() - .iter() - .filter(|message| matches!(message, edge_actor::ActorMessage::OpaqueHandle { .. })) - .count(); - rx.observe(edge_actor::RxEvent::ObjectLoaded { - edge_id: edge_id(), - object_id: edge_actor::ObjectId(9001), - sequence: 1, - handle: edge_actor::OpaqueHandle::new(43), - }); - let loaded_after = rx - .messages() - .iter() - .filter(|message| matches!(message, edge_actor::ActorMessage::OpaqueHandle { .. })) - .count(); - assert_eq!(loaded_after, loaded_before); -} - -// This proves StopEdge moves actors toward stopped, stale events after stop are -// ignored, and mismatched edge ids reject or fault according to policy. -#[test] -fn stop_and_mismatched_edge_events_do_not_create_run_work() { - // Ready Tx and Rx actors, then stop them. - let mut tx = new_tx(); - let mut rx = new_rx(); - tx.observe(edge_actor::TxEvent::EdgeReady { edge_id: edge_id() }); - rx.observe(edge_actor::RxEvent::EdgeReady { edge_id: edge_id() }); - tx.observe(edge_actor::TxEvent::StopEdge { edge_id: edge_id() }); - rx.observe(edge_actor::RxEvent::StopEdge { edge_id: edge_id() }); - - // Stale post-stop object events must be ignored. - tx.observe(edge_actor::TxEvent::ObjectProduced { - edge_id: edge_id(), - object_id: edge_actor::ObjectId(9000), - sequence: 0, - }); - rx.observe(edge_actor::RxEvent::ObjectLoaded { - edge_id: edge_id(), - object_id: edge_actor::ObjectId(9000), - sequence: 0, - handle: edge_actor::OpaqueHandle::new(42), - }); - assert!( - !tx.messages() - .iter() - .any(|message| { matches!(message, edge_actor::ActorMessage::ObjectIdentity { .. }) }) - ); - assert!( - !rx.messages() - .iter() - .any(|message| { matches!(message, edge_actor::ActorMessage::OpaqueHandle { .. }) }) - ); - - // A mismatched edge id must reject or fault, not create work on this actor. - let mut mismatched = new_tx(); - mismatched.observe(edge_actor::TxEvent::EdgeReady { - edge_id: edge_actor::EdgeId(9999), - }); - assert!(mismatched.messages().iter().any(|message| { - matches!( - message, - edge_actor::ActorMessage::CoarseFault { - reason: edge_actor::ActorFaultReason::MismatchedEdgeId, - .. - } - ) - })); -} diff --git a/crates/mvp-system/src/tests/weight_lifecycle_guarantees.rs b/crates/mvp-system/src/tests/weight_lifecycle_guarantees.rs deleted file mode 100644 index d227f67..0000000 --- a/crates/mvp-system/src/tests/weight_lifecycle_guarantees.rs +++ /dev/null @@ -1,227 +0,0 @@ -//! Black-box contract tests for MVP stage-local weight lifecycle. -//! -//! These tests intentionally know only the public weight-work surface: -//! -//! - `ProvisionStage` assignment in -//! - artifact, parse, allocation, binding, and cache outcomes in -//! - `WeightsReady`, `StageReady`, and `StageFault` out -//! -//! They assert the guarantees in -//! `specs/mvp_system/weight_lifecycle_contract.md`. - -use mvp_system::staging::weight_lifecycle as weights; - -// A valid assignment gives the stage exactly one layer range and one source. -// Tests vary only source or failure outcome so the assignment contract remains -// visible. -fn valid_assignment() -> weights::WeightAssignment { - weights::WeightAssignment { - run_id: weights::RunId(7), - stage_index: 1, - plan_layer_range: weights::LayerRange { - start: 12, - end_exclusive: 24, - }, - assigned_layer_range: weights::LayerRange { - start: 12, - end_exclusive: 24, - }, - source: weights::WeightSource::WholeGguf { - uri: "test://model.gguf".into(), - }, - } -} - -// Weight loading is intentionally opaque. The harness accepts public loader and -// worker outcomes and records only stage-visible events and commands. -fn new_weight_harness() -> weights::WeightLifecycleHarness { - weights::WeightLifecycleHarness::new(weights::NodeId(11)) -} - -// The success facts represent the observable prerequisites for WeightsReady: -// artifact bytes exist, the assigned layer range is valid, and the worker has -// loaded or bound that range. -fn successful_load_events() -> Vec { - vec![ - weights::WeightEvent::ArtifactAvailable { - bytes: weights::ArtifactBytes::Local, - }, - weights::WeightEvent::LayerRangeValidated, - weights::WeightEvent::WorkerRangeBound, - ] -} - -// Each failure case maps one public loader or worker failure to the stable -// stage fault reason expected at the control boundary. -fn failure_cases() -> Vec<(weights::WeightEvent, weights::StageFaultReason)> { - vec![ - ( - weights::WeightEvent::DownloadFailed, - weights::StageFaultReason::WeightDownloadFailed, - ), - ( - weights::WeightEvent::ParseFailed, - weights::StageFaultReason::WeightParseFailed, - ), - ( - weights::WeightEvent::DeviceAllocationFailed, - weights::StageFaultReason::DeviceAllocationFailed, - ), - ( - weights::WeightEvent::BindingFailed, - weights::StageFaultReason::WeightBindingFailed, - ), - ( - weights::WeightEvent::InvalidLayerRange, - weights::StageFaultReason::InvalidLayerRange, - ), - ] -} - -// This proves a stage receives weight source and exactly one assigned layer -// range from provisioning, validates it against the plan, and does not claim -// graph-visible ownership outside that range. -#[test] -fn assignment_is_stage_local_and_range_limited() { - // Start weight work from the provisioned assignment. - let mut harness = new_weight_harness(); - harness.observe(weights::WeightEvent::Provisioned(valid_assignment())); - - // The load command may use the physical source, but its graph-visible layer - // range must be the assigned range. - for command in harness.commands() { - if let weights::WeightCommand::LoadOrBindRange { range, .. } = command { - assert_eq!( - *range, - weights::LayerRange { - start: 12, - end_exclusive: 24, - } - ); - } - } - - // There must be no command claiming ownership of neighboring layers. - assert!(!harness.commands().iter().any(|command| { - matches!( - command, - weights::WeightCommand::AdvertiseLoadedLayerRange { - range, - .. - } if range.start < 12 || range.end_exclusive > 24 - ) - })); -} - -// This proves whole GGUF download, shard download, and cache use are physical -// mechanisms with the same public outcome: WeightsReady or StageFault. -#[test] -fn supported_physical_sources_have_same_visible_success_contract() { - // Exercise every supported source without asserting how bytes are obtained. - let sources = vec![ - weights::WeightSource::WholeGguf { - uri: "test://model.gguf".into(), - }, - weights::WeightSource::ShardSet { - uris: vec!["test://model.layers.12-24.gguf".into()], - }, - weights::WeightSource::CachedArtifact { - cache_key: "model:layers:12-24".into(), - }, - ]; - - for source in sources { - // Install the source in an otherwise valid assignment. - let mut assignment = valid_assignment(); - assignment.source = source; - let mut harness = new_weight_harness(); - harness.observe(weights::WeightEvent::Provisioned(assignment)); - - // Drive the same public success facts for every source. - for event in successful_load_events() { - harness.observe(event); - } - - // The system-visible success outcome is WeightsReady. - assert!(harness.events().iter().any(|event| { - matches!( - event, - weights::WeightLifecycleEvent::WeightsReady { - run_id: weights::RunId(7), - stage_index: 1, - } - ) - })); - } -} - -// This proves WeightsReady requires artifact availability, layer validation, -// and worker bind/load completion, and that WeightsReady precedes StageReady. -#[test] -fn weights_ready_requires_all_weight_facts_and_precedes_stage_ready() { - // Start from a valid assignment. - let mut harness = new_weight_harness(); - harness.observe(weights::WeightEvent::Provisioned(valid_assignment())); - - // Feed every success fact except the final one and prove no prefix is - // sufficient for WeightsReady. - let mut events = successful_load_events(); - let final_event = events.pop().expect("fixture has final weight event"); - for event in events { - harness.observe(event); - assert!( - !harness.events().iter().any(|event| { - matches!(event, weights::WeightLifecycleEvent::WeightsReady { .. }) - }) - ); - } - - // The final weight prerequisite emits WeightsReady. - harness.observe(final_event); - let weights_ready_pos = harness - .events() - .iter() - .position(|event| matches!(event, weights::WeightLifecycleEvent::WeightsReady { .. })) - .expect("WeightsReady must be emitted"); - - // StageReady may occur only after the StageController observes WeightsReady - // and the other local setup prerequisites. - harness.observe(weights::WeightEvent::OtherStagePrerequisitesReady); - let stage_ready_pos = harness - .events() - .iter() - .position(|event| matches!(event, weights::WeightLifecycleEvent::StageReady { .. })) - .expect("StageReady must be emitted after prerequisites"); - assert!(weights_ready_pos < stage_ready_pos); -} - -// This proves every weight failure source faults the stage and suppresses both -// WeightsReady and StageReady. -#[test] -fn weight_failures_emit_stage_fault_without_readiness() { - // Each failure source gets an isolated attempt. - for (failure_event, expected_reason) in failure_cases() { - let mut harness = new_weight_harness(); - harness.observe(weights::WeightEvent::Provisioned(valid_assignment())); - - // Deliver the public failure outcome from weight work. - harness.observe(failure_event); - - // The stable fault reason must be observable. - assert!(harness.events().iter().any(|event| { - matches!( - event, - weights::WeightLifecycleEvent::StageFault { - reason, - .. - } if *reason == expected_reason - ) - })); - - // Readiness cannot also be emitted after a faulted weight attempt. - assert!(!harness.events().iter().any(|event| { - matches!(event, weights::WeightLifecycleEvent::WeightsReady { .. }) - || matches!(event, weights::WeightLifecycleEvent::StageReady { .. }) - })); - } -} diff --git a/crates/mvp-system/src/tests/weight_shards_guarantees.rs b/crates/mvp-system/src/tests/weight_shards_guarantees.rs deleted file mode 100644 index 3d663f8..0000000 --- a/crates/mvp-system/src/tests/weight_shards_guarantees.rs +++ /dev/null @@ -1,134 +0,0 @@ -use mvp_system::staging::weight_shards as shards; - -fn model_ref() -> shards::ModelArtifactRef { - shards::ModelArtifactRef::parse("hf://org/repo@abcdef123456/model.gguf").unwrap() -} - -fn assignment() -> shards::ShardAssignment { - let model_ref = model_ref(); - let split_scheme = shards::SplitScheme::GgufLayerContiguousV1; - let split_id = shards::SplitId::derive(&model_ref, split_scheme); - shards::ShardAssignment::new( - model_ref, - split_id, - split_scheme, - 3, - 8, - shards::LayerRange::new(12, 16).unwrap(), - ) - .unwrap() -} - -fn content_hash() -> shards::ContentHash { - shards::ContentHash::literal("sha256:test-content").unwrap() -} - -#[test] -fn model_ref_canonicalization_is_stable() { - let parsed = shards::ModelArtifactRef::parse("hf://org/repo@abcdef123456/model.gguf").unwrap(); - let from_parts = - shards::ModelArtifactRef::hugging_face("/org/repo/", "abcdef123456", "/model.gguf") - .unwrap(); - - assert_eq!(parsed, from_parts); - assert_eq!(parsed.as_str(), "hf://org/repo@abcdef123456/model.gguf"); - assert_eq!(parsed.repo(), "org/repo"); - assert_eq!(parsed.revision(), "abcdef123456"); - assert_eq!(parsed.path(), "model.gguf"); -} - -#[test] -fn split_id_is_deterministic_and_model_sensitive() { - let first = model_ref(); - let same = shards::ModelArtifactRef::parse("hf://org/repo@abcdef123456/model.gguf").unwrap(); - let different = - shards::ModelArtifactRef::parse("hf://org/repo@fedcba654321/model.gguf").unwrap(); - let scheme = shards::SplitScheme::GgufLayerContiguousV1; - - assert_eq!( - shards::SplitId::derive(&first, scheme), - shards::SplitId::derive(&same, scheme) - ); - assert_ne!( - shards::SplitId::derive(&first, scheme), - shards::SplitId::derive(&different, scheme) - ); -} - -#[test] -fn assignment_rejects_invalid_stage_shape_and_ranges() { - let model_ref = model_ref(); - let scheme = shards::SplitScheme::GgufLayerContiguousV1; - let split_id = shards::SplitId::derive(&model_ref, scheme); - let range = shards::LayerRange::new(1, 2).unwrap(); - - assert_eq!( - shards::ShardAssignment::new(model_ref.clone(), split_id.clone(), scheme, 0, 0, range), - Err(shards::ShardAssignmentError::EmptyStageCount) - ); - assert_eq!( - shards::ShardAssignment::new(model_ref, split_id, scheme, 2, 2, range), - Err(shards::ShardAssignmentError::StageIndexOutOfRange) - ); - assert_eq!( - shards::LayerRange::new(4, 4), - Err(shards::LayerRangeError::EmptyOrInverted) - ); - assert_eq!( - shards::LayerRange::new(5, 4), - Err(shards::LayerRangeError::EmptyOrInverted) - ); -} - -#[test] -fn validator_accepts_matching_manifest() { - let assignment = assignment(); - let manifest = shards::ShardManifest::for_assignment(&assignment, content_hash()); - - assert_eq!( - shards::ShardValidator::validate(&assignment, &manifest), - Ok(()) - ); - assert!(shards::ValidatedShard::new(assignment, manifest, "/cache/stage-00003.gguf").is_ok()); -} - -#[test] -fn validator_rejects_mismatched_manifest() { - let assignment = assignment(); - let matching = shards::ShardManifest::for_assignment(&assignment, content_hash()); - - let mut wrong_model = matching.clone(); - wrong_model.model_digest = shards::ModelDigest::literal("wrong-model").unwrap(); - assert_eq!( - shards::ShardValidator::validate(&assignment, &wrong_model), - Err(shards::ShardValidationError::ModelDigestMismatch) - ); - - let mut wrong_split = matching.clone(); - wrong_split.split_id = shards::SplitId::literal("split-wrong").unwrap(); - assert_eq!( - shards::ShardValidator::validate(&assignment, &wrong_split), - Err(shards::ShardValidationError::SplitIdMismatch) - ); - - let mut wrong_stage = matching.clone(); - wrong_stage.stage_index = 4; - assert_eq!( - shards::ShardValidator::validate(&assignment, &wrong_stage), - Err(shards::ShardValidationError::StageIndexMismatch) - ); - - let mut wrong_count = matching.clone(); - wrong_count.stage_count = 9; - assert_eq!( - shards::ShardValidator::validate(&assignment, &wrong_count), - Err(shards::ShardValidationError::StageCountMismatch) - ); - - let mut wrong_range = matching; - wrong_range.layer_range = shards::LayerRange::new(16, 20).unwrap(); - assert_eq!( - shards::ShardValidator::validate(&assignment, &wrong_range), - Err(shards::ShardValidationError::LayerRangeMismatch) - ); -} diff --git a/crates/mvp-system/src/tests/worker_edge_adapter_guarantees.rs b/crates/mvp-system/src/tests/worker_edge_adapter_guarantees.rs deleted file mode 100644 index 8efea88..0000000 --- a/crates/mvp-system/src/tests/worker_edge_adapter_guarantees.rs +++ /dev/null @@ -1,340 +0,0 @@ -use crate::node_actor::{ - NodeAgentActor, NodeAgentMsg, NodeAgentReport, StageCommandWire, StageEdgeKindWire, - StageInboundEdgeWire, StageObjectSpecWire, StageOutboundEdgeWire, StageProvisionWire, - StageRingSpecWire, -}; -use crate::run_plan::{self, GgufSource, TokenizerSource}; -use data_plane::object_record as ingress; -use mvp_system::staging as stage; -use swactor::actor::ActorAddress; -use swactor::config::RuntimeConfig; -use swactor::runtime::Runtime; - -fn inbound_edge() -> StageInboundEdgeWire { - StageInboundEdgeWire { - edge_id: 7001, - kind: StageEdgeKindWire::TokenIn, - object_spec: StageObjectSpecWire { - max_extent: 2048, - alignment: 4, - }, - ring_spec: StageRingSpecWire { - data_capacity: 2088, - alignment: 64, - }, - } -} - -fn outbound_edge() -> StageOutboundEdgeWire { - StageOutboundEdgeWire { - edge_id: 7002, - kind: StageEdgeKindWire::Activation, - consumer_node_id: 43, - consumer_endpoint: None, - object_spec: StageObjectSpecWire { - max_extent: 589_824, - alignment: 128, - }, - ring_spec: StageRingSpecWire { - data_capacity: 589_864, - alignment: 256, - }, - } -} - -fn provision_wire() -> StageProvisionWire { - StageProvisionWire { - run_id: 55, - authorized_orchestrator: 9, - node_id: 11, - stage_index: 1, - stage_count: 3, - layer_start: 10, - layer_end_exclusive: 20, - inbound_edge_id: 7001, - outbound_edge_id: 7002, - inbound_edge: Some(inbound_edge()), - outbound_edge: Some(outbound_edge()), - model_id: "smollm2-135m-q4".to_owned(), - gguf_source: GgufSource::LocalPath("/models/smollm.gguf".to_owned()), - tokenizer: TokenizerSource::EmbeddedGguf, - stage_shard_plan: None, - } -} - -fn plan_fixture(stage_count: u32) -> run_plan::RunPlan { - let placements = (0..stage_count) - .map(|stage_index| run_plan::StagePlacement { - stage_index, - node_id: run_plan::NodeId(11 + u64::from(stage_index)), - }) - .collect::>(); - let candidate_pool = placements - .iter() - .map(|placement| placement.node_id) - .collect::>(); - - run_plan::plan_run(run_plan::PlannerInput { - run_id: run_plan::RunId(55), - orchestrator_node_id: run_plan::NodeId(1), - model: run_plan::ModelFacts { - model_id: "fixture-model".to_owned(), - gguf_source: GgufSource::LocalPath("/models/fixture.gguf".to_owned()), - num_layers: 7, - hidden_dim: 13, - dtype_family: run_plan::DTypeFamily::BFloat, - dtype_width_bytes: 2, - max_seq_len: 32, - eos_token_id: 2, - tokenizer: TokenizerSource::EmbeddedGguf, - }, - runtime: run_plan::RuntimeConfig::test_default(), - candidate_pool, - stage_count, - placement: run_plan::PlacementInput::FixedLinear(placements), - activation_ring: run_plan::RingSpec { - data_capacity: 4096, - alignment: 64, - direction: run_plan::RingDirection::Egress, - host_pinning: run_plan::HostPinning::Pageable, - wake_coalescing: run_plan::WakeCoalescing::PendingBit, - }, - token_ring: run_plan::RingSpec { - data_capacity: 4096, - alignment: 64, - direction: run_plan::RingDirection::Egress, - host_pinning: run_plan::HostPinning::Pageable, - wake_coalescing: run_plan::WakeCoalescing::PendingBit, - }, - }) - .expect("plan fixture builds") -} - -fn provision_wire_from_plan(plan: &run_plan::RunPlan, stage_index: u32) -> StageProvisionWire { - let provision = run_plan::derive_stage_provision(plan, stage_index) - .expect("stage provision derives from plan"); - StageProvisionWire { - run_id: provision.run_id.0, - authorized_orchestrator: 9, - node_id: provision.node_id.0, - stage_index: provision.stage_index, - stage_count: provision.stage_count, - layer_start: provision.layer_start, - layer_end_exclusive: provision.layer_end_exclusive, - inbound_edge_id: provision.inbound.edge_id.0, - outbound_edge_id: provision.outbound.edge_id.0, - inbound_edge: Some(StageInboundEdgeWire { - edge_id: provision.inbound.edge_id.0, - kind: edge_kind_wire(provision.inbound.kind), - object_spec: object_spec_wire(provision.inbound.object_spec), - ring_spec: ring_spec_wire(provision.inbound.ring_spec), - }), - outbound_edge: Some(StageOutboundEdgeWire { - edge_id: provision.outbound.edge_id.0, - kind: edge_kind_wire(provision.outbound.kind), - consumer_node_id: provision.outbound.consumer_node_id.0, - consumer_endpoint: None, - object_spec: object_spec_wire(provision.outbound.object_spec), - ring_spec: ring_spec_wire(provision.outbound.ring_spec), - }), - model_id: provision.model.model_id, - gguf_source: provision.gguf_source, - tokenizer: provision.tokenizer, - stage_shard_plan: None, - } -} - -fn edge_kind_wire(kind: run_plan::EdgeKind) -> StageEdgeKindWire { - match kind { - run_plan::EdgeKind::TokenIn => StageEdgeKindWire::TokenIn, - run_plan::EdgeKind::Activation => StageEdgeKindWire::Activation, - run_plan::EdgeKind::TokenOut => StageEdgeKindWire::TokenOut, - } -} - -fn object_spec_wire(spec: run_plan::ObjectSpec) -> StageObjectSpecWire { - StageObjectSpecWire { - max_extent: spec.max_extent, - alignment: spec.alignment, - } -} - -fn ring_spec_wire(spec: run_plan::RingSpec) -> StageRingSpecWire { - StageRingSpecWire { - data_capacity: spec.data_capacity, - alignment: spec.alignment, - } -} - -fn token_spec() -> ingress::ObjectSpec { - ingress::ObjectSpec { - max_extent: 16, - alignment: 4, - layout: ingress::ObjectLayout::Token, - } -} - -fn record(sequence: u64, extent: u64) -> Vec { - ingress::ObjectRecordBuilder::new(token_spec()) - .object_id(ingress::ObjectId(9000 + sequence)) - .sequence(sequence) - .extent(extent) - .payload(vec![sequence as u8; extent as usize]) - .encode() -} - -#[test] -fn stage_provision_wire_round_trips_edge_object_and_ring_facts() { - let wire = provision_wire(); - - let encoded = serde_json::to_string(&NodeAgentMsg::ProvisionStage(wire.clone())) - .expect("serialize provision wire"); - let decoded: NodeAgentMsg = serde_json::from_str(&encoded).expect("deserialize provision wire"); - - let NodeAgentMsg::ProvisionStage(decoded) = decoded else { - panic!("decoded message must remain a provision"); - }; - assert_eq!(decoded.inbound_edge, Some(inbound_edge())); - assert_eq!(decoded.outbound_edge, Some(outbound_edge())); -} - -#[test] -fn node_agent_establish_edge_commands_preserve_provisioned_runtime_facts() { - let runtime = Runtime::new(RuntimeConfig::default()); - let reports = runtime - .new_inbox::() - .expect("node report inbox"); - let actor = runtime - .spawn(NodeAgentActor::new( - stage::NodeId(11), - ActorAddress::new_random(), - Some(*reports.addr()), - )) - .expect("spawn node agent"); - - runtime - .send_to(actor, NodeAgentMsg::ProvisionStage(provision_wire())) - .expect("send provision"); - runtime.tick(); - - let commands = [ - reports.try_recv().expect("inbound command"), - reports.try_recv().expect("outbound command"), - reports.try_recv().expect("configure command"), - reports.try_recv().expect("load command"), - ]; - - assert!(commands.iter().any(|report| { - matches!( - report, - NodeAgentReport::Command(StageCommandWire::EstablishInboundEdge { edge_id: 7001, edge }) - if edge == &inbound_edge() - ) - })); - assert!(commands.iter().any(|report| { - matches!( - report, - NodeAgentReport::Command(StageCommandWire::EstablishOutboundEdge { edge_id: 7002, edge }) - if edge == &outbound_edge() - ) - })); -} - -#[test] -fn node_agent_commands_preserve_plan_derived_single_stage_edge_contract() { - let plan = plan_fixture(1); - let wire = provision_wire_from_plan(&plan, 0); - assert_eq!(wire.stage_count, 1); - assert_eq!(wire.layer_start, 0); - assert_eq!(wire.layer_end_exclusive, plan.model.num_layers); - assert_eq!( - wire.inbound_edge.as_ref().expect("inbound edge").kind, - StageEdgeKindWire::TokenIn - ); - assert_eq!( - wire.outbound_edge.as_ref().expect("outbound edge").kind, - StageEdgeKindWire::TokenOut - ); - - let runtime = Runtime::new(RuntimeConfig::default()); - let reports = runtime - .new_inbox::() - .expect("node report inbox"); - let actor = runtime - .spawn(NodeAgentActor::new( - stage::NodeId(wire.node_id), - ActorAddress::new_random(), - Some(*reports.addr()), - )) - .expect("spawn node agent"); - - runtime - .send_to(actor, NodeAgentMsg::ProvisionStage(wire.clone())) - .expect("send provision"); - runtime.tick(); - - let commands = [ - reports.try_recv().expect("inbound command"), - reports.try_recv().expect("outbound command"), - reports.try_recv().expect("configure command"), - reports.try_recv().expect("load command"), - ]; - assert!(commands.iter().any(|report| { - matches!( - report, - NodeAgentReport::Command(StageCommandWire::EstablishInboundEdge { edge_id, edge }) - if *edge_id == wire.inbound_edge_id - && edge.kind == StageEdgeKindWire::TokenIn - && edge.object_spec.max_extent == 128 - && edge.object_spec.alignment == 4 - ) - })); - assert!(commands.iter().any(|report| { - matches!( - report, - NodeAgentReport::Command(StageCommandWire::EstablishOutboundEdge { edge_id, edge }) - if *edge_id == wire.outbound_edge_id - && edge.kind == StageEdgeKindWire::TokenOut - && edge.object_spec.max_extent == 128 - && edge.object_spec.alignment == 4 - ) - })); -} - -#[test] -fn mo01_reader_uses_total_len_to_split_records_and_faults_truncated_eof() { - let first = record(0, 8); - let second = record(1, 4); - let mut joined = first.clone(); - joined.extend_from_slice(&second); - - let parsed = ingress::read_object_record(&joined, token_spec(), false) - .expect("joined stream starts with a complete record"); - let ingress::ObjectRecordRead::Complete(first_record) = parsed else { - panic!("first record must be complete"); - }; - assert_eq!(first_record.object_id, ingress::ObjectId(9000)); - assert_eq!(first_record.sequence, 0); - assert_eq!(first_record.extent, 8); - assert_eq!(first_record.total_len, first.len()); - - let parsed_second = - ingress::read_object_record(&joined[first_record.total_len..], token_spec(), true) - .expect("split cursor points at the next complete record"); - let ingress::ObjectRecordRead::Complete(second_record) = parsed_second else { - panic!("second record must be complete"); - }; - assert_eq!(second_record.object_id, ingress::ObjectId(9001)); - assert_eq!(second_record.sequence, 1); - assert_eq!(second_record.extent, 4); - assert_eq!(second_record.total_len, second.len()); - - assert_eq!( - ingress::read_object_record(&joined[..ingress::HEADER_LEN - 1], token_spec(), false), - Ok(ingress::ObjectRecordRead::Incomplete) - ); - assert_eq!( - ingress::read_object_record(&joined[..ingress::HEADER_LEN - 1], token_spec(), true), - Err(ingress::ObjectFailureReason::EofBeforeFullPayload) - ); -} diff --git a/crates/mvp-system/src/transport/endpoint_advertisement.rs b/crates/mvp-system/src/transport/endpoint_advertisement.rs index 3a8b279..4242c99 100644 --- a/crates/mvp-system/src/transport/endpoint_advertisement.rs +++ b/crates/mvp-system/src/transport/endpoint_advertisement.rs @@ -62,43 +62,3 @@ fn relay_only_endpoint(endpoint: EndpointAddr) -> Result { } Ok(out) } - -#[cfg(test)] -mod tests { - use std::net::{Ipv4Addr, SocketAddr}; - - use super::*; - - #[test] - fn relay_only_mask_preserves_relay_urls_and_removes_direct_addresses() { - let relay = "http://relay.example.com" - .parse::() - .expect("relay URL parses"); - let endpoint = EndpointAddr::new(iroh::SecretKey::from_bytes(&[9; 32]).public()) - .with_relay_url(relay.clone()) - .with_ip_addr(SocketAddr::from((Ipv4Addr::LOCALHOST, 7777))); - - let masked = advertised_endpoint(endpoint, EndpointAddrMask::RelayOnly) - .expect("relay-only endpoint builds"); - - assert_eq!(masked.ip_addrs().count(), 0); - assert_eq!( - masked.relay_urls().next().map(ToString::to_string), - Some(relay.to_string()) - ); - } - - #[test] - fn relay_only_mask_rejects_endpoint_without_relay_url() { - let endpoint = EndpointAddr::new(iroh::SecretKey::from_bytes(&[7; 32]).public()) - .with_ip_addr(SocketAddr::from((Ipv4Addr::LOCALHOST, 7777))); - - let error = advertised_endpoint(endpoint, EndpointAddrMask::RelayOnly) - .expect_err("missing relay URL fails"); - - assert!( - error.contains("requires an endpoint relay URL"), - "unexpected error: {error}" - ); - } -} diff --git a/crates/mvp-system/tests/mvp_chat_mock.rs b/crates/mvp-system/tests/mvp_chat_mock.rs deleted file mode 100644 index 3994f49..0000000 --- a/crates/mvp-system/tests/mvp_chat_mock.rs +++ /dev/null @@ -1,7 +0,0 @@ -fn main() -> std::process::ExitCode { - let args: Vec = std::env::args().skip(1).collect(); - if args.is_empty() { - return std::process::ExitCode::SUCCESS; - } - mvp_system::run_chat_from_args(args) -} diff --git a/crates/mvp-system/tests/python_worker_protocol.rs b/crates/mvp-system/tests/python_worker_protocol.rs deleted file mode 100755 index 3e313b3..0000000 --- a/crates/mvp-system/tests/python_worker_protocol.rs +++ /dev/null @@ -1,132 +0,0 @@ -#![cfg(target_os = "linux")] - -use std::io::{BufRead, BufReader, Write}; -use std::process::{Child, ChildStdin, Command, Stdio}; - -use serde_json::{Value, json}; - -struct WorkerProcess { - child: Child, - stdin: ChildStdin, - stdout: BufReader, -} - -impl WorkerProcess { - fn spawn() -> Self { - let script = worker_script(); - let mut child = Command::new("python3") - .arg(&script) - .env("DEV", "CPU") - .env("MVP_RUN_ID", "9") - .env("MVP_LOGICAL_NODE_ID", "3") - .env("MVP_STAGE_INDEX", "2") - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .unwrap_or_else(|e| panic!("spawn {}: {e}", script.display())); - let stdin = child.stdin.take().expect("worker stdin"); - let stdout = BufReader::new(child.stdout.take().expect("worker stdout")); - Self { - child, - stdin, - stdout, - } - } - - fn send_collect_until(&mut self, command: Value, expected_type: &str) -> Vec { - writeln!(self.stdin, "{command}").expect("write worker command"); - self.stdin.flush().expect("flush worker command"); - let mut events = Vec::new(); - loop { - let mut line = String::new(); - let read = self.stdout.read_line(&mut line).expect("read worker event"); - assert_ne!(read, 0, "worker exited before {expected_type}"); - let event: Value = serde_json::from_str(line.trim_end()).expect("worker event JSON"); - let actual_type = event.get("type").and_then(Value::as_str).unwrap_or(""); - assert_ne!( - actual_type, "WorkerFatal", - "worker fatal while waiting for {expected_type}: {event}" - ); - let done = actual_type == expected_type; - events.push(event); - if done { - return events; - } - } - } -} - -impl Drop for WorkerProcess { - fn drop(&mut self) { - let _ = self.child.kill(); - let _ = self.child.wait(); - } -} - -#[test] -fn benchmark_observability_worker_ready_includes_stamps_and_identity() { - if !tinygrad_available() { - eprintln!("skipping Python worker protocol check: tinygrad is unavailable"); - return; - } - - let mut worker = WorkerProcess::spawn(); - let events = worker.send_collect_until( - json!({"type":"InitializeWorker","helper_abi_version":1,"backend":{"device":"CPU"}}), - "WorkerReady", - ); - - assert!( - events - .iter() - .any(|event| event.get("type").and_then(Value::as_str) == Some("TinygradImportStarted")), - "expected tinygrad import milestone in {events:?}" - ); - assert!( - events - .iter() - .any(|event| event.get("type").and_then(Value::as_str) == Some("WorkerReady")), - "expected WorkerReady in {events:?}" - ); - for event in &events { - assert_eq!( - event - .get("benchmark") - .and_then(|benchmark| benchmark.get("schema")) - .and_then(Value::as_u64), - Some(1), - "{event}" - ); - assert_eq!( - event.get("run_id").and_then(Value::as_u64), - Some(9), - "{event}" - ); - assert_eq!( - event.get("node_id").and_then(Value::as_u64), - Some(3), - "{event}" - ); - assert_eq!( - event.get("stage_index").and_then(Value::as_u64), - Some(2), - "{event}" - ); - } -} - -fn tinygrad_available() -> bool { - Command::new("python3") - .args(["-c", "import tinygrad"]) - .status() - .is_ok_and(|status| status.success()) -} - -fn worker_script() -> std::path::PathBuf { - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../..") - .join("apps/mvp-node/tinygrad_worker.py") - .canonicalize() - .expect("tinygrad worker script exists") -}