//! Sim cross-pollination — spec §"Sim cross-pollination" in //! `examples/pipeline-parallel-inference/N3_OBSERVABILITY_UPGRADE_SPEC.md`. //! //! A simulated node's snapshots and events must conform to the same //! shape as a real node's: the bundle reader should not be able to //! tell from data shape alone whether a given snapshot came from a //! real deployment or the sim. We probe by deserialising the sim's //! snapshot bytes through the production types directly — if they //! round-trip cleanly, the shapes match. //! //! Covers: //! - F1: the sim's stage host can install a subprocess fake; its //! snapshot block satisfies production's `Tier3SubprocessState`, //! and the typed `SubprocessSpawned` event lands in the bundle's //! event stream wrapped in the existing `diag_event` envelope. //! The spec-named "stage's worker never came up" scenario //! (Spawned + no worker_ready) is verifiable in one path. //! - F2: the sim's stage host carries a tunnel-status field in its //! snapshot under the production-shape `Tier2RelaySession`, //! defaulting to `unknown / derived` so honesty-under-absence //! (§2) holds even with no scenario config. use distribution::diagnostics::{Tier2RelaySession, Tier3SubprocessState}; use distribution::types::NodeId; use serde_json::Value; use simulation::host::{Action, Host}; use simulation::stage_host::{InferenceFakeSpec, StageHost, SubprocessFakeSpec}; #[test] fn stage_host_snapshot_always_carries_tier2_relay_session_with_unknown_default() { let mut host = StageHost::new("stage-x", "name-x", "addr-x"); let _ = host.tick(0); let snap = host.snapshot(); let parsed: Value = serde_json::from_slice(&snap).expect("snapshot is JSON"); let tier2 = parsed .get("tier2_relay_session") .cloned() .expect("snapshot must always carry tier2_relay_session for shape compatibility"); let typed: Tier2RelaySession = serde_json::from_value(tier2) .expect("tier2_relay_session must round-trip through production's Tier2RelaySession"); assert_eq!( typed.status, "unknown", "default tunnel status must be `unknown` under §2 honesty-under-absence", ); assert_eq!( typed.status_source, "derived", "default status_source must be `derived` so readers know it's synthesized", ); } #[test] fn stage_host_relay_session_override_round_trips_through_production_type() { let mut host = StageHost::new("stage-r", "name-r", "addr-r"); host.set_relay_session(Tier2RelaySession { relay_url: Some("https://relay.example/".into()), status: "connected".into(), status_source: "iroh".into(), status_changed_at_ms: Some(10), status_entered_at_ms: Some(10), last_send_at_ms: Some(20), last_recv_at_ms: Some(30), tx_bytes_total: Some(1024), rx_bytes_total: Some(2048), }); let _ = host.tick(0); let snap = host.snapshot(); let parsed: Value = serde_json::from_slice(&snap).unwrap(); let typed: Tier2RelaySession = serde_json::from_value(parsed["tier2_relay_session"].clone()) .expect("override round-trips through Tier2RelaySession"); assert_eq!(typed.status, "connected"); assert_eq!(typed.status_source, "iroh"); assert_eq!(typed.tx_bytes_total, Some(1024)); } #[test] fn subprocess_fake_emits_typed_spawn_and_carries_production_shape_snapshot_block() { let mut host = StageHost::new("stage-fake", "name-f", "addr-f"); host.set_subprocess_fake(SubprocessFakeSpec { label: "fake-worker".into(), pid: 31000, command: "/bin/synthetic --x".into(), never_ready: false, exit_after_ns: None, exit_code: None, exit_signal: None, }); let actions = host.tick(0); let diag_events = collect_diag_events(&actions); let saw_spawned = diag_events.iter().any(|p| { p.get("type").and_then(|v| v.as_str()) == Some("SubprocessSpawned") && p.get("label").and_then(|v| v.as_str()) == Some("fake-worker") && p.get("pid").and_then(|v| v.as_u64()) == Some(31000) }); assert!( saw_spawned, "SubprocessSpawned must reach the bundle's diag_event stream; got {diag_events:#?}", ); // When never_ready is false, the worker_ready Custom companion // event fires so the bundle reader can distinguish "spawned and // running, ready" from the never-ready bucket. let saw_ready = diag_events.iter().any(|p| { p.get("type").and_then(|v| v.as_str()) == Some("Custom") && p.get("kind").and_then(|v| v.as_str()) == Some("worker_ready") }); assert!(saw_ready, "worker_ready Custom companion must fire when never_ready=false"); // Snapshot block is production-shape. let snap = host.snapshot(); let parsed: Value = serde_json::from_slice(&snap).unwrap(); let tier3: Tier3SubprocessState = serde_json::from_value( parsed["tier3_subprocess"].clone(), ) .expect("tier3_subprocess must round-trip through Tier3SubprocessState"); assert_eq!(tier3.subprocesses.len(), 1); let entry = &tier3.subprocesses[0]; assert_eq!(entry.label, "fake-worker"); assert_eq!(entry.pid, 31000); assert_eq!(entry.status, "running"); assert_eq!(entry.cmdline.as_deref(), Some("/bin/synthetic --x")); } #[test] fn never_ready_subprocess_fake_emits_spawned_without_worker_ready() { // The spec calls out the "stage's worker never came up" bucket // explicitly: a SubprocessSpawned with no following worker_ready // Custom event. The sim fake must be able to reproduce it so // scenarios can model that failure case. let mut host = StageHost::new("stage-stuck", "name-s", "addr-s"); host.set_subprocess_fake(SubprocessFakeSpec { label: "stuck-worker".into(), pid: 31001, command: "/bin/python startup_hangs.py".into(), never_ready: true, exit_after_ns: None, exit_code: None, exit_signal: None, }); let actions = host.tick(0); let diag_events = collect_diag_events(&actions); let saw_spawned = diag_events .iter() .any(|p| p.get("type").and_then(|v| v.as_str()) == Some("SubprocessSpawned")); let saw_ready = diag_events.iter().any(|p| { p.get("type").and_then(|v| v.as_str()) == Some("Custom") && p.get("kind").and_then(|v| v.as_str()) == Some("worker_ready") }); assert!(saw_spawned, "SubprocessSpawned must still fire"); assert!( !saw_ready, "never_ready=true suppresses worker_ready (spec §4 stuck-worker bucket)", ); } #[test] fn exit_after_ns_emits_typed_exited_with_correct_uptime() { let mut host = StageHost::new("stage-exit", "name-e", "addr-e"); host.set_subprocess_fake(SubprocessFakeSpec { label: "ephemeral".into(), pid: 31002, command: "/bin/true".into(), never_ready: false, exit_after_ns: Some(5_000_000), exit_code: Some(0), exit_signal: None, }); // First tick at t=0 spawns + emits worker_ready. let _ = host.tick(0); // Tick at t=6ms is past the 5ms exit_after_ns threshold — // SubprocessExited must fire with uptime_ms = 6. let actions = host.tick(6_000_000); let diag_events = collect_diag_events(&actions); let exit = diag_events .iter() .find(|p| p.get("type").and_then(|v| v.as_str()) == Some("SubprocessExited")) .expect("SubprocessExited must fire past exit_after_ns"); assert_eq!(exit["pid"].as_u64(), Some(31002)); assert_eq!(exit["exit_code"].as_i64(), Some(0)); assert_eq!(exit["uptime_ms"].as_u64(), Some(6)); // The post-exit snapshot must show status="exited" with the // exit code on the snapshot side too — spec cross-cutting §2 // requires both channels for §4 subprocess facts. let snap = host.snapshot(); let parsed: Value = serde_json::from_slice(&snap).unwrap(); let tier3: Tier3SubprocessState = serde_json::from_value(parsed["tier3_subprocess"].clone()).unwrap(); assert_eq!(tier3.subprocesses[0].status, "exited"); assert_eq!(tier3.subprocesses[0].exit_code, Some(0)); } // ────────────────────────────────────────────────────────────────────── // Coverage 2.4 — inference response-leg send-outcome event // ────────────────────────────────────────────────────────────────────── #[test] fn inference_fake_emits_typed_response_sent_with_timeout_outcome() { // Coverage 2.4 close-criterion shape: the last stage emits // exactly one `InferenceResponseSent` carrying target / request / // size / outcome when its outbound to the orchestrator fails. // The `1779733878` failure attribution — "last stage could not // deliver the response" — is now a single typed read, not a // triangulation against dial timeouts. let mut host = StageHost::new("stage-last", "pp-stage-last", "10.0.0.20:7700"); let orch_node_id = NodeId([0xAB; 32]); host.set_inference_fake(InferenceFakeSpec { fire_at_ns: 5_000_000, target_peer_node_id: orch_node_id, request_id: "req-7f3c".into(), byte_size: 4_096, send_outcome: "timeout".into(), }); // Drive into Running. let _ = host.tick(0); // Past the fire time: the event lands. let actions = host.tick(5_500_000); let diag_events = collect_diag_events(&actions); let sent = diag_events .iter() .find(|p| p.get("type").and_then(|v| v.as_str()) == Some("InferenceResponseSent")) .expect("InferenceResponseSent must fire past fire_at_ns"); assert_eq!(sent["request_id"].as_str(), Some("req-7f3c")); assert_eq!(sent["byte_size"].as_u64(), Some(4_096)); assert_eq!(sent["send_outcome"].as_str(), Some("timeout")); // target_peer round-trips through the production NodeId schema. let target: NodeId = serde_json::from_value(sent["target_peer"].clone()) .expect("target_peer must deserialize as NodeId"); assert_eq!(target, orch_node_id); } #[test] fn inference_fake_fires_at_most_once_across_many_ticks() { // Spec §2.4 says "exactly one" event per response send. A stage // host that re-emitted on every tick past `fire_at_ns` would // produce double-counting in the bundle. let mut host = StageHost::new("stage-once", "pp-stage-once", "10.0.0.21:7700"); host.set_inference_fake(InferenceFakeSpec { fire_at_ns: 1_000_000, target_peer_node_id: NodeId([0xCD; 32]), request_id: "req-dedupe".into(), byte_size: 128, send_outcome: "success".into(), }); let _ = host.tick(0); let mut seen = 0usize; for t in [1_000_000u64, 2_000_000, 3_000_000, 10_000_000] { let actions = host.tick(t); for p in collect_diag_events(&actions) { if p.get("type").and_then(|v| v.as_str()) == Some("InferenceResponseSent") { seen += 1; } } } assert_eq!( seen, 1, "InferenceResponseSent must fire exactly once across many ticks past fire_at_ns", ); } #[test] fn inference_fake_unset_emits_no_response_event() { // Honesty-under-absence: a stage with no inference fake produces // no InferenceResponseSent. The bundle reader sees the absence // (the postproc renders the gap-2.4 absence-line); a silent // synthesized event would break the discriminator contract. let mut host = StageHost::new("stage-quiet", "pp-stage-quiet", "10.0.0.22:7700"); let _ = host.tick(0); for t in [1_000_000u64, 5_000_000, 50_000_000] { let actions = host.tick(t); for p in collect_diag_events(&actions) { assert_ne!( p.get("type").and_then(|v| v.as_str()), Some("InferenceResponseSent"), "unsetting the inference fake must suppress InferenceResponseSent", ); } } } // ─── helpers ────────────────────────────────────────────────────────── fn collect_diag_events(actions: &[Action]) -> Vec { actions .iter() .filter_map(|a| match a { Action::RecordEvent { event, .. } => { let v: Value = serde_json::from_slice(event).ok()?; if v.get("kind").and_then(|x| x.as_str()) == Some("diag_event") { v.get("payload").cloned() } else { None } } _ => None, }) .collect() }