swactor/crates/simulation/tests/parity-bar/t_lifecycle.rs

//! TESTING_SPEC §11 — Lifecycle observability.
//!
//! Each spec-declared lifecycle event (`start_at_ms`, `restart_at_ms`,
//! `crash_at_ms`, `stop_at_ms`) must produce its corresponding boot
//! / finalize / `Custom` records in the bundle. Living nodes at
//! end-of-run receive a synthetic clean shutdown so no node ends
//! without either an organic finalize, a crash mark, or an
//! `end_of_run` finalize.

#[path = "common.rs"]
mod common;

use common::{flatten_events, load_record_files, run_engine, run_reference_scenario};
use std::collections::BTreeMap;

// ── §11.1 — Boot record presence ───────────────────────────────────

#[test]
fn start_emits_boot() {
    let bundle = run_reference_scenario();
    for node in ["orchestrator", "stage-0", "stage-1", "stage-2", "collector"] {
        let boot = bundle.root.join(node).join("boot.json");
        assert!(
            boot.is_file(),
            "{node}: expected boot.json under {}",
            boot.display()
        );
        let parsed = read_json(&boot);
        assert_eq!(
            parsed["boot_sequence"].as_i64(),
            Some(0),
            "{node}: first boot must have boot_sequence=0"
        );
    }
}

// ── §11.2 — Restart sequence integrity ─────────────────────────────

#[test]
fn restart_increments_boot_sequence() {
    let spec = restart_scenario();
    let bundle = run_engine(&spec, 17);
    let node_dir = bundle.root.join("alpha");

    // Find every boot.json — `boot.json`, `boot-001.json`, etc.
    // The engine emits one per epoch; the test grabs every file
    // matching `boot*.json` and confirms boot_sequence values are
    // 0, 1, 2 in order.
    let mut boots: Vec<i64> = Vec::new();
    common::walk_files(&node_dir, &mut |p| {
        let name = p.file_name().and_then(|s| s.to_str()).unwrap_or("");
        if name.starts_with("boot") && name.ends_with(".json") {
            let v = read_json(p);
            if let Some(seq) = v["boot_sequence"].as_i64() {
                boots.push(seq);
            }
        }
    });
    boots.sort();
    assert_eq!(
        boots,
        vec![0, 1, 2],
        "alpha should boot 3 times (seq=0,1,2) given start_at_ms=0, \
         restart_at_ms=[1000, 2000]; observed {boots:?}"
    );

    // Each epoch except the last must have a finalize.json (the
    // last epoch is still running at end-of-run — covered by §11.5).
    let mut finalizes: Vec<i64> = Vec::new();
    common::walk_files(&node_dir, &mut |p| {
        let name = p.file_name().and_then(|s| s.to_str()).unwrap_or("");
        if name.starts_with("finalize") && name.ends_with(".json") {
            let v = read_json(p);
            if let Some(seq) = v["boot_sequence"].as_i64() {
                finalizes.push(seq);
            }
        }
    });
    finalizes.sort();
    assert!(
        finalizes.contains(&0) && finalizes.contains(&1),
        "alpha's first two epochs must have finalize.json; observed {finalizes:?}"
    );
}

// ── §11.3 — Crash distinguishability ───────────────────────────────

#[test]
fn crash_omits_finalize() {
    let spec = crash_scenario();
    let bundle = run_engine(&spec, 23);
    let node_dir = bundle.root.join("crasher");

    assert!(
        node_dir.join("boot.json").is_file(),
        "crasher must produce boot.json"
    );
    assert!(
        !node_dir.join("finalize.json").exists(),
        "crasher must NOT produce finalize.json (it crashed)"
    );

    // The engine must mark the crash with a sim-only `Custom` record
    // so the post-processor can distinguish crash from "incomplete
    // capture".
    let events = flatten_events(&node_dir);
    let crash_marked = events.iter().any(|r| {
        r.get("variant").and_then(|v| v.as_str()) == Some("Custom")
            && r.get("user_kind").and_then(|v| v.as_str()) == Some("crash")
    });
    assert!(
        crash_marked,
        "crashed node must emit a Custom(kind=\"crash\") record"
    );
}

// ── §11.4 — Clean shutdown ─────────────────────────────────────────

#[test]
fn stop_emits_clean_finalize() {
    let spec = stop_scenario();
    let bundle = run_engine(&spec, 29);
    let finalize = bundle.root.join("clean").join("finalize.json");
    assert!(
        finalize.is_file(),
        "clean-stop must produce finalize.json"
    );
    let value = read_json(&finalize);
    assert_eq!(
        value["shutdown_reason"].as_str(),
        Some("clean"),
        "clean-stop's shutdown_reason must be \"clean\""
    );
}

// ── §11.5 — Mid-run finalization ───────────────────────────────────

#[test]
fn end_of_run_finalizes_all_living() {
    // Every node that's still running when virtual time ends must
    // receive a synthetic `end_of_run` finalize.
    let spec = end_of_run_scenario();
    let bundle = run_engine(&spec, 31);

    let mut finalize_reasons: BTreeMap<String, String> = BTreeMap::new();
    let mut crash_marks: BTreeMap<String, bool> = BTreeMap::new();

    for entry in std::fs::read_dir(&bundle.root)
        .expect("bundle root readable")
        .flatten()
    {
        let path = entry.path();
        if !path.is_dir() {
            continue;
        }
        let Some(node) = path.file_name().and_then(|n| n.to_str()) else {
            continue;
        };
        if node == "sim" {
            continue;
        }
        let finalize = path.join("finalize.json");
        if finalize.is_file() {
            let reason = read_json(&finalize)["shutdown_reason"]
                .as_str()
                .expect("finalize.shutdown_reason is a string")
                .to_string();
            finalize_reasons.insert(node.to_string(), reason);
        }
        let events = flatten_events(&path);
        let crashed = events.iter().any(|r| {
            r.get("variant").and_then(|v| v.as_str()) == Some("Custom")
                && r.get("user_kind").and_then(|v| v.as_str()) == Some("crash")
        });
        crash_marks.insert(node.to_string(), crashed);
    }

    for node in finalize_reasons.keys().chain(crash_marks.keys()) {
        let has_organic = finalize_reasons.get(node) == Some(&"clean".to_string());
        let has_end_of_run =
            finalize_reasons.get(node) == Some(&"end_of_run".to_string());
        let crashed = crash_marks.get(node).copied().unwrap_or(false);
        assert!(
            has_organic || has_end_of_run || crashed,
            "node {node} ended the run with neither finalize nor crash mark"
        );
    }
}

// ── Helpers ────────────────────────────────────────────────────────

fn read_json(path: &std::path::Path) -> serde_json::Value {
    let text = std::fs::read_to_string(path)
        .unwrap_or_else(|e| panic!("read {}: {e}", path.display()));
    serde_json::from_str(&text)
        .unwrap_or_else(|e| panic!("parse JSON {}: {e}", path.display()))
}

fn restart_scenario() -> String {
    r#"
run_id = "lifecycle-restart"
seed = 17
duration_ms = 3000

[[hosts]]
name = "alpha"
role = "stage"
stage_index = 0
start_at_ms = 0
restart_at_ms = [1000, 2000]
stop_at_ms = 3000
"#
    .to_string()
}

fn crash_scenario() -> String {
    r#"
run_id = "lifecycle-crash"
seed = 23
duration_ms = 3000

[[hosts]]
name = "crasher"
role = "stage"
stage_index = 0
start_at_ms = 0
crash_at_ms = 1500
"#
    .to_string()
}

fn stop_scenario() -> String {
    r#"
run_id = "lifecycle-stop"
seed = 29
duration_ms = 2000

[[hosts]]
name = "clean"
role = "stage"
stage_index = 0
start_at_ms = 0
stop_at_ms = 1500
"#
    .to_string()
}

fn end_of_run_scenario() -> String {
    r#"
run_id = "lifecycle-end-of-run"
seed = 31
duration_ms = 2000

[[hosts]]
name = "still_running"
role = "stage"
stage_index = 0
start_at_ms = 0
"#
    .to_string()
}