swactor/crates/distribution/tests/telemetry.rs

//! Distribution observability contracts.
//!
//! These tests are intentionally narrower than protocol tests: they pin public record/snapshot
//! shapes and assert telemetry remains observational rather than protocol-driving.
//!
//! Behavioral/correctness guarantees:
//! - Distribution telemetry records and snapshot shapes are stable wire contracts.
//! - Telemetry is observational: it reports protocol/cache/registry state without feeding back
//!   into behavior.
//! - Transition causes and recent probe targets are externally visible when promised.

mod telemetry_records {
    //! Distribution-owned telemetry records keep their channels and JSON payloads stable through
    //! the mux.

    use distribution::telemetry::{
        CacheEntryRec, DIST_STATE, DistributionState, MEMBERSHIP, MembershipTransition,
        RegistryEntryRec, SWIM_PROBES, SwimProbeEvent, TRANSPORT_INTERNALS, TransportInternals,
    };
    use telemetry::frame::{Lifetime, NodeId, StreamId};
    use telemetry::{ChannelId, Mux, Position, Record};

    #[test]
    fn distribution_state_record_round_trips_from_owner_crate() {
        let state = DistributionState {
            cache_size: 2,
            cache_entries: vec![CacheEntryRec {
                actor_addr: "actor-a".into(),
                node_id: "node-a".into(),
            }],
            directory_route_count: 7,
            registry_size: 3,
            registry_tombstones: 1,
            registry_entries: vec![RegistryEntryRec {
                name: "svc".into(),
                actor_addr: "actor-b".into(),
                node_id: "node-b".into(),
                tombstone: false,
            }],
            recent_probe_targets: vec!["node-c".into()],
            peer_auth_mode: "allow-list".into(),
            authorized_peer_count: 4,
        };

        assert_eq!(DistributionState::CHANNEL, DIST_STATE);
        assert_eq!(DistributionState::decode(&state.encode()).unwrap(), state);
    }

    #[test]
    fn distribution_emits_owned_channel_through_telemetry_mux() {
        let stream = StreamId::new(NodeId::new("dist-node"), Lifetime(1));
        let mux = Mux::unbounded(stream);
        let state = DistributionState {
            registry_size: 9,
            ..Default::default()
        };

        let channel = ChannelId(1);
        assert!(mux.submit(channel, state.encode()));
        let frames = mux.drain();

        assert_eq!(frames.len(), 1);
        assert_eq!(frames[0].channel, channel);
        assert_eq!(frames[0].position, Position(0));
        assert_eq!(
            DistributionState::decode(&frames[0].payload)
                .unwrap()
                .registry_size,
            9
        );
    }

    #[test]
    fn transport_internals_record_round_trips_probe_rtt() {
        let internals = TransportInternals {
            relay_connected: true,
            direct_peers: 2,
            relay_peers: 1,
            rtt_ms_p50: 405,
        };

        assert_eq!(TransportInternals::CHANNEL, TRANSPORT_INTERNALS);
        assert_eq!(
            TransportInternals::decode(&internals.encode()).unwrap(),
            internals
        );
    }

    #[test]
    fn membership_transition_record_round_trips_from_owner_crate() {
        let transition = MembershipTransition {
            peer: "peer-a".into(),
            from: "alive".into(),
            to: "suspect".into(),
            reason: "probe timeout".into(),
            last_ack_age_ms: Some(15_000),
            consecutive_timeouts: 2,
            recent_probe_targets: vec!["peer-a".into(), "peer-b".into()],
            member_state: Some("Suspect".into()),
        };

        assert_eq!(MembershipTransition::CHANNEL, MEMBERSHIP);
        assert_eq!(
            MembershipTransition::decode(&transition.encode()).unwrap(),
            transition
        );
    }

    #[test]
    fn swim_probe_event_record_round_trips_from_owner_crate() {
        let event = SwimProbeEvent {
            event: "timed_out".into(),
            target: "peer-a".into(),
            sequence: 9,
            kind: "direct".into(),
            rtt_ms: None,
            budget_ms: Some(15_000),
            budget_ticks: Some(15_000),
            last_ack_age_ms: Some(45_000),
            consecutive_timeouts: 3,
            recent_probe_targets: vec!["peer-a".into()],
            member_state: Some("Suspect".into()),
            local_phase: "weights_loaded_wait".into(),
            probe_interval_ms: 200,
            probe_timeout_ms: 15_000,
            indirect_probes: 2,
            suspicion_timeout_ms: 45_000,
            dead_reprobe_interval_ms: 1_000,
            probe_mode: "Periodic".into(),
            lifeguard_enabled: false,
        };

        assert_eq!(SwimProbeEvent::CHANNEL, SWIM_PROBES);
        assert_eq!(SwimProbeEvent::decode(&event.encode()).unwrap(), event);
    }
}

mod snapshot_and_swim_telemetry {
    //! Snapshot and SWIM telemetry readouts: public empty-state shape, recent probe targets, and
    //! transition causes.

    use distribution::snapshot::DistributionNodeSnapshot;
    use distribution::swim::node::{SwimObservation, SwimObserver};
    use distribution::swim::telemetry::SwimTelemetry;
    use distribution::types::{MemberState, NodeId};
    use std::time::Duration;

    fn id(byte: u8) -> NodeId {
        NodeId([byte; 32])
    }

    #[test]
    fn empty_distribution_snapshot_keeps_the_public_wire_shape_stable() {
        // Correctness: external distribution consumers can deserialize the retained
        // snapshot shape, and the empty constructor reports explicit zero/open defaults.
        let snapshot = DistributionNodeSnapshot::empty(id(0xAB));
        let json = serde_json::to_string(&snapshot).unwrap();
        let decoded: DistributionNodeSnapshot = serde_json::from_str(&json).unwrap();

        assert_eq!(
            decoded.node_id,
            "abababababababababababababababababababababababababababababababab"
        );
        assert_eq!(decoded.peer_auth_mode, "open");
        assert_eq!(decoded.alive_count, 0);
        assert!(decoded.members.is_empty());
        assert!(decoded.registry_entries.is_empty());
        assert!(decoded.authorized_peer_count.is_none());
    }

    #[test]
    fn swim_telemetry_reports_probe_targets_and_transition_causes_without_state_feedback() {
        // Correctness: telemetry is an observer side channel. It records recent probes
        // and transition causes for readers, and draining transitions only affects the
        // telemetry buffer, not protocol state.
        let telemetry = SwimTelemetry::new();
        let peer = id(4);

        telemetry.observe(SwimObservation::ProbeSent {
            target: peer,
            sequence: 1,
            kind: "direct",
        });
        telemetry.observe(SwimObservation::Transition {
            peer,
            from: Some(MemberState::Alive),
            to: MemberState::Suspect,
            reason: "probe-timeout",
        });

        assert_eq!(telemetry.rtt_ms_p50(), 0);
        assert_eq!(telemetry.recent_targets(), vec![peer]);
        assert_eq!(telemetry.last_reasons().get(&peer), Some(&"probe-timeout"));

        let drained = telemetry.drain_transitions();
        assert_eq!(drained.len(), 1);
        assert_eq!(drained[0].peer, peer);
        assert_eq!(drained[0].from, Some(MemberState::Alive));
        assert_eq!(drained[0].to, MemberState::Suspect);
        assert_eq!(drained[0].reason, "probe-timeout");
        assert_eq!(drained[0].consecutive_timeouts, 0);
        assert!(drained[0].last_ack_age.is_none());
        assert!(telemetry.drain_transitions().is_empty());
        assert_eq!(telemetry.recent_targets(), vec![peer]);
    }

    #[test]
    fn swim_telemetry_keeps_recent_probe_targets_bounded_and_ordered() {
        let telemetry = SwimTelemetry::new();

        for byte in 0..17 {
            telemetry.observe(SwimObservation::ProbeSent {
                target: id(byte),
                sequence: byte as u64,
                kind: "direct",
            });
        }

        let expected = (1..17).map(id).collect::<Vec<_>>();
        assert_eq!(telemetry.recent_targets(), expected);
    }

    #[test]
    fn swim_telemetry_records_probe_events_and_timeout_state_without_fabricating_rtt() {
        let telemetry = SwimTelemetry::new();
        let peer = id(8);

        telemetry.observe(SwimObservation::ProbeSent {
            target: peer,
            sequence: 1,
            kind: "direct",
        });
        std::thread::sleep(Duration::from_millis(1));
        telemetry.observe(SwimObservation::ProbeAcked {
            target: peer,
            sequence: 1,
            kind: "direct",
        });
        telemetry.observe(SwimObservation::ProbeSent {
            target: peer,
            sequence: 2,
            kind: "direct",
        });
        telemetry.observe(SwimObservation::ProbeTimedOut {
            target: peer,
            sequence: 2,
            kind: "direct",
            budget_ticks: 15_000,
        });

        let events = telemetry.drain_probe_events();
        assert_eq!(events.len(), 1, "only timeout remains immediate");
        assert_eq!(events[0].event, "timed_out");
        assert_eq!(events[0].budget_ms, Some(15_000));
        assert_eq!(events[0].rtt_ms, None);
        assert_eq!(events[0].consecutive_timeouts, 1);
        assert!(telemetry.drain_probe_events().is_empty());

        std::thread::sleep(Duration::from_millis(1_010));
        let summary = telemetry
            .drain_probe_summary()
            .expect("one-second routine probe aggregate");
        assert_eq!(summary.sent, 2);
        assert_eq!(summary.acked, 1);
        assert_eq!(summary.direct_sent, 2);
        assert_eq!(summary.rtt_samples, 1);
        assert!(summary.rtt_ms_p50.is_some());
    }
}