//! Tier-2 connection-cache + NodeMap delta tracking (S7). //! //! Verifies two things the iroh driver promises after S7: //! //! 1. **Every peer-addr handoff to iroh is surfaced.** Each time the //! driver feeds iroh a relay URL (or a direct address) for a peer, //! a [`Event::NodeMapUpdate`] event is emitted with the source the //! addr came from (`join_seed`, `swim_metadata`, //! `explicit_relay_cache`, `home_relay_fallback`). //! //! 2. **Tier-2 snapshots carry the per-peer cache lifecycle.** Once //! a connection has been opened and used, the snapshot's //! `body.iroh.connection_cache` block contains an entry for the //! peer with `generation >= 1`, `created_at_ms` populated, and //! `last_successful_send_at_ms` populated. //! //! These are behavioural checks: the test asserts on the shape of //! the event stream and snapshot, not on exact timestamps or //! generation counts (which can wobble with iroh's dynamics). #![cfg(feature = "iroh")] mod common; use std::sync::Arc; use std::time::Duration; use common::iroh::*; use distribution::diagnostics::{ Aggregator, Event, Identity, InMemorySink, Role, SnapshotTrigger, }; use distribution::diagnostics::iroh_introspect::IntrospectConfig; use distribution::iroh_driver::IrohDriver; use distribution::types::NodeId; use iroh::PublicKey; fn install_full_diagnostics( driver: &mut IrohDriver, run_id: &str, ) -> (Arc, Arc>>) { let sink = Arc::new(InMemorySink::new()); let identity = Identity::new(driver.node_id(), Role::stage(), run_id); let aggregator = Arc::new(Aggregator::new(identity, sink.clone())); driver.install_diagnostics_with_config( aggregator.clone(), IntrospectConfig { scrape_interval: Duration::from_millis(50), }, ); (sink, aggregator) } fn node_hex(id: &NodeId) -> String { let mut s = String::with_capacity(64); for b in id.0 { s.push_str(&format!("{:02x}", b)); } s } #[test] fn join_emits_node_map_update_for_seed_addr() { let mut a = make_driver(); let mut b = make_driver(); let (sink_a, _agg_a) = install_full_diagnostics(&mut a, "run-s7-join"); let _ = install_full_diagnostics(&mut b, "run-s7-join"); let b_id = b.node_id(); let b_addr = b.endpoint_addr(); a.join(&[b_addr]); // No pumping required — the NodeMapUpdate emit happens inside // join() before any async work is kicked off. let saw = sink_a.records().iter().any(|r| { matches!( &r.event, Event::NodeMapUpdate { peer, from_source, accepted } if peer == &b_id && (from_source == "join_seed" || from_source == "join_seed_direct") && *accepted ) }); a.shutdown(); b.shutdown(); assert!( saw, "join() should emit NodeMapUpdate with from_source in \ {{join_seed, join_seed_direct}} for the seed peer", ); } #[test] fn join_with_bare_seed_emits_no_node_map_update() { let mut a = make_driver(); let (sink_a, _agg_a) = install_full_diagnostics(&mut a, "run-s7-bare"); // Build a peer addr with neither relay nor direct addresses — // just the public key. iroh has nothing new to learn here, so // we shouldn't emit a NodeMapUpdate either. let bare_key = iroh::SecretKey::from_bytes(&[0xcd; 32]).public(); let bare_addr = iroh::EndpointAddr::new(bare_key); let bare_id = NodeId(*bare_key.as_bytes()); a.join(&[bare_addr]); let saw = sink_a .records() .iter() .any(|r| matches!(&r.event, Event::NodeMapUpdate { peer, .. } if peer == &bare_id)); a.shutdown(); assert!( !saw, "bare-key join() should NOT emit NodeMapUpdate — iroh has \ no new address to learn", ); } #[test] fn cache_aggregate_appears_in_tier2_snapshot_after_successful_send() { let mut a = make_driver(); let mut b = make_driver(); let (sink_a, agg_a) = install_full_diagnostics(&mut a, "run-s7-cache"); let _ = install_full_diagnostics(&mut b, "run-s7-cache"); let a_id = a.node_id(); let b_id = b.node_id(); let b_hex = node_hex(&b_id); let b_addr = b.endpoint_addr(); a.join(&[b_addr]); let converged = pump_until_pair( &mut a, &mut b, Duration::from_secs(5), |a, b| { let a_key = PublicKey::from_bytes(&a.node_id().0).unwrap(); let b_key = PublicKey::from_bytes(&b.node_id().0).unwrap(); sees_alive(a, &b_key) && sees_alive(b, &a_key) }, ); assert!(converged, "nodes did not converge within 5s"); // Pump a few more rounds so SWIM probes (which exercise // send_message → note_send_success on the cached connection) // run beyond the initial join handshake. for _ in 0..10 { pump_one(&mut a); pump_one(&mut b); std::thread::sleep(Duration::from_millis(20)); } a.force_iroh_introspect_refresh(); let snap = agg_a.snapshot(SnapshotTrigger::OnDemand); a.shutdown(); b.shutdown(); let iroh = snap .body .iroh .as_ref() .expect("snapshot must include tier-2 iroh block"); let entry = iroh .connection_cache .iter() .find(|e| e.peer_node_id_hex == b_hex) .unwrap_or_else(|| { panic!( "tier-2 snapshot connection_cache should include B \ (= {}); got entries: {:#?}", b_hex, iroh.connection_cache, ); }); assert!( entry.generation >= 1, "expected cache generation >= 1 after a successful dial; \ got {} (entry: {:#?})", entry.generation, entry, ); assert!( entry.created_at_ms.is_some(), "expected created_at_ms to be populated after a successful \ dial; got {:#?}", entry, ); assert!( entry.last_successful_send_at_ms.is_some(), "expected last_successful_send_at_ms to be populated after a \ message round-trip; got {:#?}", entry, ); // Sanity: a's own id should not appear (the cache only tracks // outbound peers). let a_hex = node_hex(&a_id); assert!( !iroh.connection_cache.iter().any(|e| e.peer_node_id_hex == a_hex), "connection_cache should not list the local node itself", ); // Confirm that the NodeMapUpdate emit fired on the get_or_connect // dial path too — once SWIM probes start, the driver feeds iroh // a relay URL resolved from one of its caches. let saw_dial_path_nmu = sink_a.records().iter().any(|r| { matches!( &r.event, Event::NodeMapUpdate { peer, from_source, accepted } if peer == &b_id && (from_source == "explicit_relay_cache" || from_source == "swim_metadata" || from_source == "home_relay_fallback") && *accepted ) }); // Without a relay configured (RelayMode::Disabled) the home-relay // path is not available; if all caches are empty we'd skip the // emit entirely. Accept either: a NodeMapUpdate from the dial // path *or* the join-seed one we already asserted on. let saw_join_seed_nmu = sink_a.records().iter().any(|r| { matches!( &r.event, Event::NodeMapUpdate { peer, from_source, .. } if peer == &b_id && (from_source == "join_seed" || from_source == "join_seed_direct") ) }); assert!( saw_dial_path_nmu || saw_join_seed_nmu, "expected at least one NodeMapUpdate event for B (either from \ join() or from the dial path); got {:#?}", sink_a .records() .iter() .filter_map(|r| match &r.event { Event::NodeMapUpdate { peer, from_source, accepted } => Some((peer, from_source.clone(), *accepted)), _ => None, }) .collect::>(), ); } #[test] fn generation_in_events_matches_snapshot_aggregate() { let mut a = make_driver(); let mut b = make_driver(); let (sink_a, agg_a) = install_full_diagnostics(&mut a, "run-s7-gen"); let _ = install_full_diagnostics(&mut b, "run-s7-gen"); let b_id = b.node_id(); let b_hex = node_hex(&b_id); let b_addr = b.endpoint_addr(); a.join(&[b_addr]); let converged = pump_until_pair( &mut a, &mut b, Duration::from_secs(5), |a, b| { let a_key = PublicKey::from_bytes(&a.node_id().0).unwrap(); let b_key = PublicKey::from_bytes(&b.node_id().0).unwrap(); sees_alive(a, &b_key) && sees_alive(b, &a_key) }, ); assert!(converged, "nodes did not converge within 5s"); // A few more rounds so we get a ConnectionCacheHit event recorded // against the established connection. for _ in 0..10 { pump_one(&mut a); pump_one(&mut b); std::thread::sleep(Duration::from_millis(20)); } a.force_iroh_introspect_refresh(); let snap = agg_a.snapshot(SnapshotTrigger::OnDemand); a.shutdown(); b.shutdown(); // Find the highest generation reported in any per-touch event // for B and compare against the snapshot aggregate. They should // agree — the post-processor depends on this invariant when // stitching events to snapshots. let max_event_gen = sink_a .records() .iter() .filter_map(|r| match &r.event { Event::ConnectionCacheHit { peer, generation } | Event::ConnectionCacheInvalidated { peer, generation, .. } if peer == &b_id => { Some(*generation) } _ => None, }) .max() .expect("expected at least one cache-touch event for B"); let iroh = snap.body.iroh.as_ref().expect("tier-2 iroh block"); let entry = iroh .connection_cache .iter() .find(|e| e.peer_node_id_hex == b_hex) .expect("snapshot connection_cache must include B"); assert!( entry.generation >= max_event_gen, "snapshot generation ({}) should be >= the highest generation \ seen on the event stream ({})", entry.generation, max_event_gen, ); }