//! LAN cluster integration tests — nodes across two physical machines. //! //! These tests run a 5-node cluster split across devuan-hpz (192.168.1.106) //! and thinkpad (192.168.1.102) communicating over a real LAN. //! //! Run with: `cargo test -p docker-tests -- --ignored lan_` //! Requires: Docker on both machines, SSH access to thinkpad use std::time::Duration; use docker_tests::*; // ──────────────────────────────────────────────────────────────────────────── // Test 1: Cross-machine cluster converges // ──────────────────────────────────────────────────────────────────────────── #[test] #[ignore] fn lan_cluster_converges() { // Given: 5 nodes split across two physical machines on a LAN let mut cluster = LanClusterHandle::start(); // When: we wait for convergence let result = wait_for_lan_convergence( &LAN_ENDPOINTS, 4, // each node sees at least 4 alive Duration::from_secs(30), ); // Then: all 5 nodes discover each other across the LAN match result { Ok(()) => { for &(host, port) in &LAN_ENDPOINTS { let snap = poll_distribution_at(host, port) .unwrap_or_else(|| panic!("{host}:{port} unreachable after convergence")); assert!( snap.alive_count >= 4, "{host}:{port} should see >= 4 alive, got {}", snap.alive_count ); assert!( snap.routing_table_size >= 3, "{host}:{port} should have >= 3 routing entries, got {}", snap.routing_table_size ); } } Err(diag) => { cluster.stop(); panic!("LAN cluster did not converge: {diag}"); } } cluster.stop(); } // ──────────────────────────────────────────────────────────────────────────── // Test 2: Death of a remote node is detected across the LAN // ──────────────────────────────────────────────────────────────────────────── #[test] #[ignore] fn lan_remote_node_death_detected() { // Given: a converged LAN cluster let mut cluster = LanClusterHandle::start(); wait_for_lan_convergence(&LAN_ENDPOINTS, 4, Duration::from_secs(30)) .expect("LAN cluster did not converge before kill test"); // When: we kill node-3 on the thinkpad kill_remote_node("node-3"); // Then: surviving nodes detect the death // Survivors: hpz seed(9091), hpz node-2(9092), thinkpad node-4(9094), thinkpad node-5(9095) let survivor_endpoints = [ ("127.0.0.1", 9091_u16), ("127.0.0.1", 9092), (LAN_THINKPAD_IP, 9094), (LAN_THINKPAD_IP, 9095), ]; let result = wait_for_death_detection_at(&survivor_endpoints, 4, Duration::from_secs(30)); match result { Ok(()) => { let any_sees_dead = survivor_endpoints.iter().any(|&(host, port)| { poll_distribution_at(host, port) .map(|snap| snap.dead_count >= 1) .unwrap_or(false) }); assert!(any_sees_dead, "at least one survivor should see a dead member"); } Err(diag) => { cluster.stop(); panic!("remote node death not detected: {diag}"); } } cluster.stop(); } // ──────────────────────────────────────────────────────────────────────────── // Test 3: A killed remote node can rejoin across the LAN // ──────────────────────────────────────────────────────────────────────────── #[test] #[ignore] fn lan_killed_remote_node_rejoins() { // Given: a converged cluster with node-3 killed and detected dead let mut cluster = LanClusterHandle::start(); wait_for_lan_convergence(&LAN_ENDPOINTS, 4, Duration::from_secs(30)) .expect("LAN cluster did not converge before rejoin test"); kill_remote_node("node-3"); let survivor_endpoints = [ ("127.0.0.1", 9091_u16), ("127.0.0.1", 9092), (LAN_THINKPAD_IP, 9094), (LAN_THINKPAD_IP, 9095), ]; wait_for_death_detection_at(&survivor_endpoints, 4, Duration::from_secs(30)) .expect("node death not detected before rejoin"); // When: we restart node-3 on the thinkpad restart_remote_node("node-3"); // Then: node-3 rejoins the cluster across the LAN let result = wait_for_lan_convergence( &[(LAN_THINKPAD_IP, 9093)], 1, Duration::from_secs(30), ); match result { Ok(()) => { let snap = poll_distribution_at(LAN_THINKPAD_IP, 9093) .expect("node-3 unreachable after rejoin"); assert!( snap.alive_count >= 1, "rejoined node should see >= 1 alive, got {}", snap.alive_count ); } Err(diag) => { cluster.stop(); panic!("killed remote node did not rejoin: {diag}"); } } cluster.stop(); } // ──────────────────────────────────────────────────────────────────────────── // Test 4: Actors are resolvable across machines // ──────────────────────────────────────────────────────────────────────────── #[test] #[ignore] fn lan_actors_resolvable_cross_machine() { // Given: a converged 5-node LAN cluster, each with 2 registered actors let mut cluster = LanClusterHandle::start(); wait_for_lan_convergence(&LAN_ENDPOINTS, 4, Duration::from_secs(30)) .expect("LAN cluster did not converge before actor resolution test"); // When: we query each node's snapshot let mut total_directory_entries = 0; let mut total_cache_size = 0; for &(host, port) in &LAN_ENDPOINTS { let snap = poll_distribution_at(host, port) .unwrap_or_else(|| panic!("{host}:{port} unreachable")); // Then: each node has its own 2 actors in the directory assert!( snap.directory_entry_count >= 2, "{host}:{port} should have >= 2 directory entries, got {}", snap.directory_entry_count ); total_directory_entries += snap.directory_entry_count; total_cache_size += snap.cache_size; } // Total actors across cluster: 10 (5 nodes * 2 actors) assert!( total_directory_entries >= 10, "total directory entries should be >= 10, got {}", total_directory_entries ); // Nodes should cache remote actor locations (including cross-machine) assert!( total_cache_size >= 5, "total cache entries should be >= 5, got {}", total_cache_size ); cluster.stop(); }