Zachery Aaron Shores-Chmielewski
02701280ed
Stream bounded provider logs and surface provisioning phases through telemetry and the dashboard. Preserve offer-search criteria, use live relay endpoints for remote bootstrap, and add GPU smoke coverage. Exercise orchestrator crash recovery and clean completed checkpoints without dropping active work.
609 lines
17 KiB
Rust
609 lines
17 KiB
Rust
/// Calls with these resolved item names can create wall-clock waits or
|
|
/// failure-producing deadlines. They are denied in every workspace role unless
|
|
/// the exact compiler-resolved call site is registered below.
|
|
const GUARDED_TIMING_ITEMS: &[&str] = &[
|
|
"park_timeout",
|
|
"recv_deadline",
|
|
"set_read_timeout",
|
|
"recv_timeout",
|
|
"sleep",
|
|
"sleep_until",
|
|
"timeout",
|
|
"timeout_at",
|
|
"wait_timeout",
|
|
"wait_timeout_while",
|
|
];
|
|
|
|
#[derive(Clone, Copy)]
|
|
struct TimingAllowance {
|
|
package: &'static str,
|
|
caller: &'static str,
|
|
callee: &'static str,
|
|
calls: usize,
|
|
purpose: &'static str,
|
|
}
|
|
|
|
const fn timing(
|
|
package: &'static str,
|
|
caller: &'static str,
|
|
callee: &'static str,
|
|
calls: usize,
|
|
purpose: &'static str,
|
|
) -> TimingAllowance {
|
|
TimingAllowance {
|
|
package,
|
|
caller,
|
|
callee,
|
|
calls,
|
|
purpose,
|
|
}
|
|
}
|
|
|
|
/// Exact audited boundaries. A rename, a different API, or one additional call
|
|
/// exceeds this registry and fails compilation. Never register a generic
|
|
/// timeout wrapper: callers of a timing API must remain visible to this guard.
|
|
const TIMING_ALLOWANCES: &[TimingAllowance] = &[
|
|
// Runtime and external-resource boundaries.
|
|
timing(
|
|
"swactor-engine",
|
|
"<tokio::LazySleep as std::future::Future>::poll",
|
|
"tokio::time::sleep",
|
|
1,
|
|
"Tokio execution-backend adapter",
|
|
),
|
|
timing(
|
|
"swactor-process",
|
|
"<operations::FollowProcessFile as std::io::Read>::read",
|
|
"std::thread::sleep",
|
|
1,
|
|
"external process-file polling",
|
|
),
|
|
timing(
|
|
"swactor-process",
|
|
"operations::find_process_identities_with_retry",
|
|
"std::thread::sleep",
|
|
1,
|
|
"external process discovery retry",
|
|
),
|
|
timing(
|
|
"swactor-process",
|
|
"operations::spawn_identity_exit_wait::{closure#0}",
|
|
"std::thread::sleep",
|
|
1,
|
|
"external process exit polling",
|
|
),
|
|
timing(
|
|
"swactor-process",
|
|
"operations::spawn_shared_child_wait::{closure#0}",
|
|
"std::thread::sleep",
|
|
1,
|
|
"external child exit polling",
|
|
),
|
|
timing(
|
|
"swactor-process",
|
|
"operations::terminate_process_group",
|
|
"std::thread::sleep",
|
|
1,
|
|
"process termination grace period",
|
|
),
|
|
timing(
|
|
"swactor-process",
|
|
"operations::wait_for_path",
|
|
"std::thread::sleep",
|
|
1,
|
|
"external path publication polling",
|
|
),
|
|
timing(
|
|
"swactor-process",
|
|
"operations::wait_shared_child_or_kill",
|
|
"std::thread::sleep",
|
|
1,
|
|
"child termination grace period",
|
|
),
|
|
timing(
|
|
"iroh-driver",
|
|
"edge_transport::spawn_edge_send_pump",
|
|
"std::sync::mpsc::Receiver::recv_timeout",
|
|
1,
|
|
"edge sender startup handoff",
|
|
),
|
|
timing(
|
|
"iroh-driver",
|
|
"edge_transport::spawn_edge_send_pump::{closure#0}::{closure#0}",
|
|
"swactor_engine::EngineHandle::timeout",
|
|
2,
|
|
"edge stream write and finish deadlines",
|
|
),
|
|
timing(
|
|
"iroh-driver",
|
|
"iroh_driver::AdapterPump::spawn_connect::{closure#0}",
|
|
"swactor_engine::EngineHandle::timeout",
|
|
1,
|
|
"peer connection deadline",
|
|
),
|
|
timing(
|
|
"iroh-driver",
|
|
"iroh_driver::IrohDriver::send_tagged_gossip::{closure#0}",
|
|
"swactor_engine::EngineHandle::timeout",
|
|
1,
|
|
"gossip connection deadline",
|
|
),
|
|
timing(
|
|
"iroh-driver",
|
|
"iroh_driver::IrohDriver::connect_peer::{closure#0}",
|
|
"swactor_engine::EngineHandle::timeout",
|
|
1,
|
|
"peer connection deadline",
|
|
),
|
|
timing(
|
|
"iroh-driver",
|
|
"iroh_driver::IrohDriver::spawn_join_request::{closure#0}",
|
|
"swactor_engine::EngineHandle::timeout",
|
|
1,
|
|
"seed connection deadline",
|
|
),
|
|
timing(
|
|
"swactor-vastai",
|
|
"blocking::BlockingVastClient::list_by_label_with_retry",
|
|
"std::thread::sleep",
|
|
1,
|
|
"provider API retry backoff",
|
|
),
|
|
timing(
|
|
"swactor-vastai",
|
|
"client::VastClient::with_base_url",
|
|
"reqwest::ClientBuilder::timeout",
|
|
1,
|
|
"provider HTTP request deadline",
|
|
),
|
|
timing(
|
|
"swactor-vastai",
|
|
"lease::provision_fleet::{closure#0}",
|
|
"tokio::time::sleep",
|
|
1,
|
|
"provider lease pacing",
|
|
),
|
|
timing(
|
|
"swactor-vastai",
|
|
"lease::provision_one::{closure#0}",
|
|
"tokio::time::sleep",
|
|
1,
|
|
"provider retry backoff",
|
|
),
|
|
timing(
|
|
"swactor-vastai",
|
|
"monitor::wait_for_running_with_policy::{closure#0}",
|
|
"tokio::time::sleep",
|
|
2,
|
|
"provider lifecycle polling",
|
|
),
|
|
timing(
|
|
"swactor-vastai",
|
|
"monitor::wait_for_ssh_endpoint_with_policy::{closure#0}",
|
|
"tokio::time::sleep",
|
|
2,
|
|
"provider endpoint polling",
|
|
),
|
|
timing(
|
|
"swactor-vastai",
|
|
"teardown::destroy_instance_with_retry_policy::{closure#0}",
|
|
"tokio::time::sleep",
|
|
1,
|
|
"provider teardown backoff",
|
|
),
|
|
timing(
|
|
"telemetry",
|
|
"endpoint::TelemetrySubscription::recv_timeout",
|
|
"crossbeam_channel::Receiver::recv_timeout",
|
|
1,
|
|
"bounded external telemetry observation",
|
|
),
|
|
// Test and harness safety fuses. These bound real external concurrency;
|
|
// virtual-time behavior continues to use the stepping backend.
|
|
timing(
|
|
"dashboard",
|
|
"control::properties::generated_concurrent_bridge_commands_forward_once_and_shutdown::{closure#1}",
|
|
"std::sync::mpsc::Receiver::recv_timeout",
|
|
1,
|
|
"test completion fuse",
|
|
),
|
|
timing(
|
|
"dashboard",
|
|
"server::tests::send_control_request::{closure#0}",
|
|
"tokio::time::timeout",
|
|
1,
|
|
"test HTTP completion fuse",
|
|
),
|
|
timing(
|
|
"data-plane",
|
|
"cancellation_and_transport_faults_reclaim_waiters_and_preserve_unrelated_progress::{closure#0}",
|
|
"std::thread::sleep",
|
|
1,
|
|
"test cleanup observation poll",
|
|
),
|
|
timing(
|
|
"data-plane",
|
|
"cancelled_write_open_releases_queued_grant",
|
|
"std::thread::sleep",
|
|
2,
|
|
"test cancellation observation poll",
|
|
),
|
|
timing(
|
|
"data-plane",
|
|
"host_read_resolves_file_source_and_seals_final_arena_lease",
|
|
"std::thread::sleep",
|
|
4,
|
|
"test lease reclamation observation poll",
|
|
),
|
|
timing(
|
|
"data-plane",
|
|
"live_view_prevents_reclaim_until_last_guard_drops",
|
|
"std::thread::sleep",
|
|
1,
|
|
"test lease reclamation observation poll",
|
|
),
|
|
timing(
|
|
"distribution",
|
|
"snapshot_and_swim_telemetry::swim_telemetry_records_probe_events_and_timeout_state_without_fabricating_rtt",
|
|
"std::thread::sleep",
|
|
2,
|
|
"wall-clock telemetry contract",
|
|
),
|
|
timing(
|
|
"iroh-driver",
|
|
"common::iroh::IrohTestCluster::pump_until",
|
|
"std::thread::sleep",
|
|
1,
|
|
"live network convergence fuse",
|
|
),
|
|
timing(
|
|
"iroh-driver",
|
|
"common::iroh::pump_until",
|
|
"std::thread::sleep",
|
|
1,
|
|
"live network convergence fuse",
|
|
),
|
|
timing(
|
|
"iroh-driver",
|
|
"common::iroh::pump_until_pair",
|
|
"std::thread::sleep",
|
|
1,
|
|
"live network convergence fuse",
|
|
),
|
|
timing(
|
|
"iroh-driver",
|
|
"endpoint_addr_includes_home_relay",
|
|
"std::thread::sleep",
|
|
1,
|
|
"live relay convergence fuse",
|
|
),
|
|
timing(
|
|
"iroh-driver",
|
|
"pull_collector_cancellation_interrupts_inflight_io",
|
|
"std::sync::mpsc::Receiver::recv_timeout",
|
|
1,
|
|
"test setup completion fuse",
|
|
),
|
|
timing(
|
|
"iroh-driver",
|
|
"pull_collector_cancellation_interrupts_inflight_io",
|
|
"std::thread::sleep",
|
|
2,
|
|
"live network cancellation observation",
|
|
),
|
|
timing(
|
|
"iroh-driver",
|
|
"real_iroh_transfer_delivers_exact_file_bytes",
|
|
"std::sync::mpsc::Receiver::recv_timeout",
|
|
2,
|
|
"live transfer completion fuse",
|
|
),
|
|
timing(
|
|
"iroh-driver",
|
|
"real_iroh_transfer_delivers_exact_file_bytes",
|
|
"std::thread::sleep",
|
|
2,
|
|
"live transfer progress poll",
|
|
),
|
|
timing(
|
|
"iroh-driver",
|
|
"same_runtime_transfer_bypasses_iroh_self_connection",
|
|
"std::sync::mpsc::Receiver::recv_timeout",
|
|
4,
|
|
"local transfer completion fuse",
|
|
),
|
|
timing(
|
|
"iroh-driver",
|
|
"edge_transport::EdgeSendHandle::finish",
|
|
"std::sync::mpsc::Receiver::recv_timeout",
|
|
1,
|
|
"edge transport completion fuse",
|
|
),
|
|
timing(
|
|
"iroh-driver",
|
|
"recv_until",
|
|
"std::sync::mpsc::Receiver::recv_timeout",
|
|
1,
|
|
"live transport observation fuse",
|
|
),
|
|
timing(
|
|
"myelin",
|
|
"node::worker_node_runtime::control_flow_properties::agent_cpu_sampler_emits_decodable_host_stats",
|
|
"std::thread::sleep",
|
|
1,
|
|
"hardware sampler observation",
|
|
),
|
|
timing(
|
|
"myelin",
|
|
"orchestration::app::hardware_telemetry_tests::orchestrator_emits_all_host_hardware_channels",
|
|
"std::thread::sleep",
|
|
1,
|
|
"hardware sampler observation",
|
|
),
|
|
timing(
|
|
"myelin",
|
|
"provisioning::tests::local_process_stop_releases_all_observer_actors",
|
|
"std::thread::sleep",
|
|
1,
|
|
"process shutdown observation",
|
|
),
|
|
timing(
|
|
"myelin",
|
|
"provisioning::tests::recv_provider_event",
|
|
"std::sync::mpsc::Receiver::recv_timeout",
|
|
1,
|
|
"provider observation fuse",
|
|
),
|
|
timing(
|
|
"myelin",
|
|
"tests::data_namespace_guarantees::wait_until",
|
|
"std::thread::sleep",
|
|
1,
|
|
"live namespace convergence fuse",
|
|
),
|
|
timing(
|
|
"myelin",
|
|
"tests::engine_composition::poll_connect",
|
|
"std::thread::sleep",
|
|
1,
|
|
"live server connection fuse",
|
|
),
|
|
timing(
|
|
"myelin",
|
|
"tests::engine_composition::recv_within",
|
|
"std::thread::sleep",
|
|
1,
|
|
"engine observation fuse",
|
|
),
|
|
timing(
|
|
"myelin-e2e-fuzz",
|
|
"harness::control::<impl harness::ClusterHarness>::kill_node",
|
|
"std::thread::sleep",
|
|
1,
|
|
"external node termination polling",
|
|
),
|
|
timing(
|
|
"myelin-e2e-fuzz",
|
|
"harness::convergence::<impl harness::ClusterHarness>::provision_nodes",
|
|
"std::thread::sleep",
|
|
1,
|
|
"external provisioning polling",
|
|
),
|
|
timing(
|
|
"myelin-e2e-fuzz",
|
|
"harness::ClusterHarness::teardown",
|
|
"std::thread::sleep",
|
|
2,
|
|
"external resource and process teardown polling",
|
|
),
|
|
timing(
|
|
"myelin-e2e-fuzz",
|
|
"harness::ClusterHarness::verify_missing_resource_recovery",
|
|
"std::thread::sleep",
|
|
1,
|
|
"external provider recovery convergence polling",
|
|
),
|
|
timing(
|
|
"myelin-e2e-fuzz",
|
|
"harness::convergence::<impl harness::ClusterHarness>::wait_contextual_control",
|
|
"std::thread::sleep",
|
|
1,
|
|
"external control-plane polling",
|
|
),
|
|
timing(
|
|
"myelin-e2e-fuzz",
|
|
"harness::control::<impl harness::ClusterHarness>::wait_execution",
|
|
"std::thread::sleep",
|
|
1,
|
|
"external execution polling",
|
|
),
|
|
timing(
|
|
"myelin-e2e-fuzz",
|
|
"harness::convergence::<impl harness::ClusterHarness>::wait_for_dashboard",
|
|
"std::thread::sleep",
|
|
1,
|
|
"external dashboard readiness polling",
|
|
),
|
|
timing(
|
|
"myelin-e2e-fuzz",
|
|
"harness::control::<impl harness::ClusterHarness>::stop_orchestrator_injection",
|
|
"std::thread::sleep",
|
|
1,
|
|
"external orchestrator termination polling",
|
|
),
|
|
timing(
|
|
"myelin-e2e-fuzz",
|
|
"harness::control::<impl harness::ClusterHarness>::wait_for_no_workload_processes",
|
|
"std::thread::sleep",
|
|
1,
|
|
"external native-process census polling",
|
|
),
|
|
timing(
|
|
"myelin-e2e-fuzz",
|
|
"harness::ClusterHarness::assert_healthy",
|
|
"std::thread::sleep",
|
|
1,
|
|
"external resource cleanup polling",
|
|
),
|
|
timing(
|
|
"myelin-e2e-fuzz",
|
|
"resources::http_json",
|
|
"ureq::Request::timeout",
|
|
2,
|
|
"external HTTP request fuse",
|
|
),
|
|
timing(
|
|
"swactor-engine",
|
|
"common::wait_for",
|
|
"std::thread::sleep",
|
|
1,
|
|
"engine test completion fuse",
|
|
),
|
|
timing(
|
|
"swactor-engine",
|
|
"engine_interval_recurs",
|
|
"std::sync::mpsc::Receiver::recv_timeout",
|
|
1,
|
|
"engine test completion fuse",
|
|
),
|
|
timing(
|
|
"swactor-engine",
|
|
"engine_runs_without_an_ambient_tokio_runtime",
|
|
"std::sync::mpsc::Receiver::recv_timeout",
|
|
1,
|
|
"engine test completion fuse",
|
|
),
|
|
timing(
|
|
"swactor-engine",
|
|
"engine_timer_can_be_created_off_runtime",
|
|
"std::sync::mpsc::Receiver::recv_timeout",
|
|
1,
|
|
"engine test completion fuse",
|
|
),
|
|
timing(
|
|
"swactor-engine",
|
|
"engine_timer_fires",
|
|
"std::sync::mpsc::Receiver::recv_timeout",
|
|
1,
|
|
"engine test completion fuse",
|
|
),
|
|
timing(
|
|
"swactor-engine",
|
|
"idle_core_observes_late_external_work_within_idle_interval",
|
|
"std::thread::sleep",
|
|
1,
|
|
"real idle-driver timing contract",
|
|
),
|
|
timing(
|
|
"swactor-engine",
|
|
"spawned_supporting_work_runs",
|
|
"std::sync::mpsc::Receiver::recv_timeout",
|
|
1,
|
|
"engine test completion fuse",
|
|
),
|
|
timing(
|
|
"swactor-engine",
|
|
"stepping_blocking_work_runs_isolated",
|
|
"std::thread::sleep",
|
|
1,
|
|
"blocking worker completion poll",
|
|
),
|
|
timing(
|
|
"swactor-engine",
|
|
"stepping_blocking_work_runs_isolated::{closure#0}",
|
|
"std::thread::sleep",
|
|
1,
|
|
"blocking worker timing probe",
|
|
),
|
|
timing(
|
|
"swactor-process",
|
|
"drive_once",
|
|
"std::thread::sleep",
|
|
1,
|
|
"process integration driver pacing",
|
|
),
|
|
timing(
|
|
"swactor-process",
|
|
"supervisor::tests::collect_until_finished",
|
|
"std::thread::sleep",
|
|
1,
|
|
"process supervisor completion fuse",
|
|
),
|
|
timing(
|
|
"swactor-process",
|
|
"supervisor::tests::join_finished",
|
|
"std::thread::sleep",
|
|
1,
|
|
"process supervisor completion fuse",
|
|
),
|
|
timing(
|
|
"swactor-process",
|
|
"supervisor::tests::supervisor_stop_escalates_to_kill_after_deadline",
|
|
"std::thread::sleep",
|
|
1,
|
|
"process escalation timing contract",
|
|
),
|
|
timing(
|
|
"swactor-vastai",
|
|
"monitor::tests::loading_state_remains_slow_progress_before_terminal_evidence::{closure#0}",
|
|
"tokio::time::timeout",
|
|
1,
|
|
"provider test non-completion fuse",
|
|
),
|
|
timing(
|
|
"telemetry",
|
|
"hardware::tests::blocking_sampler_runs_sequentially_with_monotonic_sequences",
|
|
"std::sync::mpsc::Receiver::recv_timeout",
|
|
2,
|
|
"sampler test completion fuse",
|
|
),
|
|
timing(
|
|
"telemetry",
|
|
"submit_text_owned_queues_owned_string",
|
|
"telemetry::TelemetrySubscription::recv_timeout",
|
|
1,
|
|
"telemetry test observation fuse",
|
|
),
|
|
timing(
|
|
"xtask",
|
|
"demo::feed::properties::generated_supervisor_transitions_are_once_only_nonblocking_and_clean::{closure#1}",
|
|
"std::sync::mpsc::Receiver::recv_timeout",
|
|
1,
|
|
"blocking-provider test completion fuse",
|
|
),
|
|
timing(
|
|
"actor-lint-test-wait-pass",
|
|
"bounded_observation_wait_is_allowed",
|
|
"std::sync::mpsc::Receiver::recv_timeout",
|
|
1,
|
|
"compiler policy pass fixture",
|
|
),
|
|
timing(
|
|
"actor-lint-timing-overage",
|
|
"exceeds_registered_timing_budget",
|
|
"std::sync::mpsc::Receiver::recv_timeout",
|
|
1,
|
|
"compiler policy overage fixture",
|
|
),
|
|
timing(
|
|
"swactor-vastai",
|
|
"test_http::serve",
|
|
"std::net::TcpStream::set_read_timeout",
|
|
1,
|
|
"test HTTP request completion fuse",
|
|
),
|
|
timing(
|
|
"telemetry",
|
|
"carry_over_real_socket",
|
|
"std::net::UdpSocket::set_read_timeout",
|
|
1,
|
|
"live telemetry socket completion fuse",
|
|
),
|
|
timing(
|
|
"python",
|
|
"context::PyTestDataPlaneHost::run_contextual_process",
|
|
"std::thread::sleep",
|
|
1,
|
|
"debug test-host contextual output poll interval",
|
|
),
|
|
];
|