Got a ton of benchmarking and stress testing code spit out by the LLM. Did a not-so-thorough vetting, not ready for merge into master, but it mostly makes sense. Needs to actually fail on the stress tests, not anticipate failure and call it success. The benchmarks also need a more thorough going over, in order to validate that they make sense. Then we can work on improving our metrics.
299 lines
8.8 KiB
Rust
299 lines
8.8 KiB
Rust
//! Saturation stress tests - find where the runtime breaks.
|
|
//!
|
|
//! These tests intentionally push past limits to document failure modes.
|
|
|
|
use super::{BlackHole, Counter, Msg, Stress, StressResult};
|
|
use std::time::Duration;
|
|
use swactor::runtime::{Runtime, RuntimeConfig};
|
|
|
|
/// FIXME: does this even make sense to test?
|
|
/// Blast the router inbox until it overflows.
|
|
/// Documents: What happens when router can't keep up?
|
|
#[test]
|
|
#[cfg(feature = "stress")]
|
|
fn router_inbox_overflow() {
|
|
println!("\n>>> STRESS: Router Inbox Overflow");
|
|
|
|
let config = RuntimeConfig {
|
|
max_actors: 10,
|
|
router_max_messages: 100, // Tiny buffer
|
|
actor_max_messages: 1000,
|
|
num_threads: 1,
|
|
};
|
|
let runtime = Runtime::new(config);
|
|
let sink = runtime.spawn(BlackHole).unwrap();
|
|
|
|
// Blast messages without processing
|
|
let mut result = StressResult::new("router_inbox_overflow");
|
|
let start = std::time::Instant::now();
|
|
|
|
for _ in 0..10_000 {
|
|
result.operations += 1;
|
|
if runtime.send_to::<Msg>(sink, Msg).is_ok() {
|
|
result.successes += 1;
|
|
} else {
|
|
result.failures += 1;
|
|
}
|
|
}
|
|
|
|
result.duration = start.elapsed();
|
|
result.note(format!("Router buffer: 100, Messages sent: 10,000"));
|
|
result.note(format!(
|
|
"Expected: ~99% failure rate (buffer fills immediately)"
|
|
));
|
|
result.print();
|
|
|
|
// Verify we actually saw failures
|
|
assert!(result.failures > 0, "Expected router to reject messages");
|
|
println!(">>> PASS: Router correctly rejects messages when full\n");
|
|
}
|
|
|
|
/// FIXME: This test makes no sense until we make sure panics happen when
|
|
/// actor inbox buffers are full.
|
|
/// Blast a single actor's inbox until it overflows.
|
|
/// Documents: What happens when actor can't keep up?
|
|
#[test]
|
|
#[cfg(feature = "stress")]
|
|
fn actor_inbox_overflow() {
|
|
println!("\n>>> STRESS: Actor Inbox Overflow");
|
|
|
|
let config = RuntimeConfig {
|
|
max_actors: 10,
|
|
router_max_messages: 100_000, // Large router buffer
|
|
actor_max_messages: 100, // Tiny actor inbox
|
|
num_threads: 1,
|
|
};
|
|
let runtime = Runtime::new(config);
|
|
let sink = runtime.spawn(BlackHole).unwrap();
|
|
|
|
// Process router registration
|
|
runtime.tick();
|
|
|
|
// Now blast messages - router will accept them but actor inbox will fill
|
|
let mut sent = 0u64;
|
|
for _ in 0..10_000 {
|
|
if runtime.send_to::<Msg>(sink, Msg).is_ok() {
|
|
sent += 1;
|
|
}
|
|
// Tick occasionally to let router deliver
|
|
if sent % 100 == 0 {
|
|
runtime.tick();
|
|
}
|
|
}
|
|
|
|
// The router accepted messages, but many were dropped at actor inbox
|
|
// We can't easily count these drops from outside, but we can document the behavior
|
|
println!(" Router accepted {} messages", sent);
|
|
println!(" Actor inbox capacity: 100");
|
|
println!(" Note: Messages beyond inbox capacity are silently dropped");
|
|
println!(">>> This is a known limitation - bounded queues drop overflow\n");
|
|
}
|
|
|
|
/// FIXME: Does it make sense to have this as a test? Yes the runtime
|
|
/// fails if you try and spawn actors when the queue is full.
|
|
/// Spawn actors until the queue rejects.
|
|
/// Documents: What happens when actor queue fills?
|
|
#[test]
|
|
#[cfg(feature = "stress")]
|
|
fn actor_queue_overflow() {
|
|
println!("\n>>> STRESS: Actor Queue Overflow");
|
|
|
|
let config = RuntimeConfig {
|
|
max_actors: 100, // Small actor queue
|
|
router_max_messages: 10_000,
|
|
actor_max_messages: 100,
|
|
num_threads: 1,
|
|
};
|
|
let runtime = Runtime::new(config);
|
|
|
|
let mut result = StressResult::new("actor_queue_overflow");
|
|
let start = std::time::Instant::now();
|
|
|
|
// Try to spawn 500 actors into 100-slot queue
|
|
for _ in 0..500 {
|
|
result.operations += 1;
|
|
match runtime.spawn(BlackHole) {
|
|
Ok(_) => result.successes += 1,
|
|
Err(_) => result.failures += 1,
|
|
}
|
|
}
|
|
|
|
result.duration = start.elapsed();
|
|
result.note(format!("Queue capacity: 100, Spawn attempts: 500"));
|
|
result.note(format!(
|
|
"Expected: ~80% failure rate (queue fills after ~100)"
|
|
));
|
|
result.print();
|
|
|
|
// Note: Router also takes a slot, so we expect ~99 actors max
|
|
assert!(
|
|
result.successes <= 100,
|
|
"Spawned more actors than queue capacity"
|
|
);
|
|
assert!(result.failures > 0, "Expected spawn failures");
|
|
println!(">>> PASS: Actor queue correctly rejects when full\n");
|
|
}
|
|
|
|
/// FIXME: IS this actually testing what it should be?
|
|
/// Sustained overload - run at 2x capacity for extended period.
|
|
/// Documents: Does the system degrade gracefully or crash?
|
|
#[test]
|
|
#[cfg(feature = "stress")]
|
|
fn sustained_overload() {
|
|
println!("\n>>> STRESS: Sustained Overload");
|
|
|
|
let config = RuntimeConfig {
|
|
max_actors: 100,
|
|
router_max_messages: 1000,
|
|
actor_max_messages: 100,
|
|
num_threads: 1,
|
|
};
|
|
let runtime = Runtime::new(config);
|
|
|
|
// Spawn some actors
|
|
let mut actors = Vec::new();
|
|
for _ in 0..50 {
|
|
if let Ok(addr) = runtime.spawn(Counter::new()) {
|
|
actors.push(addr);
|
|
}
|
|
}
|
|
|
|
// Process registrations
|
|
for _ in 0..200 {
|
|
runtime.tick();
|
|
}
|
|
|
|
let result = Stress::new("sustained_overload")
|
|
.for_duration(Duration::from_secs(2))
|
|
.run(|| {
|
|
// Send to random actor
|
|
let idx = (std::time::Instant::now().elapsed().as_nanos() as usize) % actors.len();
|
|
let success = runtime.send_to::<Msg>(actors[idx], Msg).is_ok();
|
|
|
|
// Process some (but not all) - simulating overload
|
|
runtime.tick();
|
|
|
|
success
|
|
});
|
|
|
|
result.print();
|
|
println!(">>> System survived sustained overload without panic\n");
|
|
}
|
|
|
|
/// FIXME: The logic for timing recovery does not make sense
|
|
/// Burst traffic - idle to 100x normal, back to idle.
|
|
/// Documents: Recovery behavior after traffic spikes.
|
|
#[test]
|
|
#[cfg(feature = "stress")]
|
|
fn burst_traffic() {
|
|
println!("\n>>> STRESS: Burst Traffic");
|
|
|
|
let config = RuntimeConfig {
|
|
max_actors: 100,
|
|
router_max_messages: 10_000,
|
|
actor_max_messages: 1000,
|
|
num_threads: 1,
|
|
};
|
|
let runtime = Runtime::new(config);
|
|
let sink = runtime.spawn(Counter::new()).unwrap();
|
|
|
|
// Process registration
|
|
for _ in 0..10 {
|
|
runtime.tick();
|
|
}
|
|
|
|
let mut total_sent = 0u64;
|
|
let mut total_failed = 0u64;
|
|
|
|
// 5 burst cycles
|
|
for cycle in 0..5 {
|
|
// Burst: send 1000 messages as fast as possible
|
|
let mut burst_sent = 0;
|
|
let mut burst_failed = 0;
|
|
for _ in 0..1000 {
|
|
if runtime.send_to::<Msg>(sink, Msg).is_ok() {
|
|
burst_sent += 1;
|
|
} else {
|
|
burst_failed += 1;
|
|
}
|
|
}
|
|
total_sent += burst_sent;
|
|
total_failed += burst_failed;
|
|
|
|
// Recovery: process until queue is drained
|
|
let recovery_start = std::time::Instant::now();
|
|
for _ in 0..5000 {
|
|
runtime.tick();
|
|
}
|
|
let recovery_time = recovery_start.elapsed();
|
|
|
|
println!(
|
|
" Cycle {}: sent={}, failed={}, recovery={:?}",
|
|
cycle + 1,
|
|
burst_sent,
|
|
burst_failed,
|
|
recovery_time
|
|
);
|
|
}
|
|
|
|
println!(
|
|
"\n Total sent: {}, Total failed: {}",
|
|
total_sent, total_failed
|
|
);
|
|
println!(">>> Burst traffic test complete\n");
|
|
}
|
|
|
|
/// Find the message drop cliff - at what load factor do drops spike?
|
|
#[test]
|
|
#[cfg(feature = "stress")]
|
|
fn message_drop_curve() {
|
|
println!("\n>>> STRESS: Message Drop Curve");
|
|
println!(" Testing drop rate at various load factors...\n");
|
|
|
|
// Test at different load factors (messages per tick)
|
|
for msgs_per_tick in [1, 5, 10, 20, 50, 100] {
|
|
let config = RuntimeConfig {
|
|
max_actors: 10,
|
|
router_max_messages: 1000,
|
|
actor_max_messages: 500,
|
|
num_threads: 1,
|
|
};
|
|
let runtime = Runtime::new(config);
|
|
let sink = runtime.spawn(BlackHole).unwrap();
|
|
|
|
// Warmup
|
|
for _ in 0..10 {
|
|
runtime.tick();
|
|
}
|
|
|
|
let mut sent = 0u64;
|
|
let mut failed = 0u64;
|
|
|
|
// Run for fixed iterations
|
|
for _ in 0..100 {
|
|
// Send burst
|
|
for _ in 0..msgs_per_tick {
|
|
if runtime.send_to::<Msg>(sink, Msg).is_ok() {
|
|
sent += 1;
|
|
} else {
|
|
failed += 1;
|
|
}
|
|
}
|
|
// Process one tick
|
|
runtime.tick();
|
|
}
|
|
|
|
let drop_rate = if sent + failed > 0 {
|
|
(failed as f64 / (sent + failed) as f64) * 100.0
|
|
} else {
|
|
0.0
|
|
};
|
|
|
|
println!(
|
|
" msgs/tick={:3} sent={:5} failed={:5} drop_rate={:.1}%",
|
|
msgs_per_tick, sent, failed, drop_rate
|
|
);
|
|
}
|
|
|
|
println!("\n>>> Message drop curve test complete\n");
|
|
}
|