From 3e504d3969af61ba30c676775c6222a1ad4fb0c5 Mon Sep 17 00:00:00 2001 From: Zachery Aaron Shores-Chmielewski Date: Tue, 27 Jan 2026 20:01:35 +0700 Subject: [PATCH 1/9] feat(wip): add to design pages Ran through a few passes of the 'happy path' with the debugger, ran flamegraph analysis: we are very much overdoing the atomic ring buffer channels. They may be fast for concurrency across threads, but we don't necessarily need that for every occasion. --- .gitignore | 1 + DESIGN.md | 32 +++++--------------------------- TODOs.md | 44 ++++++++++++++++++++++++++++++++++++++++++++ src/actor.rs | 1 + src/router.rs | 8 ++++++++ 5 files changed, 59 insertions(+), 27 deletions(-) create mode 100644 TODOs.md diff --git a/.gitignore b/.gitignore index ea8c4bf..5f9ff5d 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /target +.vscode/ \ No newline at end of file diff --git a/DESIGN.md b/DESIGN.md index c43a407..93f57e6 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -6,6 +6,11 @@ memory allocator and threading provided by the rust standard library. We are not building a new erlang/BEAM. Minimal feature set means spawning actor processes, not having supervisiors, lots of process monitoring tools, prempting, etc. +# FIXME +This is slightly obsolete. Was necessary in order to concentrate on getting the basic skeleton up, now it's a distraction and unreliable. +After getting the benches/etc finished, move this into an `ARCHITECTURE.md` file, have it make sense. + + ## Actor model An actor has: @@ -48,30 +53,3 @@ The router is the engine for message delivery. It posesses: - Its own inbox: The router possesses its own mpsc queue where references to messages are stored. The router will process this queue by dereferencing and writing directly into the recipient's inbox buffer. -### Misc - -A means of providing an emergency overflow without adding much more code complexity. The mutex means -this will not be `no_std` however. - -```rust -struct HybridChannel { - // Start with lock-free ring buffer - ring: AtomicRingBuffer, - - // When full, spill into a Mutex> - overflow: parking_lot::Mutex>, - - // Track overflow frequency to resize ring proactively - overflow_count: AtomicUsize, -} - -impl HybridChannel { - fn push(&self, value: T) { - if self.ring.push(value).is_err() { - self.overflow.lock().push_back(value); - self.overflow_count.fetch_add(1, Relaxed); - // Optionally: if overflow_count > threshold, grow ring - } - } -} -``` diff --git a/TODOs.md b/TODOs.md new file mode 100644 index 0000000..4e9885c --- /dev/null +++ b/TODOs.md @@ -0,0 +1,44 @@ +### Profiling and Benchmarking + +- Research as to modern art on benching and profiling + - Implement an MVP here. + - Bench/profile against a suite of tests selected for generality across actor framework usecases + +- Identify hot paths and bottlenecks + - e.g. pretty sure the router is a major bottleneck, what else + - follow through the entire message cycle: + Parent process -> Convert to swactor::Message/Envelope -> Router -> Delivery -> Processing -> etc. + Identify every small detail on which you may be able to improve, any unneeded processing or branching + +- (optional) Visualization tools: + - make some pretty stuff for tracing messages, actor activity, router activity, etc. + +### Usage + +- After benching and profiling, cleaning up the most egregious wrongdoings we will: + - actually implement our own projects in the framework, ones I actually find useful personally + +- Optimization pipeline: + - Once we have well-established benches and profiles for general cases, build a set of tools that can auto-optimize + for given use cases. Tuning, for example, the channel buffers, router behavior, message consumption behavior, etc. + +### Chores + +- go over all the FIXMEs littered about. Add comments. +- add misc features as they come up. Prefer tools for understanding execution flows, visualizing flows, and adding +robustness, over ergonomics. Better to be slightly clunky but fast and optimized, than vice versa. + +### Far future + +- make language bindings. e.g., an npm package, python bindings, etc. + +### Optimization + +- Localize actors and inboxes: + - Because the entire runtime is message driven, the happy path must be fast. Even lock free, when we have to go through + several calls of an atomic ring buffer in order to process a single message, its unnecessary. + Parent process -> router -> actor -> router -> inbox -> parent process; every transfer going through an atomic buffer. + + - to do this, design heavily around a localized worker thread. Actors on a working thread should have their inbox localized, they + should be 'sticky' to that thread (FILO queue?), and we should route messages based on core locality. Future optimizations can include + a tunable algorithm that puts actors that frequently communicate together on the same thread. \ No newline at end of file diff --git a/src/actor.rs b/src/actor.rs index 3450bee..c3938ea 100644 --- a/src/actor.rs +++ b/src/actor.rs @@ -87,6 +87,7 @@ where fn tick(&mut self, ctx: &Runtime) { // TODO: WATERLEVEL is hard coded, and so is this message handling scheme. We should // make it so both are more flexible, with sane defaults. + // FIXME: lots of indirection just to get length on a hot path let total_messages = self.inbox.len(); let messages_to_process = if total_messages < WATERLEVEL { total_messages diff --git a/src/router.rs b/src/router.rs index 55296f0..34157f4 100644 --- a/src/router.rs +++ b/src/router.rs @@ -19,6 +19,14 @@ pub(crate) trait SenderT: Send + Sync { impl SenderT for Sender { fn try_send(&self, envelope: Envelope) { if let Some(msg) = envelope.downcast_ref::() { + // FIXME: we are directly cloning the contents of the Arc pointer here + // Do we want to? Should we provide another way? + // + // The standard concept of an actor has message and state + // isolation, so we should leave this as is. However, we should + // make it clear and obvious this pathway is the heavy, contained + // pathway, and include a shared memory pathway for logic that + // may need it. let _ = Sender::try_send(self, msg.clone()); } } -- 2.45.2 From ed26f0c0800394fc2561befa9d5cfba6c67be5fe Mon Sep 17 00:00:00 2001 From: Zachery Aaron Shores-Chmielewski Date: Fri, 30 Jan 2026 22:52:38 +0700 Subject: [PATCH 2/9] feat(wip): rewrite Our first implementation is incredibly slow, and failed to scale with increased threads. In order to fix this, we are implementing cache-aware batching of message processing, and an epoch-based actor rebalancing using shard pools. --- CACHE.md | 1050 ++++++++++++++++++++++++++++ Cargo.lock | 1 + Cargo.toml | 1 + WORKER.md | 113 +++ src/{channel.rs => channel/mod.rs} | 2 + src/channel/spsc.rs | 198 ++++++ src/lib.rs | 1 + src/worker.rs | 46 ++ 8 files changed, 1412 insertions(+) create mode 100644 CACHE.md create mode 100644 WORKER.md rename src/{channel.rs => channel/mod.rs} (99%) create mode 100644 src/channel/spsc.rs create mode 100644 src/worker.rs diff --git a/CACHE.md b/CACHE.md new file mode 100644 index 0000000..bbb982e --- /dev/null +++ b/CACHE.md @@ -0,0 +1,1050 @@ +# Cache Locality Design Guidelines for Actor Runtimes + +## Table of Contents +1. [Foundational Concepts](#foundational-concepts) +2. [Actor Data Structure Layout](#actor-data-structure-layout) +3. [Mailbox Design for Cache Efficiency](#mailbox-design-for-cache-efficiency) +4. [Memory Allocation Strategies](#memory-allocation-strategies) +5. [Scheduler Design for Locality](#scheduler-design-for-locality) +6. [NUMA-Aware Considerations](#numa-aware-considerations) +7. [Post-Design Tuning Strategies](#post-design-tuning-strategies) +8. [Framework-Specific Examples](#framework-specific-examples) + +--- + +## Foundational Concepts + +### Cache Hierarchy Understanding + +Before designing your actor runtime, you must internalize how modern CPUs access memory: + +``` +┌─────────────────────────────────────────────────────────────┐ +│ CPU Core │ +│ ┌─────────┐ │ +│ │ Registers│ ~0.5 cycles │ +│ └────┬────┘ │ +│ ▼ │ +│ ┌─────────┐ │ +│ │ L1 Cache│ 32-64KB, ~4 cycles, per-core │ +│ └────┬────┘ │ +│ ▼ │ +│ ┌─────────┐ │ +│ │ L2 Cache│ 256KB-1MB, ~12 cycles, per-core │ +│ └────┬────┘ │ +└───────┼─────────────────────────────────────────────────────┘ + ▼ + ┌─────────┐ + │ L3 Cache│ 8-64MB, ~40 cycles, shared across cores + └────┬────┘ + ▼ + ┌─────────┐ + │ RAM │ ~100-300 cycles + └─────────┘ +``` + +**Key insight**: A cache miss at L1 that propagates to RAM can be 50-100x slower than a cache hit. Your actor runtime design should minimize these misses. + +### Cache Lines: The Atomic Unit of Memory Transfer + +Modern CPUs don't transfer individual bytes—they transfer **cache lines**, typically 64 bytes on x86-64 and ARM (though 128 bytes on some ARM implementations like Apple M-series for certain operations). + +```cpp +// On most x86-64 systems: +constexpr size_t CACHE_LINE_SIZE = 64; + +// C++17 provides portable constants: +#include +// std::hardware_destructive_interference_size (typically 64) +// std::hardware_constructive_interference_size (typically 64) +``` + +This has two critical implications: + +1. **Spatial Locality**: When you access one byte, the CPU fetches 63 more. Design your data structures so related data lives nearby. + +2. **False Sharing**: When two threads write to different variables on the same cache line, the CPU invalidates that line for all cores, causing severe performance degradation. + +--- + +## Actor Data Structure Layout + +### Principle 1: Keep Hot Data Together + +The most frequently accessed data during message processing should fit within one or two cache lines. Here's how CAF (C++ Actor Framework) approaches this: + +```cpp +// Conceptual layout inspired by CAF's actor design +// CAF actors are designed to be only a few hundred bytes + +struct alignas(64) Actor { + // === CACHE LINE 1: Critical scheduling data === + std::atomic state; // 4 bytes: running/waiting/done + uint32_t flags; // 4 bytes: various flags + MailboxPtr mailbox; // 8 bytes: pointer to mailbox + SchedulerPtr home_scheduler; // 8 bytes: affinity hint + ActorId id; // 8 bytes: unique identifier + RefCount ref_count; // 8 bytes: reference counting + BehaviorPtr current_behavior; // 8 bytes: message handler + // Remaining: 16 bytes for future use + char padding1[16]; + + // === CACHE LINE 2: Less frequently accessed === + ActorPtr parent; // 8 bytes: supervision + ChildList children; // 16 bytes: supervised actors + ErrorHandler on_error; // 8 bytes: error handling + // ... additional metadata +}; + +static_assert(offsetof(Actor, parent) == 64, + "Second cache line should start at offset 64"); +``` + +**Why this matters**: When the scheduler decides whether to run an actor, it only needs the first cache line. The message handler only needs the first two cache lines for most operations. + +### Principle 2: Separate Hot and Cold Data + +Actors often have data that's accessed frequently (hot) and data accessed rarely (cold). Separating these prevents cold data from evicting hot data from cache: + +```cpp +// HOT: Accessed on every message dispatch +struct ActorHotData { + alignas(64) std::atomic mailbox_count; + BehaviorFunction* behavior; + void* user_state; // Pointer to actual actor state + SchedulerWorker* affinity; +}; + +// COLD: Accessed only during lifecycle events or errors +struct ActorColdData { + std::string name; + ActorAddress parent; + std::vector children; + std::function error_handler; + MonitorList monitors; + LinkList links; + CreationTimestamp created_at; +}; + +struct Actor { + ActorHotData hot; // First cache line(s) + ActorColdData* cold; // Pointer to cold data, allocated separately +}; +``` + +### Principle 3: Prevent False Sharing in Scheduler Structures + +When you have per-worker data structures, false sharing is your enemy. Here's the pattern used in work-stealing schedulers: + +```cpp +// BAD: Dense packing causes false sharing +struct BadWorkerArray { + std::atomic work_count[NUM_WORKERS]; // All on adjacent cache lines! +}; + +// GOOD: Cache-line padding prevents false sharing +struct alignas(64) PaddedCounter { + std::atomic count; + char padding[64 - sizeof(std::atomic)]; +}; + +struct GoodWorkerArray { + PaddedCounter work_count[NUM_WORKERS]; // Each on its own cache line +}; +``` + +The Rust ecosystem provides `cache-padded` (now deprecated but instructive): + +```rust +// From the cache-padded crate (conceptually) +use std::sync::atomic::AtomicUsize; + +// Concurrent queue with cache-padded indices +struct Queue { + head: CachePadded, // Own cache line + tail: CachePadded, // Own cache line + buffer: *mut T, +} +``` + +CAF's work-stealing scheduler applies this pattern to its worker queues, ensuring that each worker's deque head/tail pointers don't share cache lines with other workers. + +--- + +## Mailbox Design for Cache Efficiency + +### MPSC Queue Optimizations (Multiple Producers, Single Consumer) + +Actor mailboxes are typically MPSC queues. The design significantly impacts cache performance: + +```cpp +// Intrusive linked-list approach (good for variable message sizes) +// Used conceptually by many actor frameworks + +struct Message { + Message* next; // 8 bytes + MessageType type; // 4 bytes + uint32_t payload_size; // 4 bytes + alignas(16) char payload[]; // Flexible array member +}; + +class IntrusiveMailbox { + // Producer side (multiple threads write here) + alignas(64) std::atomic tail; + + // Consumer side (single thread reads here) + alignas(64) Message* head; + + // Sentinel node to simplify empty-check + Message stub; +public: + void push(Message* msg) { + msg->next = nullptr; + Message* prev = tail.exchange(msg, std::memory_order_acq_rel); + prev->next = msg; // Linearization point + } + + Message* pop() { + Message* h = head; + Message* next = h->next; + if (next) { + head = next; + // Return the message that was after stub, or actual message + return (h == &stub) ? pop() : h; + } + return nullptr; + } +}; +``` + +**Cache analysis**: +- `tail` is on its own cache line (written by producers) +- `head` is on its own cache line (written by consumer) +- This prevents false sharing between producers and consumer + +### Bounded Ring Buffer Approach (better cache locality for small messages) + +```cpp +template +class alignas(64) BoundedMailbox { + static_assert((Capacity & (Capacity - 1)) == 0, + "Capacity must be power of 2"); + + // Producer state - own cache line + alignas(64) std::atomic write_pos{0}; + + // Consumer state - own cache line + alignas(64) size_t read_pos{0}; + size_t cached_write{0}; // Cached write_pos to reduce atomic reads + + // Buffer - contiguous for spatial locality + alignas(64) T buffer[Capacity]; + +public: + bool try_push(const T& item) { + size_t wp = write_pos.load(std::memory_order_relaxed); + size_t next = (wp + 1) & (Capacity - 1); + + // Check if full (would need to load read_pos from consumer) + if (next == read_pos) return false; + + buffer[wp] = item; + write_pos.store(next, std::memory_order_release); + return true; + } + + bool try_pop(T& item) { + if (read_pos == cached_write) { + cached_write = write_pos.load(std::memory_order_acquire); + if (read_pos == cached_write) return false; + } + + item = buffer[read_pos]; + read_pos = (read_pos + 1) & (Capacity - 1); + return true; + } +}; +``` + +**Cache benefits**: +- Sequential access patterns maximize hardware prefetching +- Bounded size means working set fits in cache +- The `cached_write` optimization reduces atomic operations + +### CAF's Dual-Queue Strategy + +CAF uses a double-ended queue with interesting cache locality properties: + +```cpp +// Simplified version of CAF's mailbox concept +class CafStyleMailbox { + // Internal jobs (from same worker) - LIFO for cache locality + alignas(64) std::atomic internal_head; + spinlock internal_lock; + + // External jobs (from other workers) - FIFO for fairness + alignas(64) std::atomic external_tail; + spinlock external_lock; + +public: + // Called by the worker that owns this actor + void internal_enqueue(Job* job) { + // LIFO: new jobs go to front + // This maximizes temporal locality - recently touched + // actor state is still warm in cache + std::lock_guard lock(internal_lock); + job->next = internal_head.load(std::memory_order_relaxed); + internal_head.store(job, std::memory_order_release); + } + + // Called by other workers sending messages + void external_enqueue(Job* job) { + // FIFO: maintains message ordering guarantees + std::lock_guard lock(external_lock); + job->next = nullptr; + Job* prev = external_tail.exchange(job, std::memory_order_acq_rel); + if (prev) prev->next = job; + } +}; +``` + +The LIFO internal queue is crucial: when an actor sends a message to another actor and that message creates a response, processing the response immediately means the original actor's state is still cache-hot. + +--- + +## Memory Allocation Strategies + +### Arena Allocators: The Foundation of Cache-Friendly Allocation + +Standard `malloc` has several cache-hostile properties: +1. Metadata overhead (typically 8-16 bytes per allocation) +2. Memory fragmentation scatters related objects +3. System call overhead for large allocations + +Arena allocators solve these problems: + +```cpp +class Arena { + char* buffer; + size_t capacity; + size_t offset; + +public: + Arena(size_t size) + : buffer(static_cast(aligned_alloc(64, size))) + , capacity(size) + , offset(0) {} + + void* allocate(size_t size, size_t alignment = alignof(std::max_align_t)) { + // Align the offset + size_t aligned_offset = (offset + alignment - 1) & ~(alignment - 1); + + if (aligned_offset + size > capacity) { + return nullptr; // Or grow/chain arenas + } + + void* ptr = buffer + aligned_offset; + offset = aligned_offset + size; + return ptr; + } + + void reset() { offset = 0; } // "Free" everything at once + + ~Arena() { free(buffer); } +}; +``` + +### Per-Actor Arenas + +Each actor can have its own arena, ensuring that: +1. Actor state and its heap-allocated data are contiguous +2. GC is simplified (just reset the arena when actor dies) +3. No cross-thread allocation/deallocation (thread-local arenas) + +```cpp +class ActorArena { + static constexpr size_t INITIAL_SIZE = 4096; // One page + + struct Block { + Block* next; + size_t size; + size_t used; + alignas(16) char data[]; + }; + + Block* current; + Block* blocks; // Linked list for cleanup + +public: + void* allocate(size_t size) { + size = (size + 15) & ~15; // Align to 16 bytes + + if (current->used + size > current->size) { + grow(size); + } + + void* ptr = current->data + current->used; + current->used += size; + return ptr; + } + + // For actor-local allocations that follow message processing + template + T* create(Args&&... args) { + void* mem = allocate(sizeof(T)); + return new (mem) T(std::forward(args)...); + } +}; +``` + +### Message Pool Allocators + +Messages often have predictable sizes. Pool allocators exploit this: + +```cpp +template +class MessagePool { + static_assert(SlotSize >= sizeof(void*), "Slot must fit a pointer"); + + struct alignas(64) Block { + Block* next; + char slots[SlotSize * SlotsPerBlock]; + }; + + Block* blocks = nullptr; + void* free_list = nullptr; + +public: + void* allocate() { + if (!free_list) { + grow(); + } + void* slot = free_list; + free_list = *static_cast(slot); + return slot; + } + + void deallocate(void* ptr) { + *static_cast(ptr) = free_list; + free_list = ptr; + } + +private: + void grow() { + Block* b = new Block; + b->next = blocks; + blocks = b; + + // Thread all slots through free list + for (size_t i = 0; i < SlotsPerBlock; ++i) { + void* slot = b->slots + i * SlotSize; + *static_cast(slot) = free_list; + free_list = slot; + } + } +}; + +// Usage: Pool for common message sizes +MessagePool<64> small_messages; // ≤48 byte payload +MessagePool<256> medium_messages; // ≤240 byte payload +MessagePool<1024> large_messages; // ≤1008 byte payload +``` + +### Pony's Per-Actor Heap Approach + +Pony takes an extreme approach: each actor has its own heap, and the garbage collector (ORCA) runs per-actor without stop-the-world pauses: + +``` +// Pony's conceptual memory model +Actor { + local_heap: Heap, // Only this actor allocates here + reference_counts: Map, // Track foreign references + message_queue: Queue, // Incoming messages +} + +// Key insight: GC only runs when actor is not executing a behavior +// This means: +// 1. No stack to scan (no stack map needed) +// 2. No safepoints required +// 3. No synchronization with other actors during GC +``` + +--- + +## Scheduler Design for Locality + +### Work-Stealing with Locality Awareness + +The basic work-stealing algorithm is cache-oblivious. Here's how to make it cache-aware: + +```cpp +class LocalityAwareScheduler { + struct alignas(64) WorkerState { + std::deque local_queue; + std::atomic is_active{true}; + uint32_t numa_node; + uint32_t core_id; + + // Steal victims ordered by locality (nearest first) + std::vector steal_order; + }; + + std::vector workers; + +public: + void initialize_steal_order(uint32_t worker_id) { + WorkerState& w = workers[worker_id]; + + // Build steal order: same NUMA node first, then others + for (uint32_t i = 0; i < workers.size(); ++i) { + if (i == worker_id) continue; + + if (workers[i].numa_node == w.numa_node) { + // Same NUMA node - insert at front + w.steal_order.insert(w.steal_order.begin(), i); + } else { + // Different NUMA node - append at end + w.steal_order.push_back(i); + } + } + } + + Actor* try_steal(uint32_t worker_id) { + WorkerState& w = workers[worker_id]; + + // Try stealing in locality order + for (uint32_t victim_id : w.steal_order) { + WorkerState& victim = workers[victim_id]; + + // Steal from back (FIFO) to maintain parent-child locality + if (!victim.local_queue.empty()) { + Actor* stolen = victim.local_queue.back(); + victim.local_queue.pop_back(); + return stolen; + } + } + return nullptr; + } +}; +``` + +### CAF's Scheduler Design + +CAF's scheduler uses work-stealing with specific optimizations for cache locality: + +```cpp +// From CAF's conceptual design +class CafScheduler { + // Each worker has its own deque + // Internal enqueue: LIFO (front) - maximizes cache reuse + // External enqueue: FIFO (back) - fair ordering + // Stealing: FIFO (back) - steals "cold" work + + void worker_loop(WorkerId id) { + while (running) { + Actor* actor = try_get_local(id); + + if (!actor) { + actor = try_steal_from_random(id); + } + + if (actor) { + // Process messages until actor blocks or quota exhausted + run_actor(actor); + + // If actor has more work, re-enqueue locally (LIFO) + // This keeps hot actors on the same core + if (actor->has_pending_messages()) { + internal_enqueue(id, actor); + } + } else { + // No work available - sleep briefly + sleep_or_poll(id); + } + } + } +}; +``` + +### Affinity-Based Scheduling + +For maximum cache efficiency, actors can have "home" workers: + +```cpp +struct Actor { + std::atomic home_worker{NO_AFFINITY}; + + // ... other fields +}; + +class AffinityScheduler { + void enqueue(Actor* actor, WorkerId sender) { + WorkerId home = actor->home_worker.load(std::memory_order_relaxed); + + if (home == NO_AFFINITY) { + // First run: assign to sender's worker for initial locality + actor->home_worker.store(sender, std::memory_order_relaxed); + workers[sender].enqueue(actor); + } else { + // Subsequent runs: prefer home worker + workers[home].enqueue(actor); + } + } + + // Periodically rebalance to prevent hot spots + void rebalance() { + // Move actors with high message rates to dedicated workers + // Consider communication patterns: actors that communicate + // frequently should be on the same core/NUMA node + } +}; +``` + +--- + +## NUMA-Aware Considerations + +### Understanding NUMA Topology + +On multi-socket systems, memory access time depends on which CPU socket allocated the memory: + +``` +┌─────────────────┐ QPI/UPI ┌─────────────────┐ +│ Socket 0 │◄────────────────►│ Socket 1 │ +│ ┌───────────┐ │ │ ┌───────────┐ │ +│ │ 8 Cores │ │ │ │ 8 Cores │ │ +│ └───────────┘ │ │ └───────────┘ │ +│ ┌───────────┐ │ │ ┌───────────┐ │ +│ │ Local RAM │ │ │ │ Local RAM │ │ +│ │ 32GB │ │ │ │ 32GB │ │ +│ └───────────┘ │ │ └───────────┘ │ +└─────────────────┘ └─────────────────┘ + +Local memory access: ~100ns +Remote memory access: ~300ns (3x slower!) +``` + +### NUMA-Aware Actor Placement + +```cpp +class NumaAwareScheduler { + struct NumaNode { + std::vector workers; + Arena* local_arena; // Memory allocated on this node + std::atomic actor_count{0}; + }; + + std::vector nodes; + +public: + void spawn_actor(ActorFactory factory, NumaHint hint = HINT_LOCAL) { + int target_node; + + switch (hint) { + case HINT_LOCAL: + target_node = current_numa_node(); + break; + case HINT_NEAR_PARENT: + target_node = parent_actor->numa_node; + break; + case HINT_LEAST_LOADED: + target_node = find_least_loaded_node(); + break; + } + + // Allocate actor memory on target NUMA node + NumaNode& node = nodes[target_node]; + void* mem = node.local_arena->allocate(sizeof(Actor)); + Actor* actor = new (mem) Actor(factory); + actor->numa_node = target_node; + + // Assign to a worker on the same node + WorkerId worker = node.workers[round_robin++ % node.workers.size()]; + actor->home_worker = worker; + } + + // Work stealing prefers same-node victims + Actor* steal(WorkerId thief) { + int thief_node = worker_to_node[thief]; + + // First: try workers on same NUMA node + for (WorkerId victim : nodes[thief_node].workers) { + if (Actor* a = try_steal_from(victim)) { + return a; + } + } + + // Then: try other nodes (expensive - remote memory!) + for (int n = 0; n < nodes.size(); ++n) { + if (n == thief_node) continue; + for (WorkerId victim : nodes[n].workers) { + if (Actor* a = try_steal_from(victim)) { + return a; + } + } + } + + return nullptr; + } +}; +``` + +### Research-Backed NUMA Strategies + +The paper "Work-Stealing, Locality-Aware Actor Scheduling" by Barghi et al. identifies key strategies: + +1. **Hub Actor Identification**: Actors that spawn many short-lived children should keep those children on the same NUMA node. + +2. **Communication Locality**: Actors that frequently message each other should be co-located. + +3. **Hierarchical Stealing**: Steal from local workers first, then socket-local, then remote. + +```cpp +// Implementing hierarchical stealing based on the research +class HierarchicalStealer { + // Topology levels: Core → L3 Cache Group → Socket → System + struct Level { + std::vector peers; + double steal_probability; // Higher for closer levels + }; + + std::vector> topology; // [worker][level] + + Actor* steal(WorkerId thief) { + for (const Level& level : topology[thief]) { + // Probabilistic stealing at each level + if (random() < level.steal_probability) { + WorkerId victim = level.peers[random() % level.peers.size()]; + if (Actor* a = try_steal_from(victim)) { + return a; + } + } + } + return nullptr; + } +}; +``` + +--- + +## Post-Design Tuning Strategies + +### Profiling for Cache Misses + +Once your runtime is working, these tools help identify cache problems: + +```bash +# Linux perf for cache miss analysis +perf stat -e cache-references,cache-misses,L1-dcache-load-misses \ + ./your_actor_runtime + +# Detailed cache analysis +perf record -e cache-misses ./your_actor_runtime +perf report + +# NUMA statistics +numastat -p $(pgrep your_actor_runtime) +``` + +### Tunable Parameters + +Design your runtime with these tunable knobs: + +```cpp +struct RuntimeConfig { + // Scheduler tuning + size_t messages_per_actor_run = 10; // Batch size before yielding + size_t steal_batch_size = 1; // How many actors to steal + double steal_probability = 0.5; // Work-stealing aggressiveness + + // Memory tuning + size_t actor_arena_initial = 4096; // Initial arena size + size_t actor_arena_max = 1024 * 1024; // Maximum before GC + size_t message_pool_slots = 1024; // Pool allocator size + + // Cache tuning + size_t prefetch_distance = 3; // Mailbox prefetch depth + bool enable_numa_awareness = true; // NUMA-local allocation + bool enable_affinity = true; // Worker-actor affinity +}; +``` + +### Prefetching Strategies + +When you know you'll access data soon, tell the CPU: + +```cpp +void process_mailbox(Actor* actor) { + Message* current = actor->mailbox.head; + + while (current) { + Message* next = current->next; + + // Prefetch next message while processing current + if (next) { + __builtin_prefetch(next, 0, 3); // Read, high temporal locality + __builtin_prefetch(next->payload, 0, 3); + } + + process_message(actor, current); + current = next; + } +} +``` + +### Batch Processing for Cache Warmth + +Process multiple messages while actor state is cache-hot: + +```cpp +void run_actor(Actor* actor, size_t max_messages = 10) { + auto& behavior = actor->current_behavior; + + // Keep processing while mailbox non-empty and quota not exhausted + size_t processed = 0; + while (processed < max_messages) { + Message* msg = actor->mailbox.try_pop(); + if (!msg) break; + + // Actor state is cache-hot from previous iteration + behavior->handle(actor, msg); + ++processed; + } + + // Record for tuning: if we hit quota, actor is "hot" + if (processed == max_messages) { + actor->hot_count++; + } +} +``` + +### Adaptive Granularity (ElasticActor Approach) + +Research shows that optimal batch size depends on workload: + +```cpp +class AdaptiveScheduler { + struct ActorStation { + std::vector actors; + Mailbox shared_mailbox; + + // Actors in same station share a mailbox + // Coarser granularity = better cache utilization + // Finer granularity = better parallelism + }; + + std::vector stations; + std::atomic station_count; + + void adjust_granularity() { + double avg_latency = measure_message_latency(); + double target_latency = 1000; // 1 microsecond target + + if (avg_latency > target_latency * 1.5) { + // Latency too high: split stations for more parallelism + split_hottest_station(); + } else if (avg_latency < target_latency * 0.5) { + // Latency low: merge stations for better cache use + merge_coldest_stations(); + } + } +}; +``` + +--- + +## Framework-Specific Examples + +### CAF (C++ Actor Framework) + +CAF's key cache optimizations: + +```cpp +// 1. Minimal actor footprint (~240 bytes) +// From CAF's design philosophy: +// "CAF actors consist of only a few hundred bytes" + +// 2. Work-stealing with LIFO local queuing +// internal_enqueue adds to front (LIFO) for cache locality +// external_enqueue adds to back (FIFO) for fairness + +// 3. Type-safe messaging reduces runtime overhead +template +void typed_send(typed_actor& receiver, Args... args) { + // Compile-time type checking eliminates runtime type dispatch + // Smaller message headers, better cache utilization +} + +// 4. Configurable scheduler +auto cfg = actor_system_config{} + .set("caf.scheduler.max-threads", 8) + .set("caf.scheduler.policy", "stealing"); // or "sharing" +``` + +### Actix (Rust) + +Actix leverages Rust's ownership for cache-friendly design: + +```rust +use actix::prelude::*; + +// Actors own their state - no shared mutable state +struct MyActor { + // State is exclusively owned, always cache-local to processing + counter: u64, + buffer: Vec, // Can pre-allocate for known workloads +} + +impl Actor for MyActor { + type Context = Context; + + fn started(&mut self, ctx: &mut Self::Context) { + // Set mailbox capacity to control memory usage + ctx.set_mailbox_capacity(16); // Default is 16 + } +} + +// Messages are moved, not copied - zero-copy when possible +struct DataMessage { + payload: Vec, // Ownership transferred, not cloned +} + +impl Handler for MyActor { + type Result = (); + + fn handle(&mut self, msg: DataMessage, _ctx: &mut Context) { + // msg.payload is now owned by this actor + // No cache invalidation from other threads + self.buffer = msg.payload; + } +} +``` + +### Pony + +Pony's extreme approach to cache locality: + +```pony +// Each actor has isolated heap - perfect cache locality for GC +actor Counter + var _count: U64 = 0 + + // All state is actor-local, always cache-hot when processing + be increment() => + _count = _count + 1 + + be get(main: Main) => + // Sending is zero-copy for immutable data + main.print(_count) + +// Reference capabilities ensure no data races +// This allows GC without read/write barriers +// GC runs only between behaviors - no stack scanning needed +``` + +Key Pony cache insights: +- Per-actor heaps mean allocation is always thread-local +- No stop-the-world GC means no cache thrashing from GC threads +- Zero-copy messaging through capability-based types + +--- + +## Benchmarking Your Optimizations + +### Micro-benchmarks + +```cpp +// Message passing latency +void benchmark_mailbox_latency() { + Actor sender, receiver; + auto start = high_resolution_clock::now(); + + for (int i = 0; i < 1'000'000; ++i) { + sender.send(receiver, PingMessage{}); + receiver.process_one(); + } + + auto end = high_resolution_clock::now(); + auto ns_per_message = duration_cast(end - start).count() / 1'000'000; + std::cout << "Latency: " << ns_per_message << " ns/msg\n"; +} + +// Actor creation throughput +void benchmark_actor_creation() { + auto start = high_resolution_clock::now(); + + for (int i = 0; i < 100'000; ++i) { + auto actor = spawn(); + // Actor immediately becomes garbage + } + + auto end = high_resolution_clock::now(); + auto actors_per_sec = 100'000.0 / duration_cast(end - start).count(); + std::cout << "Creation: " << actors_per_sec << " actors/sec\n"; +} +``` + +### Monitoring Production Performance + +```cpp +class SchedulerMetrics { + // Per-worker metrics (cache-padded to avoid false sharing) + struct alignas(64) WorkerMetrics { + std::atomic messages_processed{0}; + std::atomic actors_run{0}; + std::atomic steal_attempts{0}; + std::atomic steal_successes{0}; + std::atomic cache_misses_estimate{0}; // From perf counters + }; + + std::vector workers; + +public: + void report() { + double steal_success_rate = total_steal_successes / total_steal_attempts; + // Low steal success rate + high CPU might indicate poor locality + + double messages_per_actor = total_messages / total_actors_run; + // High value indicates good batching + } +}; +``` + +--- + +## Summary: Cache Locality Checklist + +### Data Structure Design +- [ ] Actor hot data fits in 1-2 cache lines +- [ ] Scheduler per-worker state is cache-line padded +- [ ] Mailbox head/tail are on separate cache lines +- [ ] Cold data is allocated separately from hot data + +### Memory Allocation +- [ ] Per-actor or per-worker arenas for locality +- [ ] Pool allocators for common message sizes +- [ ] NUMA-aware allocation on multi-socket systems + +### Scheduling +- [ ] LIFO local scheduling for cache reuse +- [ ] Batch processing (multiple messages per actor run) +- [ ] Locality-aware work stealing (same NUMA node first) +- [ ] Actor affinity to workers + +### Tuning +- [ ] Profile with `perf` for cache miss rates +- [ ] Configurable batch sizes and steal policies +- [ ] Prefetching for predictable access patterns +- [ ] Adaptive granularity based on latency metrics + +--- + +## References + +1. CAF: The C++ Actor Framework - https://actor-framework.org/ +2. Barghi et al. "Work-Stealing, Locality-Aware Actor Scheduling" (IPDPS 2018) +3. Clebsch et al. "Orca: GC and Type System Co-Design for Actor Languages" (OOPSLA 2017) +4. Actix - Rust Actor Framework - https://actix.rs/ +5. Pony Language - https://www.ponylang.io/ +6. "Cache Line Alignment in C++" - Various Medium articles +7. Protocol Buffers Arena Allocation - https://protobuf.dev/reference/cpp/arenas/ \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index bb8e0dc..1e6027b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -45,6 +45,7 @@ name = "swactor" version = "0.1.0" dependencies = [ "crossbeam-queue", + "crossbeam-utils", "getrandom", ] diff --git a/Cargo.toml b/Cargo.toml index c8963bf..d2ebf83 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ stress = [] # Enable stress tests [dependencies] getrandom = { version = "0.2", optional = true } crossbeam-queue = "0.3.12" +crossbeam-utils = "0.8.21" [[bin]] name = "bench" diff --git a/WORKER.md b/WORKER.md new file mode 100644 index 0000000..9000a8d --- /dev/null +++ b/WORKER.md @@ -0,0 +1,113 @@ +struct MessageRing: + buffer: [u8; 4096] + head: AtomicUsize # Sender writes (Release) + tail: AtomicUsize # Worker reads (Acquire) + +struct LocalArena: + buffer: [u8; 262144] # Raw message bytes from rings + bump: usize + +# Key addition: Bucket buffer (indices into arena, not copies) +struct BucketBuffer: + # Pre-allocated array of slices. Max 64K actors, resize if needed. + # bucket[i] contains indices of messages for actor i. + buckets: Vec> # Or flat Vec with head/tail if arena-allocated + actor_order: Vec # Which actors have messages (for iteration) + +struct Worker: + worker_id: ID + inbox_rings: Vec # Per-sender rings + arena: LocalArena # Contiguous message storage + buckets: BucketBuffer # Grouped by actor + actor_table: Vec # ActorId -> Actor + +# Sender side (unchanged, 30 cycles) +function send_message(sender, target_worker, actor_id, payload): + ring = sender.rings[target_worker] + offset = reserve_in_ring(ring, 8 + len(payload)) + serialize(ring.buffer[offset:], actor_id, len(payload), payload) + ring.head.store(offset + 8 + len(payload), Release) + +# Worker side: Three-phase pipeline +function worker_run(worker): + while true: + # PHASE 1: DRAIN (same as before, ~5 cycles per message) + # ----------------------------------------------- + for ring in worker.inbox_rings: + head = ring.head.load(Acquire) + tail = ring.tail.load(Relaxed) + if head == tail: continue + + # Copy sequential chunk from ring -> arena (hardware prefetch) + size = head - tail + memcpy(worker.arena.buffer[worker.arena.bump:], + ring.buffer[tail:], size) + + # Parse boundaries while copying to avoid second pass + parse_and_bucket(worker.arena, worker.arena.bump, size, worker.buckets) + + worker.arena.bump += size + ring.tail.store(head, Relaxed) + + if worker.arena.bump == 0: + cpu_relax() + continue + + # PHASE 2: RADIX BUCKET (O(N), deterministic ~300 cycles) + # ------------------------------------------------------ + # We already built buckets during parse_and_bucket above, + # but if we deferred parsing, do it now: + + # Option A: If parsed during drain (optimal) + # Buckets already filled with (offset, len) pairs pointing into arena + + # Option B: Linear scan to build buckets (if raw bytes in arena) + offset = 0 + while offset < worker.arena.bump: + actor_id = read_u32(arena[offset:]) + msg_len = read_u32(arena[offset+4:]) + + # Append to actor's bucket (Vec push, amortized O(1)) + # Each bucket entry: (offset, msg_len) = 16 bytes + worker.buckets.buckets[actor_id].append((offset+8, msg_len)) + + # Track unique actors (optional, avoids empty bucket scans) + if worker.buckets.buckets[actor_id].len() == 1: + worker.buckets.actor_order.append(actor_id) + + offset += 8 + msg_len + + # PHASE 3: PROCESS BY ACTOR (hidden message fetch, hot actor state) + # ---------------------------------------------------------------- + for actor_id in worker.buckets.actor_order: + actor = worker.actor_table[actor_id] # First access: L3 miss (250 cycles) + + # Prefetch next actor's state while processing current (optional) + prefetch_actor(worker.buckets.actor_order, worker.actor_table) + + # Process all messages for this actor + # arena[slice] is L1 hit (12 cycles) - scanned sequentially within actor + for (msg_offset, msg_len) in worker.buckets.buckets[actor_id]: + msg_data = worker.arena.buffer[msg_offset : msg_offset+msg_len] + actor.process(msg_data) # 100 cycles work + + # actor state stays in L1 for entire inner loop + + # PHASE 4: RESET (zero cost) + worker.arena.bump = 0 + clear_buckets(worker.buckets) # Just reset lengths, don't free + +# Helper: Parse during drain to avoid touching bytes twice +function parse_and_bucket(arena, base_offset, size, buckets): + ptr = 0 + while ptr < size: + actor_id = read_u32(arena[base_offset + ptr:]) + msg_len = read_u32(arena[base_offset + ptr + 4:]) + + # Append metadata to bucket (16 bytes: offset, len) + buckets.buckets[actor_id].append((base_offset + ptr + 8, msg_len)) + + if buckets.buckets[actor_id].len() == 1: + buckets.actor_order.append(actor_id) + + ptr += 8 + msg_len \ No newline at end of file diff --git a/src/channel.rs b/src/channel/mod.rs similarity index 99% rename from src/channel.rs rename to src/channel/mod.rs index a210a93..93b865e 100644 --- a/src/channel.rs +++ b/src/channel/mod.rs @@ -1,3 +1,5 @@ +pub mod spsc; + use std::{collections::VecDeque, sync::{Arc, Mutex}}; use crossbeam_queue::ArrayQueue; diff --git a/src/channel/spsc.rs b/src/channel/spsc.rs new file mode 100644 index 0000000..a44c87e --- /dev/null +++ b/src/channel/spsc.rs @@ -0,0 +1,198 @@ +use crossbeam_utils::CachePadded; +use std::cell::UnsafeCell; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +const CACHELINE: usize = 64; + +struct Ring { + head: CachePadded, + tail: CachePadded, + buf: UnsafeCell<[u8; N]>, +} + +unsafe impl Send for Ring {} +unsafe impl Sync for Ring {} + +pub struct Producer { + inner: Arc>, + head: usize, // cached local copy + tail: usize, // cached local copy +} + +pub struct Consumer { + inner: Arc>, + head: usize, + tail: usize, +} + +unsafe impl Send for Producer {} +unsafe impl Send for Consumer {} + +pub fn channel() -> (Producer, Consumer) { + assert!(N.is_power_of_two(), "capacity must be power of 2"); + + let inner = Arc::new(Ring { + head: CachePadded::new(AtomicUsize::new(0)), + tail: CachePadded::new(AtomicUsize::new(0)), + buf: UnsafeCell::new([0u8; N]), + }); + + let producer = Producer { + inner: inner.clone(), + head: 0, + tail: 0, + }; + let consumer = Consumer { + inner, + head: 0, + tail: 0, + }; + (producer, consumer) +} + +impl Producer { + const MASK: usize = N - 1; + + /// Copy `src` into circular buffer at `pos`, handling wraparound. + #[inline] + fn write_at(buf: &mut [u8; N], pos: usize, src: &[u8]) { + let start = pos & Self::MASK; + let end = start + src.len(); + + if end <= N { + // No wrap: single copy + buf[start..end].copy_from_slice(src); + } else { + // Wrap: split into two copies + let first = N - start; + buf[start..].copy_from_slice(&src[..first]); + buf[..src.len() - first].copy_from_slice(&src[first..]); + } + } + + /// Attempts to write a byte slice into the buffer, returning the number of bytes written + pub fn try_write(&mut self, data: &[u8]) -> usize { + let needed = 4 + data.len(); + + // Refresh cached tail if we think we're full + let available = N - self.head.wrapping_sub(self.tail); + if available < needed { + self.tail = self.inner.tail.load(Ordering::Acquire); + let available = N - self.head.wrapping_sub(self.tail); + if available < needed { + return 0; + } + } + + let buf = unsafe { &mut *self.inner.buf.get() }; + + // Write length prefix (4 bytes LE) + Self::write_at(buf, self.head, &(data.len() as u32).to_le_bytes()); + + // Write payload + Self::write_at(buf, self.head + 4, data); + + self.head = self.head.wrapping_add(needed); + self.inner.head.store(self.head, Ordering::Release); + + data.len() + } +} + +impl Consumer { + const MASK: usize = N - 1; + + #[inline] + fn read_at(buf: &[u8; N], pos: usize, dst: &mut [u8]) { + let start = pos & Self::MASK; + let end = start + dst.len(); + + if end <= N { + dst.copy_from_slice(&buf[start..end]); + } else { + let first = N - start; + let second = end - N; + dst[..first].copy_from_slice(&buf[start..]); + dst[first..].copy_from_slice(&buf[..second]); + } + } + + /// Pop next message into provided buffer. Returns message length, or None if empty. + /// Panics if buffer is too small for the message. + pub fn pop_into(&mut self, dst: &mut [u8]) -> Option { + let filled = self.head.wrapping_sub(self.tail); + if filled < 4 { + self.head = self.inner.head.load(Ordering::Acquire); + let filled = self.head.wrapping_sub(self.tail); + if filled < 4 { + return None; + } + } + + let buf = unsafe { &*self.inner.buf.get() }; + + let mut len_bytes = [0u8; 4]; + Self::read_at(buf, self.tail, &mut len_bytes); + let len = u32::from_le_bytes(len_bytes) as usize; + + let filled = self.head.wrapping_sub(self.tail); + if filled < 4 + len { + self.head = self.inner.head.load(Ordering::Acquire); + let filled = self.head.wrapping_sub(self.tail); + if filled < 4 + len { + return None; + } + } + + Self::read_at(buf, self.tail + 4, &mut dst[..len]); + + self.tail = self.tail.wrapping_add(4 + len); + self.inner.tail.store(self.tail, Ordering::Release); + + Some(len) + } + + pub fn is_empty(&self) -> bool { + let head = self.inner.head.load(Ordering::Acquire); + head == self.tail + } +} + +// ============ Demo ============ +#[test] +fn sanity() { + use std::thread; + + let (mut tx, mut rx) = channel::<4096>(); + + let num_messages = 1000; + let producer = thread::spawn(move || { + for i in 0..num_messages { + let msg = format!("message {}", i); + while tx.try_write(msg.as_bytes()) == 0 { + std::hint::spin_loop(); + } + } + tx.try_write(b"DONE"); + }); + + let consumer = thread::spawn(move || { + let mut count = 0; + loop { + let mut buf = vec![]; + if let Some(_) = rx.pop_into(&mut buf) { + if buf == b"DONE" { + break; + } + count += 1; + } else { + std::hint::spin_loop(); + } + } + assert_eq!(num_messages, count, "failed to process all messages"); + }); + + producer.join().unwrap(); + consumer.join().unwrap(); +} diff --git a/src/lib.rs b/src/lib.rs index 1d67009..0a3122c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,6 +6,7 @@ pub use error::Error; mod router; pub mod runtime; +mod worker; #[cfg(feature = "getrandom")] pub(crate) fn get_random(buf: &mut [u8]) { diff --git a/src/worker.rs b/src/worker.rs new file mode 100644 index 0000000..4a0efd5 --- /dev/null +++ b/src/worker.rs @@ -0,0 +1,46 @@ +type WorkerId = usize; +type ActorId = usize; + +struct Actor(u8); + +const MESSAGE_RING_BUFFER_SIZE: usize = 4096; +const LOCAL_ARENA_BUFFER_SIZE: usize = 262144; +// hitting the maximum would imply reading nothing but length prefixes from the ring channel +const MAX_MESSAGES_PER_DRAIN: usize = MESSAGE_RING_BUFFER_SIZE / 4; + +use crate::channel::spsc::Consumer as RingBuffer; + +#[repr(align(64))] +struct LocalArena { + data: [u8; LOCAL_ARENA_BUFFER_SIZE], + offsets: [u32; MAX_MESSAGES_PER_DRAIN], +} +struct Worker { + id: WorkerId, + inbox_rings: Vec>, + arena: LocalArena, + actor_table: Vec, +} + +impl Worker { + pub fn run(&mut self) { + // cannot overflow the arena buffer + debug_assert!(self.inbox_rings.len() * MESSAGE_RING_BUFFER_SIZE < LOCAL_ARENA_BUFFER_SIZE); + + loop { + // drain messages into our local memory arena buffer + for ring in &self.inbox_rings { + // logic here + } + } + } +} + + + +struct BucketBuffer { + // Pre-allocated array of slices. Max 64K actors, resize if needed. + // bucket[i] contains indices of messages for actor i. + buckets: Vec>, // Or flat Vec with head/tail if arena-allocated + actor_order: Vec, // Which actors have messages (for iteration) +} -- 2.45.2 From abedaf6307cc130faf1e7d0a9cf2c8a731f37b30 Mon Sep 17 00:00:00 2001 From: Zachery Aaron Shores-Chmielewski Date: Thu, 5 Feb 2026 10:44:48 +0700 Subject: [PATCH 3/9] feat: workstealing, less contention for multithreaded --- Cargo.lock | 27 ++++++ Cargo.toml | 2 + DESIGN.md | 55 ------------ WORKER.md | 113 ------------------------- src/channel/mod.rs | 25 ++++-- src/channel/spsc.rs | 198 -------------------------------------------- src/lib.rs | 1 - src/router.rs | 2 +- src/runtime.rs | 166 +++++++++++++++++++++++++++++-------- src/worker.rs | 46 ---------- 10 files changed, 180 insertions(+), 455 deletions(-) delete mode 100644 DESIGN.md delete mode 100644 WORKER.md delete mode 100644 src/channel/spsc.rs delete mode 100644 src/worker.rs diff --git a/Cargo.lock b/Cargo.lock index 1e6027b..561715c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,25 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.12" @@ -40,13 +59,21 @@ version = "0.2.180" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + [[package]] name = "swactor" version = "0.1.0" dependencies = [ + "crossbeam-deque", "crossbeam-queue", "crossbeam-utils", "getrandom", + "smallvec", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d2ebf83..6471309 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,8 +15,10 @@ stress = [] # Enable stress tests [dependencies] getrandom = { version = "0.2", optional = true } +crossbeam-deque = "0.8" crossbeam-queue = "0.3.12" crossbeam-utils = "0.8.21" +smallvec = "1.13" [[bin]] name = "bench" diff --git a/DESIGN.md b/DESIGN.md deleted file mode 100644 index 93f57e6..0000000 --- a/DESIGN.md +++ /dev/null @@ -1,55 +0,0 @@ -# Design goals -Get as much usability and speed as possible while keeping line count low. Aim for no footguns, ability to plug in -logic easily, and run near anywhere. We may make this a `![no_std]` library, but the MVP will use the -memory allocator and threading provided by the rust standard library. - -We are not building a new erlang/BEAM. Minimal feature set means spawning actor processes, not having supervisiors, lots of process -monitoring tools, prempting, etc. - -# FIXME -This is slightly obsolete. Was necessary in order to concentrate on getting the basic skeleton up, now it's a distraction and unreliable. -After getting the benches/etc finished, move this into an `ARCHITECTURE.md` file, have it make sense. - - -## Actor model - -An actor has: - - - An inbox: - this is a mpsc channel that the runtime/router dumps messages into and the actor consumes when the runtime loads it - Implemented as a barebones atomic ring buffer. The router is responsible for inserting messages. - - - an outbox channel connection: - this is a mpmc channel that is implemented by the runtime and router. Actors on this specific channel put responses and outgoing messages into this channel, to be routed to the given address. - - - a growable and mutable state: - An actor owns some, from the runtime perspective, type erased bytes. The actor when processing messages can access its own state, but no other task can. This includes viewing. - - - a set of functions for processing messages: - When the runtime loads the actor, it locks the inbox and attempts to process the messages therein. - -## Runtime - -In order for an actor to consume and send messages, it is processed by a runtime. The runtime, in order to negotiate messages between -actors, possesses a router. - -A runtime has: - - An actor processing thread(s): - the processor will mark an actor as busy, load its state and inbox, and begin consuming messages from the inbox. The number of messages consumed is determined by the runtime. A good start is a backpressure strategy: after loading, process messages until mailbox is empty or size drops below a threshold (e.g., "drain to 50%"). - - - A message router: - the router is responsible for ensuring messages posted by actors get delivered to the appropriate inbox. - - - An atomic ring buffer containing thread-safe references to actors that are not currently loaded. Actors are popped off the buffer, messages are - processed, and the reference is returned to the buffer/queue before the next actor is loaded. - -## Router - -The router is the engine for message delivery. It posesses: - - - An actor address book: - The address book maps actor ids to `Sender` references that can be used to deliver messages to the actor inbox. - - - Its own inbox: - The router possesses its own mpsc queue where references to messages are stored. The router will process this queue by dereferencing and writing directly into the recipient's inbox buffer. - diff --git a/WORKER.md b/WORKER.md deleted file mode 100644 index 9000a8d..0000000 --- a/WORKER.md +++ /dev/null @@ -1,113 +0,0 @@ -struct MessageRing: - buffer: [u8; 4096] - head: AtomicUsize # Sender writes (Release) - tail: AtomicUsize # Worker reads (Acquire) - -struct LocalArena: - buffer: [u8; 262144] # Raw message bytes from rings - bump: usize - -# Key addition: Bucket buffer (indices into arena, not copies) -struct BucketBuffer: - # Pre-allocated array of slices. Max 64K actors, resize if needed. - # bucket[i] contains indices of messages for actor i. - buckets: Vec> # Or flat Vec with head/tail if arena-allocated - actor_order: Vec # Which actors have messages (for iteration) - -struct Worker: - worker_id: ID - inbox_rings: Vec # Per-sender rings - arena: LocalArena # Contiguous message storage - buckets: BucketBuffer # Grouped by actor - actor_table: Vec # ActorId -> Actor - -# Sender side (unchanged, 30 cycles) -function send_message(sender, target_worker, actor_id, payload): - ring = sender.rings[target_worker] - offset = reserve_in_ring(ring, 8 + len(payload)) - serialize(ring.buffer[offset:], actor_id, len(payload), payload) - ring.head.store(offset + 8 + len(payload), Release) - -# Worker side: Three-phase pipeline -function worker_run(worker): - while true: - # PHASE 1: DRAIN (same as before, ~5 cycles per message) - # ----------------------------------------------- - for ring in worker.inbox_rings: - head = ring.head.load(Acquire) - tail = ring.tail.load(Relaxed) - if head == tail: continue - - # Copy sequential chunk from ring -> arena (hardware prefetch) - size = head - tail - memcpy(worker.arena.buffer[worker.arena.bump:], - ring.buffer[tail:], size) - - # Parse boundaries while copying to avoid second pass - parse_and_bucket(worker.arena, worker.arena.bump, size, worker.buckets) - - worker.arena.bump += size - ring.tail.store(head, Relaxed) - - if worker.arena.bump == 0: - cpu_relax() - continue - - # PHASE 2: RADIX BUCKET (O(N), deterministic ~300 cycles) - # ------------------------------------------------------ - # We already built buckets during parse_and_bucket above, - # but if we deferred parsing, do it now: - - # Option A: If parsed during drain (optimal) - # Buckets already filled with (offset, len) pairs pointing into arena - - # Option B: Linear scan to build buckets (if raw bytes in arena) - offset = 0 - while offset < worker.arena.bump: - actor_id = read_u32(arena[offset:]) - msg_len = read_u32(arena[offset+4:]) - - # Append to actor's bucket (Vec push, amortized O(1)) - # Each bucket entry: (offset, msg_len) = 16 bytes - worker.buckets.buckets[actor_id].append((offset+8, msg_len)) - - # Track unique actors (optional, avoids empty bucket scans) - if worker.buckets.buckets[actor_id].len() == 1: - worker.buckets.actor_order.append(actor_id) - - offset += 8 + msg_len - - # PHASE 3: PROCESS BY ACTOR (hidden message fetch, hot actor state) - # ---------------------------------------------------------------- - for actor_id in worker.buckets.actor_order: - actor = worker.actor_table[actor_id] # First access: L3 miss (250 cycles) - - # Prefetch next actor's state while processing current (optional) - prefetch_actor(worker.buckets.actor_order, worker.actor_table) - - # Process all messages for this actor - # arena[slice] is L1 hit (12 cycles) - scanned sequentially within actor - for (msg_offset, msg_len) in worker.buckets.buckets[actor_id]: - msg_data = worker.arena.buffer[msg_offset : msg_offset+msg_len] - actor.process(msg_data) # 100 cycles work - - # actor state stays in L1 for entire inner loop - - # PHASE 4: RESET (zero cost) - worker.arena.bump = 0 - clear_buckets(worker.buckets) # Just reset lengths, don't free - -# Helper: Parse during drain to avoid touching bytes twice -function parse_and_bucket(arena, base_offset, size, buckets): - ptr = 0 - while ptr < size: - actor_id = read_u32(arena[base_offset + ptr:]) - msg_len = read_u32(arena[base_offset + ptr + 4:]) - - # Append metadata to bucket (16 bytes: offset, len) - buckets.buckets[actor_id].append((base_offset + ptr + 8, msg_len)) - - if buckets.buckets[actor_id].len() == 1: - buckets.actor_order.append(actor_id) - - ptr += 8 + msg_len \ No newline at end of file diff --git a/src/channel/mod.rs b/src/channel/mod.rs index 93b865e..fbccdb5 100644 --- a/src/channel/mod.rs +++ b/src/channel/mod.rs @@ -1,19 +1,21 @@ -pub mod spsc; -use std::{collections::VecDeque, sync::{Arc, Mutex}}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; -use crossbeam_queue::ArrayQueue; +use crossbeam_queue::{ArrayQueue, SegQueue}; pub struct HybridChannel { ring: ArrayQueue, - overflow: Mutex>, + overflow: SegQueue, + overflow_len: AtomicUsize, } impl HybridChannel { pub fn new(capacity: usize) -> Self { Self { ring: ArrayQueue::new(capacity), - overflow: Mutex::new(VecDeque::new()), + overflow: SegQueue::new(), + overflow_len: AtomicUsize::new(0), } } @@ -21,7 +23,8 @@ impl HybridChannel { match self.ring.push(value) { Ok(()) => Ok(()), Err(v) => { - self.overflow.lock().unwrap().push_back(v); + self.overflow.push(v); + self.overflow_len.fetch_add(1, Ordering::Relaxed); Ok(()) } } @@ -32,11 +35,17 @@ impl HybridChannel { return Some(value); } - self.overflow.lock().unwrap().pop_front() + match self.overflow.pop() { + Some(value) => { + self.overflow_len.fetch_sub(1, Ordering::Relaxed); + Some(value) + } + None => None, + } } pub fn len(&self) -> usize { - self.ring.len() + self.overflow.lock().unwrap().len() + self.ring.len() + self.overflow_len.load(Ordering::Relaxed) } } diff --git a/src/channel/spsc.rs b/src/channel/spsc.rs deleted file mode 100644 index a44c87e..0000000 --- a/src/channel/spsc.rs +++ /dev/null @@ -1,198 +0,0 @@ -use crossbeam_utils::CachePadded; -use std::cell::UnsafeCell; -use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering}; - -const CACHELINE: usize = 64; - -struct Ring { - head: CachePadded, - tail: CachePadded, - buf: UnsafeCell<[u8; N]>, -} - -unsafe impl Send for Ring {} -unsafe impl Sync for Ring {} - -pub struct Producer { - inner: Arc>, - head: usize, // cached local copy - tail: usize, // cached local copy -} - -pub struct Consumer { - inner: Arc>, - head: usize, - tail: usize, -} - -unsafe impl Send for Producer {} -unsafe impl Send for Consumer {} - -pub fn channel() -> (Producer, Consumer) { - assert!(N.is_power_of_two(), "capacity must be power of 2"); - - let inner = Arc::new(Ring { - head: CachePadded::new(AtomicUsize::new(0)), - tail: CachePadded::new(AtomicUsize::new(0)), - buf: UnsafeCell::new([0u8; N]), - }); - - let producer = Producer { - inner: inner.clone(), - head: 0, - tail: 0, - }; - let consumer = Consumer { - inner, - head: 0, - tail: 0, - }; - (producer, consumer) -} - -impl Producer { - const MASK: usize = N - 1; - - /// Copy `src` into circular buffer at `pos`, handling wraparound. - #[inline] - fn write_at(buf: &mut [u8; N], pos: usize, src: &[u8]) { - let start = pos & Self::MASK; - let end = start + src.len(); - - if end <= N { - // No wrap: single copy - buf[start..end].copy_from_slice(src); - } else { - // Wrap: split into two copies - let first = N - start; - buf[start..].copy_from_slice(&src[..first]); - buf[..src.len() - first].copy_from_slice(&src[first..]); - } - } - - /// Attempts to write a byte slice into the buffer, returning the number of bytes written - pub fn try_write(&mut self, data: &[u8]) -> usize { - let needed = 4 + data.len(); - - // Refresh cached tail if we think we're full - let available = N - self.head.wrapping_sub(self.tail); - if available < needed { - self.tail = self.inner.tail.load(Ordering::Acquire); - let available = N - self.head.wrapping_sub(self.tail); - if available < needed { - return 0; - } - } - - let buf = unsafe { &mut *self.inner.buf.get() }; - - // Write length prefix (4 bytes LE) - Self::write_at(buf, self.head, &(data.len() as u32).to_le_bytes()); - - // Write payload - Self::write_at(buf, self.head + 4, data); - - self.head = self.head.wrapping_add(needed); - self.inner.head.store(self.head, Ordering::Release); - - data.len() - } -} - -impl Consumer { - const MASK: usize = N - 1; - - #[inline] - fn read_at(buf: &[u8; N], pos: usize, dst: &mut [u8]) { - let start = pos & Self::MASK; - let end = start + dst.len(); - - if end <= N { - dst.copy_from_slice(&buf[start..end]); - } else { - let first = N - start; - let second = end - N; - dst[..first].copy_from_slice(&buf[start..]); - dst[first..].copy_from_slice(&buf[..second]); - } - } - - /// Pop next message into provided buffer. Returns message length, or None if empty. - /// Panics if buffer is too small for the message. - pub fn pop_into(&mut self, dst: &mut [u8]) -> Option { - let filled = self.head.wrapping_sub(self.tail); - if filled < 4 { - self.head = self.inner.head.load(Ordering::Acquire); - let filled = self.head.wrapping_sub(self.tail); - if filled < 4 { - return None; - } - } - - let buf = unsafe { &*self.inner.buf.get() }; - - let mut len_bytes = [0u8; 4]; - Self::read_at(buf, self.tail, &mut len_bytes); - let len = u32::from_le_bytes(len_bytes) as usize; - - let filled = self.head.wrapping_sub(self.tail); - if filled < 4 + len { - self.head = self.inner.head.load(Ordering::Acquire); - let filled = self.head.wrapping_sub(self.tail); - if filled < 4 + len { - return None; - } - } - - Self::read_at(buf, self.tail + 4, &mut dst[..len]); - - self.tail = self.tail.wrapping_add(4 + len); - self.inner.tail.store(self.tail, Ordering::Release); - - Some(len) - } - - pub fn is_empty(&self) -> bool { - let head = self.inner.head.load(Ordering::Acquire); - head == self.tail - } -} - -// ============ Demo ============ -#[test] -fn sanity() { - use std::thread; - - let (mut tx, mut rx) = channel::<4096>(); - - let num_messages = 1000; - let producer = thread::spawn(move || { - for i in 0..num_messages { - let msg = format!("message {}", i); - while tx.try_write(msg.as_bytes()) == 0 { - std::hint::spin_loop(); - } - } - tx.try_write(b"DONE"); - }); - - let consumer = thread::spawn(move || { - let mut count = 0; - loop { - let mut buf = vec![]; - if let Some(_) = rx.pop_into(&mut buf) { - if buf == b"DONE" { - break; - } - count += 1; - } else { - std::hint::spin_loop(); - } - } - assert_eq!(num_messages, count, "failed to process all messages"); - }); - - producer.join().unwrap(); - consumer.join().unwrap(); -} diff --git a/src/lib.rs b/src/lib.rs index 0a3122c..1d67009 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,7 +6,6 @@ pub use error::Error; mod router; pub mod runtime; -mod worker; #[cfg(feature = "getrandom")] pub(crate) fn get_random(buf: &mut [u8]) { diff --git a/src/router.rs b/src/router.rs index 34157f4..d015fe1 100644 --- a/src/router.rs +++ b/src/router.rs @@ -21,7 +21,7 @@ impl SenderT for Sender { if let Some(msg) = envelope.downcast_ref::() { // FIXME: we are directly cloning the contents of the Arc pointer here // Do we want to? Should we provide another way? - // + // // The standard concept of an actor has message and state // isolation, so we should leave this as is. However, we should // make it clear and obvious this pathway is the heavy, contained diff --git a/src/runtime.rs b/src/runtime.rs index 7a9f768..1a5df3b 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -1,6 +1,9 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::thread::{self, JoinHandle}; +use std::time::Duration; + +use crossbeam_deque::{Injector, Steal, Stealer, Worker}; use crate::channel::HybridChannel; use crate::{ @@ -56,10 +59,24 @@ impl Default for RuntimeConfig { } } +/// Per-worker state for work-stealing scheduler +struct WorkerState { + local: Worker>, // FIFO for fairness + id: usize, +} + +/// Shared scheduler state for work-stealing +struct SchedulerState { + injector: Injector>, // For spawn() + stealers: Vec>>, // For work-stealing + is_running: AtomicBool, +} + /// The `Runtime` struct is the primary gateway for interacting with the framework. pub struct Runtime { config: RuntimeConfig, - actor_queue: HybridChannel>, + actor_queue: HybridChannel>, // Used for single-threaded mode + scheduler: Option>, // Used for multi-threaded mode router_interface: Sender, router: Option>, // `None` if single-threaded @@ -113,6 +130,7 @@ impl Runtime { Self { config, actor_queue, + scheduler: None, // Initialized in run() for multi-threaded mode router_interface: router_sender, is_running: AtomicBool::new(false), router: router_option, @@ -133,9 +151,16 @@ impl Runtime { Error::from("Runtime error: failed to add actor to router. Router inbox full") })?; - self.actor_queue - .push(Box::new(Actor::new(inbox, actor))) - .map_err(|_| Error::from("Runtime error: Failed to spawn actor. Queue full."))?; + let boxed_actor: Box = Box::new(Actor::new(inbox, actor)); + + // Multi-threaded: use injector, single-threaded: use shared queue + if let Some(ref scheduler) = self.scheduler { + scheduler.injector.push(boxed_actor); + } else { + self.actor_queue + .push(boxed_actor) + .map_err(|_| Error::from("Runtime error: Failed to spawn actor. Queue full."))?; + } Ok(addr) } @@ -193,48 +218,56 @@ impl Runtime { .take() .expect("Router must be present for multi-threaded runtime"); + let num_workers = self.config.num_threads - 1; + + // Create workers and collect stealers for work-stealing + let mut workers = Vec::with_capacity(num_workers); + let mut stealers = Vec::with_capacity(num_workers); + for id in 0..num_workers { + let worker = WorkerState { + local: Worker::new_fifo(), // FIFO for fairness + id, + }; + stealers.push(worker.local.stealer()); + workers.push(worker); + } + + let scheduler = Arc::new(SchedulerState { + injector: Injector::new(), + stealers, + is_running: AtomicBool::new(true), + }); + + // Transfer any actors spawned before run() to the injector + while let Some(actor) = self.actor_queue.pop() { + scheduler.injector.push(actor); + } + + self.scheduler = Some(scheduler.clone()); let rt = Arc::new(self); let mut handles: Vec> = vec![]; // Router thread owns the router directly - no synchronization needed + // Process multiple ticks per cycle to maximize throughput + const ROUTER_BATCH_SIZE: usize = 64; let router_handle = { let ctx = rt.clone(); + let sched = scheduler.clone(); thread::spawn(move || { - while ctx.is_running.load(Ordering::Acquire) { - router.tick(&ctx); - thread::yield_now(); + while sched.is_running.load(Ordering::Acquire) { + for _ in 0..ROUTER_BATCH_SIZE { + router.tick(&ctx); + } } }) }; handles.push(router_handle); - // Spawn worker threads - let num_workers = rt.config.num_threads - 1; - for _ in 0..num_workers { + // Spawn worker threads, each owns its WorkerState + for worker in workers { let ctx = rt.clone(); - let handle = thread::spawn(move || { - while ctx.is_running.load(Ordering::Acquire) { - if let Some(mut actor) = ctx.actor_queue.pop() { - actor.tick(&ctx); - // FIXME: Justify this loop. It is here to prevent panics when the - // actor queue is full, but results in a spinlock. - loop { - match ctx.actor_queue.push(actor) { - Ok(()) => break, - Err(a) => { - actor = a; - if !ctx.is_running.load(Ordering::Acquire) { - break; - } - thread::yield_now(); - } - } - } - } else { - thread::yield_now(); - } - } - }); + let sched = scheduler.clone(); + let handle = thread::spawn(move || worker_loop(ctx, sched, worker)); handles.push(handle); } @@ -256,5 +289,72 @@ impl Runtime { /// Signal all workers to stop pub fn shutdown(&self) { self.is_running.store(false, Ordering::Release); + // Also stop the scheduler if multi-threaded + if let Some(ref scheduler) = self.scheduler { + scheduler.is_running.store(false, Ordering::Release); + } + } +} + +/// Work-stealing worker loop for multi-threaded runtime +fn worker_loop(ctx: Arc, scheduler: Arc, worker: WorkerState) { + const TICKS_PER_ACTOR: usize = 4; + const INJECTOR_CHECK_INTERVAL: usize = 64; // Check injector every N iterations + let mut spin_count: usize = 0; + let mut iteration: usize = 0; + + while scheduler.is_running.load(Ordering::Acquire) { + iteration = iteration.wrapping_add(1); + + // Priority 1: Local queue + let mut actor = worker.local.pop(); + + // Priority 2: Periodically check injector for new actors + // This ensures newly spawned actors get picked up even when workers have local work + if actor.is_none() || (iteration % INJECTOR_CHECK_INTERVAL == 0) { + if let Steal::Success(a) = scheduler.injector.steal_batch_and_pop(&worker.local) { + if actor.is_some() { + // We already had an actor, push the stolen one to local + worker.local.push(a); + } else { + actor = Some(a); + } + } + } + + // Priority 3: Steal from peer workers when idle + if actor.is_none() { + for (i, stealer) in scheduler.stealers.iter().enumerate() { + if i == worker.id { + continue; + } + if let Steal::Success(a) = stealer.steal_batch_and_pop(&worker.local) { + actor = Some(a); + break; + } + } + } + + // Process or backoff + match actor { + Some(mut actor) => { + spin_count = 0; + for _ in 0..TICKS_PER_ACTOR { + actor.tick(&ctx); + } + worker.local.push(actor); + } + None => { + // Exponential backoff + spin_count = spin_count.saturating_add(1); + if spin_count < 10 { + std::hint::spin_loop(); + } else if spin_count < 100 { + thread::yield_now(); + } else { + thread::sleep(Duration::from_micros(10)); + } + } + } } } diff --git a/src/worker.rs b/src/worker.rs deleted file mode 100644 index 4a0efd5..0000000 --- a/src/worker.rs +++ /dev/null @@ -1,46 +0,0 @@ -type WorkerId = usize; -type ActorId = usize; - -struct Actor(u8); - -const MESSAGE_RING_BUFFER_SIZE: usize = 4096; -const LOCAL_ARENA_BUFFER_SIZE: usize = 262144; -// hitting the maximum would imply reading nothing but length prefixes from the ring channel -const MAX_MESSAGES_PER_DRAIN: usize = MESSAGE_RING_BUFFER_SIZE / 4; - -use crate::channel::spsc::Consumer as RingBuffer; - -#[repr(align(64))] -struct LocalArena { - data: [u8; LOCAL_ARENA_BUFFER_SIZE], - offsets: [u32; MAX_MESSAGES_PER_DRAIN], -} -struct Worker { - id: WorkerId, - inbox_rings: Vec>, - arena: LocalArena, - actor_table: Vec, -} - -impl Worker { - pub fn run(&mut self) { - // cannot overflow the arena buffer - debug_assert!(self.inbox_rings.len() * MESSAGE_RING_BUFFER_SIZE < LOCAL_ARENA_BUFFER_SIZE); - - loop { - // drain messages into our local memory arena buffer - for ring in &self.inbox_rings { - // logic here - } - } - } -} - - - -struct BucketBuffer { - // Pre-allocated array of slices. Max 64K actors, resize if needed. - // bucket[i] contains indices of messages for actor i. - buckets: Vec>, // Or flat Vec with head/tail if arena-allocated - actor_order: Vec, // Which actors have messages (for iteration) -} -- 2.45.2 From 172e78a50bb42ad594c13963efa5d6079a171a5d Mon Sep 17 00:00:00 2001 From: Zachery Aaron Shores-Chmielewski Date: Thu, 5 Feb 2026 21:22:36 +0700 Subject: [PATCH 4/9] feat: mailbox In the middle of a vibe-coded infra change. Modularizing the components into APIs amenable to lots of testing and optimization. --- ARCHITECTURE.md | 198 ++++++++ CACHE.md | 1050 ---------------------------------------- README.md | 58 ++- src/lib.rs | 1 + src/worker/mailbox.rs | 54 +++ src/worker/mod.rs | 1 + tests/mailbox_tests.rs | 226 +++++++++ 7 files changed, 537 insertions(+), 1051 deletions(-) create mode 100644 ARCHITECTURE.md delete mode 100644 CACHE.md create mode 100644 src/worker/mailbox.rs create mode 100644 src/worker/mod.rs create mode 100644 tests/mailbox_tests.rs diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..8c6a0cb --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,198 @@ +# Swactor Architecture + +## System Diagram + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ Runtime │ +│ (composes everything) │ +│ │ +│ ┌───────────────────────────────────────────────────────────────────────┐ │ +│ │ Address Map │ │ +│ │ ActorAddress → WorkerId │ │ +│ │ (shared across all workers, read-heavy) │ │ +│ └──────┬──────────────────┬──────────────────────┬─────────────────────┘ │ +│ │ │ │ │ +│ ▼ ▼ ▼ │ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ +│ │ Worker 0 │ │ Worker 1 │ ... │ Worker N │ │ +│ │ (thread) │ │ (thread) │ │ (thread) │ │ +│ │ │ │ │ │ │ │ +│ │ ┌─────────┐ │ │ ┌─────────┐ │ │ ┌─────────┐ │ │ +│ │ │ Actor A │ │ │ │ Actor C │ │ │ │ Actor E │ │ │ +│ │ │ [═════] │ │ │ │ [═════] │ │ │ │ [═════] │ │ │ +│ │ │ mailbox │ │ │ │ mailbox │ │ │ │ mailbox │ │ │ +│ │ └─────────┘ │ │ └─────────┘ │ │ └─────────┘ │ │ +│ │ ┌─────────┐ │ │ ┌─────────┐ │ │ ┌─────────┐ │ │ +│ │ │ Actor B │ │ │ │ Actor D │ │ │ │ Actor F │ │ │ +│ │ │ [═════] │ │ │ │ [═════] │ │ │ │ [═════] │ │ │ +│ │ │ mailbox │ │ │ │ mailbox │ │ │ │ mailbox │ │ │ +│ │ └─────────┘ │ │ └─────────┘ │ │ └─────────┘ │ │ +│ │ │ │ │ │ │ │ +│ │ ┌─────────┐ │ │ ┌─────────┐ │ │ ┌─────────┐ │ │ +│ │ │Transfer │◄├────├─┤Transfer │◄├───────├─┤Transfer │ │ │ +│ │ │ Queue │ │ │ │ Queue │ │ │ │ Queue │ │ │ +│ │ │ (MPSC) │─├────├►│ (MPSC) │─├───────├►│ (MPSC) │ │ │ +│ │ └─────────┘ │ │ └─────────┘ │ │ └─────────┘ │ │ +│ └─────────────┘ └─────────────┘ └─────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────────────┘ + + + ══ = VecDeque (no atomics) +``` + +## Message Flow + +``` + SAME WORKER (fast path — zero atomics) + ═══════════════════════════════════════ + + Actor A Actor B + handle() { mailbox (VecDeque) + ctx.send(addr_B, msg) ▲ + │ │ + ├─ address_map[addr_B] │ + │ → Worker 0 (that's me!) │ + │ │ + └─ mailbox_B.push(msg) ──────┘ + } no atomics, no envelope + + + CROSS-WORKER (one atomic hop) + ═══════════════════════════════ + + Actor A (Worker 0) Worker 1 Actor C (Worker 1) + handle() { transfer queue mailbox (VecDeque) + ctx.send(addr_C, msg) ▲ ▲ + │ │ │ + ├─ address_map[addr_C] │ │ + │ → Worker 1 (not me) │ │ + │ │ │ + └─ envelope(addr_C, msg) ──────┘ │ + (atomic push) │ │ + └── worker 1 pops ─────┘ + and distributes + (local, no atomic) + } +``` + +## Worker Loop + +``` + ┌─────────────────────────────────────────────┐ + │ Worker Thread │ + │ │ + │ loop { │ + │ ┌──────────────────────────────────────┐ │ + │ │ 1. DRAIN TRANSFER QUEUE │ │ + │ │ while let Some((addr, env)) = │ │ + │ │ transfer_queue.pop() │ │ + │ │ { │ │ + │ │ local_actors[addr].mailbox │ │ + │ │ .push(env.unpack()) │ │ + │ │ } │ │ + │ └──────────────────────────────────────┘ │ + │ ┌──────────────────────────────────────┐ │ + │ │ 2. TICK ACTORS │ │ + │ │ for actor in &mut actor_pool { │ │ + │ │ let n = drain_count(actor); │ │ + │ │ for _ in 0..n { │ │ + │ │ let msg = actor.mailbox.pop();│ │ + │ │ actor.handle(&ctx, msg); │ │ + │ │ } │ │ + │ │ } │ │ + │ └──────────────────────────────────────┘ │ + │ ┌──────────────────────────────────────┐ │ + │ │ 3. IDLE? │ │ + │ │ if no messages processed: │ │ + │ │ spin → yield → park │ │ + │ └──────────────────────────────────────┘ │ + │ } │ + └───────────────────────────────────────────────┘ +``` + +## File Tree + +``` +src/ +├── lib.rs # crate root, feature flags, public exports +├── error.rs # Error type +│ +├── actor.rs # Message trait, ActorInterface trait, ActorAddress +│ # - ActorInterface::handle(&mut self, ctx: &dyn Context, msg) +│ # - actors depend ONLY on Context, nothing else +│ +├── context.rs # Context trait — the "syscall interface" for actors +│ # - send(), self_addr(), spawn() +│ # - this is ALL actors can see of the framework +│ +├── envelope.rs # Envelope type — type erasure for cross-thread messages +│ # - wraps typed messages for the transfer queue +│ # - unwraps back to concrete type at destination +│ +├── address_map.rs # ActorAddress → WorkerId mapping +│ # - shared read-heavy structure +│ # - written on spawn, read on every send +│ +├── transfer.rs # Transfer queue — per-worker MPSC +│ # - the ONE concurrent data structure on the hot path +│ # - carries (ActorAddress, Envelope) pairs +│ +├── worker/ +│ ├── mod.rs # Worker struct and worker loop +│ │ # - owns actor pool + transfer queue +│ │ # - the thread boundary: concurrent outside, local inside +│ │ # - drain transfer queue → tick actors → backoff +│ │ +│ ├── mailbox.rs # VecDeque-based local mailbox +│ │ # - NO atomics, NO Arc, NO crossbeam +│ │ # - only touched by the owning worker thread +│ │ +│ └── pool.rs # Actor pool — stores actors assigned to this worker +│ # - local HashMap or Vec for ActorAddress → Actor lookup +│ # - insert on spawn, remove on shutdown +│ +├── runtime.rs # Runtime — the composition point +│ # - creates workers, address map +│ # - implements Context (delegates to address map + transfer queues) +│ # - public API: new(), spawn(), send_to(), run(), tick(), shutdown() +│ +├── config.rs # RuntimeConfig — tuning knobs +│ # - num_threads, max_actors, mailbox capacity +│ # - drain strategy, backoff policy +│ # - placement strategy (round-robin, caller-affinity, etc.) +│ +└── placement.rs # Actor placement strategy + # - decides which worker a new actor goes to + # - round-robin, least-loaded, caller-affinity +``` + +## Components + +| Component | File(s) | What It Does | Concurrent? | +|---|---|---|---| +| **Worker** | `worker/mod.rs` | Owns a thread, a pool of actors, their mailboxes, and a transfer queue. Runs the tick loop. Everything inside is single-threaded. | No (that's the point) | +| **Mailbox** | `worker/mailbox.rs` | `VecDeque` per actor. Zero atomics. Only the owning worker reads/writes. | No | +| **Actor Pool** | `worker/pool.rs` | Stores actors on this worker. Local lookup by address. | No | +| **Transfer Queue** | `transfer.rs` | MPSC queue per worker. The only atomic boundary. Other workers push, this worker pops. | Yes (the ONE place) | +| **Address Map** | `address_map.rs` | Maps ActorAddress → WorkerId. Read on every cross-thread send, written on spawn. | Yes (read-heavy) | +| **Envelope** | `envelope.rs` | Type-erases messages for the transfer queue. Unwrapped at destination. | No (data format) | +| **Context** | `context.rs` | Trait that actors see. `send()`, `self_addr()`, `spawn()`. Hides all framework internals. | N/A (trait) | +| **Runtime** | `runtime.rs` | Wires it all together. Creates workers, holds address map, exposes public API. | Minimal (delegates) | +| **Placement** | `placement.rs` | Decides which worker gets a new actor. | No (called at spawn time) | + +## Single-Threaded / WASM Mode + +One worker. No transfer queue needed. No address map needed (everything is local). The system collapses to: + +``` + Worker 0 + ┌───────────────────────┐ + │ Actor A [mailbox] │ + │ Actor B [mailbox] │ All sends are local. + │ Actor C [mailbox] │ All mailboxes are VecDeque. + │ │ Zero atomics anywhere. + │ tick() drives loop │ + └───────────────────────┘ +``` diff --git a/CACHE.md b/CACHE.md deleted file mode 100644 index bbb982e..0000000 --- a/CACHE.md +++ /dev/null @@ -1,1050 +0,0 @@ -# Cache Locality Design Guidelines for Actor Runtimes - -## Table of Contents -1. [Foundational Concepts](#foundational-concepts) -2. [Actor Data Structure Layout](#actor-data-structure-layout) -3. [Mailbox Design for Cache Efficiency](#mailbox-design-for-cache-efficiency) -4. [Memory Allocation Strategies](#memory-allocation-strategies) -5. [Scheduler Design for Locality](#scheduler-design-for-locality) -6. [NUMA-Aware Considerations](#numa-aware-considerations) -7. [Post-Design Tuning Strategies](#post-design-tuning-strategies) -8. [Framework-Specific Examples](#framework-specific-examples) - ---- - -## Foundational Concepts - -### Cache Hierarchy Understanding - -Before designing your actor runtime, you must internalize how modern CPUs access memory: - -``` -┌─────────────────────────────────────────────────────────────┐ -│ CPU Core │ -│ ┌─────────┐ │ -│ │ Registers│ ~0.5 cycles │ -│ └────┬────┘ │ -│ ▼ │ -│ ┌─────────┐ │ -│ │ L1 Cache│ 32-64KB, ~4 cycles, per-core │ -│ └────┬────┘ │ -│ ▼ │ -│ ┌─────────┐ │ -│ │ L2 Cache│ 256KB-1MB, ~12 cycles, per-core │ -│ └────┬────┘ │ -└───────┼─────────────────────────────────────────────────────┘ - ▼ - ┌─────────┐ - │ L3 Cache│ 8-64MB, ~40 cycles, shared across cores - └────┬────┘ - ▼ - ┌─────────┐ - │ RAM │ ~100-300 cycles - └─────────┘ -``` - -**Key insight**: A cache miss at L1 that propagates to RAM can be 50-100x slower than a cache hit. Your actor runtime design should minimize these misses. - -### Cache Lines: The Atomic Unit of Memory Transfer - -Modern CPUs don't transfer individual bytes—they transfer **cache lines**, typically 64 bytes on x86-64 and ARM (though 128 bytes on some ARM implementations like Apple M-series for certain operations). - -```cpp -// On most x86-64 systems: -constexpr size_t CACHE_LINE_SIZE = 64; - -// C++17 provides portable constants: -#include -// std::hardware_destructive_interference_size (typically 64) -// std::hardware_constructive_interference_size (typically 64) -``` - -This has two critical implications: - -1. **Spatial Locality**: When you access one byte, the CPU fetches 63 more. Design your data structures so related data lives nearby. - -2. **False Sharing**: When two threads write to different variables on the same cache line, the CPU invalidates that line for all cores, causing severe performance degradation. - ---- - -## Actor Data Structure Layout - -### Principle 1: Keep Hot Data Together - -The most frequently accessed data during message processing should fit within one or two cache lines. Here's how CAF (C++ Actor Framework) approaches this: - -```cpp -// Conceptual layout inspired by CAF's actor design -// CAF actors are designed to be only a few hundred bytes - -struct alignas(64) Actor { - // === CACHE LINE 1: Critical scheduling data === - std::atomic state; // 4 bytes: running/waiting/done - uint32_t flags; // 4 bytes: various flags - MailboxPtr mailbox; // 8 bytes: pointer to mailbox - SchedulerPtr home_scheduler; // 8 bytes: affinity hint - ActorId id; // 8 bytes: unique identifier - RefCount ref_count; // 8 bytes: reference counting - BehaviorPtr current_behavior; // 8 bytes: message handler - // Remaining: 16 bytes for future use - char padding1[16]; - - // === CACHE LINE 2: Less frequently accessed === - ActorPtr parent; // 8 bytes: supervision - ChildList children; // 16 bytes: supervised actors - ErrorHandler on_error; // 8 bytes: error handling - // ... additional metadata -}; - -static_assert(offsetof(Actor, parent) == 64, - "Second cache line should start at offset 64"); -``` - -**Why this matters**: When the scheduler decides whether to run an actor, it only needs the first cache line. The message handler only needs the first two cache lines for most operations. - -### Principle 2: Separate Hot and Cold Data - -Actors often have data that's accessed frequently (hot) and data accessed rarely (cold). Separating these prevents cold data from evicting hot data from cache: - -```cpp -// HOT: Accessed on every message dispatch -struct ActorHotData { - alignas(64) std::atomic mailbox_count; - BehaviorFunction* behavior; - void* user_state; // Pointer to actual actor state - SchedulerWorker* affinity; -}; - -// COLD: Accessed only during lifecycle events or errors -struct ActorColdData { - std::string name; - ActorAddress parent; - std::vector children; - std::function error_handler; - MonitorList monitors; - LinkList links; - CreationTimestamp created_at; -}; - -struct Actor { - ActorHotData hot; // First cache line(s) - ActorColdData* cold; // Pointer to cold data, allocated separately -}; -``` - -### Principle 3: Prevent False Sharing in Scheduler Structures - -When you have per-worker data structures, false sharing is your enemy. Here's the pattern used in work-stealing schedulers: - -```cpp -// BAD: Dense packing causes false sharing -struct BadWorkerArray { - std::atomic work_count[NUM_WORKERS]; // All on adjacent cache lines! -}; - -// GOOD: Cache-line padding prevents false sharing -struct alignas(64) PaddedCounter { - std::atomic count; - char padding[64 - sizeof(std::atomic)]; -}; - -struct GoodWorkerArray { - PaddedCounter work_count[NUM_WORKERS]; // Each on its own cache line -}; -``` - -The Rust ecosystem provides `cache-padded` (now deprecated but instructive): - -```rust -// From the cache-padded crate (conceptually) -use std::sync::atomic::AtomicUsize; - -// Concurrent queue with cache-padded indices -struct Queue { - head: CachePadded, // Own cache line - tail: CachePadded, // Own cache line - buffer: *mut T, -} -``` - -CAF's work-stealing scheduler applies this pattern to its worker queues, ensuring that each worker's deque head/tail pointers don't share cache lines with other workers. - ---- - -## Mailbox Design for Cache Efficiency - -### MPSC Queue Optimizations (Multiple Producers, Single Consumer) - -Actor mailboxes are typically MPSC queues. The design significantly impacts cache performance: - -```cpp -// Intrusive linked-list approach (good for variable message sizes) -// Used conceptually by many actor frameworks - -struct Message { - Message* next; // 8 bytes - MessageType type; // 4 bytes - uint32_t payload_size; // 4 bytes - alignas(16) char payload[]; // Flexible array member -}; - -class IntrusiveMailbox { - // Producer side (multiple threads write here) - alignas(64) std::atomic tail; - - // Consumer side (single thread reads here) - alignas(64) Message* head; - - // Sentinel node to simplify empty-check - Message stub; -public: - void push(Message* msg) { - msg->next = nullptr; - Message* prev = tail.exchange(msg, std::memory_order_acq_rel); - prev->next = msg; // Linearization point - } - - Message* pop() { - Message* h = head; - Message* next = h->next; - if (next) { - head = next; - // Return the message that was after stub, or actual message - return (h == &stub) ? pop() : h; - } - return nullptr; - } -}; -``` - -**Cache analysis**: -- `tail` is on its own cache line (written by producers) -- `head` is on its own cache line (written by consumer) -- This prevents false sharing between producers and consumer - -### Bounded Ring Buffer Approach (better cache locality for small messages) - -```cpp -template -class alignas(64) BoundedMailbox { - static_assert((Capacity & (Capacity - 1)) == 0, - "Capacity must be power of 2"); - - // Producer state - own cache line - alignas(64) std::atomic write_pos{0}; - - // Consumer state - own cache line - alignas(64) size_t read_pos{0}; - size_t cached_write{0}; // Cached write_pos to reduce atomic reads - - // Buffer - contiguous for spatial locality - alignas(64) T buffer[Capacity]; - -public: - bool try_push(const T& item) { - size_t wp = write_pos.load(std::memory_order_relaxed); - size_t next = (wp + 1) & (Capacity - 1); - - // Check if full (would need to load read_pos from consumer) - if (next == read_pos) return false; - - buffer[wp] = item; - write_pos.store(next, std::memory_order_release); - return true; - } - - bool try_pop(T& item) { - if (read_pos == cached_write) { - cached_write = write_pos.load(std::memory_order_acquire); - if (read_pos == cached_write) return false; - } - - item = buffer[read_pos]; - read_pos = (read_pos + 1) & (Capacity - 1); - return true; - } -}; -``` - -**Cache benefits**: -- Sequential access patterns maximize hardware prefetching -- Bounded size means working set fits in cache -- The `cached_write` optimization reduces atomic operations - -### CAF's Dual-Queue Strategy - -CAF uses a double-ended queue with interesting cache locality properties: - -```cpp -// Simplified version of CAF's mailbox concept -class CafStyleMailbox { - // Internal jobs (from same worker) - LIFO for cache locality - alignas(64) std::atomic internal_head; - spinlock internal_lock; - - // External jobs (from other workers) - FIFO for fairness - alignas(64) std::atomic external_tail; - spinlock external_lock; - -public: - // Called by the worker that owns this actor - void internal_enqueue(Job* job) { - // LIFO: new jobs go to front - // This maximizes temporal locality - recently touched - // actor state is still warm in cache - std::lock_guard lock(internal_lock); - job->next = internal_head.load(std::memory_order_relaxed); - internal_head.store(job, std::memory_order_release); - } - - // Called by other workers sending messages - void external_enqueue(Job* job) { - // FIFO: maintains message ordering guarantees - std::lock_guard lock(external_lock); - job->next = nullptr; - Job* prev = external_tail.exchange(job, std::memory_order_acq_rel); - if (prev) prev->next = job; - } -}; -``` - -The LIFO internal queue is crucial: when an actor sends a message to another actor and that message creates a response, processing the response immediately means the original actor's state is still cache-hot. - ---- - -## Memory Allocation Strategies - -### Arena Allocators: The Foundation of Cache-Friendly Allocation - -Standard `malloc` has several cache-hostile properties: -1. Metadata overhead (typically 8-16 bytes per allocation) -2. Memory fragmentation scatters related objects -3. System call overhead for large allocations - -Arena allocators solve these problems: - -```cpp -class Arena { - char* buffer; - size_t capacity; - size_t offset; - -public: - Arena(size_t size) - : buffer(static_cast(aligned_alloc(64, size))) - , capacity(size) - , offset(0) {} - - void* allocate(size_t size, size_t alignment = alignof(std::max_align_t)) { - // Align the offset - size_t aligned_offset = (offset + alignment - 1) & ~(alignment - 1); - - if (aligned_offset + size > capacity) { - return nullptr; // Or grow/chain arenas - } - - void* ptr = buffer + aligned_offset; - offset = aligned_offset + size; - return ptr; - } - - void reset() { offset = 0; } // "Free" everything at once - - ~Arena() { free(buffer); } -}; -``` - -### Per-Actor Arenas - -Each actor can have its own arena, ensuring that: -1. Actor state and its heap-allocated data are contiguous -2. GC is simplified (just reset the arena when actor dies) -3. No cross-thread allocation/deallocation (thread-local arenas) - -```cpp -class ActorArena { - static constexpr size_t INITIAL_SIZE = 4096; // One page - - struct Block { - Block* next; - size_t size; - size_t used; - alignas(16) char data[]; - }; - - Block* current; - Block* blocks; // Linked list for cleanup - -public: - void* allocate(size_t size) { - size = (size + 15) & ~15; // Align to 16 bytes - - if (current->used + size > current->size) { - grow(size); - } - - void* ptr = current->data + current->used; - current->used += size; - return ptr; - } - - // For actor-local allocations that follow message processing - template - T* create(Args&&... args) { - void* mem = allocate(sizeof(T)); - return new (mem) T(std::forward(args)...); - } -}; -``` - -### Message Pool Allocators - -Messages often have predictable sizes. Pool allocators exploit this: - -```cpp -template -class MessagePool { - static_assert(SlotSize >= sizeof(void*), "Slot must fit a pointer"); - - struct alignas(64) Block { - Block* next; - char slots[SlotSize * SlotsPerBlock]; - }; - - Block* blocks = nullptr; - void* free_list = nullptr; - -public: - void* allocate() { - if (!free_list) { - grow(); - } - void* slot = free_list; - free_list = *static_cast(slot); - return slot; - } - - void deallocate(void* ptr) { - *static_cast(ptr) = free_list; - free_list = ptr; - } - -private: - void grow() { - Block* b = new Block; - b->next = blocks; - blocks = b; - - // Thread all slots through free list - for (size_t i = 0; i < SlotsPerBlock; ++i) { - void* slot = b->slots + i * SlotSize; - *static_cast(slot) = free_list; - free_list = slot; - } - } -}; - -// Usage: Pool for common message sizes -MessagePool<64> small_messages; // ≤48 byte payload -MessagePool<256> medium_messages; // ≤240 byte payload -MessagePool<1024> large_messages; // ≤1008 byte payload -``` - -### Pony's Per-Actor Heap Approach - -Pony takes an extreme approach: each actor has its own heap, and the garbage collector (ORCA) runs per-actor without stop-the-world pauses: - -``` -// Pony's conceptual memory model -Actor { - local_heap: Heap, // Only this actor allocates here - reference_counts: Map, // Track foreign references - message_queue: Queue, // Incoming messages -} - -// Key insight: GC only runs when actor is not executing a behavior -// This means: -// 1. No stack to scan (no stack map needed) -// 2. No safepoints required -// 3. No synchronization with other actors during GC -``` - ---- - -## Scheduler Design for Locality - -### Work-Stealing with Locality Awareness - -The basic work-stealing algorithm is cache-oblivious. Here's how to make it cache-aware: - -```cpp -class LocalityAwareScheduler { - struct alignas(64) WorkerState { - std::deque local_queue; - std::atomic is_active{true}; - uint32_t numa_node; - uint32_t core_id; - - // Steal victims ordered by locality (nearest first) - std::vector steal_order; - }; - - std::vector workers; - -public: - void initialize_steal_order(uint32_t worker_id) { - WorkerState& w = workers[worker_id]; - - // Build steal order: same NUMA node first, then others - for (uint32_t i = 0; i < workers.size(); ++i) { - if (i == worker_id) continue; - - if (workers[i].numa_node == w.numa_node) { - // Same NUMA node - insert at front - w.steal_order.insert(w.steal_order.begin(), i); - } else { - // Different NUMA node - append at end - w.steal_order.push_back(i); - } - } - } - - Actor* try_steal(uint32_t worker_id) { - WorkerState& w = workers[worker_id]; - - // Try stealing in locality order - for (uint32_t victim_id : w.steal_order) { - WorkerState& victim = workers[victim_id]; - - // Steal from back (FIFO) to maintain parent-child locality - if (!victim.local_queue.empty()) { - Actor* stolen = victim.local_queue.back(); - victim.local_queue.pop_back(); - return stolen; - } - } - return nullptr; - } -}; -``` - -### CAF's Scheduler Design - -CAF's scheduler uses work-stealing with specific optimizations for cache locality: - -```cpp -// From CAF's conceptual design -class CafScheduler { - // Each worker has its own deque - // Internal enqueue: LIFO (front) - maximizes cache reuse - // External enqueue: FIFO (back) - fair ordering - // Stealing: FIFO (back) - steals "cold" work - - void worker_loop(WorkerId id) { - while (running) { - Actor* actor = try_get_local(id); - - if (!actor) { - actor = try_steal_from_random(id); - } - - if (actor) { - // Process messages until actor blocks or quota exhausted - run_actor(actor); - - // If actor has more work, re-enqueue locally (LIFO) - // This keeps hot actors on the same core - if (actor->has_pending_messages()) { - internal_enqueue(id, actor); - } - } else { - // No work available - sleep briefly - sleep_or_poll(id); - } - } - } -}; -``` - -### Affinity-Based Scheduling - -For maximum cache efficiency, actors can have "home" workers: - -```cpp -struct Actor { - std::atomic home_worker{NO_AFFINITY}; - - // ... other fields -}; - -class AffinityScheduler { - void enqueue(Actor* actor, WorkerId sender) { - WorkerId home = actor->home_worker.load(std::memory_order_relaxed); - - if (home == NO_AFFINITY) { - // First run: assign to sender's worker for initial locality - actor->home_worker.store(sender, std::memory_order_relaxed); - workers[sender].enqueue(actor); - } else { - // Subsequent runs: prefer home worker - workers[home].enqueue(actor); - } - } - - // Periodically rebalance to prevent hot spots - void rebalance() { - // Move actors with high message rates to dedicated workers - // Consider communication patterns: actors that communicate - // frequently should be on the same core/NUMA node - } -}; -``` - ---- - -## NUMA-Aware Considerations - -### Understanding NUMA Topology - -On multi-socket systems, memory access time depends on which CPU socket allocated the memory: - -``` -┌─────────────────┐ QPI/UPI ┌─────────────────┐ -│ Socket 0 │◄────────────────►│ Socket 1 │ -│ ┌───────────┐ │ │ ┌───────────┐ │ -│ │ 8 Cores │ │ │ │ 8 Cores │ │ -│ └───────────┘ │ │ └───────────┘ │ -│ ┌───────────┐ │ │ ┌───────────┐ │ -│ │ Local RAM │ │ │ │ Local RAM │ │ -│ │ 32GB │ │ │ │ 32GB │ │ -│ └───────────┘ │ │ └───────────┘ │ -└─────────────────┘ └─────────────────┘ - -Local memory access: ~100ns -Remote memory access: ~300ns (3x slower!) -``` - -### NUMA-Aware Actor Placement - -```cpp -class NumaAwareScheduler { - struct NumaNode { - std::vector workers; - Arena* local_arena; // Memory allocated on this node - std::atomic actor_count{0}; - }; - - std::vector nodes; - -public: - void spawn_actor(ActorFactory factory, NumaHint hint = HINT_LOCAL) { - int target_node; - - switch (hint) { - case HINT_LOCAL: - target_node = current_numa_node(); - break; - case HINT_NEAR_PARENT: - target_node = parent_actor->numa_node; - break; - case HINT_LEAST_LOADED: - target_node = find_least_loaded_node(); - break; - } - - // Allocate actor memory on target NUMA node - NumaNode& node = nodes[target_node]; - void* mem = node.local_arena->allocate(sizeof(Actor)); - Actor* actor = new (mem) Actor(factory); - actor->numa_node = target_node; - - // Assign to a worker on the same node - WorkerId worker = node.workers[round_robin++ % node.workers.size()]; - actor->home_worker = worker; - } - - // Work stealing prefers same-node victims - Actor* steal(WorkerId thief) { - int thief_node = worker_to_node[thief]; - - // First: try workers on same NUMA node - for (WorkerId victim : nodes[thief_node].workers) { - if (Actor* a = try_steal_from(victim)) { - return a; - } - } - - // Then: try other nodes (expensive - remote memory!) - for (int n = 0; n < nodes.size(); ++n) { - if (n == thief_node) continue; - for (WorkerId victim : nodes[n].workers) { - if (Actor* a = try_steal_from(victim)) { - return a; - } - } - } - - return nullptr; - } -}; -``` - -### Research-Backed NUMA Strategies - -The paper "Work-Stealing, Locality-Aware Actor Scheduling" by Barghi et al. identifies key strategies: - -1. **Hub Actor Identification**: Actors that spawn many short-lived children should keep those children on the same NUMA node. - -2. **Communication Locality**: Actors that frequently message each other should be co-located. - -3. **Hierarchical Stealing**: Steal from local workers first, then socket-local, then remote. - -```cpp -// Implementing hierarchical stealing based on the research -class HierarchicalStealer { - // Topology levels: Core → L3 Cache Group → Socket → System - struct Level { - std::vector peers; - double steal_probability; // Higher for closer levels - }; - - std::vector> topology; // [worker][level] - - Actor* steal(WorkerId thief) { - for (const Level& level : topology[thief]) { - // Probabilistic stealing at each level - if (random() < level.steal_probability) { - WorkerId victim = level.peers[random() % level.peers.size()]; - if (Actor* a = try_steal_from(victim)) { - return a; - } - } - } - return nullptr; - } -}; -``` - ---- - -## Post-Design Tuning Strategies - -### Profiling for Cache Misses - -Once your runtime is working, these tools help identify cache problems: - -```bash -# Linux perf for cache miss analysis -perf stat -e cache-references,cache-misses,L1-dcache-load-misses \ - ./your_actor_runtime - -# Detailed cache analysis -perf record -e cache-misses ./your_actor_runtime -perf report - -# NUMA statistics -numastat -p $(pgrep your_actor_runtime) -``` - -### Tunable Parameters - -Design your runtime with these tunable knobs: - -```cpp -struct RuntimeConfig { - // Scheduler tuning - size_t messages_per_actor_run = 10; // Batch size before yielding - size_t steal_batch_size = 1; // How many actors to steal - double steal_probability = 0.5; // Work-stealing aggressiveness - - // Memory tuning - size_t actor_arena_initial = 4096; // Initial arena size - size_t actor_arena_max = 1024 * 1024; // Maximum before GC - size_t message_pool_slots = 1024; // Pool allocator size - - // Cache tuning - size_t prefetch_distance = 3; // Mailbox prefetch depth - bool enable_numa_awareness = true; // NUMA-local allocation - bool enable_affinity = true; // Worker-actor affinity -}; -``` - -### Prefetching Strategies - -When you know you'll access data soon, tell the CPU: - -```cpp -void process_mailbox(Actor* actor) { - Message* current = actor->mailbox.head; - - while (current) { - Message* next = current->next; - - // Prefetch next message while processing current - if (next) { - __builtin_prefetch(next, 0, 3); // Read, high temporal locality - __builtin_prefetch(next->payload, 0, 3); - } - - process_message(actor, current); - current = next; - } -} -``` - -### Batch Processing for Cache Warmth - -Process multiple messages while actor state is cache-hot: - -```cpp -void run_actor(Actor* actor, size_t max_messages = 10) { - auto& behavior = actor->current_behavior; - - // Keep processing while mailbox non-empty and quota not exhausted - size_t processed = 0; - while (processed < max_messages) { - Message* msg = actor->mailbox.try_pop(); - if (!msg) break; - - // Actor state is cache-hot from previous iteration - behavior->handle(actor, msg); - ++processed; - } - - // Record for tuning: if we hit quota, actor is "hot" - if (processed == max_messages) { - actor->hot_count++; - } -} -``` - -### Adaptive Granularity (ElasticActor Approach) - -Research shows that optimal batch size depends on workload: - -```cpp -class AdaptiveScheduler { - struct ActorStation { - std::vector actors; - Mailbox shared_mailbox; - - // Actors in same station share a mailbox - // Coarser granularity = better cache utilization - // Finer granularity = better parallelism - }; - - std::vector stations; - std::atomic station_count; - - void adjust_granularity() { - double avg_latency = measure_message_latency(); - double target_latency = 1000; // 1 microsecond target - - if (avg_latency > target_latency * 1.5) { - // Latency too high: split stations for more parallelism - split_hottest_station(); - } else if (avg_latency < target_latency * 0.5) { - // Latency low: merge stations for better cache use - merge_coldest_stations(); - } - } -}; -``` - ---- - -## Framework-Specific Examples - -### CAF (C++ Actor Framework) - -CAF's key cache optimizations: - -```cpp -// 1. Minimal actor footprint (~240 bytes) -// From CAF's design philosophy: -// "CAF actors consist of only a few hundred bytes" - -// 2. Work-stealing with LIFO local queuing -// internal_enqueue adds to front (LIFO) for cache locality -// external_enqueue adds to back (FIFO) for fairness - -// 3. Type-safe messaging reduces runtime overhead -template -void typed_send(typed_actor& receiver, Args... args) { - // Compile-time type checking eliminates runtime type dispatch - // Smaller message headers, better cache utilization -} - -// 4. Configurable scheduler -auto cfg = actor_system_config{} - .set("caf.scheduler.max-threads", 8) - .set("caf.scheduler.policy", "stealing"); // or "sharing" -``` - -### Actix (Rust) - -Actix leverages Rust's ownership for cache-friendly design: - -```rust -use actix::prelude::*; - -// Actors own their state - no shared mutable state -struct MyActor { - // State is exclusively owned, always cache-local to processing - counter: u64, - buffer: Vec, // Can pre-allocate for known workloads -} - -impl Actor for MyActor { - type Context = Context; - - fn started(&mut self, ctx: &mut Self::Context) { - // Set mailbox capacity to control memory usage - ctx.set_mailbox_capacity(16); // Default is 16 - } -} - -// Messages are moved, not copied - zero-copy when possible -struct DataMessage { - payload: Vec, // Ownership transferred, not cloned -} - -impl Handler for MyActor { - type Result = (); - - fn handle(&mut self, msg: DataMessage, _ctx: &mut Context) { - // msg.payload is now owned by this actor - // No cache invalidation from other threads - self.buffer = msg.payload; - } -} -``` - -### Pony - -Pony's extreme approach to cache locality: - -```pony -// Each actor has isolated heap - perfect cache locality for GC -actor Counter - var _count: U64 = 0 - - // All state is actor-local, always cache-hot when processing - be increment() => - _count = _count + 1 - - be get(main: Main) => - // Sending is zero-copy for immutable data - main.print(_count) - -// Reference capabilities ensure no data races -// This allows GC without read/write barriers -// GC runs only between behaviors - no stack scanning needed -``` - -Key Pony cache insights: -- Per-actor heaps mean allocation is always thread-local -- No stop-the-world GC means no cache thrashing from GC threads -- Zero-copy messaging through capability-based types - ---- - -## Benchmarking Your Optimizations - -### Micro-benchmarks - -```cpp -// Message passing latency -void benchmark_mailbox_latency() { - Actor sender, receiver; - auto start = high_resolution_clock::now(); - - for (int i = 0; i < 1'000'000; ++i) { - sender.send(receiver, PingMessage{}); - receiver.process_one(); - } - - auto end = high_resolution_clock::now(); - auto ns_per_message = duration_cast(end - start).count() / 1'000'000; - std::cout << "Latency: " << ns_per_message << " ns/msg\n"; -} - -// Actor creation throughput -void benchmark_actor_creation() { - auto start = high_resolution_clock::now(); - - for (int i = 0; i < 100'000; ++i) { - auto actor = spawn(); - // Actor immediately becomes garbage - } - - auto end = high_resolution_clock::now(); - auto actors_per_sec = 100'000.0 / duration_cast(end - start).count(); - std::cout << "Creation: " << actors_per_sec << " actors/sec\n"; -} -``` - -### Monitoring Production Performance - -```cpp -class SchedulerMetrics { - // Per-worker metrics (cache-padded to avoid false sharing) - struct alignas(64) WorkerMetrics { - std::atomic messages_processed{0}; - std::atomic actors_run{0}; - std::atomic steal_attempts{0}; - std::atomic steal_successes{0}; - std::atomic cache_misses_estimate{0}; // From perf counters - }; - - std::vector workers; - -public: - void report() { - double steal_success_rate = total_steal_successes / total_steal_attempts; - // Low steal success rate + high CPU might indicate poor locality - - double messages_per_actor = total_messages / total_actors_run; - // High value indicates good batching - } -}; -``` - ---- - -## Summary: Cache Locality Checklist - -### Data Structure Design -- [ ] Actor hot data fits in 1-2 cache lines -- [ ] Scheduler per-worker state is cache-line padded -- [ ] Mailbox head/tail are on separate cache lines -- [ ] Cold data is allocated separately from hot data - -### Memory Allocation -- [ ] Per-actor or per-worker arenas for locality -- [ ] Pool allocators for common message sizes -- [ ] NUMA-aware allocation on multi-socket systems - -### Scheduling -- [ ] LIFO local scheduling for cache reuse -- [ ] Batch processing (multiple messages per actor run) -- [ ] Locality-aware work stealing (same NUMA node first) -- [ ] Actor affinity to workers - -### Tuning -- [ ] Profile with `perf` for cache miss rates -- [ ] Configurable batch sizes and steal policies -- [ ] Prefetching for predictable access patterns -- [ ] Adaptive granularity based on latency metrics - ---- - -## References - -1. CAF: The C++ Actor Framework - https://actor-framework.org/ -2. Barghi et al. "Work-Stealing, Locality-Aware Actor Scheduling" (IPDPS 2018) -3. Clebsch et al. "Orca: GC and Type System Co-Design for Actor Languages" (OOPSLA 2017) -4. Actix - Rust Actor Framework - https://actix.rs/ -5. Pony Language - https://www.ponylang.io/ -6. "Cache Line Alignment in C++" - Various Medium articles -7. Protocol Buffers Arena Allocation - https://protobuf.dev/reference/cpp/arenas/ \ No newline at end of file diff --git a/README.md b/README.md index bc88d5a..181a624 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,58 @@ # swactor -Small wasm-compatible actor library +(S)mall (W)ASM-compatible (actor) library + +## Quick example + +```rust +use swactor::{ + actor::{ActorAddress, ActorInterface}, + runtime::{Runtime, RuntimeConfig}, +}; + +#[derive(Debug, Default)] +struct Greeter { num_greeted: usize } + +#[derive(Debug, Default, Clone)] +struct GreetMessage { who: String, return_addr: ActorAddress } + +#[derive(Debug, Default, Clone)] +struct GreetResponse(String); + +impl ActorInterface for Greeter { + type Incoming = GreetMessage; + type Response = GreetResponse; + + fn handle(&mut self, ctx: &Runtime, msg: GreetMessage) { + let res = GreetResponse(format!("Hello, {}!", msg.who)); + self.num_greeted += 1; + if let Err(_) = ctx.send_to(msg.return_addr, res) { + self.num_greeted -= 1; + } + } +} + +fn main() { + let rt = Runtime::new(RuntimeConfig::default()); + let addr = rt.spawn(Greeter::default()).expect("failed to spawn"); + + let inbox = rt.new_inbox::().unwrap(); + rt.send_to(addr, GreetMessage { + who: "world".into(), + return_addr: *inbox.addr(), + }).unwrap(); + + for _ in 0..3 { rt.tick(); } + let resp = inbox.try_recv().expect("should have response"); + println!("{}", resp.0); // "Hello, world!" +} +``` + +## Build & test + +```sh +cargo build +cargo test +cargo test --features stress # stress tests +cargo run --bin bench --release # benchmarks +cargo run --example hello +``` diff --git a/src/lib.rs b/src/lib.rs index 1d67009..ae7d08a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,5 @@ pub mod actor; +pub mod worker; mod channel; pub(crate) mod error; diff --git a/src/worker/mailbox.rs b/src/worker/mailbox.rs new file mode 100644 index 0000000..82ef233 --- /dev/null +++ b/src/worker/mailbox.rs @@ -0,0 +1,54 @@ +use std::collections::VecDeque; + +use crate::actor::Message; + +const DEFAULT_WATERLEVEL: usize = 10; + +pub struct Mailbox { + queue: VecDeque, + waterlevel: usize, +} + +impl Mailbox { + pub fn new() -> Self { + Self { + queue: VecDeque::new(), + waterlevel: DEFAULT_WATERLEVEL, + } + } + + pub fn with_waterlevel(waterlevel: usize) -> Self { + Self { + queue: VecDeque::new(), + waterlevel, + } + } + + pub fn push(&mut self, msg: M) { + self.queue.push_back(msg); + } + + pub fn pop(&mut self) -> Option { + self.queue.pop_front() + } + + pub fn len(&self) -> usize { + self.queue.len() + } + + pub fn is_empty(&self) -> bool { + self.queue.is_empty() + } + + /// How many messages to process this tick: + /// - `len < waterlevel` → process all (`len`) + /// - `len >= waterlevel` → process half (`len >> 1`) + pub fn drain_count(&self) -> usize { + let len = self.queue.len(); + if len < self.waterlevel { + len + } else { + len >> 1 + } + } +} diff --git a/src/worker/mod.rs b/src/worker/mod.rs new file mode 100644 index 0000000..808ccf2 --- /dev/null +++ b/src/worker/mod.rs @@ -0,0 +1 @@ +pub mod mailbox; diff --git a/tests/mailbox_tests.rs b/tests/mailbox_tests.rs new file mode 100644 index 0000000..8c5aa93 --- /dev/null +++ b/tests/mailbox_tests.rs @@ -0,0 +1,226 @@ +use swactor::worker::mailbox::Mailbox; + +// ── Basic operations ── + +#[test] +fn push_and_pop() { + let mut mb = Mailbox::new(); + mb.push(42i32); + assert_eq!(mb.pop(), Some(42)); +} + +#[test] +fn fifo_ordering() { + let mut mb = Mailbox::new(); + mb.push(1); + mb.push(2); + mb.push(3); + assert_eq!(mb.pop(), Some(1)); + assert_eq!(mb.pop(), Some(2)); + assert_eq!(mb.pop(), Some(3)); +} + +#[test] +fn pop_empty() { + let mut mb: Mailbox = Mailbox::new(); + assert_eq!(mb.pop(), None); +} + +#[test] +fn multiple_messages() { + let mut mb = Mailbox::new(); + for i in 0..100 { + mb.push(i); + } + for i in 0..100 { + assert_eq!(mb.pop(), Some(i)); + } + assert_eq!(mb.pop(), None); +} + +#[test] +fn interleaved_push_pop() { + let mut mb = Mailbox::new(); + mb.push(1); + mb.push(2); + assert_eq!(mb.pop(), Some(1)); + mb.push(3); + assert_eq!(mb.pop(), Some(2)); + assert_eq!(mb.pop(), Some(3)); + assert_eq!(mb.pop(), None); +} + +// ── Drain count / watermark logic ── + +#[test] +fn drain_count_empty() { + let mb: Mailbox = Mailbox::new(); + assert_eq!(mb.drain_count(), 0); +} + +#[test] +fn drain_count_below_waterlevel() { + let mut mb = Mailbox::new(); + for i in 0..5 { + mb.push(i); + } + // 5 < 10 (default waterlevel) → process all + assert_eq!(mb.drain_count(), 5); +} + +#[test] +fn drain_count_at_waterlevel() { + let mut mb = Mailbox::new(); + for i in 0..10 { + mb.push(i); + } + // 10 >= 10 → process half → 5 + assert_eq!(mb.drain_count(), 5); +} + +#[test] +fn drain_count_above_waterlevel() { + let mut mb = Mailbox::new(); + for i in 0..20 { + mb.push(i); + } + // 20 >= 10 → 20 >> 1 = 10 + assert_eq!(mb.drain_count(), 10); +} + +#[test] +fn drain_count_one_message() { + let mut mb = Mailbox::new(); + mb.push(1i32); + // 1 < 10 → process all → 1 + assert_eq!(mb.drain_count(), 1); +} + +#[test] +fn drain_count_just_below_waterlevel() { + let mut mb = Mailbox::new(); + for i in 0..9 { + mb.push(i); + } + // 9 < 10 → process all → 9 + assert_eq!(mb.drain_count(), 9); +} + +#[test] +fn drain_count_large() { + let mut mb = Mailbox::new(); + for i in 0..1000 { + mb.push(i); + } + // 1000 >= 10 → 1000 >> 1 = 500 + assert_eq!(mb.drain_count(), 500); +} + +#[test] +fn drain_count_custom_waterlevel() { + let mut mb = Mailbox::with_waterlevel(4); + for i in 0..3 { + mb.push(i); + } + // 3 < 4 → process all → 3 + assert_eq!(mb.drain_count(), 3); + + mb.push(99); + // 4 >= 4 → 4 >> 1 = 2 + assert_eq!(mb.drain_count(), 2); +} + +#[test] +fn drain_count_updates_after_pop() { + let mut mb = Mailbox::new(); + for i in 0..20 { + mb.push(i); + } + // 20 >= 10 → 10 + assert_eq!(mb.drain_count(), 10); + + // pop 15, leaving 5 + for _ in 0..15 { + mb.pop(); + } + // 5 < 10 → process all → 5 + assert_eq!(mb.drain_count(), 5); +} + +// ── Properties ── + +#[test] +fn len_tracks_pushes() { + let mut mb = Mailbox::new(); + assert_eq!(mb.len(), 0); + mb.push(1); + assert_eq!(mb.len(), 1); + mb.push(2); + assert_eq!(mb.len(), 2); + mb.push(3); + assert_eq!(mb.len(), 3); +} + +#[test] +fn len_tracks_pops() { + let mut mb = Mailbox::new(); + mb.push(1); + mb.push(2); + mb.push(3); + assert_eq!(mb.len(), 3); + mb.pop(); + assert_eq!(mb.len(), 2); + mb.pop(); + assert_eq!(mb.len(), 1); + mb.pop(); + assert_eq!(mb.len(), 0); +} + +#[test] +fn is_empty_on_new() { + let mb: Mailbox = Mailbox::new(); + assert!(mb.is_empty()); +} + +#[test] +fn is_empty_after_drain() { + let mut mb = Mailbox::new(); + mb.push(1); + mb.push(2); + mb.push(3); + assert!(!mb.is_empty()); + mb.pop(); + mb.pop(); + mb.pop(); + assert!(mb.is_empty()); +} + +// ── Type tests ── + +#[test] +fn works_with_primitive_types() { + let mut mb_i32 = Mailbox::new(); + mb_i32.push(42i32); + assert_eq!(mb_i32.pop(), Some(42)); + + let mut mb_string = Mailbox::new(); + mb_string.push(String::from("hello")); + assert_eq!(mb_string.pop(), Some(String::from("hello"))); +} + +#[test] +fn works_with_custom_structs() { + #[derive(Debug, Clone, PartialEq)] + struct MyMsg { + id: u64, + payload: String, + } + + let mut mb = Mailbox::new(); + let msg = MyMsg { + id: 1, + payload: "test".into(), + }; + mb.push(msg.clone()); + assert_eq!(mb.pop(), Some(msg)); +} -- 2.45.2 From 7c7947c823c77285a8275c11cbd426cf4c0a60e4 Mon Sep 17 00:00:00 2001 From: Zachery Aaron Shores-Chmielewski Date: Thu, 5 Feb 2026 22:54:03 +0700 Subject: [PATCH 5/9] feat: actor addresses, Ctx abstraction, pools, and more Fleshing out the architecture before abstracting into components amenable to api-based test harnesses for fuzz and optimization working loops. --- Cargo.lock | 20 -- Cargo.toml | 1 - README.md | 5 +- benches/scaling.rs | 18 +- benches/throughput.rs | 32 +- examples/hello.rs | 5 +- examples/ring.rs | 5 +- src/actor.rs | 69 +++-- src/address_map.rs | 90 ++++++ src/config.rs | 24 ++ src/context.rs | 45 +++ src/envelope.rs | 49 +++ src/lib.rs | 22 +- src/placement.rs | 37 +++ src/router.rs | 83 ----- src/runtime.rs | 585 +++++++++++++++++------------------- src/worker/mod.rs | 137 +++++++++ src/worker/pool.rs | 51 ++++ tests/runtime_tests.rs | 10 +- tests/stress/concurrency.rs | 7 +- tests/stress/mod.rs | 6 +- tests/stress/saturation.rs | 51 ++-- 22 files changed, 824 insertions(+), 528 deletions(-) create mode 100644 src/address_map.rs create mode 100644 src/config.rs create mode 100644 src/context.rs create mode 100644 src/envelope.rs create mode 100644 src/placement.rs delete mode 100644 src/router.rs create mode 100644 src/worker/pool.rs diff --git a/Cargo.lock b/Cargo.lock index 561715c..a25b4f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,25 +8,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" -[[package]] -name = "crossbeam-deque" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "crossbeam-queue" version = "0.3.12" @@ -69,7 +50,6 @@ checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" name = "swactor" version = "0.1.0" dependencies = [ - "crossbeam-deque", "crossbeam-queue", "crossbeam-utils", "getrandom", diff --git a/Cargo.toml b/Cargo.toml index 6471309..78f8ddb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,6 @@ stress = [] # Enable stress tests [dependencies] getrandom = { version = "0.2", optional = true } -crossbeam-deque = "0.8" crossbeam-queue = "0.3.12" crossbeam-utils = "0.8.21" smallvec = "1.13" diff --git a/README.md b/README.md index 181a624..bcce6fe 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ ```rust use swactor::{ + Ctx, actor::{ActorAddress, ActorInterface}, runtime::{Runtime, RuntimeConfig}, }; @@ -22,10 +23,10 @@ impl ActorInterface for Greeter { type Incoming = GreetMessage; type Response = GreetResponse; - fn handle(&mut self, ctx: &Runtime, msg: GreetMessage) { + fn handle(&mut self, ctx: &Ctx, msg: GreetMessage) { let res = GreetResponse(format!("Hello, {}!", msg.who)); self.num_greeted += 1; - if let Err(_) = ctx.send_to(msg.return_addr, res) { + if let Err(_) = ctx.send(msg.return_addr, res) { self.num_greeted -= 1; } } diff --git a/benches/scaling.rs b/benches/scaling.rs index e54bf64..85440cd 100644 --- a/benches/scaling.rs +++ b/benches/scaling.rs @@ -8,6 +8,7 @@ use crate::harness::{black_box, Bench, BenchSuite}; use std::thread; use swactor::{ + Ctx, actor::ActorInterface, runtime::{Runtime, RuntimeConfig}, }; @@ -34,7 +35,7 @@ impl ActorInterface for CounterActor { type Incoming = Increment; type Response = (); - fn handle(&mut self, _ctx: &Runtime, _msg: Increment) { + fn handle(&mut self, _ctx: &Ctx, _msg: Increment) { self.count += 1; } } @@ -47,7 +48,7 @@ impl ActorInterface for SharedCounter { type Incoming = Increment; type Response = (); - fn handle(&mut self, _ctx: &Runtime, _msg: Increment) { + fn handle(&mut self, _ctx: &Ctx, _msg: Increment) { self.count .fetch_add(1, std::sync::atomic::Ordering::Relaxed); } @@ -71,7 +72,7 @@ impl ActorInterface for PayloadActor { type Incoming = Payload; type Response = (); - fn handle(&mut self, _ctx: &Runtime, msg: Payload) { + fn handle(&mut self, _ctx: &Ctx, msg: Payload) { self.bytes_received += msg.0.len(); black_box(&msg.0); } @@ -97,8 +98,7 @@ pub fn bench_actor_count_scaling(suite: &mut BenchSuite) { || { let config = RuntimeConfig { max_actors: (actor_count as usize) + 100, - router_max_messages: (total_messages as usize) * 3, - actor_max_messages: (messages_per_actor as usize) * 2, + actor_max_messages: (total_messages as usize) * 3, num_threads: 1, }; let runtime = Runtime::new(config); @@ -110,7 +110,7 @@ pub fn bench_actor_count_scaling(suite: &mut BenchSuite) { actors.push(addr); } - // Process registrations + // Process spawns for _ in 0..(actor_count * 2) { runtime.tick(); } @@ -156,8 +156,7 @@ pub fn bench_thread_count_scaling(suite: &mut BenchSuite) { let counter = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); let config = RuntimeConfig { max_actors: (actor_count as usize) + 100, - router_max_messages: (total_messages as usize) * 3, - actor_max_messages: (messages_per_actor as usize) * 2, + actor_max_messages: (total_messages as usize) * 3, num_threads: thread_count, }; let runtime = Runtime::new(config); @@ -233,14 +232,13 @@ pub fn bench_payload_size_scaling(suite: &mut BenchSuite) { || { let config = RuntimeConfig { max_actors: 10, - router_max_messages: (message_count as usize) * 2, actor_max_messages: (message_count as usize) * 2, num_threads: 1, }; let runtime = Runtime::new(config); let sink = runtime.spawn(PayloadActor::new()).unwrap(); - // Process registration + // Process spawn for _ in 0..10 { runtime.tick(); } diff --git a/benches/throughput.rs b/benches/throughput.rs index 58c67b3..001b0a1 100644 --- a/benches/throughput.rs +++ b/benches/throughput.rs @@ -8,6 +8,7 @@ use crate::harness::{black_box, Bench, BenchSuite}; use swactor::{ + Ctx, actor::{ActorAddress, ActorInterface}, runtime::{Runtime, RuntimeConfig}, }; @@ -34,7 +35,7 @@ impl ActorInterface for SinkActor { type Incoming = Ping; type Response = (); - fn handle(&mut self, _ctx: &Runtime, _msg: Ping) { + fn handle(&mut self, _ctx: &Ctx, _msg: Ping) { self.count += 1; } } @@ -58,9 +59,9 @@ impl ActorInterface for ForwardActor { type Incoming = Ping; type Response = Ping; - fn handle(&mut self, ctx: &Runtime, msg: Ping) { + fn handle(&mut self, ctx: &Ctx, msg: Ping) { if let Some(next) = self.next { - let _ = ctx.send_to(next, msg); + let _ = ctx.send(next, msg); } } } @@ -69,7 +70,7 @@ impl ActorInterface for ForwardActor { // Benchmarks // ============================================================================ -/// Benchmark: Messages sent through the router to a single sink actor +/// Benchmark: Messages sent through the runtime to a single sink actor pub fn bench_message_throughput(suite: &mut BenchSuite) { for msg_count in [1_000u64, 10_000, 100_000] { let name = format!("message_throughput_{}", msg_count); @@ -83,7 +84,6 @@ pub fn bench_message_throughput(suite: &mut BenchSuite) { // Setup: create runtime and sink actor let config = RuntimeConfig { max_actors: 100, - router_max_messages: (msg_count as usize) * 2, actor_max_messages: (msg_count as usize) * 2, num_threads: 1, }; @@ -97,8 +97,6 @@ pub fn bench_message_throughput(suite: &mut BenchSuite) { let _ = runtime.send_to::(sink, Ping); } // Process until done - // Tick enough times to process all messages - // (router tick + actor tick) * messages / WATERLEVEL for _ in 0..(count * 3) { runtime.tick(); } @@ -123,7 +121,6 @@ pub fn bench_spawn_rate(suite: &mut BenchSuite) { || { let config = RuntimeConfig { max_actors: 1000, - router_max_messages: 10_000, actor_max_messages: 100, num_threads: 1, }; @@ -133,7 +130,7 @@ pub fn bench_spawn_rate(suite: &mut BenchSuite) { for _ in 0..actor_count { let _ = runtime.spawn(SinkActor::new()); } - // Process router messages to register all actors + // Process spawns for _ in 0..(actor_count * 2) { runtime.tick(); } @@ -159,10 +156,9 @@ pub fn bench_fanout(suite: &mut BenchSuite) { || { let config = RuntimeConfig { max_actors: (fan_count as usize) + 10, - router_max_messages: (fan_count as usize) + actor_max_messages: (fan_count as usize) * (messages_per_receiver as usize) * 2, - actor_max_messages: (messages_per_receiver as usize) * 2, num_threads: 1, }; let runtime = Runtime::new(config); @@ -174,7 +170,7 @@ pub fn bench_fanout(suite: &mut BenchSuite) { sinks.push(addr); } - // Process router registrations + // Process spawns for _ in 0..(fan_count * 2) { runtime.tick(); } @@ -217,8 +213,7 @@ pub fn bench_fanin(suite: &mut BenchSuite) { let total_messages = (sender_count * messages_per_sender) as usize; let config = RuntimeConfig { max_actors: (sender_count as usize) + 10, - router_max_messages: total_messages * 3, - actor_max_messages: total_messages * 2, + actor_max_messages: total_messages * 3, num_threads: 1, }; let runtime = Runtime::new(config); @@ -233,7 +228,7 @@ pub fn bench_fanin(suite: &mut BenchSuite) { senders.push(addr); } - // Process router registrations + // Process spawns for _ in 0..((sender_count + 1) * 2) { runtime.tick(); } @@ -275,8 +270,7 @@ pub fn bench_ring(suite: &mut BenchSuite) { || { let config = RuntimeConfig { max_actors: (ring_size as usize) + 10, - router_max_messages: 10_000, - actor_max_messages: 1_000, + actor_max_messages: 10_000, num_threads: 1, }; let runtime = Runtime::new(config); @@ -292,7 +286,7 @@ pub fn bench_ring(suite: &mut BenchSuite) { // so instead we'll use an inbox to receive the final message // For now, we'll just measure message passing through a chain - // Process registrations + // Process spawns for _ in 0..(ring_size * 2) { runtime.tick(); } @@ -301,7 +295,7 @@ pub fn bench_ring(suite: &mut BenchSuite) { }, |(runtime, actors, laps)| { // Send to first actor (even though they don't forward, we're - // measuring the router + inbox overhead) + // measuring the transfer queue + inbox overhead) for _ in 0..laps { for actor in &actors { let _ = runtime.send_to::(*actor, Ping); diff --git a/examples/hello.rs b/examples/hello.rs index d67c439..18378d9 100644 --- a/examples/hello.rs +++ b/examples/hello.rs @@ -1,4 +1,5 @@ use swactor::{ + Ctx, actor::{ActorAddress, ActorInterface}, runtime::{Runtime, RuntimeConfig}, }; @@ -24,10 +25,10 @@ impl ActorInterface for Greeter { type Incoming = GreetMessage; type Response = GreetResponse; - fn handle(&mut self, ctx: &Runtime, msg: GreetMessage) { + fn handle(&mut self, ctx: &Ctx, msg: GreetMessage) { let res = GreetResponse(format!("Hello, {}!", msg.who)); self.num_greeted += 1; - if let Err(_) = ctx.send_to(msg.return_addr, res) { + if let Err(_) = ctx.send(msg.return_addr, res) { // no error handling self.num_greeted -= 1; } diff --git a/examples/ring.rs b/examples/ring.rs index d3c6031..703a3be 100644 --- a/examples/ring.rs +++ b/examples/ring.rs @@ -1,4 +1,5 @@ use swactor::{ + Ctx, actor::{ActorAddress, ActorInterface}, runtime::{Inbox, Runtime, RuntimeConfig}, }; @@ -30,8 +31,8 @@ impl RingActor { impl ActorInterface for RingActor { type Incoming = RingMessage; type Response = (); - fn handle(&mut self, ctx: &Runtime, msg: Self::Incoming) { - if let Err(_) = ctx.send_to(self.next, msg.next()) { + fn handle(&mut self, ctx: &Ctx, msg: Self::Incoming) { + if let Err(_) = ctx.send(self.next, msg.next()) { // do nothing } } diff --git a/src/actor.rs b/src/actor.rs index c3938ea..e14cf82 100644 --- a/src/actor.rs +++ b/src/actor.rs @@ -1,4 +1,7 @@ -use crate::{WATERLEVEL, channel::Receiver, get_random, runtime::Runtime}; +use std::any::Any; + +use crate::{get_random, worker::mailbox::Mailbox}; +use crate::context::{ContextInner, Ctx}; /// The primary trait defining data that can be passed to and from actor processes pub trait Message: 'static + Sized + Clone + Send + Sync {} @@ -14,7 +17,7 @@ impl Message for T {} /// /// # Example /// ``` -/// use swactor::{actor::{ActorAddress, ActorInterface}, runtime::Runtime}; +/// use swactor::{Ctx, actor::{ActorAddress, ActorInterface}}; /// /// struct Greeter { /// num_greeted: usize, @@ -32,10 +35,10 @@ impl Message for T {} /// impl ActorInterface for Greeter { /// type Incoming = GreetMessage; /// type Response = GreetResponse; -/// -/// fn handle(&mut self, ctx: &Runtime, msg: Self::Incoming) { +/// +/// fn handle(&mut self, ctx: &Ctx, msg: Self::Incoming) { /// let response = GreetResponse(format!("Hello, {}!", msg.who).to_string()); -/// if let Ok(_) = ctx.send_to(msg.return_addr, response) { +/// if let Ok(_) = ctx.send(msg.return_addr, response) { /// self.num_greeted += 1; /// } /// } @@ -44,7 +47,7 @@ impl Message for T {} pub trait ActorInterface: 'static + Send { type Incoming: Message; type Response: Message; - fn handle(&mut self, ctx: &Runtime, msg: Self::Incoming); + fn handle(&mut self, ctx: &Ctx, msg: Self::Incoming); } /// A unique address for this actor. 32 bytes is overkill for a small application, @@ -60,48 +63,58 @@ impl ActorAddress { } } -/// The actor process as represented in the Runtime, with the actor state stored with it's inbox. +/// The actor process as represented in the Runtime, with the actor state stored with its mailbox. pub(crate) struct Actor where A: ActorInterface, { - inbox: Receiver, + addr: ActorAddress, + mailbox: Mailbox, inner: A, } impl Actor { - pub(crate) fn new(inbox: Receiver, inner: A) -> Self { - Self { inbox, inner } + pub(crate) fn new(addr: ActorAddress, mailbox: Mailbox, inner: A) -> Self { + Self { + addr, + mailbox, + inner, + } } } /// Trait for type-erased actors pub(crate) trait AnyActor: Send { - fn tick(&mut self, ctx: &Runtime); + /// Tick the actor, processing pending messages. Returns `true` if any work was done. + fn tick(&mut self, inner: &dyn ContextInner) -> bool; + /// Deliver a type-erased message into this actor's mailbox. + /// Returns `true` if the downcast succeeded. + fn deliver(&mut self, msg: Box) -> bool; } impl AnyActor for Actor where A: ActorInterface, { - fn tick(&mut self, ctx: &Runtime) { - // TODO: WATERLEVEL is hard coded, and so is this message handling scheme. We should - // make it so both are more flexible, with sane defaults. - // FIXME: lots of indirection just to get length on a hot path - let total_messages = self.inbox.len(); - let messages_to_process = if total_messages < WATERLEVEL { - total_messages - } else { - total_messages >> 1 - }; - - for _ in 0..messages_to_process { - match self.inbox.try_recv() { - Some(msg) => self.inner.handle(ctx, msg), - None => unreachable!( - "We checked number of unprocessed messages in the queue ahead of processing" - ), + fn tick(&mut self, inner: &dyn ContextInner) -> bool { + let n = self.mailbox.drain_count(); + if n > 0 { + let ctx = Ctx::new(inner, self.addr); + for _ in 0..n { + if let Some(msg) = self.mailbox.pop() { + self.inner.handle(&ctx, msg); + } } } + n > 0 + } + + fn deliver(&mut self, msg: Box) -> bool { + if let Ok(typed) = msg.downcast::() { + self.mailbox.push(*typed); + true + } else { + false + } } } diff --git a/src/address_map.rs b/src/address_map.rs new file mode 100644 index 0000000..adabe3a --- /dev/null +++ b/src/address_map.rs @@ -0,0 +1,90 @@ +use std::collections::HashMap; +use std::sync::RwLock; + +use crate::actor::ActorAddress; + +/// Identifies a worker thread. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) struct WorkerId(pub(crate) usize); + +impl WorkerId { + pub fn as_usize(self) -> usize { + self.0 + } +} + +/// Maps actor addresses to the worker that owns them. +/// +/// `RwLock` — zero contention for parallel reads, write-rare (only on spawn). +pub(crate) struct AddressMap { + inner: RwLock>, +} + +impl AddressMap { + pub fn new() -> Self { + Self { + inner: RwLock::new(HashMap::new()), + } + } + + pub fn with_capacity(cap: usize) -> Self { + Self { + inner: RwLock::new(HashMap::with_capacity(cap)), + } + } + + pub fn insert(&self, addr: ActorAddress, worker: WorkerId) { + self.inner.write().unwrap().insert(addr, worker); + } + + pub fn remove(&self, addr: &ActorAddress) { + self.inner.write().unwrap().remove(addr); + } + + pub fn lookup(&self, addr: &ActorAddress) -> Option { + self.inner.read().unwrap().get(addr).copied() + } + + pub fn len(&self) -> usize { + self.inner.read().unwrap().len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn insert_and_lookup() { + let map = AddressMap::new(); + let addr = ActorAddress::default(); + let wid = WorkerId(3); + map.insert(addr, wid); + assert_eq!(map.lookup(&addr), Some(wid)); + } + + #[test] + fn lookup_missing_returns_none() { + let map = AddressMap::new(); + let addr = ActorAddress::default(); + assert_eq!(map.lookup(&addr), None); + } + + #[test] + fn remove_works() { + let map = AddressMap::new(); + let addr = ActorAddress::default(); + map.insert(addr, WorkerId(0)); + map.remove(&addr); + assert_eq!(map.lookup(&addr), None); + } + + #[test] + fn len_tracks_entries() { + let map = AddressMap::with_capacity(10); + assert_eq!(map.len(), 0); + let addr1 = ActorAddress::default(); + map.insert(addr1, WorkerId(0)); + assert_eq!(map.len(), 1); + } +} diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000..f6d2a8b --- /dev/null +++ b/src/config.rs @@ -0,0 +1,24 @@ +/// The tunable settings for the runtime. +pub struct RuntimeConfig { + pub max_actors: usize, + pub actor_max_messages: usize, + pub num_threads: usize, +} + +/// 8kB for the `Box<..>` before counting the rest of the memory +const DEFAULT_MAX_ACTORS: usize = 1_000; + +/// 16kB PER ACTOR to alloc space for storing messages. +/// With default setting of [DEFAULT_MAX_ACTORS] this is: +/// 1_000 * 16kB = 16MB +const DEFAULT_ACTOR_MAX_MESSAGES: usize = 1_000; + +impl Default for RuntimeConfig { + fn default() -> Self { + Self { + max_actors: DEFAULT_MAX_ACTORS, + actor_max_messages: DEFAULT_ACTOR_MAX_MESSAGES, + num_threads: 1, + } + } +} diff --git a/src/context.rs b/src/context.rs new file mode 100644 index 0000000..6bd2387 --- /dev/null +++ b/src/context.rs @@ -0,0 +1,45 @@ +use std::any::Any; + +use crate::actor::{Actor, ActorAddress, ActorInterface, AnyActor, Message}; +use crate::worker::mailbox::Mailbox; +use crate::Error; + +/// Object-safe inner trait for sending type-erased messages. +pub(crate) trait ContextInner { + fn send_any(&self, addr: ActorAddress, msg: Box) -> Result<(), Error>; + fn spawn_any(&self, addr: ActorAddress, actor: Box) -> Result<(), Error>; +} + +/// Actor syscall interface — passed to `ActorInterface::handle()`. +/// +/// Wraps a `&dyn ContextInner` to solve the object-safety problem while +/// providing a typed public API. +pub struct Ctx<'a> { + inner: &'a dyn ContextInner, + self_addr: ActorAddress, +} + +impl<'a> Ctx<'a> { + pub(crate) fn new(inner: &'a dyn ContextInner, self_addr: ActorAddress) -> Self { + Self { inner, self_addr } + } + + /// Returns the address of the actor currently being ticked. + pub fn self_addr(&self) -> ActorAddress { + self.self_addr + } + + /// Send a typed message to an actor address. + pub fn send(&self, addr: ActorAddress, msg: M) -> Result<(), Error> { + self.inner.send_any(addr, Box::new(msg)) + } + + /// Spawn a new actor, returning its address. + pub fn spawn(&self, actor: A) -> Result { + let addr = ActorAddress::new_random(); + let actor = Actor::new(addr, Mailbox::new(), actor); + let boxed: Box = Box::new(actor); + self.inner.spawn_any(addr, boxed)?; + Ok(addr) + } +} diff --git a/src/envelope.rs b/src/envelope.rs new file mode 100644 index 0000000..d55fb8a --- /dev/null +++ b/src/envelope.rs @@ -0,0 +1,49 @@ +use std::any::Any; + +use crate::actor::ActorAddress; + +/// A type-erased message envelope for cross-worker delivery. +/// +/// Uses `Box` (no atomic refcount) and move semantics (no clone). +pub(crate) struct Envelope { + dest: ActorAddress, + payload: Box, +} + +impl Envelope { + pub fn new(dest: ActorAddress, payload: Box) -> Self { + Self { dest, payload } + } + + pub fn dest(&self) -> ActorAddress { + self.dest + } + + pub fn downcast(self) -> Option { + self.payload.downcast::().ok().map(|b| *b) + } + + pub fn into_payload(self) -> Box { + self.payload + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn envelope_roundtrip() { + let addr = ActorAddress::default(); + let env = Envelope::new(addr, Box::new(42u64)); + assert_eq!(env.dest(), addr); + assert_eq!(env.downcast::(), Some(42u64)); + } + + #[test] + fn envelope_wrong_type_returns_none() { + let addr = ActorAddress::default(); + let env = Envelope::new(addr, Box::new(42u64)); + assert!(env.downcast::().is_none()); + } +} diff --git a/src/lib.rs b/src/lib.rs index ae7d08a..08f7b07 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,11 +1,18 @@ pub mod actor; pub mod worker; -mod channel; +pub(crate) mod channel; pub(crate) mod error; pub use error::Error; -mod router; +pub mod context; +pub use context::Ctx; + +pub(crate) mod envelope; +pub(crate) mod address_map; +pub(crate) mod placement; +pub mod config; + pub mod runtime; #[cfg(feature = "getrandom")] @@ -26,14 +33,3 @@ pub(crate) fn get_random(buf: &mut [u8]) { *byte = bytes[i % core::mem::size_of::()]; } } - -/// FIXME: remove hard coded defaults -/// The strategy for message processing is such: -/// -/// ```ignore -/// if total_messages < WATERLEVEL: -/// process all -/// else -/// process total_messages >> 1 -/// ``` -const WATERLEVEL: usize = 10; diff --git a/src/placement.rs b/src/placement.rs new file mode 100644 index 0000000..d88e63d --- /dev/null +++ b/src/placement.rs @@ -0,0 +1,37 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; + +use crate::address_map::WorkerId; + +/// Round-robin actor placement strategy. +pub(crate) struct Placement { + next: AtomicUsize, + num_workers: usize, +} + +impl Placement { + pub fn new(num_workers: usize) -> Self { + Self { + next: AtomicUsize::new(0), + num_workers, + } + } + + pub fn next_worker(&self) -> WorkerId { + let id = self.next.fetch_add(1, Ordering::Relaxed) % self.num_workers; + WorkerId(id) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_robin() { + let p = Placement::new(3); + assert_eq!(p.next_worker(), WorkerId(0)); + assert_eq!(p.next_worker(), WorkerId(1)); + assert_eq!(p.next_worker(), WorkerId(2)); + assert_eq!(p.next_worker(), WorkerId(0)); + } +} diff --git a/src/router.rs b/src/router.rs deleted file mode 100644 index d015fe1..0000000 --- a/src/router.rs +++ /dev/null @@ -1,83 +0,0 @@ -use std::{collections::HashMap, sync::Arc}; - -use crate::{ - actor::{ActorAddress, ActorInterface, Message}, - channel::Sender, - runtime::Runtime, -}; - -/// FIXME: Go over with a fine-toothed comb and reassure yourself this typing -/// makes sense, that we are not doing loads of indirection on a hot path. -/// -/// A type erased `Message` to be routed between actor processes. -pub(crate) type Envelope = Arc; - -pub(crate) trait SenderT: Send + Sync { - fn try_send(&self, envelope: Envelope); -} - -impl SenderT for Sender { - fn try_send(&self, envelope: Envelope) { - if let Some(msg) = envelope.downcast_ref::() { - // FIXME: we are directly cloning the contents of the Arc pointer here - // Do we want to? Should we provide another way? - // - // The standard concept of an actor has message and state - // isolation, so we should leave this as is. However, we should - // make it clear and obvious this pathway is the heavy, contained - // pathway, and include a shared memory pathway for logic that - // may need it. - let _ = Sender::try_send(self, msg.clone()); - } - } -} - -/// Internal messages for the Router's own inbox -#[derive(Clone)] -pub(crate) enum RouterMessage { - /// register addrs with sender - AddAddr(ActorAddress, Arc), - - /// FIXME: this will be active when we allow actors to shut themselves - /// down. For now, disable the warning. - #[allow(dead_code)] - /// remove an actor from the address book - RemoveAddr(ActorAddress), - - /// send to - SendToAddr { addr: ActorAddress, msg: Envelope }, -} - -/// The `Router` is responsible for taking in and delivering all messages in the runtime. -pub(crate) struct Router { - directory: HashMap>, -} - -impl Router { - pub fn new() -> Self { - Self { - directory: HashMap::new(), - } - } -} - -impl ActorInterface for Router { - type Incoming = RouterMessage; - type Response = (); - - fn handle(&mut self, _ctx: &Runtime, msg: Self::Incoming) { - match msg { - RouterMessage::AddAddr(addr, sender) => { - self.directory.insert(addr, sender); - } - RouterMessage::RemoveAddr(addr) => { - self.directory.remove(&addr); - } - RouterMessage::SendToAddr { addr, msg } => { - if let Some(sender) = self.directory.get(&addr) { - sender.try_send(msg); - } - } - } - } -} diff --git a/src/runtime.rs b/src/runtime.rs index 1a5df3b..e602db2 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -1,17 +1,71 @@ +use std::any::Any; +use std::cell::RefCell; +use std::collections::HashMap; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; +use std::sync::{Arc, RwLock}; use std::thread::{self, JoinHandle}; -use std::time::Duration; -use crossbeam_deque::{Injector, Steal, Stealer, Worker}; +use crate::actor::{Actor, ActorAddress, ActorInterface, AnyActor, Message}; +use crate::address_map::{AddressMap, WorkerId}; +use crate::channel::{Receiver, Sender}; +// Re-export RuntimeConfig so existing code using `runtime::RuntimeConfig` still works +pub use crate::config::RuntimeConfig; +use crate::context::ContextInner; +use crate::envelope::Envelope; +use crate::placement::Placement; +use crate::worker::mailbox::Mailbox; +use crate::worker::Worker; +use crate::Error; -use crate::channel::HybridChannel; -use crate::{ - actor::{Actor, ActorAddress, ActorInterface, AnyActor, Message}, - channel::{Receiver, Sender}, - router::{Router, RouterMessage}, - Error, -}; +// ─── SenderT trait (moved from router.rs) ──────────────────────────────────── + +/// Type-erased sender for external inboxes. +pub(crate) trait SenderT: Send + Sync { + fn try_send_any(&self, msg: Box); +} + +impl SenderT for Sender { + fn try_send_any(&self, msg: Box) { + if let Ok(typed) = msg.downcast::() { + let _ = Sender::try_send(self, *typed); + } + } +} + +// ─── InboxRegistry ─────────────────────────────────────────────────────────── + +/// Registry of external inboxes — replaces the Router's role for non-actor receivers. +pub(crate) struct InboxRegistry { + senders: RwLock>>, +} + +impl InboxRegistry { + pub fn new() -> Self { + Self { + senders: RwLock::new(HashMap::new()), + } + } + + pub fn register(&self, addr: ActorAddress, sender: Arc) { + self.senders.write().unwrap().insert(addr, sender); + } + + pub fn try_deliver( + &self, + addr: ActorAddress, + msg: Box, + ) -> Result<(), Error> { + let senders = self.senders.read().unwrap(); + if let Some(sender) = senders.get(&addr) { + sender.try_send_any(msg); + Ok(()) + } else { + Err(Error::from("Address not found")) + } + } +} + +// ─── Inbox ─────────────────────────────────────────────────────────────────── /// Generic message inbox for receiving messages outside of the runtime. pub struct Inbox { @@ -29,59 +83,210 @@ impl Inbox { } } -/// The tunable settings for the runtime. -pub struct RuntimeConfig { - pub max_actors: usize, - pub router_max_messages: usize, - pub actor_max_messages: usize, - pub num_threads: usize, -} - -/// 8kB for the `Box<..>` before counting the rest of the memory -const DEFAULT_MAX_ACTORS: usize = 1_000; - -/// 160kB for the `Arc<..>` before counting the rest of the memory -const DEFAULT_ROUTER_MAX_MESSAGES: usize = 10_000; - -/// 16kB PER ACTOR to alloc space for storing the `Arc<..>` pointers -/// With default setting of [DEFAULT_MAX_ACTORS] this is: -/// 1_000 * 16kB = 16MB -const DEFAULT_ACTOR_MAX_MESSAGES: usize = 1_000; - -impl Default for RuntimeConfig { - fn default() -> Self { - Self { - max_actors: DEFAULT_MAX_ACTORS, - router_max_messages: DEFAULT_ROUTER_MAX_MESSAGES, - actor_max_messages: DEFAULT_ACTOR_MAX_MESSAGES, - num_threads: 1, - } - } -} - -/// Per-worker state for work-stealing scheduler -struct WorkerState { - local: Worker>, // FIFO for fairness - id: usize, -} - -/// Shared scheduler state for work-stealing -struct SchedulerState { - injector: Injector>, // For spawn() - stealers: Vec>>, // For work-stealing - is_running: AtomicBool, -} +// ─── Runtime ───────────────────────────────────────────────────────────────── /// The `Runtime` struct is the primary gateway for interacting with the framework. pub struct Runtime { config: RuntimeConfig, - actor_queue: HybridChannel>, // Used for single-threaded mode - scheduler: Option>, // Used for multi-threaded mode - router_interface: Sender, - router: Option>, // `None` if single-threaded - - // for multithreaded contexts + address_map: Arc, + inbox_registry: Arc, + transfer_txs: Vec>, + spawn_txs: Vec)>>, + placement: Placement, is_running: AtomicBool, + /// Single-threaded mode: worker stored inline + single_worker: Option>, + /// Multi-threaded mode: workers waiting to be assigned to threads by run() + pending_workers: Option>, +} + +// Safety: RefCell is only accessed from the thread that owns the Runtime +// in single-threaded mode. In multi-threaded mode, single_worker is None and +// pending_workers is consumed by run() before Arc sharing. +unsafe impl Sync for Runtime {} + +impl Runtime { + /// Builds a new `Runtime` struct, but does not yet run anything. If multithreaded, call + /// `run()`, if single threaded, needs to be driven by calls to the `tick()` method. + pub fn new(config: RuntimeConfig) -> Self { + let num_workers = if config.num_threads < 2 { + 1 + } else { + config.num_threads + }; + + let address_map = Arc::new(AddressMap::with_capacity(config.max_actors)); + let inbox_registry = Arc::new(InboxRegistry::new()); + let placement = Placement::new(num_workers); + + let mut transfer_txs = Vec::with_capacity(num_workers); + let mut spawn_txs = Vec::with_capacity(num_workers); + let mut workers = Vec::with_capacity(num_workers); + + for i in 0..num_workers { + let transfer_rx = Receiver::::new(config.actor_max_messages); + let transfer_tx = transfer_rx.new_sender(); + transfer_txs.push(transfer_tx); + + let spawn_rx = + Receiver::<(ActorAddress, Box)>::new(config.max_actors); + let spawn_tx = spawn_rx.new_sender(); + spawn_txs.push(spawn_tx); + + workers.push(Worker::new(WorkerId(i), transfer_rx, spawn_rx)); + } + + if config.num_threads < 2 { + // Single-threaded: store one worker inline + let worker = workers.remove(0); + Self { + config, + address_map, + inbox_registry, + transfer_txs, + spawn_txs, + placement, + is_running: AtomicBool::new(false), + single_worker: Some(RefCell::new(worker)), + pending_workers: None, + } + } else { + // Multi-threaded: stash workers for run() + Self { + config, + address_map, + inbox_registry, + transfer_txs, + spawn_txs, + placement, + is_running: AtomicBool::new(false), + single_worker: None, + pending_workers: Some(workers), + } + } + } + + /// Spawn an actor, returns its address + pub fn spawn(&self, actor: A) -> Result { + let addr = ActorAddress::new_random(); + let worker_id = self.placement.next_worker(); + self.address_map.insert(addr, worker_id); + let actor = Actor::new(addr, Mailbox::new(), actor); + let boxed: Box = Box::new(actor); + self.spawn_txs[worker_id.as_usize()] + .try_send((addr, boxed)) + .map_err(|_| Error::from("Runtime error: spawn queue full"))?; + Ok(addr) + } + + /// Send a message to an actor address + pub fn send_to(&self, addr: ActorAddress, msg: M) -> Result<(), Error> { + let msg_box: Box = Box::new(msg); + match self.address_map.lookup(&addr) { + Some(wid) => self.transfer_txs[wid.as_usize()] + .try_send(Envelope::new(addr, msg_box)) + .map_err(|_| Error::from("Transfer queue full")), + None => self.inbox_registry.try_deliver(addr, msg_box), + } + } + + /// Create an external inbox for receiving messages in the outer process containing the runtime + pub fn new_inbox(&self) -> Result, Error> { + let addr = ActorAddress::new_random(); + let receiver = Receiver::::new(self.config.actor_max_messages); + let sender = receiver.new_sender(); + self.inbox_registry.register(addr, Arc::new(sender)); + Ok(Inbox { + addr, + inner: receiver, + }) + } + + /// Drive one tick of the single-threaded worker. + pub fn tick(&self) { + if let Some(ref worker) = self.single_worker { + worker.borrow_mut().tick_once( + &self.address_map, + &self.transfer_txs, + &self.spawn_txs, + &self.placement, + &self.inbox_registry, + ); + } + } + + /// Spawn worker threads and start processing, returning a set of handles and + /// a Runtime object to interface with. + /// + /// ### WARN: + /// ##### Returns an error if the configuration is set as single threaded + /// `config.num_threads == 1` + pub fn run(mut self) -> Result { + if self.config.num_threads < 2 { + return Err(Error::from( + "Runtime error: cannot call `Runtime::run()` from a single-threaded context.", + )); + } + + self.is_running.store(true, Ordering::Release); + + let workers = self + .pending_workers + .take() + .expect("Workers must be present for multi-threaded runtime"); + + let rt = Arc::new(self); + let mut handles: Vec> = Vec::with_capacity(workers.len()); + + for mut worker in workers { + let rt_clone = rt.clone(); + let handle = thread::spawn(move || { + worker_loop(&mut worker, &rt_clone); + }); + handles.push(handle); + } + + Ok(RuntimeHandle { + runtime: rt, + threads: handles, + }) + } + + /// Signal all workers to stop + pub fn shutdown(&self) { + self.is_running.store(false, Ordering::Release); + } +} + +/// Worker thread loop for multi-threaded runtime. +/// Uses spin → yield → park backoff to reduce CPU usage when idle. +fn worker_loop(worker: &mut Worker, rt: &Runtime) { + let mut idle_count: u32 = 0; + + while rt.is_running.load(Ordering::Acquire) { + let did_work = worker.tick_once( + &rt.address_map, + &rt.transfer_txs, + &rt.spawn_txs, + &rt.placement, + &rt.inbox_registry, + ); + + if did_work { + idle_count = 0; + } else { + idle_count = idle_count.saturating_add(1); + if idle_count < 64 { + core::hint::spin_loop(); + } else if idle_count < 256 { + thread::yield_now(); + } else { + // Park: sleep briefly, cap at 1ms + let micros = std::cmp::min((idle_count - 256) as u64 * 50, 1000); + thread::sleep(std::time::Duration::from_micros(micros)); + } + } + } } /// Handle for dealing with a runtime that has started via the `Runtime::run()` method. @@ -103,258 +308,22 @@ impl RuntimeHandle { } } -impl Runtime { - /// Builds a new `Runtime` struct, but does not yet run anything. If multithreaded, call - /// `run()`, if single threaded, needs to be driven by calls to the `tick()` method. - pub fn new(config: RuntimeConfig) -> Self { - let actor_queue = HybridChannel::new(config.max_actors); - - // router is a unique actor in that the runtime needs access to it's `Sender` handle - let router_inner = Router::new(); - let router_inbox: Receiver = - Receiver::<::Incoming>::new(config.router_max_messages); - let router_sender = router_inbox.new_sender(); - let router = Actor::new(router_inbox, router_inner); - - // Single-threaded: router goes in queue. Multi-threaded: stays in Option - let router_option = if config.num_threads < 2 { - actor_queue - .push(Box::new(router) as Box) - .map_err(|_| "failed to add router to actor queue") - .expect("failed to spawn router at runtime initialization."); - None - } else { - Some(router) - }; - - Self { - config, - actor_queue, - scheduler: None, // Initialized in run() for multi-threaded mode - router_interface: router_sender, - is_running: AtomicBool::new(false), - router: router_option, +impl ContextInner for Runtime { + fn send_any(&self, addr: ActorAddress, msg: Box) -> Result<(), Error> { + match self.address_map.lookup(&addr) { + Some(wid) => { + let _ = self.transfer_txs[wid.as_usize()].try_send(Envelope::new(addr, msg)); + Ok(()) + } + None => self.inbox_registry.try_deliver(addr, msg), } } - /// Spawn an actor, returns its address - pub fn spawn(&self, actor: A) -> Result { - // assign a stochastic - let addr = ActorAddress::new_random(); - let inbox = Receiver::::new(self.config.actor_max_messages); - let sender = inbox.new_sender(); - - // Register the sender with the router - self.router_interface - .try_send(RouterMessage::AddAddr(addr, Arc::new(sender))) - .map_err(|_| { - Error::from("Runtime error: failed to add actor to router. Router inbox full") - })?; - - let boxed_actor: Box = Box::new(Actor::new(inbox, actor)); - - // Multi-threaded: use injector, single-threaded: use shared queue - if let Some(ref scheduler) = self.scheduler { - scheduler.injector.push(boxed_actor); - } else { - self.actor_queue - .push(boxed_actor) - .map_err(|_| Error::from("Runtime error: Failed to spawn actor. Queue full."))?; - } - - Ok(addr) - } - - /// Send a message to an actor address - pub fn send_to(&self, addr: ActorAddress, msg: M) -> Result<(), Error> { - self.router_interface - .try_send(RouterMessage::SendToAddr { - addr, - msg: Arc::new(msg), - }) - .map_err(|_| Error::from("Failed to send message to router.")) - } - - /// Create an external inbox for receiving messages in the outer process containing the runtime - pub fn new_inbox(&self) -> Result, Error> { - let addr = ActorAddress::new_random(); - - let receiver = Receiver::::new(self.config.actor_max_messages); - let sender = receiver.new_sender(); - - // Register the sender with the router - self.router_interface - .try_send(RouterMessage::AddAddr(addr, Arc::new(sender))) - .map_err(|_| { - Error::from( - "Runtime error: failed to add a new inbox channel. Router inbox is full.", - ) - })?; - - Ok(Inbox { - addr, - inner: receiver, - }) - } - - /// Spawn worker threads and start processing, returning a set of handles and - /// a Runtime object to interface with. - /// - /// ### WARN: - /// ##### This function panics if the configuration is set as single threaded - /// `config.num_threads == 1` - pub fn run(mut self) -> Result { - if self.config.num_threads < 2 { - return Err(Error::from( - "Runtime error: cannot call `Runtime::run()` from a single-threaded context.", - )); - } - - self.is_running.store(true, Ordering::Release); - - // Take router out before wrapping in Arc - it will be owned by router thread - let mut router = self - .router - .take() - .expect("Router must be present for multi-threaded runtime"); - - let num_workers = self.config.num_threads - 1; - - // Create workers and collect stealers for work-stealing - let mut workers = Vec::with_capacity(num_workers); - let mut stealers = Vec::with_capacity(num_workers); - for id in 0..num_workers { - let worker = WorkerState { - local: Worker::new_fifo(), // FIFO for fairness - id, - }; - stealers.push(worker.local.stealer()); - workers.push(worker); - } - - let scheduler = Arc::new(SchedulerState { - injector: Injector::new(), - stealers, - is_running: AtomicBool::new(true), - }); - - // Transfer any actors spawned before run() to the injector - while let Some(actor) = self.actor_queue.pop() { - scheduler.injector.push(actor); - } - - self.scheduler = Some(scheduler.clone()); - let rt = Arc::new(self); - let mut handles: Vec> = vec![]; - - // Router thread owns the router directly - no synchronization needed - // Process multiple ticks per cycle to maximize throughput - const ROUTER_BATCH_SIZE: usize = 64; - let router_handle = { - let ctx = rt.clone(); - let sched = scheduler.clone(); - thread::spawn(move || { - while sched.is_running.load(Ordering::Acquire) { - for _ in 0..ROUTER_BATCH_SIZE { - router.tick(&ctx); - } - } - }) - }; - handles.push(router_handle); - - // Spawn worker threads, each owns its WorkerState - for worker in workers { - let ctx = rt.clone(); - let sched = scheduler.clone(); - let handle = thread::spawn(move || worker_loop(ctx, sched, worker)); - handles.push(handle); - } - - Ok(RuntimeHandle { - runtime: rt, - threads: handles, - }) - } - - /// Pop the actor off the top of the queue and process it's messages, returning it to the back of - /// the queue upon completion. - pub fn tick(&self) { - if let Some(mut actor) = self.actor_queue.pop() { - actor.tick(&self); - let _ = self.actor_queue.push(actor); - } - } - - /// Signal all workers to stop - pub fn shutdown(&self) { - self.is_running.store(false, Ordering::Release); - // Also stop the scheduler if multi-threaded - if let Some(ref scheduler) = self.scheduler { - scheduler.is_running.store(false, Ordering::Release); - } - } -} - -/// Work-stealing worker loop for multi-threaded runtime -fn worker_loop(ctx: Arc, scheduler: Arc, worker: WorkerState) { - const TICKS_PER_ACTOR: usize = 4; - const INJECTOR_CHECK_INTERVAL: usize = 64; // Check injector every N iterations - let mut spin_count: usize = 0; - let mut iteration: usize = 0; - - while scheduler.is_running.load(Ordering::Acquire) { - iteration = iteration.wrapping_add(1); - - // Priority 1: Local queue - let mut actor = worker.local.pop(); - - // Priority 2: Periodically check injector for new actors - // This ensures newly spawned actors get picked up even when workers have local work - if actor.is_none() || (iteration % INJECTOR_CHECK_INTERVAL == 0) { - if let Steal::Success(a) = scheduler.injector.steal_batch_and_pop(&worker.local) { - if actor.is_some() { - // We already had an actor, push the stolen one to local - worker.local.push(a); - } else { - actor = Some(a); - } - } - } - - // Priority 3: Steal from peer workers when idle - if actor.is_none() { - for (i, stealer) in scheduler.stealers.iter().enumerate() { - if i == worker.id { - continue; - } - if let Steal::Success(a) = stealer.steal_batch_and_pop(&worker.local) { - actor = Some(a); - break; - } - } - } - - // Process or backoff - match actor { - Some(mut actor) => { - spin_count = 0; - for _ in 0..TICKS_PER_ACTOR { - actor.tick(&ctx); - } - worker.local.push(actor); - } - None => { - // Exponential backoff - spin_count = spin_count.saturating_add(1); - if spin_count < 10 { - std::hint::spin_loop(); - } else if spin_count < 100 { - thread::yield_now(); - } else { - thread::sleep(Duration::from_micros(10)); - } - } - } + fn spawn_any(&self, addr: ActorAddress, actor: Box) -> Result<(), Error> { + let worker_id = self.placement.next_worker(); + self.address_map.insert(addr, worker_id); + self.spawn_txs[worker_id.as_usize()] + .try_send((addr, actor)) + .map_err(|_| Error::from("Spawn queue full")) } } diff --git a/src/worker/mod.rs b/src/worker/mod.rs index 808ccf2..bc56b5b 100644 --- a/src/worker/mod.rs +++ b/src/worker/mod.rs @@ -1 +1,138 @@ pub mod mailbox; +pub(crate) mod pool; + +use std::any::Any; +use std::cell::RefCell; + +use crate::actor::{ActorAddress, AnyActor}; +use crate::address_map::{AddressMap, WorkerId}; +use crate::channel::{Receiver, Sender}; +use crate::context::ContextInner; +use crate::envelope::Envelope; +use crate::placement::Placement; +use crate::runtime::InboxRegistry; +use crate::Error; +use pool::ActorPool; + +/// A worker owns a set of actors and runs them in a loop. +pub(crate) struct Worker { + id: WorkerId, + pool: ActorPool, + transfer_rx: Receiver, + spawn_rx: Receiver<(ActorAddress, Box)>, +} + +impl Worker { + pub fn new( + id: WorkerId, + transfer_rx: Receiver, + spawn_rx: Receiver<(ActorAddress, Box)>, + ) -> Self { + Self { + id, + pool: ActorPool::new(), + transfer_rx, + spawn_rx, + } + } + + /// Run one iteration of the worker loop. Returns `true` if any work was done. + pub fn tick_once( + &mut self, + address_map: &AddressMap, + transfer_txs: &[Sender], + spawn_txs: &[Sender<(ActorAddress, Box)>], + placement: &Placement, + inbox_registry: &InboxRegistry, + ) -> bool { + let mut did_work = false; + + // 1. Drain spawn queue → add actors to pool + while let Some((addr, actor)) = self.spawn_rx.try_recv() { + self.pool.insert(addr, actor); + did_work = true; + } + + // 2. Drain transfer queue → deliver envelopes to actors + while let Some(envelope) = self.transfer_rx.try_recv() { + let dest = envelope.dest(); + let payload = envelope.into_payload(); + self.pool.deliver(&dest, payload); + did_work = true; + } + + // 3. Tick all actors with WorkerContext + let pending_local: RefCell)>> = + RefCell::new(Vec::new()); + + { + let worker_ctx = WorkerContext { + worker_id: self.id, + address_map, + transfer_txs, + spawn_txs, + placement, + inbox_registry, + pending_local: &pending_local, + }; + if self.pool.tick_all(&worker_ctx) { + did_work = true; + } + } + + // 4. Drain pending_local buffer → deliver to local actors + let pending = pending_local.into_inner(); + if !pending.is_empty() { + did_work = true; + } + for (addr, msg) in pending { + self.pool.deliver(&addr, msg); + } + + did_work + } +} + +/// The `ContextInner` impl for worker threads. +/// +/// Same-worker sends are buffered in `pending_local` (delivered after current tick round). +/// Cross-worker sends go through the transfer queue. +struct WorkerContext<'a> { + worker_id: WorkerId, + address_map: &'a AddressMap, + transfer_txs: &'a [Sender], + spawn_txs: &'a [Sender<(ActorAddress, Box)>], + placement: &'a Placement, + inbox_registry: &'a InboxRegistry, + pending_local: &'a RefCell)>>, +} + +impl ContextInner for WorkerContext<'_> { + fn send_any(&self, addr: ActorAddress, msg: Box) -> Result<(), Error> { + match self.address_map.lookup(&addr) { + Some(wid) if wid == self.worker_id => { + // Same worker: buffer for local delivery (after current tick round) + self.pending_local.borrow_mut().push((addr, msg)); + Ok(()) + } + Some(wid) => { + // Cross worker: envelope through transfer queue + let envelope = Envelope::new(addr, msg); + let _ = self.transfer_txs[wid.as_usize()].try_send(envelope); + Ok(()) + } + None => { + // Try inbox registry (external inboxes) + self.inbox_registry.try_deliver(addr, msg) + } + } + } + + fn spawn_any(&self, addr: ActorAddress, actor: Box) -> Result<(), Error> { + let worker_id = self.placement.next_worker(); + self.address_map.insert(addr, worker_id); + self.spawn_txs[worker_id.as_usize()] + .try_send((addr, actor)) + .map_err(|_| Error::from("Spawn queue full")) + } +} diff --git a/src/worker/pool.rs b/src/worker/pool.rs new file mode 100644 index 0000000..95c7321 --- /dev/null +++ b/src/worker/pool.rs @@ -0,0 +1,51 @@ +use std::any::Any; +use std::collections::HashMap; + +use crate::actor::{ActorAddress, AnyActor}; +use crate::context::ContextInner; + +/// Per-worker actor storage. +pub(crate) struct ActorPool { + actors: HashMap>, +} + +impl ActorPool { + pub fn new() -> Self { + Self { + actors: HashMap::new(), + } + } + + pub fn insert(&mut self, addr: ActorAddress, actor: Box) { + self.actors.insert(addr, actor); + } + + pub fn remove(&mut self, addr: &ActorAddress) -> Option> { + self.actors.remove(addr) + } + + /// Deliver a type-erased message to the actor at `addr`. + /// Returns `true` if the actor was found and the message type matched. + pub fn deliver(&mut self, addr: &ActorAddress, msg: Box) -> bool { + if let Some(actor) = self.actors.get_mut(addr) { + actor.deliver(msg) + } else { + false + } + } + + /// Tick all actors in the pool. Returns `true` if any actor processed messages. + pub fn tick_all(&mut self, inner: &dyn ContextInner) -> bool { + let mut did_work = false; + for actor in self.actors.values_mut() { + if actor.tick(inner) { + did_work = true; + } + } + did_work + } + + pub fn len(&self) -> usize { + self.actors.len() + } +} diff --git a/tests/runtime_tests.rs b/tests/runtime_tests.rs index 56ca321..d57a514 100644 --- a/tests/runtime_tests.rs +++ b/tests/runtime_tests.rs @@ -1,4 +1,4 @@ -use swactor::{actor::{ActorAddress, ActorInterface}, runtime::{Inbox, Runtime, RuntimeConfig}}; +use swactor::{Ctx, actor::{ActorAddress, ActorInterface}, runtime::{Inbox, Runtime, RuntimeConfig}}; #[derive(Clone)] struct PingMessage { @@ -14,8 +14,8 @@ impl ActorInterface for PongActor { type Incoming = PingMessage; type Response = PongMessage; - fn handle(&mut self, ctx: &Runtime, msg: PingMessage) { - let _ = ctx.send_to(msg.reply_to, PongMessage); + fn handle(&mut self, ctx: &Ctx, msg: PingMessage) { + let _ = ctx.send(msg.reply_to, PongMessage); } } @@ -31,8 +31,8 @@ impl ActorInterface for ForwarderActor { type Incoming = ForwardMessage; type Response = (); - fn handle(&mut self, ctx: &Runtime, msg: ForwardMessage) { - let _ = ctx.send_to(self.target, msg); + fn handle(&mut self, ctx: &Ctx, msg: ForwardMessage) { + let _ = ctx.send(self.target, msg); } } diff --git a/tests/stress/concurrency.rs b/tests/stress/concurrency.rs index 2511745..acb3cf4 100644 --- a/tests/stress/concurrency.rs +++ b/tests/stress/concurrency.rs @@ -25,7 +25,6 @@ fn shutdown_under_load() { let result = std::panic::catch_unwind(|| { let config = RuntimeConfig { max_actors: 100, - router_max_messages: 10_000, actor_max_messages: 1000, num_threads: 4, }; @@ -100,7 +99,6 @@ fn send_to_newborn() { for _ in 0..100 { let config = RuntimeConfig { max_actors: 1000, - router_max_messages: 10_000, actor_max_messages: 100, num_threads: 4, }; @@ -149,7 +147,6 @@ fn rapid_spawn_churn() { let config = RuntimeConfig { max_actors: 100, - router_max_messages: 10_000, actor_max_messages: 100, num_threads: 4, }; @@ -210,8 +207,7 @@ fn inbox_contention() { let config = RuntimeConfig { max_actors: 10, - router_max_messages: 100_000, - actor_max_messages: 10_000, + actor_max_messages: 100_000, num_threads: 4, }; let runtime = Runtime::new(config); @@ -284,7 +280,6 @@ fn shutdown_timing_fuzz() { let result = std::panic::catch_unwind(|| { let config = RuntimeConfig { max_actors: 50, - router_max_messages: 1000, actor_max_messages: 100, num_threads: 4, }; diff --git a/tests/stress/mod.rs b/tests/stress/mod.rs index be625c4..87cfdd7 100644 --- a/tests/stress/mod.rs +++ b/tests/stress/mod.rs @@ -165,7 +165,7 @@ impl Stress { } // Test actors used across stress tests -use swactor::{actor::ActorInterface, runtime::Runtime}; +use swactor::{Ctx, actor::ActorInterface}; /// An actor that just absorbs messages pub struct BlackHole; @@ -176,7 +176,7 @@ pub struct Msg; impl ActorInterface for BlackHole { type Incoming = Msg; type Response = (); - fn handle(&mut self, _ctx: &Runtime, _msg: Msg) {} + fn handle(&mut self, _ctx: &Ctx, _msg: Msg) {} } /// An actor that counts messages received @@ -193,7 +193,7 @@ impl Counter { impl ActorInterface for Counter { type Incoming = Msg; type Response = (); - fn handle(&mut self, _ctx: &Runtime, _msg: Msg) { + fn handle(&mut self, _ctx: &Ctx, _msg: Msg) { self.count += 1; } } diff --git a/tests/stress/saturation.rs b/tests/stress/saturation.rs index a8fc97e..9345cd2 100644 --- a/tests/stress/saturation.rs +++ b/tests/stress/saturation.rs @@ -6,23 +6,25 @@ use super::{BlackHole, Counter, Msg, Stress, StressResult}; use std::time::Duration; use swactor::runtime::{Runtime, RuntimeConfig}; -/// Blast the router inbox +/// Blast the transfer queue (replaces router_inbox_overflow) #[test] #[cfg(feature = "stress")] -fn router_inbox_overflow() { - println!("\n>>> STRESS: Router Inbox Overflow"); +fn transfer_queue_overflow() { + println!("\n>>> STRESS: Transfer Queue Overflow"); let config = RuntimeConfig { max_actors: 10, - router_max_messages: 100, // Tiny buffer - actor_max_messages: 1000, + actor_max_messages: 100, // Tiny buffer num_threads: 1, }; let runtime = Runtime::new(config); let sink = runtime.spawn(BlackHole).unwrap(); + // Process spawn + runtime.tick(); + // Blast messages without processing - let mut result = StressResult::new("router_inbox_overflow"); + let mut result = StressResult::new("transfer_queue_overflow"); let start = std::time::Instant::now(); for _ in 0..10_000 { @@ -35,15 +37,15 @@ fn router_inbox_overflow() { } result.duration = start.elapsed(); - result.note(format!("Router buffer: 100, Messages sent: 10,000")); + result.note(format!("Transfer buffer: 100, Messages sent: 10,000")); - // With hybrid, no failures expected + // With hybrid channel, no failures expected assert_eq!(result.failures, 0, "Hybrid channel should not reject"); result.print(); - println!(">>> PASS: Hybrid channel prevented router overflow\n"); + println!(">>> PASS: Hybrid channel prevented transfer queue overflow\n"); } -/// Blast a single actor's inbox +/// Blast a single actor's mailbox via transfer queue #[test] #[cfg(feature = "stress")] fn actor_inbox_overflow() { @@ -51,26 +53,25 @@ fn actor_inbox_overflow() { let config = RuntimeConfig { max_actors: 10, - router_max_messages: 100_000, // Large router buffer - actor_max_messages: 100, // Tiny actor inbox + actor_max_messages: 100_000, // Large transfer buffer num_threads: 1, }; let runtime = Runtime::new(config); let sink = runtime.spawn(Counter::new()).unwrap(); - // Process router registration + // Process spawn runtime.tick(); - // Now blast messages - router will accept them but actor inbox will fill + // Now blast messages let mut sent = 0u64; - let mut router_failed = 0u64; + let mut failed = 0u64; for _ in 0..10_000 { if runtime.send_to::(sink, Msg).is_ok() { sent += 1; } else { - router_failed += 1; + failed += 1; } - // Tick occasionally to let router deliver + // Tick occasionally to let worker deliver if sent % 100 == 0 { runtime.tick(); } @@ -81,10 +82,10 @@ fn actor_inbox_overflow() { runtime.tick(); } - println!(" Router accepted: {}", sent); - println!(" Router rejected: {}", router_failed); + println!(" Sent: {}", sent); + println!(" Failed: {}", failed); - assert_eq!(router_failed, 0, "Router rejected message under load"); + assert_eq!(failed, 0, "Transfer queue rejected message under load"); println!(">>> PASS: No message loss with hybrid channel\n"); } @@ -96,7 +97,6 @@ fn actor_queue_overflow() { let config = RuntimeConfig { max_actors: 100, // Small actor queue - router_max_messages: 10_000, actor_max_messages: 100, num_threads: 1, }; @@ -118,12 +118,12 @@ fn actor_queue_overflow() { result.note(format!("Queue capacity: 100, Spawn attempts: 500")); result.print(); - // Note: Router also takes a slot, so we expect ~99 actors max + // Note: With hybrid channel (overflow to SegQueue), we expect no failures assert_eq!( result.failures, 0, "Spawned more actors than queue capacity" ); - println!(">>> PASS: Actor queue correctly rejects when full\n"); + println!(">>> PASS: Actor queue correctly handles overflow\n"); } /// FIXME: IS this actually testing what it should be? @@ -136,8 +136,7 @@ fn sustained_overload() { let config = RuntimeConfig { max_actors: 100, - router_max_messages: 1000, - actor_max_messages: 100, + actor_max_messages: 1000, num_threads: 1, }; let runtime = Runtime::new(config); @@ -150,7 +149,7 @@ fn sustained_overload() { } } - // Process registrations + // Process spawn registrations for _ in 0..200 { runtime.tick(); } -- 2.45.2 From 06566b5e0f0849d9c72d496bf709b40e98d2044e Mon Sep 17 00:00:00 2001 From: Zachery Aaron Shores-Chmielewski Date: Fri, 6 Feb 2026 13:48:51 +0700 Subject: [PATCH 6/9] feat: more fine tuning available in the config --- ARCHITECTURE.md | 31 ++++++++++---------- TODOs.md | 44 ---------------------------- benches/scaling.rs | 26 ++++++++++------- benches/throughput.rs | 46 +++++++++++++++++++---------- src/config.rs | 31 ++++++++++++++++++++ src/context.rs | 4 ++- src/runtime.rs | 58 ++++++++++++++++++++++--------------- src/worker/mailbox.rs | 11 +------ src/worker/mod.rs | 36 ++++++++++++++--------- tests/mailbox_tests.rs | 42 +++++++++++++-------------- tests/stress/concurrency.rs | 7 +++-- tests/stress/saturation.rs | 5 +++- 12 files changed, 185 insertions(+), 156 deletions(-) delete mode 100644 TODOs.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 8c6a0cb..a93831e 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -120,10 +120,11 @@ src/ ├── error.rs # Error type │ ├── actor.rs # Message trait, ActorInterface trait, ActorAddress -│ # - ActorInterface::handle(&mut self, ctx: &dyn Context, msg) -│ # - actors depend ONLY on Context, nothing else +│ # - ActorInterface::handle(&mut self, ctx: &Ctx, msg) +│ # - actors depend ONLY on Ctx, nothing else │ -├── context.rs # Context trait — the "syscall interface" for actors +├── context.rs # Ctx wrapper — the "syscall interface" for actors +│ # - wraps &dyn ContextInner (solves object-safety) │ # - send(), self_addr(), spawn() │ # - this is ALL actors can see of the framework │ @@ -135,9 +136,10 @@ src/ │ # - shared read-heavy structure │ # - written on spawn, read on every send │ -├── transfer.rs # Transfer queue — per-worker MPSC +├── channel/ +│ └── mod.rs # HybridChannel — per-worker MPSC │ # - the ONE concurrent data structure on the hot path -│ # - carries (ActorAddress, Envelope) pairs +│ # - carries Envelope (cross-worker) and spawn tuples │ ├── worker/ │ ├── mod.rs # Worker struct and worker loop @@ -150,22 +152,21 @@ src/ │ │ # - only touched by the owning worker thread │ │ │ └── pool.rs # Actor pool — stores actors assigned to this worker -│ # - local HashMap or Vec for ActorAddress → Actor lookup +│ # - local HashMap for ActorAddress → Actor lookup │ # - insert on spawn, remove on shutdown │ ├── runtime.rs # Runtime — the composition point │ # - creates workers, address map -│ # - implements Context (delegates to address map + transfer queues) +│ # - implements ContextInner (delegates to address map + transfer queues) │ # - public API: new(), spawn(), send_to(), run(), tick(), shutdown() │ ├── config.rs # RuntimeConfig — tuning knobs -│ # - num_threads, max_actors, mailbox capacity -│ # - drain strategy, backoff policy -│ # - placement strategy (round-robin, caller-affinity, etc.) +│ # - num_threads, max_actors, actor_max_messages +│ # - mailbox_waterlevel (drain threshold per actor) +│ # - BackoffPolicy (spin/yield/sleep thresholds) │ └── placement.rs # Actor placement strategy - # - decides which worker a new actor goes to - # - round-robin, least-loaded, caller-affinity + # - currently: round-robin across workers ``` ## Components @@ -175,12 +176,12 @@ src/ | **Worker** | `worker/mod.rs` | Owns a thread, a pool of actors, their mailboxes, and a transfer queue. Runs the tick loop. Everything inside is single-threaded. | No (that's the point) | | **Mailbox** | `worker/mailbox.rs` | `VecDeque` per actor. Zero atomics. Only the owning worker reads/writes. | No | | **Actor Pool** | `worker/pool.rs` | Stores actors on this worker. Local lookup by address. | No | -| **Transfer Queue** | `transfer.rs` | MPSC queue per worker. The only atomic boundary. Other workers push, this worker pops. | Yes (the ONE place) | +| **Transfer Queue** | `channel/mod.rs` | HybridChannel MPSC queue per worker. The only atomic boundary. Other workers push, this worker pops. | Yes (the ONE place) | | **Address Map** | `address_map.rs` | Maps ActorAddress → WorkerId. Read on every cross-thread send, written on spawn. | Yes (read-heavy) | | **Envelope** | `envelope.rs` | Type-erases messages for the transfer queue. Unwrapped at destination. | No (data format) | -| **Context** | `context.rs` | Trait that actors see. `send()`, `self_addr()`, `spawn()`. Hides all framework internals. | N/A (trait) | +| **Context** | `context.rs` | `Ctx` wrapper over `&dyn ContextInner`. `send()`, `self_addr()`, `spawn()`. Hides all framework internals. | N/A (trait) | | **Runtime** | `runtime.rs` | Wires it all together. Creates workers, holds address map, exposes public API. | Minimal (delegates) | -| **Placement** | `placement.rs` | Decides which worker gets a new actor. | No (called at spawn time) | +| **Placement** | `placement.rs` | Decides which worker gets a new actor. Currently round-robin. | No (called at spawn time) | ## Single-Threaded / WASM Mode diff --git a/TODOs.md b/TODOs.md deleted file mode 100644 index 4e9885c..0000000 --- a/TODOs.md +++ /dev/null @@ -1,44 +0,0 @@ -### Profiling and Benchmarking - -- Research as to modern art on benching and profiling - - Implement an MVP here. - - Bench/profile against a suite of tests selected for generality across actor framework usecases - -- Identify hot paths and bottlenecks - - e.g. pretty sure the router is a major bottleneck, what else - - follow through the entire message cycle: - Parent process -> Convert to swactor::Message/Envelope -> Router -> Delivery -> Processing -> etc. - Identify every small detail on which you may be able to improve, any unneeded processing or branching - -- (optional) Visualization tools: - - make some pretty stuff for tracing messages, actor activity, router activity, etc. - -### Usage - -- After benching and profiling, cleaning up the most egregious wrongdoings we will: - - actually implement our own projects in the framework, ones I actually find useful personally - -- Optimization pipeline: - - Once we have well-established benches and profiles for general cases, build a set of tools that can auto-optimize - for given use cases. Tuning, for example, the channel buffers, router behavior, message consumption behavior, etc. - -### Chores - -- go over all the FIXMEs littered about. Add comments. -- add misc features as they come up. Prefer tools for understanding execution flows, visualizing flows, and adding -robustness, over ergonomics. Better to be slightly clunky but fast and optimized, than vice versa. - -### Far future - -- make language bindings. e.g., an npm package, python bindings, etc. - -### Optimization - -- Localize actors and inboxes: - - Because the entire runtime is message driven, the happy path must be fast. Even lock free, when we have to go through - several calls of an atomic ring buffer in order to process a single message, its unnecessary. - Parent process -> router -> actor -> router -> inbox -> parent process; every transfer going through an atomic buffer. - - - to do this, design heavily around a localized worker thread. Actors on a working thread should have their inbox localized, they - should be 'sticky' to that thread (FILO queue?), and we should route messages based on core locality. Future optimizations can include - a tunable algorithm that puts actors that frequently communicate together on the same thread. \ No newline at end of file diff --git a/benches/scaling.rs b/benches/scaling.rs index 85440cd..33d7b23 100644 --- a/benches/scaling.rs +++ b/benches/scaling.rs @@ -84,15 +84,18 @@ impl ActorInterface for PayloadActor { /// Benchmark: How throughput scales with actor count pub fn bench_actor_count_scaling(suite: &mut BenchSuite) { - let messages_per_actor = 100u64; - - for actor_count in [10u64, 100, 500, 1000] { + for (actor_count, messages_per_actor, warmup_n, iters_n) in [ + (10u64, 200u64, 5usize, 50usize), + (100, 200, 5, 30), + (500, 200, 5, 15), + (1000, 200, 5, 12), + ] { let name = format!("scaling_{}_actors", actor_count); let total_messages = actor_count * messages_per_actor; let result = Bench::new(&name) - .warmup(2) - .iters(10) + .warmup(warmup_n) + .iters(iters_n) .elements(total_messages) .run_with_setup( || { @@ -100,6 +103,7 @@ pub fn bench_actor_count_scaling(suite: &mut BenchSuite) { max_actors: (actor_count as usize) + 100, actor_max_messages: (total_messages as usize) * 3, num_threads: 1, + ..Default::default() }; let runtime = Runtime::new(config); @@ -148,8 +152,8 @@ pub fn bench_thread_count_scaling(suite: &mut BenchSuite) { let name = format!("scaling_{}_threads", thread_count); let result = Bench::new(&name) - .warmup(1) - .iters(5) + .warmup(5) + .iters(30) .elements(total_messages) .run_with_setup( || { @@ -158,6 +162,7 @@ pub fn bench_thread_count_scaling(suite: &mut BenchSuite) { max_actors: (actor_count as usize) + 100, actor_max_messages: (total_messages as usize) * 3, num_threads: thread_count, + ..Default::default() }; let runtime = Runtime::new(config); @@ -218,15 +223,15 @@ pub fn bench_thread_count_scaling(suite: &mut BenchSuite) { /// Benchmark: How throughput scales with message payload size pub fn bench_payload_size_scaling(suite: &mut BenchSuite) { - let message_count = 1_000u64; + let message_count = 3_000u64; for payload_size in [64usize, 1024, 16384, 65536] { let name = format!("payload_{}B", payload_size); let payload = vec![0u8; payload_size]; let result = Bench::new(&name) - .warmup(2) - .iters(20) + .warmup(5) + .iters(40) .elements(message_count) .run_with_setup( || { @@ -234,6 +239,7 @@ pub fn bench_payload_size_scaling(suite: &mut BenchSuite) { max_actors: 10, actor_max_messages: (message_count as usize) * 2, num_threads: 1, + ..Default::default() }; let runtime = Runtime::new(config); let sink = runtime.spawn(PayloadActor::new()).unwrap(); diff --git a/benches/throughput.rs b/benches/throughput.rs index 001b0a1..eaed827 100644 --- a/benches/throughput.rs +++ b/benches/throughput.rs @@ -76,8 +76,8 @@ pub fn bench_message_throughput(suite: &mut BenchSuite) { let name = format!("message_throughput_{}", msg_count); let result = Bench::new(&name) - .warmup(3) - .iters(20) + .warmup(5) + .iters(100) .elements(msg_count) .run_with_setup( || { @@ -86,6 +86,7 @@ pub fn bench_message_throughput(suite: &mut BenchSuite) { max_actors: 100, actor_max_messages: (msg_count as usize) * 2, num_threads: 1, + ..Default::default() }; let runtime = Runtime::new(config); let sink = runtime.spawn(SinkActor::new()).unwrap(); @@ -114,8 +115,8 @@ pub fn bench_spawn_rate(suite: &mut BenchSuite) { let name = format!("spawn_rate_{}_actors", actor_count); let result = Bench::new(&name) - .warmup(3) - .iters(50) + .warmup(5) + .iters(100) .elements(actor_count) .run_with_setup( || { @@ -123,6 +124,7 @@ pub fn bench_spawn_rate(suite: &mut BenchSuite) { max_actors: 1000, actor_max_messages: 100, num_threads: 1, + ..Default::default() }; Runtime::new(config) }, @@ -144,13 +146,16 @@ pub fn bench_spawn_rate(suite: &mut BenchSuite) { /// Benchmark: Fan-out (1 sender to N receivers) pub fn bench_fanout(suite: &mut BenchSuite) { - for fan_count in [10u64, 100, 500] { + for (fan_count, messages_per_receiver, warmup_n, iters_n) in [ + (10u64, 500u64, 5usize, 50usize), + (100, 200, 5, 40), + (500, 100, 5, 30), + ] { let name = format!("fanout_1_to_{}", fan_count); - let messages_per_receiver = 100u64; let result = Bench::new(&name) - .warmup(2) - .iters(20) + .warmup(warmup_n) + .iters(iters_n) .elements(fan_count * messages_per_receiver) .run_with_setup( || { @@ -160,6 +165,7 @@ pub fn bench_fanout(suite: &mut BenchSuite) { * (messages_per_receiver as usize) * 2, num_threads: 1, + ..Default::default() }; let runtime = Runtime::new(config); @@ -200,13 +206,16 @@ pub fn bench_fanout(suite: &mut BenchSuite) { /// Benchmark: Fan-in (N senders to 1 receiver) pub fn bench_fanin(suite: &mut BenchSuite) { - for sender_count in [10u64, 100, 500] { + for (sender_count, messages_per_sender, warmup_n, iters_n) in [ + (10u64, 300u64, 5usize, 50usize), + (100, 100, 5, 30), + (500, 100, 5, 20), + ] { let name = format!("fanin_{}_to_1", sender_count); - let messages_per_sender = 100u64; let result = Bench::new(&name) - .warmup(2) - .iters(20) + .warmup(warmup_n) + .iters(iters_n) .elements(sender_count * messages_per_sender) .run_with_setup( || { @@ -215,6 +224,7 @@ pub fn bench_fanin(suite: &mut BenchSuite) { max_actors: (sender_count as usize) + 10, actor_max_messages: total_messages * 3, num_threads: 1, + ..Default::default() }; let runtime = Runtime::new(config); @@ -258,13 +268,16 @@ pub fn bench_fanin(suite: &mut BenchSuite) { /// Benchmark: Ring topology (message passed around N actors in a circle) pub fn bench_ring(suite: &mut BenchSuite) { - for ring_size in [10u64, 100, 500] { + for (ring_size, laps, warmup_n, iters_n) in [ + (10u64, 100u64, 5usize, 100usize), + (100, 20, 5, 50), + (500, 10, 5, 40), + ] { let name = format!("ring_{}_actors", ring_size); - let laps = 10u64; // How many times around the ring let result = Bench::new(&name) - .warmup(2) - .iters(20) + .warmup(warmup_n) + .iters(iters_n) .elements(ring_size * laps) .run_with_setup( || { @@ -272,6 +285,7 @@ pub fn bench_ring(suite: &mut BenchSuite) { max_actors: (ring_size as usize) + 10, actor_max_messages: 10_000, num_threads: 1, + ..Default::default() }; let runtime = Runtime::new(config); diff --git a/src/config.rs b/src/config.rs index f6d2a8b..95eab7b 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,8 +1,35 @@ +/// Backoff policy for worker threads when idle. +/// +/// Workers spin → yield → sleep with increasing delay when no work is available. +pub struct BackoffPolicy { + /// Number of idle ticks before switching from spin to yield. + pub spin_threshold: u32, + /// Number of idle ticks before switching from yield to sleep. + pub yield_threshold: u32, + /// Microseconds added per tick beyond the yield threshold. + pub sleep_increment_us: u64, + /// Maximum sleep duration in microseconds. + pub sleep_max_us: u64, +} + +impl Default for BackoffPolicy { + fn default() -> Self { + Self { + spin_threshold: 64, + yield_threshold: 256, + sleep_increment_us: 50, + sleep_max_us: 1000, + } + } +} + /// The tunable settings for the runtime. pub struct RuntimeConfig { pub max_actors: usize, pub actor_max_messages: usize, pub num_threads: usize, + pub mailbox_waterlevel: usize, + pub backoff_policy: BackoffPolicy, } /// 8kB for the `Box<..>` before counting the rest of the memory @@ -13,12 +40,16 @@ const DEFAULT_MAX_ACTORS: usize = 1_000; /// 1_000 * 16kB = 16MB const DEFAULT_ACTOR_MAX_MESSAGES: usize = 1_000; +const DEFAULT_MAILBOX_WATERLEVEL: usize = 10; + impl Default for RuntimeConfig { fn default() -> Self { Self { max_actors: DEFAULT_MAX_ACTORS, actor_max_messages: DEFAULT_ACTOR_MAX_MESSAGES, num_threads: 1, + mailbox_waterlevel: DEFAULT_MAILBOX_WATERLEVEL, + backoff_policy: BackoffPolicy::default(), } } } diff --git a/src/context.rs b/src/context.rs index 6bd2387..f5e5bb6 100644 --- a/src/context.rs +++ b/src/context.rs @@ -8,6 +8,7 @@ use crate::Error; pub(crate) trait ContextInner { fn send_any(&self, addr: ActorAddress, msg: Box) -> Result<(), Error>; fn spawn_any(&self, addr: ActorAddress, actor: Box) -> Result<(), Error>; + fn mailbox_waterlevel(&self) -> usize; } /// Actor syscall interface — passed to `ActorInterface::handle()`. @@ -37,7 +38,8 @@ impl<'a> Ctx<'a> { /// Spawn a new actor, returning its address. pub fn spawn(&self, actor: A) -> Result { let addr = ActorAddress::new_random(); - let actor = Actor::new(addr, Mailbox::new(), actor); + let waterlevel = self.inner.mailbox_waterlevel(); + let actor = Actor::new(addr, Mailbox::new(waterlevel), actor); let boxed: Box = Box::new(actor); self.inner.spawn_any(addr, boxed)?; Ok(addr) diff --git a/src/runtime.rs b/src/runtime.rs index e602db2..8c2e7ab 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -8,13 +8,13 @@ use std::thread::{self, JoinHandle}; use crate::actor::{Actor, ActorAddress, ActorInterface, AnyActor, Message}; use crate::address_map::{AddressMap, WorkerId}; use crate::channel::{Receiver, Sender}; -// Re-export RuntimeConfig so existing code using `runtime::RuntimeConfig` still works -pub use crate::config::RuntimeConfig; +// Re-export config types so existing code using `runtime::RuntimeConfig` still works +pub use crate::config::{BackoffPolicy, RuntimeConfig}; use crate::context::ContextInner; use crate::envelope::Envelope; use crate::placement::Placement; use crate::worker::mailbox::Mailbox; -use crate::worker::Worker; +use crate::worker::{TickContext, Worker}; use crate::Error; // ─── SenderT trait (moved from router.rs) ──────────────────────────────────── @@ -171,7 +171,7 @@ impl Runtime { let addr = ActorAddress::new_random(); let worker_id = self.placement.next_worker(); self.address_map.insert(addr, worker_id); - let actor = Actor::new(addr, Mailbox::new(), actor); + let actor = Actor::new(addr, Mailbox::new(self.config.mailbox_waterlevel), actor); let boxed: Box = Box::new(actor); self.spawn_txs[worker_id.as_usize()] .try_send((addr, boxed)) @@ -205,13 +205,15 @@ impl Runtime { /// Drive one tick of the single-threaded worker. pub fn tick(&self) { if let Some(ref worker) = self.single_worker { - worker.borrow_mut().tick_once( - &self.address_map, - &self.transfer_txs, - &self.spawn_txs, - &self.placement, - &self.inbox_registry, - ); + let tc = TickContext { + address_map: &self.address_map, + transfer_txs: &self.transfer_txs, + spawn_txs: &self.spawn_txs, + placement: &self.placement, + inbox_registry: &self.inbox_registry, + config: &self.config, + }; + worker.borrow_mut().tick_once(&tc); } } @@ -261,28 +263,34 @@ impl Runtime { /// Worker thread loop for multi-threaded runtime. /// Uses spin → yield → park backoff to reduce CPU usage when idle. fn worker_loop(worker: &mut Worker, rt: &Runtime) { + let tc = TickContext { + address_map: &rt.address_map, + transfer_txs: &rt.transfer_txs, + spawn_txs: &rt.spawn_txs, + placement: &rt.placement, + inbox_registry: &rt.inbox_registry, + config: &rt.config, + }; + let bp = &rt.config.backoff_policy; let mut idle_count: u32 = 0; while rt.is_running.load(Ordering::Acquire) { - let did_work = worker.tick_once( - &rt.address_map, - &rt.transfer_txs, - &rt.spawn_txs, - &rt.placement, - &rt.inbox_registry, - ); + let did_work = worker.tick_once(&tc); if did_work { idle_count = 0; } else { idle_count = idle_count.saturating_add(1); - if idle_count < 64 { - core::hint::spin_loop(); - } else if idle_count < 256 { + if idle_count < bp.spin_threshold { + // Hot spin — no hint, keep polling fast + } else if idle_count < bp.yield_threshold { thread::yield_now(); } else { - // Park: sleep briefly, cap at 1ms - let micros = std::cmp::min((idle_count - 256) as u64 * 50, 1000); + // Park: sleep briefly, cap at configured max + let micros = std::cmp::min( + (idle_count - bp.yield_threshold) as u64 * bp.sleep_increment_us, + bp.sleep_max_us, + ); thread::sleep(std::time::Duration::from_micros(micros)); } } @@ -326,4 +334,8 @@ impl ContextInner for Runtime { .try_send((addr, actor)) .map_err(|_| Error::from("Spawn queue full")) } + + fn mailbox_waterlevel(&self) -> usize { + self.config.mailbox_waterlevel + } } diff --git a/src/worker/mailbox.rs b/src/worker/mailbox.rs index 82ef233..a331e22 100644 --- a/src/worker/mailbox.rs +++ b/src/worker/mailbox.rs @@ -2,22 +2,13 @@ use std::collections::VecDeque; use crate::actor::Message; -const DEFAULT_WATERLEVEL: usize = 10; - pub struct Mailbox { queue: VecDeque, waterlevel: usize, } impl Mailbox { - pub fn new() -> Self { - Self { - queue: VecDeque::new(), - waterlevel: DEFAULT_WATERLEVEL, - } - } - - pub fn with_waterlevel(waterlevel: usize) -> Self { + pub fn new(waterlevel: usize) -> Self { Self { queue: VecDeque::new(), waterlevel, diff --git a/src/worker/mod.rs b/src/worker/mod.rs index bc56b5b..f8752a8 100644 --- a/src/worker/mod.rs +++ b/src/worker/mod.rs @@ -7,6 +7,7 @@ use std::cell::RefCell; use crate::actor::{ActorAddress, AnyActor}; use crate::address_map::{AddressMap, WorkerId}; use crate::channel::{Receiver, Sender}; +use crate::config::RuntimeConfig; use crate::context::ContextInner; use crate::envelope::Envelope; use crate::placement::Placement; @@ -14,6 +15,16 @@ use crate::runtime::InboxRegistry; use crate::Error; use pool::ActorPool; +/// Shared state passed to tick_once — single thin pointer avoids register spill. +pub(crate) struct TickContext<'a> { + pub address_map: &'a AddressMap, + pub transfer_txs: &'a [Sender], + pub spawn_txs: &'a [Sender<(ActorAddress, Box)>], + pub placement: &'a Placement, + pub inbox_registry: &'a InboxRegistry, + pub config: &'a RuntimeConfig, +} + /// A worker owns a set of actors and runs them in a loop. pub(crate) struct Worker { id: WorkerId, @@ -37,14 +48,7 @@ impl Worker { } /// Run one iteration of the worker loop. Returns `true` if any work was done. - pub fn tick_once( - &mut self, - address_map: &AddressMap, - transfer_txs: &[Sender], - spawn_txs: &[Sender<(ActorAddress, Box)>], - placement: &Placement, - inbox_registry: &InboxRegistry, - ) -> bool { + pub fn tick_once(&mut self, tc: &TickContext) -> bool { let mut did_work = false; // 1. Drain spawn queue → add actors to pool @@ -68,11 +72,12 @@ impl Worker { { let worker_ctx = WorkerContext { worker_id: self.id, - address_map, - transfer_txs, - spawn_txs, - placement, - inbox_registry, + address_map: tc.address_map, + transfer_txs: tc.transfer_txs, + spawn_txs: tc.spawn_txs, + placement: tc.placement, + inbox_registry: tc.inbox_registry, + config: tc.config, pending_local: &pending_local, }; if self.pool.tick_all(&worker_ctx) { @@ -104,6 +109,7 @@ struct WorkerContext<'a> { spawn_txs: &'a [Sender<(ActorAddress, Box)>], placement: &'a Placement, inbox_registry: &'a InboxRegistry, + config: &'a RuntimeConfig, pending_local: &'a RefCell)>>, } @@ -135,4 +141,8 @@ impl ContextInner for WorkerContext<'_> { .try_send((addr, actor)) .map_err(|_| Error::from("Spawn queue full")) } + + fn mailbox_waterlevel(&self) -> usize { + self.config.mailbox_waterlevel + } } diff --git a/tests/mailbox_tests.rs b/tests/mailbox_tests.rs index 8c5aa93..2deebba 100644 --- a/tests/mailbox_tests.rs +++ b/tests/mailbox_tests.rs @@ -4,14 +4,14 @@ use swactor::worker::mailbox::Mailbox; #[test] fn push_and_pop() { - let mut mb = Mailbox::new(); + let mut mb = Mailbox::new(10); mb.push(42i32); assert_eq!(mb.pop(), Some(42)); } #[test] fn fifo_ordering() { - let mut mb = Mailbox::new(); + let mut mb = Mailbox::new(10); mb.push(1); mb.push(2); mb.push(3); @@ -22,13 +22,13 @@ fn fifo_ordering() { #[test] fn pop_empty() { - let mut mb: Mailbox = Mailbox::new(); + let mut mb: Mailbox = Mailbox::new(10); assert_eq!(mb.pop(), None); } #[test] fn multiple_messages() { - let mut mb = Mailbox::new(); + let mut mb = Mailbox::new(10); for i in 0..100 { mb.push(i); } @@ -40,7 +40,7 @@ fn multiple_messages() { #[test] fn interleaved_push_pop() { - let mut mb = Mailbox::new(); + let mut mb = Mailbox::new(10); mb.push(1); mb.push(2); assert_eq!(mb.pop(), Some(1)); @@ -54,13 +54,13 @@ fn interleaved_push_pop() { #[test] fn drain_count_empty() { - let mb: Mailbox = Mailbox::new(); + let mb: Mailbox = Mailbox::new(10); assert_eq!(mb.drain_count(), 0); } #[test] fn drain_count_below_waterlevel() { - let mut mb = Mailbox::new(); + let mut mb = Mailbox::new(10); for i in 0..5 { mb.push(i); } @@ -70,7 +70,7 @@ fn drain_count_below_waterlevel() { #[test] fn drain_count_at_waterlevel() { - let mut mb = Mailbox::new(); + let mut mb = Mailbox::new(10); for i in 0..10 { mb.push(i); } @@ -80,7 +80,7 @@ fn drain_count_at_waterlevel() { #[test] fn drain_count_above_waterlevel() { - let mut mb = Mailbox::new(); + let mut mb = Mailbox::new(10); for i in 0..20 { mb.push(i); } @@ -90,7 +90,7 @@ fn drain_count_above_waterlevel() { #[test] fn drain_count_one_message() { - let mut mb = Mailbox::new(); + let mut mb = Mailbox::new(10); mb.push(1i32); // 1 < 10 → process all → 1 assert_eq!(mb.drain_count(), 1); @@ -98,7 +98,7 @@ fn drain_count_one_message() { #[test] fn drain_count_just_below_waterlevel() { - let mut mb = Mailbox::new(); + let mut mb = Mailbox::new(10); for i in 0..9 { mb.push(i); } @@ -108,7 +108,7 @@ fn drain_count_just_below_waterlevel() { #[test] fn drain_count_large() { - let mut mb = Mailbox::new(); + let mut mb = Mailbox::new(10); for i in 0..1000 { mb.push(i); } @@ -118,7 +118,7 @@ fn drain_count_large() { #[test] fn drain_count_custom_waterlevel() { - let mut mb = Mailbox::with_waterlevel(4); + let mut mb = Mailbox::new(4); for i in 0..3 { mb.push(i); } @@ -132,7 +132,7 @@ fn drain_count_custom_waterlevel() { #[test] fn drain_count_updates_after_pop() { - let mut mb = Mailbox::new(); + let mut mb = Mailbox::new(10); for i in 0..20 { mb.push(i); } @@ -151,7 +151,7 @@ fn drain_count_updates_after_pop() { #[test] fn len_tracks_pushes() { - let mut mb = Mailbox::new(); + let mut mb = Mailbox::new(10); assert_eq!(mb.len(), 0); mb.push(1); assert_eq!(mb.len(), 1); @@ -163,7 +163,7 @@ fn len_tracks_pushes() { #[test] fn len_tracks_pops() { - let mut mb = Mailbox::new(); + let mut mb = Mailbox::new(10); mb.push(1); mb.push(2); mb.push(3); @@ -178,13 +178,13 @@ fn len_tracks_pops() { #[test] fn is_empty_on_new() { - let mb: Mailbox = Mailbox::new(); + let mb: Mailbox = Mailbox::new(10); assert!(mb.is_empty()); } #[test] fn is_empty_after_drain() { - let mut mb = Mailbox::new(); + let mut mb = Mailbox::new(10); mb.push(1); mb.push(2); mb.push(3); @@ -199,11 +199,11 @@ fn is_empty_after_drain() { #[test] fn works_with_primitive_types() { - let mut mb_i32 = Mailbox::new(); + let mut mb_i32 = Mailbox::new(10); mb_i32.push(42i32); assert_eq!(mb_i32.pop(), Some(42)); - let mut mb_string = Mailbox::new(); + let mut mb_string = Mailbox::new(10); mb_string.push(String::from("hello")); assert_eq!(mb_string.pop(), Some(String::from("hello"))); } @@ -216,7 +216,7 @@ fn works_with_custom_structs() { payload: String, } - let mut mb = Mailbox::new(); + let mut mb = Mailbox::new(10); let msg = MyMsg { id: 1, payload: "test".into(), diff --git a/tests/stress/concurrency.rs b/tests/stress/concurrency.rs index acb3cf4..0656652 100644 --- a/tests/stress/concurrency.rs +++ b/tests/stress/concurrency.rs @@ -27,6 +27,7 @@ fn shutdown_under_load() { max_actors: 100, actor_max_messages: 1000, num_threads: 4, + ..Default::default() }; let runtime = Runtime::new(config); @@ -101,6 +102,7 @@ fn send_to_newborn() { max_actors: 1000, actor_max_messages: 100, num_threads: 4, + ..Default::default() }; let runtime = Runtime::new(config); let handle = runtime.run().unwrap(); @@ -137,7 +139,6 @@ fn send_to_newborn() { println!(">>> Test complete\n"); } -/// FIXME: This test means nothing until we allow killing off actor processes /// Rapid spawn/despawn cycles. /// Target: Queue management under churn. #[test] @@ -149,6 +150,7 @@ fn rapid_spawn_churn() { max_actors: 100, actor_max_messages: 100, num_threads: 4, + ..Default::default() }; let runtime = Runtime::new(config); let handle = runtime.run().unwrap(); @@ -209,6 +211,7 @@ fn inbox_contention() { max_actors: 10, actor_max_messages: 100_000, num_threads: 4, + ..Default::default() }; let runtime = Runtime::new(config); let target = runtime.spawn(BlackHole).unwrap(); @@ -262,7 +265,6 @@ fn inbox_contention() { println!(">>> Test complete - no panics\n"); } -/// FIXME: Not sure this test is meaningful. /// Shutdown timing fuzz - randomize when shutdown is called. /// Target: Edge cases in shutdown state machine. #[test] @@ -282,6 +284,7 @@ fn shutdown_timing_fuzz() { max_actors: 50, actor_max_messages: 100, num_threads: 4, + ..Default::default() }; let runtime = Runtime::new(config); diff --git a/tests/stress/saturation.rs b/tests/stress/saturation.rs index 9345cd2..dc2aeb2 100644 --- a/tests/stress/saturation.rs +++ b/tests/stress/saturation.rs @@ -16,6 +16,7 @@ fn transfer_queue_overflow() { max_actors: 10, actor_max_messages: 100, // Tiny buffer num_threads: 1, + ..Default::default() }; let runtime = Runtime::new(config); let sink = runtime.spawn(BlackHole).unwrap(); @@ -55,6 +56,7 @@ fn actor_inbox_overflow() { max_actors: 10, actor_max_messages: 100_000, // Large transfer buffer num_threads: 1, + ..Default::default() }; let runtime = Runtime::new(config); let sink = runtime.spawn(Counter::new()).unwrap(); @@ -99,6 +101,7 @@ fn actor_queue_overflow() { max_actors: 100, // Small actor queue actor_max_messages: 100, num_threads: 1, + ..Default::default() }; let runtime = Runtime::new(config); @@ -126,7 +129,6 @@ fn actor_queue_overflow() { println!(">>> PASS: Actor queue correctly handles overflow\n"); } -/// FIXME: IS this actually testing what it should be? /// Sustained overload - run at 2x capacity for extended period. /// Documents: Does the system degrade gracefully or crash? #[test] @@ -138,6 +140,7 @@ fn sustained_overload() { max_actors: 100, actor_max_messages: 1000, num_threads: 1, + ..Default::default() }; let runtime = Runtime::new(config); -- 2.45.2 From c80060f45317b832bbe6f28110eb6db5bf393fff Mon Sep 17 00:00:00 2001 From: Zachery Aaron Shores-Chmielewski Date: Fri, 6 Feb 2026 14:06:37 +0700 Subject: [PATCH 7/9] fix: consolidate modules Better consoliation of modules into more logical components. --- benches/scaling.rs | 3 +- benches/throughput.rs | 3 +- examples/hello.rs | 3 +- examples/ring.rs | 3 +- src/actor.rs | 3 +- src/address_map.rs | 30 +++++++++ src/{channel/mod.rs => channel.rs} | 13 ---- src/context.rs | 47 ------------- src/envelope.rs | 25 ------- src/lib.rs | 4 -- src/placement.rs | 37 ---------- src/runtime.rs | 79 ++++++++++++++++++++-- src/{worker/mod.rs => worker.rs} | 104 ++++++++++++++++++++++++++--- src/worker/mailbox.rs | 45 ------------- src/worker/pool.rs | 51 -------------- tests/mailbox_tests.rs | 2 +- tests/runtime_tests.rs | 2 +- tests/stress/mod.rs | 2 +- 18 files changed, 206 insertions(+), 250 deletions(-) rename src/{channel/mod.rs => channel.rs} (77%) delete mode 100644 src/context.rs delete mode 100644 src/placement.rs rename src/{worker/mod.rs => worker.rs} (66%) delete mode 100644 src/worker/mailbox.rs delete mode 100644 src/worker/pool.rs diff --git a/benches/scaling.rs b/benches/scaling.rs index 33d7b23..0febea5 100644 --- a/benches/scaling.rs +++ b/benches/scaling.rs @@ -8,9 +8,8 @@ use crate::harness::{black_box, Bench, BenchSuite}; use std::thread; use swactor::{ - Ctx, actor::ActorInterface, - runtime::{Runtime, RuntimeConfig}, + runtime::{Ctx, Runtime, RuntimeConfig}, }; // ============================================================================ diff --git a/benches/throughput.rs b/benches/throughput.rs index eaed827..900a4d0 100644 --- a/benches/throughput.rs +++ b/benches/throughput.rs @@ -8,9 +8,8 @@ use crate::harness::{black_box, Bench, BenchSuite}; use swactor::{ - Ctx, actor::{ActorAddress, ActorInterface}, - runtime::{Runtime, RuntimeConfig}, + runtime::{Ctx, Runtime, RuntimeConfig}, }; // ============================================================================ diff --git a/examples/hello.rs b/examples/hello.rs index 18378d9..7f9b483 100644 --- a/examples/hello.rs +++ b/examples/hello.rs @@ -1,7 +1,6 @@ use swactor::{ - Ctx, actor::{ActorAddress, ActorInterface}, - runtime::{Runtime, RuntimeConfig}, + runtime::{Ctx, Runtime, RuntimeConfig}, }; #[derive(Debug, Default)] diff --git a/examples/ring.rs b/examples/ring.rs index 703a3be..dd4e87b 100644 --- a/examples/ring.rs +++ b/examples/ring.rs @@ -1,7 +1,6 @@ use swactor::{ - Ctx, actor::{ActorAddress, ActorInterface}, - runtime::{Inbox, Runtime, RuntimeConfig}, + runtime::{Ctx, Inbox, Runtime, RuntimeConfig}, }; #[derive(Debug, Default, Clone)] diff --git a/src/actor.rs b/src/actor.rs index e14cf82..a5ae92a 100644 --- a/src/actor.rs +++ b/src/actor.rs @@ -1,7 +1,6 @@ use std::any::Any; -use crate::{get_random, worker::mailbox::Mailbox}; -use crate::context::{ContextInner, Ctx}; +use crate::{get_random, runtime::{ContextInner, Ctx}, worker::Mailbox}; /// The primary trait defining data that can be passed to and from actor processes pub trait Message: 'static + Sized + Clone + Send + Sync {} diff --git a/src/address_map.rs b/src/address_map.rs index adabe3a..48ac949 100644 --- a/src/address_map.rs +++ b/src/address_map.rs @@ -1,5 +1,6 @@ use std::collections::HashMap; use std::sync::RwLock; +use std::sync::atomic::{AtomicUsize, Ordering}; use crate::actor::ActorAddress; @@ -50,6 +51,26 @@ impl AddressMap { } } +/// Round-robin actor placement strategy. +pub(crate) struct Placement { + next: AtomicUsize, + num_workers: usize, +} + +impl Placement { + pub fn new(num_workers: usize) -> Self { + Self { + next: AtomicUsize::new(0), + num_workers, + } + } + + pub fn next_worker(&self) -> WorkerId { + let id = self.next.fetch_add(1, Ordering::Relaxed) % self.num_workers; + WorkerId(id) + } +} + #[cfg(test)] mod tests { use super::*; @@ -87,4 +108,13 @@ mod tests { map.insert(addr1, WorkerId(0)); assert_eq!(map.len(), 1); } + + #[test] + fn round_robin() { + let p = Placement::new(3); + assert_eq!(p.next_worker(), WorkerId(0)); + assert_eq!(p.next_worker(), WorkerId(1)); + assert_eq!(p.next_worker(), WorkerId(2)); + assert_eq!(p.next_worker(), WorkerId(0)); + } } diff --git a/src/channel/mod.rs b/src/channel.rs similarity index 77% rename from src/channel/mod.rs rename to src/channel.rs index fbccdb5..c487ec4 100644 --- a/src/channel/mod.rs +++ b/src/channel.rs @@ -1,13 +1,11 @@ use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering}; use crossbeam_queue::{ArrayQueue, SegQueue}; pub struct HybridChannel { ring: ArrayQueue, overflow: SegQueue, - overflow_len: AtomicUsize, } impl HybridChannel { @@ -15,7 +13,6 @@ impl HybridChannel { Self { ring: ArrayQueue::new(capacity), overflow: SegQueue::new(), - overflow_len: AtomicUsize::new(0), } } @@ -24,7 +21,6 @@ impl HybridChannel { Ok(()) => Ok(()), Err(v) => { self.overflow.push(v); - self.overflow_len.fetch_add(1, Ordering::Relaxed); Ok(()) } } @@ -37,16 +33,12 @@ impl HybridChannel { match self.overflow.pop() { Some(value) => { - self.overflow_len.fetch_sub(1, Ordering::Relaxed); Some(value) } None => None, } } - pub fn len(&self) -> usize { - self.ring.len() + self.overflow_len.load(Ordering::Relaxed) - } } pub(crate) struct Receiver { @@ -59,11 +51,6 @@ impl Receiver { Self { queue } } - - pub fn len(&self) -> usize { - self.queue.len() - } - pub fn try_recv(&self) -> Option { return self.queue.pop(); } diff --git a/src/context.rs b/src/context.rs deleted file mode 100644 index f5e5bb6..0000000 --- a/src/context.rs +++ /dev/null @@ -1,47 +0,0 @@ -use std::any::Any; - -use crate::actor::{Actor, ActorAddress, ActorInterface, AnyActor, Message}; -use crate::worker::mailbox::Mailbox; -use crate::Error; - -/// Object-safe inner trait for sending type-erased messages. -pub(crate) trait ContextInner { - fn send_any(&self, addr: ActorAddress, msg: Box) -> Result<(), Error>; - fn spawn_any(&self, addr: ActorAddress, actor: Box) -> Result<(), Error>; - fn mailbox_waterlevel(&self) -> usize; -} - -/// Actor syscall interface — passed to `ActorInterface::handle()`. -/// -/// Wraps a `&dyn ContextInner` to solve the object-safety problem while -/// providing a typed public API. -pub struct Ctx<'a> { - inner: &'a dyn ContextInner, - self_addr: ActorAddress, -} - -impl<'a> Ctx<'a> { - pub(crate) fn new(inner: &'a dyn ContextInner, self_addr: ActorAddress) -> Self { - Self { inner, self_addr } - } - - /// Returns the address of the actor currently being ticked. - pub fn self_addr(&self) -> ActorAddress { - self.self_addr - } - - /// Send a typed message to an actor address. - pub fn send(&self, addr: ActorAddress, msg: M) -> Result<(), Error> { - self.inner.send_any(addr, Box::new(msg)) - } - - /// Spawn a new actor, returning its address. - pub fn spawn(&self, actor: A) -> Result { - let addr = ActorAddress::new_random(); - let waterlevel = self.inner.mailbox_waterlevel(); - let actor = Actor::new(addr, Mailbox::new(waterlevel), actor); - let boxed: Box = Box::new(actor); - self.inner.spawn_any(addr, boxed)?; - Ok(addr) - } -} diff --git a/src/envelope.rs b/src/envelope.rs index d55fb8a..c3a716d 100644 --- a/src/envelope.rs +++ b/src/envelope.rs @@ -2,31 +2,6 @@ use std::any::Any; use crate::actor::ActorAddress; -/// A type-erased message envelope for cross-worker delivery. -/// -/// Uses `Box` (no atomic refcount) and move semantics (no clone). -pub(crate) struct Envelope { - dest: ActorAddress, - payload: Box, -} - -impl Envelope { - pub fn new(dest: ActorAddress, payload: Box) -> Self { - Self { dest, payload } - } - - pub fn dest(&self) -> ActorAddress { - self.dest - } - - pub fn downcast(self) -> Option { - self.payload.downcast::().ok().map(|b| *b) - } - - pub fn into_payload(self) -> Box { - self.payload - } -} #[cfg(test)] mod tests { diff --git a/src/lib.rs b/src/lib.rs index 08f7b07..fc57124 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,12 +5,8 @@ pub(crate) mod channel; pub(crate) mod error; pub use error::Error; -pub mod context; -pub use context::Ctx; -pub(crate) mod envelope; pub(crate) mod address_map; -pub(crate) mod placement; pub mod config; pub mod runtime; diff --git a/src/placement.rs b/src/placement.rs deleted file mode 100644 index d88e63d..0000000 --- a/src/placement.rs +++ /dev/null @@ -1,37 +0,0 @@ -use std::sync::atomic::{AtomicUsize, Ordering}; - -use crate::address_map::WorkerId; - -/// Round-robin actor placement strategy. -pub(crate) struct Placement { - next: AtomicUsize, - num_workers: usize, -} - -impl Placement { - pub fn new(num_workers: usize) -> Self { - Self { - next: AtomicUsize::new(0), - num_workers, - } - } - - pub fn next_worker(&self) -> WorkerId { - let id = self.next.fetch_add(1, Ordering::Relaxed) % self.num_workers; - WorkerId(id) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn round_robin() { - let p = Placement::new(3); - assert_eq!(p.next_worker(), WorkerId(0)); - assert_eq!(p.next_worker(), WorkerId(1)); - assert_eq!(p.next_worker(), WorkerId(2)); - assert_eq!(p.next_worker(), WorkerId(0)); - } -} diff --git a/src/runtime.rs b/src/runtime.rs index 8c2e7ab..622c0db 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -6,14 +6,11 @@ use std::sync::{Arc, RwLock}; use std::thread::{self, JoinHandle}; use crate::actor::{Actor, ActorAddress, ActorInterface, AnyActor, Message}; -use crate::address_map::{AddressMap, WorkerId}; +use crate::address_map::{AddressMap, Placement, WorkerId}; use crate::channel::{Receiver, Sender}; // Re-export config types so existing code using `runtime::RuntimeConfig` still works pub use crate::config::{BackoffPolicy, RuntimeConfig}; -use crate::context::ContextInner; -use crate::envelope::Envelope; -use crate::placement::Placement; -use crate::worker::mailbox::Mailbox; +use crate::worker::Mailbox; use crate::worker::{TickContext, Worker}; use crate::Error; @@ -32,6 +29,34 @@ impl SenderT for Sender { } } + +/// A type-erased message envelope for cross-worker delivery. +/// +/// Uses `Box` (no atomic refcount) and move semantics (no clone). +pub(crate) struct Envelope { + dest: ActorAddress, + payload: Box, +} + +impl Envelope { + pub fn new(dest: ActorAddress, payload: Box) -> Self { + Self { dest, payload } + } + + pub fn dest(&self) -> ActorAddress { + self.dest + } + + pub fn downcast(self) -> Option { + self.payload.downcast::().ok().map(|b| *b) + } + + pub fn into_payload(self) -> Box { + self.payload + } +} + + // ─── InboxRegistry ─────────────────────────────────────────────────────────── /// Registry of external inboxes — replaces the Router's role for non-actor receivers. @@ -83,6 +108,50 @@ impl Inbox { } } + +/// Object-safe inner trait for sending type-erased messages. +pub(crate) trait ContextInner { + fn send_any(&self, addr: ActorAddress, msg: Box) -> Result<(), Error>; + fn spawn_any(&self, addr: ActorAddress, actor: Box) -> Result<(), Error>; + fn mailbox_waterlevel(&self) -> usize; +} + +/// Actor syscall interface — passed to `ActorInterface::handle()`. +/// +/// Wraps a `&dyn ContextInner` to solve the object-safety problem while +/// providing a typed public API. +pub struct Ctx<'a> { + inner: &'a dyn ContextInner, + self_addr: ActorAddress, +} + +impl<'a> Ctx<'a> { + pub(crate) fn new(inner: &'a dyn ContextInner, self_addr: ActorAddress) -> Self { + Self { inner, self_addr } + } + + /// Returns the address of the actor currently being ticked. + pub fn self_addr(&self) -> ActorAddress { + self.self_addr + } + + /// Send a typed message to an actor address. + pub fn send(&self, addr: ActorAddress, msg: M) -> Result<(), Error> { + self.inner.send_any(addr, Box::new(msg)) + } + + /// Spawn a new actor, returning its address. + pub fn spawn(&self, actor: A) -> Result { + let addr = ActorAddress::new_random(); + let waterlevel = self.inner.mailbox_waterlevel(); + let actor = Actor::new(addr, Mailbox::new(waterlevel), actor); + let boxed: Box = Box::new(actor); + self.inner.spawn_any(addr, boxed)?; + Ok(addr) + } +} + + // ─── Runtime ───────────────────────────────────────────────────────────────── /// The `Runtime` struct is the primary gateway for interacting with the framework. diff --git a/src/worker/mod.rs b/src/worker.rs similarity index 66% rename from src/worker/mod.rs rename to src/worker.rs index f8752a8..c6d0e3c 100644 --- a/src/worker/mod.rs +++ b/src/worker.rs @@ -1,19 +1,13 @@ -pub mod mailbox; -pub(crate) mod pool; - use std::any::Any; use std::cell::RefCell; +use std::collections::{HashMap, VecDeque}; -use crate::actor::{ActorAddress, AnyActor}; -use crate::address_map::{AddressMap, WorkerId}; +use crate::actor::{ActorAddress, AnyActor, Message}; +use crate::address_map::{AddressMap, Placement, WorkerId}; use crate::channel::{Receiver, Sender}; use crate::config::RuntimeConfig; -use crate::context::ContextInner; -use crate::envelope::Envelope; -use crate::placement::Placement; -use crate::runtime::InboxRegistry; +use crate::runtime::{ContextInner, Envelope, InboxRegistry}; use crate::Error; -use pool::ActorPool; /// Shared state passed to tick_once — single thin pointer avoids register spill. pub(crate) struct TickContext<'a> { @@ -146,3 +140,93 @@ impl ContextInner for WorkerContext<'_> { self.config.mailbox_waterlevel } } + +/// Per-worker actor storage. +pub(crate) struct ActorPool { + actors: HashMap>, +} + +impl ActorPool { + pub fn new() -> Self { + Self { + actors: HashMap::new(), + } + } + + pub fn insert(&mut self, addr: ActorAddress, actor: Box) { + self.actors.insert(addr, actor); + } + + pub fn remove(&mut self, addr: &ActorAddress) -> Option> { + self.actors.remove(addr) + } + + /// Deliver a type-erased message to the actor at `addr`. + /// Returns `true` if the actor was found and the message type matched. + pub fn deliver(&mut self, addr: &ActorAddress, msg: Box) -> bool { + if let Some(actor) = self.actors.get_mut(addr) { + actor.deliver(msg) + } else { + false + } + } + + /// Tick all actors in the pool. Returns `true` if any actor processed messages. + pub fn tick_all(&mut self, inner: &dyn ContextInner) -> bool { + let mut did_work = false; + for actor in self.actors.values_mut() { + if actor.tick(inner) { + did_work = true; + } + } + did_work + } + + pub fn len(&self) -> usize { + self.actors.len() + } +} + + + +pub struct Mailbox { + queue: VecDeque, + waterlevel: usize, +} + +impl Mailbox { + pub fn new(waterlevel: usize) -> Self { + Self { + queue: VecDeque::new(), + waterlevel, + } + } + + pub fn push(&mut self, msg: M) { + self.queue.push_back(msg); + } + + pub fn pop(&mut self) -> Option { + self.queue.pop_front() + } + + pub fn len(&self) -> usize { + self.queue.len() + } + + pub fn is_empty(&self) -> bool { + self.queue.is_empty() + } + + /// How many messages to process this tick: + /// - `len < waterlevel` → process all (`len`) + /// - `len >= waterlevel` → process half (`len >> 1`) + pub fn drain_count(&self) -> usize { + let len = self.queue.len(); + if len < self.waterlevel { + len + } else { + len >> 1 + } + } +} diff --git a/src/worker/mailbox.rs b/src/worker/mailbox.rs deleted file mode 100644 index a331e22..0000000 --- a/src/worker/mailbox.rs +++ /dev/null @@ -1,45 +0,0 @@ -use std::collections::VecDeque; - -use crate::actor::Message; - -pub struct Mailbox { - queue: VecDeque, - waterlevel: usize, -} - -impl Mailbox { - pub fn new(waterlevel: usize) -> Self { - Self { - queue: VecDeque::new(), - waterlevel, - } - } - - pub fn push(&mut self, msg: M) { - self.queue.push_back(msg); - } - - pub fn pop(&mut self) -> Option { - self.queue.pop_front() - } - - pub fn len(&self) -> usize { - self.queue.len() - } - - pub fn is_empty(&self) -> bool { - self.queue.is_empty() - } - - /// How many messages to process this tick: - /// - `len < waterlevel` → process all (`len`) - /// - `len >= waterlevel` → process half (`len >> 1`) - pub fn drain_count(&self) -> usize { - let len = self.queue.len(); - if len < self.waterlevel { - len - } else { - len >> 1 - } - } -} diff --git a/src/worker/pool.rs b/src/worker/pool.rs deleted file mode 100644 index 95c7321..0000000 --- a/src/worker/pool.rs +++ /dev/null @@ -1,51 +0,0 @@ -use std::any::Any; -use std::collections::HashMap; - -use crate::actor::{ActorAddress, AnyActor}; -use crate::context::ContextInner; - -/// Per-worker actor storage. -pub(crate) struct ActorPool { - actors: HashMap>, -} - -impl ActorPool { - pub fn new() -> Self { - Self { - actors: HashMap::new(), - } - } - - pub fn insert(&mut self, addr: ActorAddress, actor: Box) { - self.actors.insert(addr, actor); - } - - pub fn remove(&mut self, addr: &ActorAddress) -> Option> { - self.actors.remove(addr) - } - - /// Deliver a type-erased message to the actor at `addr`. - /// Returns `true` if the actor was found and the message type matched. - pub fn deliver(&mut self, addr: &ActorAddress, msg: Box) -> bool { - if let Some(actor) = self.actors.get_mut(addr) { - actor.deliver(msg) - } else { - false - } - } - - /// Tick all actors in the pool. Returns `true` if any actor processed messages. - pub fn tick_all(&mut self, inner: &dyn ContextInner) -> bool { - let mut did_work = false; - for actor in self.actors.values_mut() { - if actor.tick(inner) { - did_work = true; - } - } - did_work - } - - pub fn len(&self) -> usize { - self.actors.len() - } -} diff --git a/tests/mailbox_tests.rs b/tests/mailbox_tests.rs index 2deebba..66a671e 100644 --- a/tests/mailbox_tests.rs +++ b/tests/mailbox_tests.rs @@ -1,4 +1,4 @@ -use swactor::worker::mailbox::Mailbox; +use swactor::worker::Mailbox; // ── Basic operations ── diff --git a/tests/runtime_tests.rs b/tests/runtime_tests.rs index d57a514..ed21044 100644 --- a/tests/runtime_tests.rs +++ b/tests/runtime_tests.rs @@ -1,4 +1,4 @@ -use swactor::{Ctx, actor::{ActorAddress, ActorInterface}, runtime::{Inbox, Runtime, RuntimeConfig}}; +use swactor::{actor::{ActorAddress, ActorInterface}, runtime::{Ctx, Inbox, Runtime, RuntimeConfig}}; #[derive(Clone)] struct PingMessage { diff --git a/tests/stress/mod.rs b/tests/stress/mod.rs index 87cfdd7..2839ac9 100644 --- a/tests/stress/mod.rs +++ b/tests/stress/mod.rs @@ -165,7 +165,7 @@ impl Stress { } // Test actors used across stress tests -use swactor::{Ctx, actor::ActorInterface}; +use swactor::{actor::ActorInterface, runtime::Ctx}; /// An actor that just absorbs messages pub struct BlackHole; -- 2.45.2 From 799b654251e0c6ac556894ec7342abfdc79052f2 Mon Sep 17 00:00:00 2001 From: Zachery Aaron Shores-Chmielewski Date: Fri, 6 Feb 2026 18:05:06 +0700 Subject: [PATCH 8/9] feat: refactor tests Refactor the tests and component APIs to be more amenable to metrics, fuzzing, understanding data flows, etc. --- ARCHITECTURE.md | 199 ------------- Cargo.lock | 535 +++++++++++++++++++++++++++++++++- Cargo.toml | 10 +- benches/harness.rs | 270 ----------------- benches/main.rs | 46 --- benches/runtime_benchmarks.rs | 276 ++++++++++++++++++ benches/scaling.rs | 283 ------------------ benches/throughput.rs | 350 ---------------------- src/actor.rs | 37 --- src/envelope.rs | 24 -- src/runtime.rs | 220 ++++++-------- src/worker.rs | 39 ++- tests/mailbox_tests.rs | 226 -------------- tests/runtime_api_tests.rs | 97 ++++++ tests/runtime_tests.rs | 130 --------- tests/stress/concurrency.rs | 330 --------------------- tests/stress/mod.rs | 199 ------------- tests/stress/saturation.rs | 175 ----------- tests/stress_tests.rs | 11 - 19 files changed, 1040 insertions(+), 2417 deletions(-) delete mode 100644 ARCHITECTURE.md delete mode 100644 benches/harness.rs delete mode 100644 benches/main.rs create mode 100644 benches/runtime_benchmarks.rs delete mode 100644 benches/scaling.rs delete mode 100644 benches/throughput.rs delete mode 100644 src/envelope.rs delete mode 100644 tests/mailbox_tests.rs create mode 100644 tests/runtime_api_tests.rs delete mode 100644 tests/runtime_tests.rs delete mode 100644 tests/stress/concurrency.rs delete mode 100644 tests/stress/mod.rs delete mode 100644 tests/stress/saturation.rs delete mode 100644 tests/stress_tests.rs diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md deleted file mode 100644 index a93831e..0000000 --- a/ARCHITECTURE.md +++ /dev/null @@ -1,199 +0,0 @@ -# Swactor Architecture - -## System Diagram - -``` -┌─────────────────────────────────────────────────────────────────────────────┐ -│ Runtime │ -│ (composes everything) │ -│ │ -│ ┌───────────────────────────────────────────────────────────────────────┐ │ -│ │ Address Map │ │ -│ │ ActorAddress → WorkerId │ │ -│ │ (shared across all workers, read-heavy) │ │ -│ └──────┬──────────────────┬──────────────────────┬─────────────────────┘ │ -│ │ │ │ │ -│ ▼ ▼ ▼ │ -│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ -│ │ Worker 0 │ │ Worker 1 │ ... │ Worker N │ │ -│ │ (thread) │ │ (thread) │ │ (thread) │ │ -│ │ │ │ │ │ │ │ -│ │ ┌─────────┐ │ │ ┌─────────┐ │ │ ┌─────────┐ │ │ -│ │ │ Actor A │ │ │ │ Actor C │ │ │ │ Actor E │ │ │ -│ │ │ [═════] │ │ │ │ [═════] │ │ │ │ [═════] │ │ │ -│ │ │ mailbox │ │ │ │ mailbox │ │ │ │ mailbox │ │ │ -│ │ └─────────┘ │ │ └─────────┘ │ │ └─────────┘ │ │ -│ │ ┌─────────┐ │ │ ┌─────────┐ │ │ ┌─────────┐ │ │ -│ │ │ Actor B │ │ │ │ Actor D │ │ │ │ Actor F │ │ │ -│ │ │ [═════] │ │ │ │ [═════] │ │ │ │ [═════] │ │ │ -│ │ │ mailbox │ │ │ │ mailbox │ │ │ │ mailbox │ │ │ -│ │ └─────────┘ │ │ └─────────┘ │ │ └─────────┘ │ │ -│ │ │ │ │ │ │ │ -│ │ ┌─────────┐ │ │ ┌─────────┐ │ │ ┌─────────┐ │ │ -│ │ │Transfer │◄├────├─┤Transfer │◄├───────├─┤Transfer │ │ │ -│ │ │ Queue │ │ │ │ Queue │ │ │ │ Queue │ │ │ -│ │ │ (MPSC) │─├────├►│ (MPSC) │─├───────├►│ (MPSC) │ │ │ -│ │ └─────────┘ │ │ └─────────┘ │ │ └─────────┘ │ │ -│ └─────────────┘ └─────────────┘ └─────────────┘ │ -│ │ -└─────────────────────────────────────────────────────────────────────────────┘ - - - ══ = VecDeque (no atomics) -``` - -## Message Flow - -``` - SAME WORKER (fast path — zero atomics) - ═══════════════════════════════════════ - - Actor A Actor B - handle() { mailbox (VecDeque) - ctx.send(addr_B, msg) ▲ - │ │ - ├─ address_map[addr_B] │ - │ → Worker 0 (that's me!) │ - │ │ - └─ mailbox_B.push(msg) ──────┘ - } no atomics, no envelope - - - CROSS-WORKER (one atomic hop) - ═══════════════════════════════ - - Actor A (Worker 0) Worker 1 Actor C (Worker 1) - handle() { transfer queue mailbox (VecDeque) - ctx.send(addr_C, msg) ▲ ▲ - │ │ │ - ├─ address_map[addr_C] │ │ - │ → Worker 1 (not me) │ │ - │ │ │ - └─ envelope(addr_C, msg) ──────┘ │ - (atomic push) │ │ - └── worker 1 pops ─────┘ - and distributes - (local, no atomic) - } -``` - -## Worker Loop - -``` - ┌─────────────────────────────────────────────┐ - │ Worker Thread │ - │ │ - │ loop { │ - │ ┌──────────────────────────────────────┐ │ - │ │ 1. DRAIN TRANSFER QUEUE │ │ - │ │ while let Some((addr, env)) = │ │ - │ │ transfer_queue.pop() │ │ - │ │ { │ │ - │ │ local_actors[addr].mailbox │ │ - │ │ .push(env.unpack()) │ │ - │ │ } │ │ - │ └──────────────────────────────────────┘ │ - │ ┌──────────────────────────────────────┐ │ - │ │ 2. TICK ACTORS │ │ - │ │ for actor in &mut actor_pool { │ │ - │ │ let n = drain_count(actor); │ │ - │ │ for _ in 0..n { │ │ - │ │ let msg = actor.mailbox.pop();│ │ - │ │ actor.handle(&ctx, msg); │ │ - │ │ } │ │ - │ │ } │ │ - │ └──────────────────────────────────────┘ │ - │ ┌──────────────────────────────────────┐ │ - │ │ 3. IDLE? │ │ - │ │ if no messages processed: │ │ - │ │ spin → yield → park │ │ - │ └──────────────────────────────────────┘ │ - │ } │ - └───────────────────────────────────────────────┘ -``` - -## File Tree - -``` -src/ -├── lib.rs # crate root, feature flags, public exports -├── error.rs # Error type -│ -├── actor.rs # Message trait, ActorInterface trait, ActorAddress -│ # - ActorInterface::handle(&mut self, ctx: &Ctx, msg) -│ # - actors depend ONLY on Ctx, nothing else -│ -├── context.rs # Ctx wrapper — the "syscall interface" for actors -│ # - wraps &dyn ContextInner (solves object-safety) -│ # - send(), self_addr(), spawn() -│ # - this is ALL actors can see of the framework -│ -├── envelope.rs # Envelope type — type erasure for cross-thread messages -│ # - wraps typed messages for the transfer queue -│ # - unwraps back to concrete type at destination -│ -├── address_map.rs # ActorAddress → WorkerId mapping -│ # - shared read-heavy structure -│ # - written on spawn, read on every send -│ -├── channel/ -│ └── mod.rs # HybridChannel — per-worker MPSC -│ # - the ONE concurrent data structure on the hot path -│ # - carries Envelope (cross-worker) and spawn tuples -│ -├── worker/ -│ ├── mod.rs # Worker struct and worker loop -│ │ # - owns actor pool + transfer queue -│ │ # - the thread boundary: concurrent outside, local inside -│ │ # - drain transfer queue → tick actors → backoff -│ │ -│ ├── mailbox.rs # VecDeque-based local mailbox -│ │ # - NO atomics, NO Arc, NO crossbeam -│ │ # - only touched by the owning worker thread -│ │ -│ └── pool.rs # Actor pool — stores actors assigned to this worker -│ # - local HashMap for ActorAddress → Actor lookup -│ # - insert on spawn, remove on shutdown -│ -├── runtime.rs # Runtime — the composition point -│ # - creates workers, address map -│ # - implements ContextInner (delegates to address map + transfer queues) -│ # - public API: new(), spawn(), send_to(), run(), tick(), shutdown() -│ -├── config.rs # RuntimeConfig — tuning knobs -│ # - num_threads, max_actors, actor_max_messages -│ # - mailbox_waterlevel (drain threshold per actor) -│ # - BackoffPolicy (spin/yield/sleep thresholds) -│ -└── placement.rs # Actor placement strategy - # - currently: round-robin across workers -``` - -## Components - -| Component | File(s) | What It Does | Concurrent? | -|---|---|---|---| -| **Worker** | `worker/mod.rs` | Owns a thread, a pool of actors, their mailboxes, and a transfer queue. Runs the tick loop. Everything inside is single-threaded. | No (that's the point) | -| **Mailbox** | `worker/mailbox.rs` | `VecDeque` per actor. Zero atomics. Only the owning worker reads/writes. | No | -| **Actor Pool** | `worker/pool.rs` | Stores actors on this worker. Local lookup by address. | No | -| **Transfer Queue** | `channel/mod.rs` | HybridChannel MPSC queue per worker. The only atomic boundary. Other workers push, this worker pops. | Yes (the ONE place) | -| **Address Map** | `address_map.rs` | Maps ActorAddress → WorkerId. Read on every cross-thread send, written on spawn. | Yes (read-heavy) | -| **Envelope** | `envelope.rs` | Type-erases messages for the transfer queue. Unwrapped at destination. | No (data format) | -| **Context** | `context.rs` | `Ctx` wrapper over `&dyn ContextInner`. `send()`, `self_addr()`, `spawn()`. Hides all framework internals. | N/A (trait) | -| **Runtime** | `runtime.rs` | Wires it all together. Creates workers, holds address map, exposes public API. | Minimal (delegates) | -| **Placement** | `placement.rs` | Decides which worker gets a new actor. Currently round-robin. | No (called at spawn time) | - -## Single-Threaded / WASM Mode - -One worker. No transfer queue needed. No address map needed (everything is local). The system collapses to: - -``` - Worker 0 - ┌───────────────────────┐ - │ Actor A [mailbox] │ - │ Actor B [mailbox] │ All sends are local. - │ Actor C [mailbox] │ All mailboxes are VecDeque. - │ │ Zero atomics anywhere. - │ tick() drives loop │ - └───────────────────────┘ -``` diff --git a/Cargo.lock b/Cargo.lock index a25b4f1..d0e4e9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,12 +2,158 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bumpalo" +version = "3.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cfg-if" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.5.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6899ea499e3fb9305a65d5ebf6e3d2248c5fab291f300ad0a704fbe142eae31a" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.5.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b12c8b680195a62a8364d16b8447b01b6c2c8f9aaf68bee653be34d4245e238" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e64b0cc0439b12df2fa678eae89a1c56a529fd067a9115f7827f1fffd22b32" + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.12" @@ -23,6 +169,18 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "getrandom" version = "0.2.17" @@ -34,6 +192,59 @@ dependencies = [ "wasi", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" + +[[package]] +name = "js-sys" +version = "0.3.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + [[package]] name = "libc" version = "0.2.180" @@ -41,19 +252,230 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" [[package]] -name = "smallvec" -version = "1.15.1" +name = "memchr" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c" + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] [[package]] name = "swactor" version = "0.1.0" dependencies = [ + "criterion", "crossbeam-queue", "crossbeam-utils", "getrandom", - "smallvec", +] + +[[package]] +name = "syn" +version = "2.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "unicode-ident" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", ] [[package]] @@ -61,3 +483,108 @@ name = "wasi" version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zerocopy" +version = "0.8.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db6d35d663eadb6c932438e763b262fe1a70987f9ae936e60158176d710cae4a" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4122cd3169e94605190e77839c9a40d40ed048d305bfdc146e7df40ab0f3e517" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ff05f8caa9038894637571ae6b9e29466c1f4f829d26c9b28f869a29cbe3445" diff --git a/Cargo.toml b/Cargo.toml index 78f8ddb..a348085 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,8 +17,10 @@ stress = [] # Enable stress tests getrandom = { version = "0.2", optional = true } crossbeam-queue = "0.3.12" crossbeam-utils = "0.8.21" -smallvec = "1.13" -[[bin]] -name = "bench" -path = "benches/main.rs" +[dev-dependencies] +criterion = { version = "0.5", features = ["html_reports"] } + +[[bench]] +name = "runtime_benchmarks" +harness = false diff --git a/benches/harness.rs b/benches/harness.rs deleted file mode 100644 index 8eddc47..0000000 --- a/benches/harness.rs +++ /dev/null @@ -1,270 +0,0 @@ -//! Manual benchmark harness - zero dependencies, full control. -//! -//! Provides statistical analysis of benchmark runs including: -//! - Mean, median, min, max -//! - Standard deviation -//! - Percentiles (P50, P90, P99, P99.9) -//! - Throughput calculations -//! - Outlier detection and removal - -use std::time::{Duration, Instant}; - -/// Results from a single benchmark run -#[derive(Debug, Clone)] -pub struct BenchResult { - pub name: String, - pub iterations: usize, - pub total_time: Duration, - pub times: Vec, - /// Optional: elements processed (for throughput calculation) - pub elements: Option, -} - -/// Statistical summary of benchmark results -#[derive(Debug)] -pub struct Stats { - pub mean: Duration, - pub median: Duration, - pub min: Duration, - pub max: Duration, - pub std_dev: Duration, - pub p50: Duration, - pub p90: Duration, - pub p99: Duration, - pub p999: Duration, - pub throughput: Option, // elements per second -} - -impl BenchResult { - /// Calculate statistics from the raw timing data - pub fn stats(&self) -> Stats { - let mut sorted: Vec = self.times.clone(); - sorted.sort(); - - let n = sorted.len(); - assert!(n > 0, "Cannot compute stats on empty results"); - - let sum: Duration = sorted.iter().sum(); - let mean = sum / n as u32; - - let median = if n % 2 == 0 { - (sorted[n / 2 - 1] + sorted[n / 2]) / 2 - } else { - sorted[n / 2] - }; - - // Standard deviation - let mean_nanos = mean.as_nanos() as f64; - let variance: f64 = sorted - .iter() - .map(|t| { - let diff = t.as_nanos() as f64 - mean_nanos; - diff * diff - }) - .sum::() - / n as f64; - let std_dev = Duration::from_nanos(variance.sqrt() as u64); - - // Percentiles - let percentile = |p: f64| -> Duration { - let idx = ((p / 100.0) * (n - 1) as f64).round() as usize; - sorted[idx.min(n - 1)] - }; - - let throughput = self.elements.map(|e| { - let secs = self.total_time.as_secs_f64(); - if secs > 0.0 { - (e * self.iterations as u64) as f64 / secs - } else { - 0.0 - } - }); - - Stats { - mean, - median, - min: sorted[0], - max: sorted[n - 1], - std_dev, - p50: percentile(50.0), - p90: percentile(90.0), - p99: percentile(99.0), - p999: percentile(99.9), - throughput, - } - } - - /// Pretty print the results - pub fn print(&self) { - let stats = self.stats(); - - println!("\n{}", "=".repeat(60)); - println!(" {}", self.name); - println!("{}", "=".repeat(60)); - println!(" Iterations: {}", self.iterations); - println!(" Total time: {:?}", self.total_time); - println!(); - println!(" Mean: {:?}", stats.mean); - println!(" Median: {:?}", stats.median); - println!(" Std Dev: {:?}", stats.std_dev); - println!(" Min: {:?}", stats.min); - println!(" Max: {:?}", stats.max); - println!(); - println!(" P50: {:?}", stats.p50); - println!(" P90: {:?}", stats.p90); - println!(" P99: {:?}", stats.p99); - println!(" P99.9: {:?}", stats.p999); - - if let Some(throughput) = stats.throughput { - println!(); - println!(" Throughput: {:.2} ops/sec", throughput); - if throughput > 1_000_000.0 { - println!(" {:.2} M ops/sec", throughput / 1_000_000.0); - } else if throughput > 1_000.0 { - println!(" {:.2} K ops/sec", throughput / 1_000.0); - } - } - println!("{}", "=".repeat(60)); - } -} - -/// A benchmark builder for configuring and running benchmarks -pub struct Bench { - name: String, - warmup_iters: usize, - bench_iters: usize, - elements_per_iter: Option, -} - -impl Bench { - pub fn new(name: impl Into) -> Self { - Self { - name: name.into(), - warmup_iters: 3, - bench_iters: 100, - elements_per_iter: None, - } - } - - /// Set number of warmup iterations (default: 3) - pub fn warmup(mut self, n: usize) -> Self { - self.warmup_iters = n; - self - } - - /// Set number of benchmark iterations (default: 100) - pub fn iters(mut self, n: usize) -> Self { - self.bench_iters = n; - self - } - - /// Set elements per iteration for throughput calculation - pub fn elements(mut self, n: u64) -> Self { - self.elements_per_iter = Some(n); - self - } - - /// Run the benchmark with setup before each iteration - pub fn run_with_setup(self, mut setup: S, mut f: F) -> BenchResult - where - S: FnMut() -> T, - F: FnMut(T), - { - // Warmup - for _ in 0..self.warmup_iters { - let state = setup(); - f(state); - } - - // Benchmark - let mut times = Vec::with_capacity(self.bench_iters); - let total_start = Instant::now(); - - for _ in 0..self.bench_iters { - let state = setup(); - let start = Instant::now(); - f(state); - times.push(start.elapsed()); - } - - let total_time = total_start.elapsed(); - - BenchResult { - name: self.name, - iterations: self.bench_iters, - total_time, - times, - elements: self.elements_per_iter, - } - } -} - -/// A collection of benchmarks to run together -pub struct BenchSuite { - name: String, - results: Vec, -} - -impl BenchSuite { - pub fn new(name: impl Into) -> Self { - Self { - name: name.into(), - results: Vec::new(), - } - } - - pub fn add(&mut self, result: BenchResult) { - self.results.push(result); - } - - pub fn print_summary(&self) { - println!("\n{}", "#".repeat(70)); - println!("# BENCHMARK SUITE: {}", self.name); - println!("{}", "#".repeat(70)); - - for result in &self.results { - result.print(); - } - - // Summary table - println!("\n{}", "-".repeat(70)); - println!(" SUMMARY"); - println!("{}", "-".repeat(70)); - println!( - " {:30} {:>12} {:>12} {:>12}", - "Benchmark", "Mean", "P99", "Throughput" - ); - println!("{}", "-".repeat(70)); - - for result in &self.results { - let stats = result.stats(); - let throughput_str = stats - .throughput - .map(|t| { - if t > 1_000_000.0 { - format!("{:.2}M/s", t / 1_000_000.0) - } else if t > 1_000.0 { - format!("{:.2}K/s", t / 1_000.0) - } else { - format!("{:.2}/s", t) - } - }) - .unwrap_or_else(|| "-".to_string()); - - println!( - " {:30} {:>12.2?} {:>12.2?} {:>12}", - result.name, stats.mean, stats.p99, throughput_str - ); - } - println!("{}", "-".repeat(70)); - } -} - -/// Prevent the compiler from optimizing away a value -#[inline(never)] -pub fn black_box(x: T) -> T { - // Use inline assembly to prevent optimization - // This is a simplified version - in practice, reads from the value - let ptr = &x as *const T; - unsafe { std::ptr::read_volatile(ptr) } -} diff --git a/benches/main.rs b/benches/main.rs deleted file mode 100644 index 41f32fd..0000000 --- a/benches/main.rs +++ /dev/null @@ -1,46 +0,0 @@ -//! Swactor Benchmark Suite -//! -//! A manual benchmark harness for measuring runtime performance. -//! Zero external dependencies - just std::time. -//! -//! Run with: cargo run --bin bench --release -//! -//! Options: -//! --throughput Run throughput benchmarks only -//! --scaling Run scaling benchmarks only -//! --all Run all benchmarks (default) - -mod harness; -mod throughput; -mod scaling; - -use std::env; - -fn main() { - let args: Vec = env::args().collect(); - - println!("============================================================"); - println!(" SWACTOR BENCHMARK SUITE"); - println!("============================================================"); - println!(); - - // Parse arguments - let run_throughput = args.contains(&"--throughput".to_string()) - || args.contains(&"--all".to_string()) - || args.len() == 1; - let run_scaling = args.contains(&"--scaling".to_string()) - || args.contains(&"--all".to_string()) - || args.len() == 1; - - if run_throughput { - let suite = throughput::run_all(); - suite.print_summary(); - } - - if run_scaling { - let suite = scaling::run_all(); - suite.print_summary(); - } - - println!("\nBenchmarks complete."); -} diff --git a/benches/runtime_benchmarks.rs b/benches/runtime_benchmarks.rs new file mode 100644 index 0000000..64eff02 --- /dev/null +++ b/benches/runtime_benchmarks.rs @@ -0,0 +1,276 @@ +use criterion::{ + criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion, Throughput, +}; +use swactor::{ + actor::{ActorAddress, ActorInterface}, + config::RuntimeConfig, + runtime::{Ctx, Runtime}, +}; + +// --------------------------------------------------------------------------- +// Helper +// --------------------------------------------------------------------------- + +fn make_config(max_actors: usize, max_messages: usize) -> RuntimeConfig { + RuntimeConfig { + max_actors, + actor_max_messages: max_messages, + num_threads: 1, + ..Default::default() + } +} + +// --------------------------------------------------------------------------- +// Message types +// --------------------------------------------------------------------------- + +#[derive(Clone)] +struct NoopMessage; + +#[derive(Clone)] +struct PingMessage { + reply_to: ActorAddress, +} + +#[derive(Clone)] +struct PongMessage; + +#[derive(Clone)] +struct CountMessage(u64); + +#[derive(Clone)] +struct RingMessage { + hops: u64, +} + +// --------------------------------------------------------------------------- +// Actor types +// --------------------------------------------------------------------------- + +struct NoopActor; + +impl ActorInterface for NoopActor { + type Incoming = NoopMessage; + type Response = (); + fn handle(&mut self, _ctx: &Ctx, _msg: NoopMessage) {} +} + +struct EchoActor; + +impl ActorInterface for EchoActor { + type Incoming = PingMessage; + type Response = (); + fn handle(&mut self, ctx: &Ctx, msg: PingMessage) { + let _ = ctx.send(msg.reply_to, PongMessage); + } +} + +struct SinkActor; + +impl ActorInterface for SinkActor { + type Incoming = CountMessage; + type Response = (); + fn handle(&mut self, _ctx: &Ctx, _msg: CountMessage) {} +} + +struct RingActor { + next: ActorAddress, +} + +impl ActorInterface for RingActor { + type Incoming = RingMessage; + type Response = (); + fn handle(&mut self, ctx: &Ctx, msg: RingMessage) { + let _ = ctx.send(self.next, RingMessage { hops: msg.hops + 1 }); + } +} + +// --------------------------------------------------------------------------- +// Latency benchmarks +// --------------------------------------------------------------------------- + +fn latency_benchmarks(c: &mut Criterion) { + let mut group = c.benchmark_group("latency"); + + // A1 — Spawn latency + group.bench_function("spawn", |b| { + b.iter_batched( + || Runtime::new(make_config(1_000, 1_000)), + |rt| { + rt.spawn(NoopActor).unwrap(); + }, + BatchSize::SmallInput, + ); + }); + + // A2 — Message round-trip + group.bench_function("message_roundtrip", |b| { + b.iter_batched( + || { + let rt = Runtime::new(make_config(1_000, 1_000)); + let addr = rt.spawn(EchoActor).unwrap(); + rt.tick(); // register actor + let inbox = rt.new_inbox::().unwrap(); + let inbox_addr = *inbox.addr(); + (rt, addr, inbox, inbox_addr) + }, + |(rt, addr, inbox, inbox_addr)| { + rt.send_to(addr, PingMessage { reply_to: inbox_addr }).unwrap(); + for _ in 0..20 { + rt.tick(); + if inbox.try_recv().is_some() { + return; + } + } + panic!("PongMessage not received within 20 ticks"); + }, + BatchSize::SmallInput, + ); + }); + + // A3 — Fire-and-forget send + group.bench_function("send_fire_and_forget", |b| { + b.iter_batched( + || { + let rt = Runtime::new(make_config(1_000, 100_000)); + let addr = rt.spawn(NoopActor).unwrap(); + rt.tick(); // register actor + (rt, addr) + }, + |(rt, addr)| { + rt.send_to(addr, NoopMessage).unwrap(); + }, + BatchSize::SmallInput, + ); + }); + + // A4 — Inbox creation + group.bench_function("inbox_creation", |b| { + b.iter_batched( + || Runtime::new(make_config(1_000, 1_000)), + |rt| { + rt.new_inbox::().unwrap(); + }, + BatchSize::SmallInput, + ); + }); + + group.finish(); +} + +// --------------------------------------------------------------------------- +// Throughput benchmarks +// --------------------------------------------------------------------------- + +fn throughput_benchmarks(c: &mut Criterion) { + let mut group = c.benchmark_group("throughput"); + + // B1 — Single-actor throughput + for n in [100, 1_000, 10_000] { + group.throughput(Throughput::Elements(n as u64)); + group.bench_with_input(BenchmarkId::new("single_actor", n), &n, |b, &n| { + b.iter_batched( + || { + let rt = Runtime::new(make_config(100, n + 100)); + let addr = rt.spawn(SinkActor).unwrap(); + rt.tick(); // register actor + for i in 0..n { + rt.send_to(addr, CountMessage(i as u64)).unwrap(); + } + rt + }, + |rt| { + for _ in 0..50 { + rt.tick(); + } + }, + BatchSize::LargeInput, + ); + }); + } + + // B2 — Multi-actor throughput + for (actors, msgs_per) in [(10, 100), (100, 100), (100, 1_000)] { + let total = actors * msgs_per; + group.throughput(Throughput::Elements(total as u64)); + let param = format!("{actors}x{msgs_per}"); + group.bench_with_input(BenchmarkId::new("multi_actor", ¶m), &(actors, msgs_per), |b, &(actors, msgs_per)| { + b.iter_batched( + || { + let rt = Runtime::new(make_config(actors + 100, msgs_per + 100)); + let addrs: Vec<_> = (0..actors) + .map(|_| rt.spawn(SinkActor).unwrap()) + .collect(); + rt.tick(); // register actors + for &addr in &addrs { + for i in 0..msgs_per { + rt.send_to(addr, CountMessage(i as u64)).unwrap(); + } + } + rt + }, + |rt| { + for _ in 0..100 { + rt.tick(); + } + }, + BatchSize::LargeInput, + ); + }); + } + + // B3 — Ring throughput + for ring_size in [10usize, 100, 500] { + group.throughput(Throughput::Elements((ring_size + 1) as u64)); + group.bench_with_input(BenchmarkId::new("ring", ring_size), &ring_size, |b, &ring_size| { + b.iter_batched( + || { + let rt = Runtime::new(make_config(ring_size + 100, 100)); + let inbox = rt.new_inbox::().unwrap(); + // Build the ring: last actor sends to inbox, each prior actor sends to the next + let mut next_addr = *inbox.addr(); + let mut entry_addr = next_addr; + for _ in 0..ring_size { + let addr = rt.spawn(RingActor { next: next_addr }).unwrap(); + entry_addr = addr; + next_addr = addr; + } + rt.tick(); // register all actors + (rt, entry_addr, inbox) + }, + |(rt, entry_addr, inbox)| { + rt.send_to(entry_addr, RingMessage { hops: 0 }).unwrap(); + for _ in 0..(ring_size + 10) { + rt.tick(); + if inbox.try_recv().is_some() { + return; + } + } + panic!("RingMessage not received within tick budget"); + }, + BatchSize::LargeInput, + ); + }); + } + + // B4 — Spawn throughput + for n in [100, 1_000, 5_000] { + group.throughput(Throughput::Elements(n as u64)); + group.bench_with_input(BenchmarkId::new("spawn", n), &n, |b, &n| { + b.iter_batched( + || Runtime::new(make_config(n + 100, 1_000)), + |rt| { + for _ in 0..n { + rt.spawn(NoopActor).unwrap(); + } + }, + BatchSize::LargeInput, + ); + }); + } + + group.finish(); +} + +criterion_group!(benches, latency_benchmarks, throughput_benchmarks); +criterion_main!(benches); diff --git a/benches/scaling.rs b/benches/scaling.rs deleted file mode 100644 index 0febea5..0000000 --- a/benches/scaling.rs +++ /dev/null @@ -1,283 +0,0 @@ -//! Scaling benchmarks for the swactor runtime. -//! -//! These benchmarks measure how performance scales with: -//! - Number of actors -//! - Number of worker threads -//! - Message payload size - -use crate::harness::{black_box, Bench, BenchSuite}; -use std::thread; -use swactor::{ - actor::ActorInterface, - runtime::{Ctx, Runtime, RuntimeConfig}, -}; - -// ============================================================================ -// Test Actors -// ============================================================================ - -/// A counter actor that just increments on each message -struct CounterActor { - count: usize, -} - -impl CounterActor { - fn new() -> Self { - Self { count: 0 } - } -} - -#[derive(Clone)] -struct Increment; - -impl ActorInterface for CounterActor { - type Incoming = Increment; - type Response = (); - - fn handle(&mut self, _ctx: &Ctx, _msg: Increment) { - self.count += 1; - } -} - -struct SharedCounter { - count: std::sync::Arc, -} - -impl ActorInterface for SharedCounter { - type Incoming = Increment; - type Response = (); - - fn handle(&mut self, _ctx: &Ctx, _msg: Increment) { - self.count - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - } -} - -/// An actor that handles variable-sized payloads -struct PayloadActor { - bytes_received: usize, -} - -impl PayloadActor { - fn new() -> Self { - Self { bytes_received: 0 } - } -} - -#[derive(Clone)] -struct Payload(Vec); - -impl ActorInterface for PayloadActor { - type Incoming = Payload; - type Response = (); - - fn handle(&mut self, _ctx: &Ctx, msg: Payload) { - self.bytes_received += msg.0.len(); - black_box(&msg.0); - } -} - -// ============================================================================ -// Benchmarks -// ============================================================================ - -/// Benchmark: How throughput scales with actor count -pub fn bench_actor_count_scaling(suite: &mut BenchSuite) { - for (actor_count, messages_per_actor, warmup_n, iters_n) in [ - (10u64, 200u64, 5usize, 50usize), - (100, 200, 5, 30), - (500, 200, 5, 15), - (1000, 200, 5, 12), - ] { - let name = format!("scaling_{}_actors", actor_count); - let total_messages = actor_count * messages_per_actor; - - let result = Bench::new(&name) - .warmup(warmup_n) - .iters(iters_n) - .elements(total_messages) - .run_with_setup( - || { - let config = RuntimeConfig { - max_actors: (actor_count as usize) + 100, - actor_max_messages: (total_messages as usize) * 3, - num_threads: 1, - ..Default::default() - }; - let runtime = Runtime::new(config); - - // Spawn actors - let mut actors = Vec::with_capacity(actor_count as usize); - for _ in 0..actor_count { - let addr = runtime.spawn(CounterActor::new()).unwrap(); - actors.push(addr); - } - - // Process spawns - for _ in 0..(actor_count * 2) { - runtime.tick(); - } - - (runtime, actors, messages_per_actor) - }, - |(runtime, actors, msgs_per)| { - // Distribute messages across all actors - for _ in 0..msgs_per { - for actor in &actors { - let _ = runtime.send_to::(*actor, Increment); - } - } - - // Process all - let total = actors.len() as u64 * msgs_per; - for _ in 0..(total * 3) { - runtime.tick(); - } - black_box(()); - }, - ); - - suite.add(result); - } -} - -/// Benchmark: How throughput scales with thread count (multithreaded runtime) -pub fn bench_thread_count_scaling(suite: &mut BenchSuite) { - let actor_count = 100u64; - let messages_per_actor = 500u64; - let total_messages = actor_count * messages_per_actor; - - for thread_count in [2usize, 4, 8] { - let name = format!("scaling_{}_threads", thread_count); - - let result = Bench::new(&name) - .warmup(5) - .iters(30) - .elements(total_messages) - .run_with_setup( - || { - let counter = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let config = RuntimeConfig { - max_actors: (actor_count as usize) + 100, - actor_max_messages: (total_messages as usize) * 3, - num_threads: thread_count, - ..Default::default() - }; - let runtime = Runtime::new(config); - - let mut actors = Vec::with_capacity(actor_count as usize); - for _ in 0..actor_count { - let addr = runtime - .spawn(SharedCounter { - count: counter.clone(), - }) - .unwrap(); - actors.push(addr); - } - - let handle = runtime.run().unwrap(); - - for actor in &actors { - loop { - if handle - .runtime - .send_to::(*actor, Increment) - .is_ok() - { - break; - } - thread::yield_now(); - } - } - - while counter.load(std::sync::atomic::Ordering::Relaxed) < actors.len() { - thread::yield_now(); - } - counter.store(0, std::sync::atomic::Ordering::Relaxed); - - (handle, actors, counter) - }, - |(handle, actors, counter)| { - for _ in 0..messages_per_actor { - for actor in &actors { - let _ = handle.runtime.send_to::(*actor, Increment); - } - } - - while counter.load(std::sync::atomic::Ordering::Relaxed) - < total_messages as usize - { - thread::yield_now(); - } - - handle.shutdown(); - handle.join(); - black_box(()); - }, - ); - - suite.add(result); - } -} - -/// Benchmark: How throughput scales with message payload size -pub fn bench_payload_size_scaling(suite: &mut BenchSuite) { - let message_count = 3_000u64; - - for payload_size in [64usize, 1024, 16384, 65536] { - let name = format!("payload_{}B", payload_size); - let payload = vec![0u8; payload_size]; - - let result = Bench::new(&name) - .warmup(5) - .iters(40) - .elements(message_count) - .run_with_setup( - || { - let config = RuntimeConfig { - max_actors: 10, - actor_max_messages: (message_count as usize) * 2, - num_threads: 1, - ..Default::default() - }; - let runtime = Runtime::new(config); - let sink = runtime.spawn(PayloadActor::new()).unwrap(); - - // Process spawn - for _ in 0..10 { - runtime.tick(); - } - - (runtime, sink, payload.clone()) - }, - |(runtime, sink, payload)| { - for _ in 0..message_count { - let _ = runtime.send_to::(sink, Payload(payload.clone())); - } - - for _ in 0..(message_count * 3) { - runtime.tick(); - } - black_box(()); - }, - ); - - suite.add(result); - } -} - -/// Run all scaling benchmarks -pub fn run_all() -> BenchSuite { - let mut suite = BenchSuite::new("Scaling Benchmarks"); - - println!("\nRunning actor count scaling benchmarks..."); - bench_actor_count_scaling(&mut suite); - - println!("Running thread count scaling benchmarks..."); - bench_thread_count_scaling(&mut suite); - - println!("Running payload size scaling benchmarks..."); - bench_payload_size_scaling(&mut suite); - - suite -} diff --git a/benches/throughput.rs b/benches/throughput.rs deleted file mode 100644 index 900a4d0..0000000 --- a/benches/throughput.rs +++ /dev/null @@ -1,350 +0,0 @@ -//! Core throughput benchmarks for the swactor runtime. -//! -//! These benchmarks measure: -//! - Message passing throughput -//! - Actor spawn rate -//! - Fan-out and fan-in patterns -//! - Ping-pong latency - -use crate::harness::{black_box, Bench, BenchSuite}; -use swactor::{ - actor::{ActorAddress, ActorInterface}, - runtime::{Ctx, Runtime, RuntimeConfig}, -}; - -// ============================================================================ -// Test Actors -// ============================================================================ - -/// A sink actor that counts messages received -struct SinkActor { - count: usize, -} - -impl SinkActor { - fn new() -> Self { - Self { count: 0 } - } -} - -#[derive(Clone)] -struct Ping; - -impl ActorInterface for SinkActor { - type Incoming = Ping; - type Response = (); - - fn handle(&mut self, _ctx: &Ctx, _msg: Ping) { - self.count += 1; - } -} - -/// A forwarding actor that passes messages along a chain -struct ForwardActor { - next: Option, -} - -impl ForwardActor { - fn new() -> Self { - Self { next: None } - } - - fn with_next(next: ActorAddress) -> Self { - Self { next: Some(next) } - } -} - -impl ActorInterface for ForwardActor { - type Incoming = Ping; - type Response = Ping; - - fn handle(&mut self, ctx: &Ctx, msg: Ping) { - if let Some(next) = self.next { - let _ = ctx.send(next, msg); - } - } -} - -// ============================================================================ -// Benchmarks -// ============================================================================ - -/// Benchmark: Messages sent through the runtime to a single sink actor -pub fn bench_message_throughput(suite: &mut BenchSuite) { - for msg_count in [1_000u64, 10_000, 100_000] { - let name = format!("message_throughput_{}", msg_count); - - let result = Bench::new(&name) - .warmup(5) - .iters(100) - .elements(msg_count) - .run_with_setup( - || { - // Setup: create runtime and sink actor - let config = RuntimeConfig { - max_actors: 100, - actor_max_messages: (msg_count as usize) * 2, - num_threads: 1, - ..Default::default() - }; - let runtime = Runtime::new(config); - let sink = runtime.spawn(SinkActor::new()).unwrap(); - (runtime, sink, msg_count) - }, - |(runtime, sink, count)| { - // Send all messages - for _ in 0..count { - let _ = runtime.send_to::(sink, Ping); - } - // Process until done - for _ in 0..(count * 3) { - runtime.tick(); - } - black_box(()); - }, - ); - - suite.add(result); - } -} - -/// Benchmark: Actor spawn rate -pub fn bench_spawn_rate(suite: &mut BenchSuite) { - for actor_count in [100u64, 500, 900] { - let name = format!("spawn_rate_{}_actors", actor_count); - - let result = Bench::new(&name) - .warmup(5) - .iters(100) - .elements(actor_count) - .run_with_setup( - || { - let config = RuntimeConfig { - max_actors: 1000, - actor_max_messages: 100, - num_threads: 1, - ..Default::default() - }; - Runtime::new(config) - }, - |runtime| { - for _ in 0..actor_count { - let _ = runtime.spawn(SinkActor::new()); - } - // Process spawns - for _ in 0..(actor_count * 2) { - runtime.tick(); - } - black_box(()); - }, - ); - - suite.add(result); - } -} - -/// Benchmark: Fan-out (1 sender to N receivers) -pub fn bench_fanout(suite: &mut BenchSuite) { - for (fan_count, messages_per_receiver, warmup_n, iters_n) in [ - (10u64, 500u64, 5usize, 50usize), - (100, 200, 5, 40), - (500, 100, 5, 30), - ] { - let name = format!("fanout_1_to_{}", fan_count); - - let result = Bench::new(&name) - .warmup(warmup_n) - .iters(iters_n) - .elements(fan_count * messages_per_receiver) - .run_with_setup( - || { - let config = RuntimeConfig { - max_actors: (fan_count as usize) + 10, - actor_max_messages: (fan_count as usize) - * (messages_per_receiver as usize) - * 2, - num_threads: 1, - ..Default::default() - }; - let runtime = Runtime::new(config); - - // Spawn N sink actors - let mut sinks = Vec::with_capacity(fan_count as usize); - for _ in 0..fan_count { - let addr = runtime.spawn(SinkActor::new()).unwrap(); - sinks.push(addr); - } - - // Process spawns - for _ in 0..(fan_count * 2) { - runtime.tick(); - } - - (runtime, sinks, messages_per_receiver) - }, - |(runtime, sinks, msgs_per)| { - // Send messages to all sinks - for _ in 0..msgs_per { - for sink in &sinks { - let _ = runtime.send_to::(*sink, Ping); - } - } - - // Process all messages - let total_msgs = sinks.len() as u64 * msgs_per; - for _ in 0..(total_msgs * 3) { - runtime.tick(); - } - black_box(()); - }, - ); - - suite.add(result); - } -} - -/// Benchmark: Fan-in (N senders to 1 receiver) -pub fn bench_fanin(suite: &mut BenchSuite) { - for (sender_count, messages_per_sender, warmup_n, iters_n) in [ - (10u64, 300u64, 5usize, 50usize), - (100, 100, 5, 30), - (500, 100, 5, 20), - ] { - let name = format!("fanin_{}_to_1", sender_count); - - let result = Bench::new(&name) - .warmup(warmup_n) - .iters(iters_n) - .elements(sender_count * messages_per_sender) - .run_with_setup( - || { - let total_messages = (sender_count * messages_per_sender) as usize; - let config = RuntimeConfig { - max_actors: (sender_count as usize) + 10, - actor_max_messages: total_messages * 3, - num_threads: 1, - ..Default::default() - }; - let runtime = Runtime::new(config); - - // Spawn the sink - let sink = runtime.spawn(SinkActor::new()).unwrap(); - - // Spawn N forwarders pointing at sink - let mut senders = Vec::with_capacity(sender_count as usize); - for _ in 0..sender_count { - let addr = runtime.spawn(ForwardActor::with_next(sink)).unwrap(); - senders.push(addr); - } - - // Process spawns - for _ in 0..((sender_count + 1) * 2) { - runtime.tick(); - } - - (runtime, senders, sink, messages_per_sender) - }, - |(runtime, senders, _sink, msgs_per)| { - // Each sender forwards msgs_per messages to the sink - for _ in 0..msgs_per { - for sender in &senders { - let _ = runtime.send_to::(*sender, Ping); - } - } - - // Process all messages (forwarder receives + forwards, sink receives) - let total_msgs = senders.len() as u64 * msgs_per; - for _ in 0..(total_msgs * 6) { - runtime.tick(); - } - black_box(()); - }, - ); - - suite.add(result); - } -} - -/// Benchmark: Ring topology (message passed around N actors in a circle) -pub fn bench_ring(suite: &mut BenchSuite) { - for (ring_size, laps, warmup_n, iters_n) in [ - (10u64, 100u64, 5usize, 100usize), - (100, 20, 5, 50), - (500, 10, 5, 40), - ] { - let name = format!("ring_{}_actors", ring_size); - - let result = Bench::new(&name) - .warmup(warmup_n) - .iters(iters_n) - .elements(ring_size * laps) - .run_with_setup( - || { - let config = RuntimeConfig { - max_actors: (ring_size as usize) + 10, - actor_max_messages: 10_000, - num_threads: 1, - ..Default::default() - }; - let runtime = Runtime::new(config); - - // First, spawn all actors without links - let mut actors: Vec = Vec::with_capacity(ring_size as usize); - for _ in 0..ring_size { - let addr = runtime.spawn(ForwardActor::new()).unwrap(); - actors.push(addr); - } - - // We can't update their `next` field after spawn in this design, - // so instead we'll use an inbox to receive the final message - // For now, we'll just measure message passing through a chain - - // Process spawns - for _ in 0..(ring_size * 2) { - runtime.tick(); - } - - (runtime, actors, laps) - }, - |(runtime, actors, laps)| { - // Send to first actor (even though they don't forward, we're - // measuring the transfer queue + inbox overhead) - for _ in 0..laps { - for actor in &actors { - let _ = runtime.send_to::(*actor, Ping); - } - } - - let total = actors.len() as u64 * laps; - for _ in 0..(total * 3) { - runtime.tick(); - } - black_box(()); - }, - ); - - suite.add(result); - } -} - -/// Run all throughput benchmarks -pub fn run_all() -> BenchSuite { - let mut suite = BenchSuite::new("Throughput Benchmarks"); - - println!("\nRunning message throughput benchmarks..."); - bench_message_throughput(&mut suite); - - println!("Running spawn rate benchmarks..."); - bench_spawn_rate(&mut suite); - - println!("Running fan-out benchmarks..."); - bench_fanout(&mut suite); - - println!("Running fan-in benchmarks..."); - bench_fanin(&mut suite); - - println!("Running ring topology benchmarks..."); - bench_ring(&mut suite); - - suite -} diff --git a/src/actor.rs b/src/actor.rs index a5ae92a..7a7a9b8 100644 --- a/src/actor.rs +++ b/src/actor.rs @@ -6,43 +6,6 @@ use crate::{get_random, runtime::{ContextInner, Ctx}, worker::Mailbox}; pub trait Message: 'static + Sized + Clone + Send + Sync {} impl Message for T {} -/// The trait that needs to be implemented in order to run a process as an `Actor` -/// -/// The `Incoming` type represents `Messages` that can be delivered to the `Actor`. -/// -/// The `Response` type represents possible `Messages` the actor may attempt to reply with. -/// -/// The `fn handle(..)` is where you implement the logic for handling `Incoming` messages -/// -/// # Example -/// ``` -/// use swactor::{Ctx, actor::{ActorAddress, ActorInterface}}; -/// -/// struct Greeter { -/// num_greeted: usize, -/// } -/// -/// #[derive(Clone)] // required to auto implement `Message` -/// struct GreetMessage { -/// who: String, -/// return_addr: ActorAddress, -/// } -/// -/// #[derive(Clone)] -/// struct GreetResponse(String); -/// -/// impl ActorInterface for Greeter { -/// type Incoming = GreetMessage; -/// type Response = GreetResponse; -/// -/// fn handle(&mut self, ctx: &Ctx, msg: Self::Incoming) { -/// let response = GreetResponse(format!("Hello, {}!", msg.who).to_string()); -/// if let Ok(_) = ctx.send(msg.return_addr, response) { -/// self.num_greeted += 1; -/// } -/// } -/// } -/// ``` pub trait ActorInterface: 'static + Send { type Incoming: Message; type Response: Message; diff --git a/src/envelope.rs b/src/envelope.rs deleted file mode 100644 index c3a716d..0000000 --- a/src/envelope.rs +++ /dev/null @@ -1,24 +0,0 @@ -use std::any::Any; - -use crate::actor::ActorAddress; - - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn envelope_roundtrip() { - let addr = ActorAddress::default(); - let env = Envelope::new(addr, Box::new(42u64)); - assert_eq!(env.dest(), addr); - assert_eq!(env.downcast::(), Some(42u64)); - } - - #[test] - fn envelope_wrong_type_returns_none() { - let addr = ActorAddress::default(); - let env = Envelope::new(addr, Box::new(42u64)); - assert!(env.downcast::().is_none()); - } -} diff --git a/src/runtime.rs b/src/runtime.rs index 622c0db..c2d034c 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -14,83 +14,6 @@ use crate::worker::Mailbox; use crate::worker::{TickContext, Worker}; use crate::Error; -// ─── SenderT trait (moved from router.rs) ──────────────────────────────────── - -/// Type-erased sender for external inboxes. -pub(crate) trait SenderT: Send + Sync { - fn try_send_any(&self, msg: Box); -} - -impl SenderT for Sender { - fn try_send_any(&self, msg: Box) { - if let Ok(typed) = msg.downcast::() { - let _ = Sender::try_send(self, *typed); - } - } -} - - -/// A type-erased message envelope for cross-worker delivery. -/// -/// Uses `Box` (no atomic refcount) and move semantics (no clone). -pub(crate) struct Envelope { - dest: ActorAddress, - payload: Box, -} - -impl Envelope { - pub fn new(dest: ActorAddress, payload: Box) -> Self { - Self { dest, payload } - } - - pub fn dest(&self) -> ActorAddress { - self.dest - } - - pub fn downcast(self) -> Option { - self.payload.downcast::().ok().map(|b| *b) - } - - pub fn into_payload(self) -> Box { - self.payload - } -} - - -// ─── InboxRegistry ─────────────────────────────────────────────────────────── - -/// Registry of external inboxes — replaces the Router's role for non-actor receivers. -pub(crate) struct InboxRegistry { - senders: RwLock>>, -} - -impl InboxRegistry { - pub fn new() -> Self { - Self { - senders: RwLock::new(HashMap::new()), - } - } - - pub fn register(&self, addr: ActorAddress, sender: Arc) { - self.senders.write().unwrap().insert(addr, sender); - } - - pub fn try_deliver( - &self, - addr: ActorAddress, - msg: Box, - ) -> Result<(), Error> { - let senders = self.senders.read().unwrap(); - if let Some(sender) = senders.get(&addr) { - sender.try_send_any(msg); - Ok(()) - } else { - Err(Error::from("Address not found")) - } - } -} - -// ─── Inbox ─────────────────────────────────────────────────────────────────── /// Generic message inbox for receiving messages outside of the runtime. pub struct Inbox { @@ -108,12 +31,23 @@ impl Inbox { } } +/// Handle for dealing with a runtime that has started via the `Runtime::run()` method. +pub struct RuntimeHandle { + pub runtime: Arc, + threads: Vec>, +} -/// Object-safe inner trait for sending type-erased messages. -pub(crate) trait ContextInner { - fn send_any(&self, addr: ActorAddress, msg: Box) -> Result<(), Error>; - fn spawn_any(&self, addr: ActorAddress, actor: Box) -> Result<(), Error>; - fn mailbox_waterlevel(&self) -> usize; +impl RuntimeHandle { + pub fn join(self) { + for handle in self.threads { + let _ = handle.join(); + } + } + + /// Simple helper, calls the inner `Runtime::shutdown()` method + pub fn shutdown(&self) { + self.runtime.shutdown(); + } } /// Actor syscall interface — passed to `ActorInterface::handle()`. @@ -151,6 +85,19 @@ impl<'a> Ctx<'a> { } } +/// Type-erased sender for external inboxes. +pub(crate) trait SenderT: Send + Sync { + fn try_send_any(&self, msg: Box); +} + +impl SenderT for Sender { + fn try_send_any(&self, msg: Box) { + if let Ok(typed) = msg.downcast::() { + let _ = Sender::try_send(self, *typed); + } + } +} + // ─── Runtime ───────────────────────────────────────────────────────────────── @@ -329,61 +276,78 @@ impl Runtime { } } -/// Worker thread loop for multi-threaded runtime. -/// Uses spin → yield → park backoff to reduce CPU usage when idle. -fn worker_loop(worker: &mut Worker, rt: &Runtime) { - let tc = TickContext { - address_map: &rt.address_map, - transfer_txs: &rt.transfer_txs, - spawn_txs: &rt.spawn_txs, - placement: &rt.placement, - inbox_registry: &rt.inbox_registry, - config: &rt.config, - }; - let bp = &rt.config.backoff_policy; - let mut idle_count: u32 = 0; - while rt.is_running.load(Ordering::Acquire) { - let did_work = worker.tick_once(&tc); - if did_work { - idle_count = 0; +/// A type-erased message envelope for cross-worker delivery. +/// +/// Uses `Box` (no atomic refcount) and move semantics (no clone). +pub(crate) struct Envelope { + dest: ActorAddress, + payload: Box, +} + +impl Envelope { + pub fn new(dest: ActorAddress, payload: Box) -> Self { + Self { dest, payload } + } + + pub fn dest(&self) -> ActorAddress { + self.dest + } + + pub fn downcast(self) -> Option { + self.payload.downcast::().ok().map(|b| *b) + } + + pub fn into_payload(self) -> Box { + self.payload + } +} + + +// ─── InboxRegistry ─────────────────────────────────────────────────────────── + +/// Registry of external inboxes — replaces the Router's role for non-actor receivers. +pub(crate) struct InboxRegistry { + senders: RwLock>>, +} + +impl InboxRegistry { + pub fn new() -> Self { + Self { + senders: RwLock::new(HashMap::new()), + } + } + + pub fn register(&self, addr: ActorAddress, sender: Arc) { + self.senders.write().unwrap().insert(addr, sender); + } + + pub fn try_deliver( + &self, + addr: ActorAddress, + msg: Box, + ) -> Result<(), Error> { + let senders = self.senders.read().unwrap(); + if let Some(sender) = senders.get(&addr) { + sender.try_send_any(msg); + Ok(()) } else { - idle_count = idle_count.saturating_add(1); - if idle_count < bp.spin_threshold { - // Hot spin — no hint, keep polling fast - } else if idle_count < bp.yield_threshold { - thread::yield_now(); - } else { - // Park: sleep briefly, cap at configured max - let micros = std::cmp::min( - (idle_count - bp.yield_threshold) as u64 * bp.sleep_increment_us, - bp.sleep_max_us, - ); - thread::sleep(std::time::Duration::from_micros(micros)); - } + Err(Error::from("Address not found")) } } } -/// Handle for dealing with a runtime that has started via the `Runtime::run()` method. -pub struct RuntimeHandle { - pub runtime: Arc, - threads: Vec>, + + + +/// Object-safe inner trait for sending type-erased messages. +pub(crate) trait ContextInner { + fn send_any(&self, addr: ActorAddress, msg: Box) -> Result<(), Error>; + fn spawn_any(&self, addr: ActorAddress, actor: Box) -> Result<(), Error>; + fn mailbox_waterlevel(&self) -> usize; } -impl RuntimeHandle { - pub fn join(self) { - for handle in self.threads { - let _ = handle.join(); - } - } - - /// Simple helper, calls the inner `Runtime::shutdown()` method - pub fn shutdown(&self) { - self.runtime.shutdown(); - } -} impl ContextInner for Runtime { fn send_any(&self, addr: ActorAddress, msg: Box) -> Result<(), Error> { diff --git a/src/worker.rs b/src/worker.rs index c6d0e3c..18a965d 100644 --- a/src/worker.rs +++ b/src/worker.rs @@ -6,7 +6,7 @@ use crate::actor::{ActorAddress, AnyActor, Message}; use crate::address_map::{AddressMap, Placement, WorkerId}; use crate::channel::{Receiver, Sender}; use crate::config::RuntimeConfig; -use crate::runtime::{ContextInner, Envelope, InboxRegistry}; +use crate::runtime::{ContextInner, Envelope, InboxRegistry, Runtime}; use crate::Error; /// Shared state passed to tick_once — single thin pointer avoids register spill. @@ -230,3 +230,40 @@ impl Mailbox { } } } + +/// Worker thread loop for multi-threaded runtime. +/// Uses spin → yield → park backoff to reduce CPU usage when idle. +fn worker_loop(worker: &mut Worker, rt: &Runtime) { + let tc = TickContext { + address_map: &rt.address_map, + transfer_txs: &rt.transfer_txs, + spawn_txs: &rt.spawn_txs, + placement: &rt.placement, + inbox_registry: &rt.inbox_registry, + config: &rt.config, + }; + let bp = &rt.config.backoff_policy; + let mut idle_count: u32 = 0; + + while rt.is_running.load(Ordering::Acquire) { + let did_work = worker.tick_once(&tc); + + if did_work { + idle_count = 0; + } else { + idle_count = idle_count.saturating_add(1); + if idle_count < bp.spin_threshold { + // Hot spin — no hint, keep polling fast + } else if idle_count < bp.yield_threshold { + thread::yield_now(); + } else { + // Park: sleep briefly, cap at configured max + let micros = std::cmp::min( + (idle_count - bp.yield_threshold) as u64 * bp.sleep_increment_us, + bp.sleep_max_us, + ); + thread::sleep(std::time::Duration::from_micros(micros)); + } + } + } +} \ No newline at end of file diff --git a/tests/mailbox_tests.rs b/tests/mailbox_tests.rs deleted file mode 100644 index 66a671e..0000000 --- a/tests/mailbox_tests.rs +++ /dev/null @@ -1,226 +0,0 @@ -use swactor::worker::Mailbox; - -// ── Basic operations ── - -#[test] -fn push_and_pop() { - let mut mb = Mailbox::new(10); - mb.push(42i32); - assert_eq!(mb.pop(), Some(42)); -} - -#[test] -fn fifo_ordering() { - let mut mb = Mailbox::new(10); - mb.push(1); - mb.push(2); - mb.push(3); - assert_eq!(mb.pop(), Some(1)); - assert_eq!(mb.pop(), Some(2)); - assert_eq!(mb.pop(), Some(3)); -} - -#[test] -fn pop_empty() { - let mut mb: Mailbox = Mailbox::new(10); - assert_eq!(mb.pop(), None); -} - -#[test] -fn multiple_messages() { - let mut mb = Mailbox::new(10); - for i in 0..100 { - mb.push(i); - } - for i in 0..100 { - assert_eq!(mb.pop(), Some(i)); - } - assert_eq!(mb.pop(), None); -} - -#[test] -fn interleaved_push_pop() { - let mut mb = Mailbox::new(10); - mb.push(1); - mb.push(2); - assert_eq!(mb.pop(), Some(1)); - mb.push(3); - assert_eq!(mb.pop(), Some(2)); - assert_eq!(mb.pop(), Some(3)); - assert_eq!(mb.pop(), None); -} - -// ── Drain count / watermark logic ── - -#[test] -fn drain_count_empty() { - let mb: Mailbox = Mailbox::new(10); - assert_eq!(mb.drain_count(), 0); -} - -#[test] -fn drain_count_below_waterlevel() { - let mut mb = Mailbox::new(10); - for i in 0..5 { - mb.push(i); - } - // 5 < 10 (default waterlevel) → process all - assert_eq!(mb.drain_count(), 5); -} - -#[test] -fn drain_count_at_waterlevel() { - let mut mb = Mailbox::new(10); - for i in 0..10 { - mb.push(i); - } - // 10 >= 10 → process half → 5 - assert_eq!(mb.drain_count(), 5); -} - -#[test] -fn drain_count_above_waterlevel() { - let mut mb = Mailbox::new(10); - for i in 0..20 { - mb.push(i); - } - // 20 >= 10 → 20 >> 1 = 10 - assert_eq!(mb.drain_count(), 10); -} - -#[test] -fn drain_count_one_message() { - let mut mb = Mailbox::new(10); - mb.push(1i32); - // 1 < 10 → process all → 1 - assert_eq!(mb.drain_count(), 1); -} - -#[test] -fn drain_count_just_below_waterlevel() { - let mut mb = Mailbox::new(10); - for i in 0..9 { - mb.push(i); - } - // 9 < 10 → process all → 9 - assert_eq!(mb.drain_count(), 9); -} - -#[test] -fn drain_count_large() { - let mut mb = Mailbox::new(10); - for i in 0..1000 { - mb.push(i); - } - // 1000 >= 10 → 1000 >> 1 = 500 - assert_eq!(mb.drain_count(), 500); -} - -#[test] -fn drain_count_custom_waterlevel() { - let mut mb = Mailbox::new(4); - for i in 0..3 { - mb.push(i); - } - // 3 < 4 → process all → 3 - assert_eq!(mb.drain_count(), 3); - - mb.push(99); - // 4 >= 4 → 4 >> 1 = 2 - assert_eq!(mb.drain_count(), 2); -} - -#[test] -fn drain_count_updates_after_pop() { - let mut mb = Mailbox::new(10); - for i in 0..20 { - mb.push(i); - } - // 20 >= 10 → 10 - assert_eq!(mb.drain_count(), 10); - - // pop 15, leaving 5 - for _ in 0..15 { - mb.pop(); - } - // 5 < 10 → process all → 5 - assert_eq!(mb.drain_count(), 5); -} - -// ── Properties ── - -#[test] -fn len_tracks_pushes() { - let mut mb = Mailbox::new(10); - assert_eq!(mb.len(), 0); - mb.push(1); - assert_eq!(mb.len(), 1); - mb.push(2); - assert_eq!(mb.len(), 2); - mb.push(3); - assert_eq!(mb.len(), 3); -} - -#[test] -fn len_tracks_pops() { - let mut mb = Mailbox::new(10); - mb.push(1); - mb.push(2); - mb.push(3); - assert_eq!(mb.len(), 3); - mb.pop(); - assert_eq!(mb.len(), 2); - mb.pop(); - assert_eq!(mb.len(), 1); - mb.pop(); - assert_eq!(mb.len(), 0); -} - -#[test] -fn is_empty_on_new() { - let mb: Mailbox = Mailbox::new(10); - assert!(mb.is_empty()); -} - -#[test] -fn is_empty_after_drain() { - let mut mb = Mailbox::new(10); - mb.push(1); - mb.push(2); - mb.push(3); - assert!(!mb.is_empty()); - mb.pop(); - mb.pop(); - mb.pop(); - assert!(mb.is_empty()); -} - -// ── Type tests ── - -#[test] -fn works_with_primitive_types() { - let mut mb_i32 = Mailbox::new(10); - mb_i32.push(42i32); - assert_eq!(mb_i32.pop(), Some(42)); - - let mut mb_string = Mailbox::new(10); - mb_string.push(String::from("hello")); - assert_eq!(mb_string.pop(), Some(String::from("hello"))); -} - -#[test] -fn works_with_custom_structs() { - #[derive(Debug, Clone, PartialEq)] - struct MyMsg { - id: u64, - payload: String, - } - - let mut mb = Mailbox::new(10); - let msg = MyMsg { - id: 1, - payload: "test".into(), - }; - mb.push(msg.clone()); - assert_eq!(mb.pop(), Some(msg)); -} diff --git a/tests/runtime_api_tests.rs b/tests/runtime_api_tests.rs new file mode 100644 index 0000000..88238a5 --- /dev/null +++ b/tests/runtime_api_tests.rs @@ -0,0 +1,97 @@ +use swactor::{ + actor::{ActorAddress, ActorInterface}, + runtime::{Ctx, Inbox, Runtime, RuntimeConfig}, +}; + +// --------------------------------------------------------------------------- +// Shared test fixtures +// --------------------------------------------------------------------------- + +#[derive(Clone)] +struct EchoMessage { + payload: usize, + reply_to: ActorAddress, +} + +#[derive(Clone, Debug, PartialEq)] +struct EchoResponse(usize); + +struct EchoActor; + +impl ActorInterface for EchoActor { + type Incoming = EchoMessage; + type Response = EchoResponse; + + fn handle(&mut self, ctx: &Ctx, msg: EchoMessage) { + let _ = ctx.send(msg.reply_to, EchoResponse(msg.payload)); + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[test] +fn test_single_thread_spawn_actor_and_inbox() { + let rt = Runtime::new(RuntimeConfig::default()); + + let actor_addr = rt.spawn(EchoActor).expect("spawn echo actor"); + let inbox: Inbox = rt.new_inbox().unwrap(); + + rt.send_to( + actor_addr, + EchoMessage { + payload: 42, + reply_to: *inbox.addr(), + }, + ) + .unwrap(); + + for _ in 0..10 { + rt.tick(); + if let Some(response) = inbox.try_recv() { + assert_eq!(response, EchoResponse(42)); + return; + } + } + + panic!("Did not receive EchoResponse"); +} + +#[test] +fn test_multi_thread_spawn_actor_and_inbox() { + let config = RuntimeConfig { + num_threads: 4, + ..Default::default() + }; + let rt = Runtime::new(config); + + let actor_addr = rt.spawn(EchoActor).expect("spawn echo actor"); + let inbox: Inbox = rt.new_inbox().unwrap(); + + rt.send_to( + actor_addr, + EchoMessage { + payload: 99, + reply_to: *inbox.addr(), + }, + ) + .unwrap(); + + let handle = rt.run().unwrap(); + + let check = std::thread::spawn(move || { + for _ in 0..100 { + std::thread::sleep(std::time::Duration::from_millis(10)); + if let Some(response) = inbox.try_recv() { + handle.shutdown(); + return Some(response); + } + } + handle.shutdown(); + None + }); + + let result = check.join().unwrap(); + assert_eq!(result, Some(EchoResponse(99))); +} diff --git a/tests/runtime_tests.rs b/tests/runtime_tests.rs deleted file mode 100644 index ed21044..0000000 --- a/tests/runtime_tests.rs +++ /dev/null @@ -1,130 +0,0 @@ -use swactor::{actor::{ActorAddress, ActorInterface}, runtime::{Ctx, Inbox, Runtime, RuntimeConfig}}; - -#[derive(Clone)] -struct PingMessage { - reply_to: ActorAddress, -} - -#[derive(Clone)] -struct PongMessage; - -struct PongActor; - -impl ActorInterface for PongActor { - type Incoming = PingMessage; - type Response = PongMessage; - - fn handle(&mut self, ctx: &Ctx, msg: PingMessage) { - let _ = ctx.send(msg.reply_to, PongMessage); - } -} - -/// An actor that forwards messages to another address -struct ForwarderActor { - target: ActorAddress, -} - -#[derive(Clone)] -struct ForwardMessage(usize); - -impl ActorInterface for ForwarderActor { - type Incoming = ForwardMessage; - type Response = (); - - fn handle(&mut self, ctx: &Ctx, msg: ForwardMessage) { - let _ = ctx.send(self.target, msg); - } -} - -#[test] -fn test_single_threaded_ping_pong() { - let rt = Runtime::new(RuntimeConfig::default()); - let inbox: Inbox = rt.new_inbox().unwrap(); - - let pong_addr = rt.spawn(PongActor).expect("spawn pong"); - - // Send ping - rt.send_to( - pong_addr, - PingMessage { - reply_to: *inbox.addr(), - }, - ) - .unwrap(); - - // Tick until we get a response - for _ in 0..10 { - rt.tick(); - if inbox.try_recv().is_some() { - return; // Success! - } - } - - panic!("Did not receive pong response"); -} - -#[test] -fn test_single_threaded_message_chain() { - let rt = Runtime::new(RuntimeConfig::default()); - let inbox: Inbox = rt.new_inbox().unwrap(); - - // Create a chain: A -> B -> C -> inbox - let c_addr = rt - .spawn(ForwarderActor { - target: *inbox.addr(), - }) - .unwrap(); - let b_addr = rt.spawn(ForwarderActor { target: c_addr }).unwrap(); - let a_addr = rt.spawn(ForwarderActor { target: b_addr }).unwrap(); - - // Send message to start of chain - rt.send_to(a_addr, ForwardMessage(42)).unwrap(); - - // Tick until message arrives - for _ in 0..20 { - rt.tick(); - if let Some(ForwardMessage(val)) = inbox.try_recv() { - assert_eq!(val, 42); - return; - } - } - - panic!("Message did not traverse the chain"); -} - -#[test] -fn test_multithreaded_message_passing() { - let config = RuntimeConfig { - num_threads: 4, - ..Default::default() - }; - let rt = Runtime::new(config); - let inbox: Inbox = rt.new_inbox().unwrap(); - - // Create a longer chain to exercise multi-threading - let mut target = *inbox.addr(); - for _ in 0..20 { - target = rt.spawn(ForwarderActor { target }).unwrap(); - } - let start_addr = target; - - // Send message - rt.send_to(start_addr, ForwardMessage(999)).unwrap(); - - // Spawn thread to check for result and shutdown - let ctx = rt.run().unwrap(); - let inbox_check = std::thread::spawn(move || { - for _ in 0..100 { - std::thread::sleep(std::time::Duration::from_millis(10)); - if let Some(ForwardMessage(val)) = inbox.try_recv() { - ctx.shutdown(); - return Some(val); - } - } - ctx.shutdown(); - None - }); - - let result = inbox_check.join().unwrap(); - assert_eq!(result, Some(999)); -} diff --git a/tests/stress/concurrency.rs b/tests/stress/concurrency.rs deleted file mode 100644 index 0656652..0000000 --- a/tests/stress/concurrency.rs +++ /dev/null @@ -1,330 +0,0 @@ -//! Concurrency stress tests - hunt for race conditions. -//! -//! These tests target the shutdown races and concurrent access patterns -//! that are most likely to expose bugs. - -use super::{BlackHole, Msg}; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::Arc; -use std::thread; -use std::time::Duration; -use swactor::runtime::{Runtime, RuntimeConfig}; - -/// Shutdown while messages are in flight. -/// Target: AtomicBool ordering bugs, use-after-shutdown. -#[test] -#[cfg(feature = "stress")] -fn shutdown_under_load() { - println!("\n>>> STRESS: Shutdown Under Load"); - - let mut panics = 0; - let mut successes = 0; - - // Run many iterations to catch rare races - for iteration in 0..100 { - let result = std::panic::catch_unwind(|| { - let config = RuntimeConfig { - max_actors: 100, - actor_max_messages: 1000, - num_threads: 4, - ..Default::default() - }; - let runtime = Runtime::new(config); - - // Spawn actors - let mut actors = Vec::new(); - for _ in 0..50 { - if let Ok(addr) = runtime.spawn(BlackHole) { - actors.push(addr); - } - } - - let handle = runtime.run().unwrap(); - let rt = handle.runtime.clone(); - - // Sender thread - blast messages - let actors_clone = actors.clone(); - let rt_send = rt.clone(); - let sender = thread::spawn(move || { - for _ in 0..1000 { - for actor in &actors_clone { - let _ = rt_send.send_to::(*actor, Msg); - } - } - }); - - // Random delay before shutdown - let delay = Duration::from_micros((iteration * 17) % 500); - thread::sleep(delay); - - // Shutdown while sender is still going - handle.shutdown(); - - // Wait for sender (it should not panic) - let _ = sender.join(); - - // Join should complete (not hang) - handle.join(); - }); - - match result { - Ok(_) => successes += 1, - Err(_) => panics += 1, - } - } - - println!(" Iterations: 100"); - println!(" Successes: {}", successes); - println!(" Panics: {}", panics); - - if panics > 0 { - println!(">>> FAIL: {} panics detected during shutdown\n", panics); - } else { - println!(">>> PASS: No panics during shutdown under load\n"); - } - - assert_eq!(panics, 0, "Shutdown under load caused panics"); -} - -/// Send to actor immediately after spawn. -/// Target: Race between spawn registration and first message. -#[test] -#[cfg(feature = "stress")] -fn send_to_newborn() { - println!("\n>>> STRESS: Send to Newborn Actor"); - - let mut total_spawned = 0; - let mut total_send_ok = 0; - let mut total_send_fail = 0; - - for _ in 0..100 { - let config = RuntimeConfig { - max_actors: 1000, - actor_max_messages: 100, - num_threads: 4, - ..Default::default() - }; - let runtime = Runtime::new(config); - let handle = runtime.run().unwrap(); - - // Immediately spawn and send - for _ in 0..50 { - if let Ok(addr) = handle.runtime.spawn(BlackHole) { - total_spawned += 1; - // Send immediately - actor may not be registered yet - if handle.runtime.send_to::(addr, Msg).is_ok() { - total_send_ok += 1; - } else { - total_send_fail += 1; - } - } - } - - handle.shutdown(); - handle.join(); - } - - println!(" Total spawned: {}", total_spawned); - println!(" Sends succeeded: {}", total_send_ok); - println!(" Sends failed: {}", total_send_fail); - - if total_send_fail > 0 { - println!(">>> FAIL: {} messages failed to send\n", total_send_fail); - } else { - println!(">>> PASS: All messages succeeded\n"); - } - - assert_eq!(total_send_fail, 0, "Race condition caused failed message delivery"); - - println!(">>> Test complete\n"); -} - -/// Rapid spawn/despawn cycles. -/// Target: Queue management under churn. -#[test] -#[cfg(feature = "stress")] -fn rapid_spawn_churn() { - println!("\n>>> STRESS: Rapid Spawn Churn"); - - let config = RuntimeConfig { - max_actors: 100, - actor_max_messages: 100, - num_threads: 4, - ..Default::default() - }; - let runtime = Runtime::new(config); - let handle = runtime.run().unwrap(); - - let spawn_count = Arc::new(AtomicUsize::new(0)); - let fail_count = Arc::new(AtomicUsize::new(0)); - - // Multiple threads spawning actors - let mut threads = Vec::new(); - for _ in 0..4 { - let rt = handle.runtime.clone(); - let spawns = spawn_count.clone(); - let fails = fail_count.clone(); - - threads.push(thread::spawn(move || { - for _ in 0..500 { - match rt.spawn(BlackHole) { - Ok(_) => { - spawns.fetch_add(1, Ordering::Relaxed); - } - Err(_) => { - fails.fetch_add(1, Ordering::Relaxed); - } - } - // Small yield to increase interleaving - thread::yield_now(); - } - })); - } - - // Let it churn - thread::sleep(Duration::from_millis(100)); - - handle.shutdown(); - - for t in threads { - let _ = t.join(); - } - handle.join(); - - let total_spawns = spawn_count.load(Ordering::Relaxed); - let total_fails = fail_count.load(Ordering::Relaxed); - - println!(" Spawn attempts: {}", total_spawns + total_fails); - println!(" Successes: {}", total_spawns); - println!(" Failures: {} (expected - queue fills)", total_fails); - println!(">>> Test complete - no panics\n"); -} - -/// Multiple threads sending to same actor. -/// Target: Inbox contention, message ordering. -#[test] -#[cfg(feature = "stress")] -fn inbox_contention() { - println!("\n>>> STRESS: Inbox Contention"); - - let config = RuntimeConfig { - max_actors: 10, - actor_max_messages: 100_000, - num_threads: 4, - ..Default::default() - }; - let runtime = Runtime::new(config); - let target = runtime.spawn(BlackHole).unwrap(); - let handle = runtime.run().unwrap(); - - // Wait for registration - thread::sleep(Duration::from_millis(10)); - - let send_count = Arc::new(AtomicUsize::new(0)); - let fail_count = Arc::new(AtomicUsize::new(0)); - - // 8 threads all sending to same actor - let mut threads = Vec::new(); - for _ in 0..8 { - let rt = handle.runtime.clone(); - let sends = send_count.clone(); - let fails = fail_count.clone(); - - threads.push(thread::spawn(move || { - for _ in 0..10_000 { - if rt.send_to::(target, Msg).is_ok() { - sends.fetch_add(1, Ordering::Relaxed); - } else { - fails.fetch_add(1, Ordering::Relaxed); - } - } - })); - } - - for t in threads { - let _ = t.join(); - } - - // Let messages process - thread::sleep(Duration::from_millis(50)); - - handle.shutdown(); - handle.join(); - - let total_sends = send_count.load(Ordering::Relaxed); - let total_fails = fail_count.load(Ordering::Relaxed); - - println!(" Threads: 8"); - println!(" Msgs per thread: 10,000"); - println!(" Total sent: {}", total_sends); - println!(" Total failed: {}", total_fails); - println!( - " Success rate: {:.1}%", - (total_sends as f64 / (total_sends + total_fails) as f64) * 100.0 - ); - println!(">>> Test complete - no panics\n"); -} - -/// Shutdown timing fuzz - randomize when shutdown is called. -/// Target: Edge cases in shutdown state machine. -#[test] -#[cfg(feature = "stress")] -fn shutdown_timing_fuzz() { - println!("\n>>> STRESS: Shutdown Timing Fuzz"); - - let mut results = Vec::new(); - - for delay_us in [0, 1, 10, 100, 1000, 5000] { - let mut ok = 0; - let mut fail = 0; - - for _ in 0..20 { - let result = std::panic::catch_unwind(|| { - let config = RuntimeConfig { - max_actors: 50, - actor_max_messages: 100, - num_threads: 4, - ..Default::default() - }; - let runtime = Runtime::new(config); - - for _ in 0..20 { - let _ = runtime.spawn(BlackHole); - } - - let handle = runtime.run().unwrap(); - - // Specific delay - if delay_us > 0 { - thread::sleep(Duration::from_micros(delay_us)); - } - - handle.shutdown(); - handle.join(); - }); - - match result { - Ok(_) => ok += 1, - Err(_) => fail += 1, - } - } - - results.push((delay_us, ok, fail)); - } - - println!(" delay_us ok fail"); - println!(" -------- -- ----"); - for (delay, ok, fail) in &results { - println!(" {:>8} {:>2} {:>4}", delay, ok, fail); - } - - let total_fails: i32 = results.iter().map(|(_, _, f)| *f).sum(); - if total_fails > 0 { - println!( - "\n>>> FAIL: {} panics across timing variations", - total_fails - ); - } else { - println!("\n>>> PASS: All timing variations succeeded"); - } -} diff --git a/tests/stress/mod.rs b/tests/stress/mod.rs deleted file mode 100644 index 2839ac9..0000000 --- a/tests/stress/mod.rs +++ /dev/null @@ -1,199 +0,0 @@ -//! Stress test utilities and result reporting. -//! -//! Provides a simple framework for stress tests with JSON + pretty output. - -#![allow(dead_code)] // Utilities may not all be used in every test - -pub mod concurrency; -pub mod saturation; - -use std::time::{Duration, Instant}; - -/// Results from a stress test -#[derive(Debug)] -pub struct StressResult { - pub name: String, - pub duration: Duration, - pub operations: u64, - pub successes: u64, - pub failures: u64, - pub notes: Vec, -} - -impl StressResult { - pub fn new(name: impl Into) -> Self { - Self { - name: name.into(), - duration: Duration::ZERO, - operations: 0, - successes: 0, - failures: 0, - notes: Vec::new(), - } - } - - pub fn failure_rate(&self) -> f64 { - if self.operations == 0 { - 0.0 - } else { - (self.failures as f64 / self.operations as f64) * 100.0 - } - } - - pub fn throughput(&self) -> f64 { - let secs = self.duration.as_secs_f64(); - if secs > 0.0 { - self.operations as f64 / secs - } else { - 0.0 - } - } - - pub fn note(&mut self, msg: impl Into) { - self.notes.push(msg.into()); - } - - pub fn print(&self) { - println!("\n{}", "=".repeat(60)); - println!(" STRESS: {}", self.name); - println!("{}", "=".repeat(60)); - println!(" Duration: {:?}", self.duration); - println!(" Operations: {}", self.operations); - println!(" Successes: {}", self.successes); - println!(" Failures: {}", self.failures); - println!(" Failure Rate: {:.2}%", self.failure_rate()); - println!(" Throughput: {:.2} ops/sec", self.throughput()); - - if !self.notes.is_empty() { - println!(); - println!(" Notes:"); - for note in &self.notes { - println!(" - {}", note); - } - } - println!("{}", "=".repeat(60)); - } - - pub fn to_json(&self) -> String { - format!( - r#"{{"name":"{}","duration_ms":{},"operations":{},"successes":{},"failures":{},"failure_rate_pct":{:.2},"throughput":{:.2},"notes":{:?}}}"#, - self.name, - self.duration.as_millis(), - self.operations, - self.successes, - self.failures, - self.failure_rate(), - self.throughput(), - self.notes - ) - } -} - -/// A simple stress test runner -pub struct Stress { - name: String, - duration: Option, - iterations: Option, -} - -impl Stress { - pub fn new(name: impl Into) -> Self { - Self { - name: name.into(), - duration: None, - iterations: None, - } - } - - /// Run for a fixed duration - pub fn for_duration(mut self, d: Duration) -> Self { - self.duration = Some(d); - self - } - - /// Run for a fixed number of iterations - pub fn for_iterations(mut self, n: u64) -> Self { - self.iterations = Some(n); - self - } - - /// Run the stress test, counting successes and failures - pub fn run(self, mut f: F) -> StressResult - where - F: FnMut() -> bool, // returns true on success, false on failure - { - let mut result = StressResult::new(&self.name); - let start = Instant::now(); - - match (self.duration, self.iterations) { - (Some(duration), _) => { - while start.elapsed() < duration { - if f() { - result.successes += 1; - } else { - result.failures += 1; - } - result.operations += 1; - } - } - (None, Some(iterations)) => { - for _ in 0..iterations { - if f() { - result.successes += 1; - } else { - result.failures += 1; - } - result.operations += 1; - } - } - (None, None) => { - // Default: 1000 iterations - for _ in 0..1000 { - if f() { - result.successes += 1; - } else { - result.failures += 1; - } - result.operations += 1; - } - } - } - - result.duration = start.elapsed(); - result - } -} - -// Test actors used across stress tests -use swactor::{actor::ActorInterface, runtime::Ctx}; - -/// An actor that just absorbs messages -pub struct BlackHole; - -#[derive(Clone)] -pub struct Msg; - -impl ActorInterface for BlackHole { - type Incoming = Msg; - type Response = (); - fn handle(&mut self, _ctx: &Ctx, _msg: Msg) {} -} - -/// An actor that counts messages received -pub struct Counter { - pub count: usize, -} - -impl Counter { - pub fn new() -> Self { - Self { count: 0 } - } -} - -impl ActorInterface for Counter { - type Incoming = Msg; - type Response = (); - fn handle(&mut self, _ctx: &Ctx, _msg: Msg) { - self.count += 1; - } -} diff --git a/tests/stress/saturation.rs b/tests/stress/saturation.rs deleted file mode 100644 index dc2aeb2..0000000 --- a/tests/stress/saturation.rs +++ /dev/null @@ -1,175 +0,0 @@ -//! Saturation stress tests - find where the runtime breaks. -//! -//! These tests intentionally push past limits to document failure modes. - -use super::{BlackHole, Counter, Msg, Stress, StressResult}; -use std::time::Duration; -use swactor::runtime::{Runtime, RuntimeConfig}; - -/// Blast the transfer queue (replaces router_inbox_overflow) -#[test] -#[cfg(feature = "stress")] -fn transfer_queue_overflow() { - println!("\n>>> STRESS: Transfer Queue Overflow"); - - let config = RuntimeConfig { - max_actors: 10, - actor_max_messages: 100, // Tiny buffer - num_threads: 1, - ..Default::default() - }; - let runtime = Runtime::new(config); - let sink = runtime.spawn(BlackHole).unwrap(); - - // Process spawn - runtime.tick(); - - // Blast messages without processing - let mut result = StressResult::new("transfer_queue_overflow"); - let start = std::time::Instant::now(); - - for _ in 0..10_000 { - result.operations += 1; - if runtime.send_to::(sink, Msg).is_ok() { - result.successes += 1; - } else { - result.failures += 1; - } - } - - result.duration = start.elapsed(); - result.note(format!("Transfer buffer: 100, Messages sent: 10,000")); - - // With hybrid channel, no failures expected - assert_eq!(result.failures, 0, "Hybrid channel should not reject"); - result.print(); - println!(">>> PASS: Hybrid channel prevented transfer queue overflow\n"); -} - -/// Blast a single actor's mailbox via transfer queue -#[test] -#[cfg(feature = "stress")] -fn actor_inbox_overflow() { - println!("\n>>> STRESS: Actor Inbox Overflow"); - - let config = RuntimeConfig { - max_actors: 10, - actor_max_messages: 100_000, // Large transfer buffer - num_threads: 1, - ..Default::default() - }; - let runtime = Runtime::new(config); - let sink = runtime.spawn(Counter::new()).unwrap(); - - // Process spawn - runtime.tick(); - - // Now blast messages - let mut sent = 0u64; - let mut failed = 0u64; - for _ in 0..10_000 { - if runtime.send_to::(sink, Msg).is_ok() { - sent += 1; - } else { - failed += 1; - } - // Tick occasionally to let worker deliver - if sent % 100 == 0 { - runtime.tick(); - } - } - - // Process all remaining messages - for _ in 0..5000 { - runtime.tick(); - } - - println!(" Sent: {}", sent); - println!(" Failed: {}", failed); - - assert_eq!(failed, 0, "Transfer queue rejected message under load"); - println!(">>> PASS: No message loss with hybrid channel\n"); -} - -/// Blast the runtime with actor spawns -#[test] -#[cfg(feature = "stress")] -fn actor_queue_overflow() { - println!("\n>>> STRESS: Actor Queue Overflow"); - - let config = RuntimeConfig { - max_actors: 100, // Small actor queue - actor_max_messages: 100, - num_threads: 1, - ..Default::default() - }; - let runtime = Runtime::new(config); - - let mut result = StressResult::new("actor_queue_overflow"); - let start = std::time::Instant::now(); - - // Try to spawn 500 actors into 100-slot queue - for _ in 0..500 { - result.operations += 1; - match runtime.spawn(BlackHole) { - Ok(_) => result.successes += 1, - Err(_) => result.failures += 1, - } - } - - result.duration = start.elapsed(); - result.note(format!("Queue capacity: 100, Spawn attempts: 500")); - result.print(); - - // Note: With hybrid channel (overflow to SegQueue), we expect no failures - assert_eq!( - result.failures, 0, - "Spawned more actors than queue capacity" - ); - println!(">>> PASS: Actor queue correctly handles overflow\n"); -} - -/// Sustained overload - run at 2x capacity for extended period. -/// Documents: Does the system degrade gracefully or crash? -#[test] -#[cfg(feature = "stress")] -fn sustained_overload() { - println!("\n>>> STRESS: Sustained Overload"); - - let config = RuntimeConfig { - max_actors: 100, - actor_max_messages: 1000, - num_threads: 1, - ..Default::default() - }; - let runtime = Runtime::new(config); - - // Spawn some actors - let mut actors = Vec::new(); - for _ in 0..50 { - if let Ok(addr) = runtime.spawn(Counter::new()) { - actors.push(addr); - } - } - - // Process spawn registrations - for _ in 0..200 { - runtime.tick(); - } - - let result = Stress::new("sustained_overload") - .for_duration(Duration::from_secs(2)) - .run(|| { - // Send to random actor - let idx = (std::time::Instant::now().elapsed().as_nanos() as usize) % actors.len(); - let success = runtime.send_to::(actors[idx], Msg).is_ok(); - - // Process some (but not all) - simulating overload - runtime.tick(); - - success - }); - - result.print(); - println!(">>> System survived sustained overload without panic\n"); -} diff --git a/tests/stress_tests.rs b/tests/stress_tests.rs deleted file mode 100644 index 216aea2..0000000 --- a/tests/stress_tests.rs +++ /dev/null @@ -1,11 +0,0 @@ -//! Stress test suite for swactor runtime. -//! -//! Run with: cargo test --features stress stress_ -- --nocapture -//! -//! These tests are hidden behind the `stress` feature flag because they: -//! - Take longer to run -//! - Intentionally push the system to failure -//! - May produce different results on different machines - -#[cfg(feature = "stress")] -mod stress; -- 2.45.2 From cf107315bc8c69ac06fe7811412cc247653ebb54 Mon Sep 17 00:00:00 2001 From: Zachery Aaron Shores-Chmielewski Date: Fri, 6 Feb 2026 18:16:09 +0700 Subject: [PATCH 9/9] feat: separation of runtime and worker thread logic Logically separate the runtime and worker thread, to make it more amenable for testing, visualization, reasoning, etc. --- src/runtime.rs | 10 +++++++- src/worker.rs | 65 +++++++++++++++++++++----------------------------- 2 files changed, 36 insertions(+), 39 deletions(-) diff --git a/src/runtime.rs b/src/runtime.rs index c2d034c..d667df6 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -259,7 +259,15 @@ impl Runtime { for mut worker in workers { let rt_clone = rt.clone(); let handle = thread::spawn(move || { - worker_loop(&mut worker, &rt_clone); + let tc = TickContext { + address_map: &rt_clone.address_map, + transfer_txs: &rt_clone.transfer_txs, + spawn_txs: &rt_clone.spawn_txs, + placement: &rt_clone.placement, + inbox_registry: &rt_clone.inbox_registry, + config: &rt_clone.config, + }; + worker.run(&tc, &rt_clone.is_running, &rt_clone.config.backoff_policy); }); handles.push(handle); } diff --git a/src/worker.rs b/src/worker.rs index 18a965d..1ca7c0f 100644 --- a/src/worker.rs +++ b/src/worker.rs @@ -1,12 +1,14 @@ use std::any::Any; use std::cell::RefCell; use std::collections::{HashMap, VecDeque}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::thread; use crate::actor::{ActorAddress, AnyActor, Message}; use crate::address_map::{AddressMap, Placement, WorkerId}; use crate::channel::{Receiver, Sender}; -use crate::config::RuntimeConfig; -use crate::runtime::{ContextInner, Envelope, InboxRegistry, Runtime}; +use crate::config::{BackoffPolicy, RuntimeConfig}; +use crate::runtime::{ContextInner, Envelope, InboxRegistry}; use crate::Error; /// Shared state passed to tick_once — single thin pointer avoids register spill. @@ -90,6 +92,29 @@ impl Worker { did_work } + + pub(crate) fn run(&mut self, tc: &TickContext, is_running: &AtomicBool, backoff: &BackoffPolicy) { + let mut idle_count: u32 = 0; + while is_running.load(Ordering::Acquire) { + let did_work = self.tick_once(tc); + if did_work { + idle_count = 0; + } else { + idle_count = idle_count.saturating_add(1); + if idle_count < backoff.spin_threshold { + // Hot spin + } else if idle_count < backoff.yield_threshold { + thread::yield_now(); + } else { + let micros = std::cmp::min( + (idle_count - backoff.yield_threshold) as u64 * backoff.sleep_increment_us, + backoff.sleep_max_us, + ); + thread::sleep(std::time::Duration::from_micros(micros)); + } + } + } + } } /// The `ContextInner` impl for worker threads. @@ -231,39 +256,3 @@ impl Mailbox { } } -/// Worker thread loop for multi-threaded runtime. -/// Uses spin → yield → park backoff to reduce CPU usage when idle. -fn worker_loop(worker: &mut Worker, rt: &Runtime) { - let tc = TickContext { - address_map: &rt.address_map, - transfer_txs: &rt.transfer_txs, - spawn_txs: &rt.spawn_txs, - placement: &rt.placement, - inbox_registry: &rt.inbox_registry, - config: &rt.config, - }; - let bp = &rt.config.backoff_policy; - let mut idle_count: u32 = 0; - - while rt.is_running.load(Ordering::Acquire) { - let did_work = worker.tick_once(&tc); - - if did_work { - idle_count = 0; - } else { - idle_count = idle_count.saturating_add(1); - if idle_count < bp.spin_threshold { - // Hot spin — no hint, keep polling fast - } else if idle_count < bp.yield_threshold { - thread::yield_now(); - } else { - // Park: sleep briefly, cap at configured max - let micros = std::cmp::min( - (idle_count - bp.yield_threshold) as u64 * bp.sleep_increment_us, - bp.sleep_max_us, - ); - thread::sleep(std::time::Duration::from_micros(micros)); - } - } - } -} \ No newline at end of file -- 2.45.2