zacheryasc/swactor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Compare commits

base: zacheryasc:3b76b54d452fbc4428bd88b51e013cd059b0f44a
Branches Tags
zacheryasc:master
zacheryasc:development
zacheryasc:backup/pre-rewrite-msg
...
compare: zacheryasc:61751f3054cc1f6f798de8f30f9d63e76776c760
Branches Tags
zacheryasc:development
zacheryasc:master
zacheryasc:backup/pre-rewrite-msg

2 commits
3b76b54d45 ... 61751f3054

Author SHA1 Message Date
Zachery Aaron Shores-Chmielewski
61751f3054 feat: process primitives 
Tools for treating groups of actors like an OS process.
 2026-02-21 00:33:06 +07:00
Zachery Aaron Shores-Chmielewski
1152e32e16 feat: cluster test improvements; datastore pools (#45) 
Co-authored-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
Co-committed-by: Zachery Aaron Shores-Chmielewski <zacheryasc@gmail.com>
 2026-02-20 17:30:37 +00:00
40 changed files with 8180 additions and 87 deletions
Show stats Expand all files Collapse all files

4
.gitignore vendored
View file

@ -24,4 +24,6 @@ deploy.toml
!.deploy/deploy.example.toml
# Local dev node state
.dev-node/
.dev-node/
.dev-cluster/
.sim-cluster/

3
Cargo.lock generated
View file

@ -6949,9 +6949,12 @@ name = "xtask"
version = "0.1.0"
dependencies = [
"clap",
"iroh-relay",
"libc",
"reqwest 0.12.28",
"serde",
"serde_json",
"tokio",
"toml 0.8.23",
]

392
crates/dashboard/src/pool_html.rs Normal file
View file

@ -0,0 +1,392 @@
pub const POOL_HTML: &str = r##"<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Swactor Runtime – Pool</title>
<style>
* { margin: 0; padding: 0; box-sizing: border-box; }
body { font-family: 'Menlo', 'Consolas', 'Monaco', monospace; background: #0f1117; color: #e0e0e0; font-size: 13px; }
.header {
display: flex; align-items: center; justify-content: space-between;
padding: 12px 20px; background: #161822; border-bottom: 1px solid #2a2d3e;
}
.header-left { display: flex; align-items: center; }
.header h1 { font-size: 16px; font-weight: 600; color: #fff; }
.status-dot {
width: 10px; height: 10px; border-radius: 50%; background: #4caf50;
display: inline-block; margin-left: 8px; vertical-align: middle;
}
.status-dot.disconnected { background: #f44336; }
.status-dot.done { background: #ff9800; }
.nav-links { display: flex; gap: 4px; margin-left: 20px; }
.nav-link {
color: #888; text-decoration: none; font-size: 12px;
padding: 4px 10px; border-radius: 3px; transition: color 0.2s;
}
.nav-link:hover { color: #e0e0e0; }
.nav-link.active { color: #fff; background: #2a2d3e; }
.header-right { display: flex; align-items: center; gap: 12px; }
.grid {
display: grid;
grid-template-columns: 1fr 1fr;
gap: 12px; padding: 12px;
}
.panel {
background: #161822; border: 1px solid #2a2d3e; border-radius: 6px;
padding: 14px; overflow: hidden;
}
.panel h2 { font-size: 12px; color: #888; text-transform: uppercase; letter-spacing: 1px; margin-bottom: 10px; }
.full-width { grid-column: 1 / -1; }
.stats-cards {
display: grid; grid-template-columns: repeat(5, 1fr); gap: 10px;
}
.stat-card {
background: #1c1f2e; border-radius: 4px; padding: 10px; text-align: center;
}
.stat-card .value { font-size: 22px; font-weight: 700; color: #fff; }
.stat-card .label { font-size: 10px; color: #888; text-transform: uppercase; margin-top: 2px; }
.members-table { max-height: 400px; overflow-y: auto; }
.members-table table { width: 100%; border-collapse: collapse; }
.members-table th, .members-table td {
padding: 4px 8px; text-align: left; border-bottom: 1px solid #1c1f2e; font-size: 11px;
white-space: nowrap;
}
.members-table th { color: #888; font-weight: 500; position: sticky; top: 0; background: #161822; }
.content-table { max-height: 400px; overflow-y: auto; }
.content-table table { width: 100%; border-collapse: collapse; }
.content-table th, .content-table td {
padding: 4px 8px; text-align: left; border-bottom: 1px solid #1c1f2e; font-size: 11px;
white-space: nowrap;
}
.content-table th { color: #888; font-weight: 500; position: sticky; top: 0; background: #161822; }
/* Capacity bars */
.cap-bar-wrap { margin-bottom: 6px; }
.cap-bar-label {
display: flex; justify-content: space-between; font-size: 10px; color: #888; margin-bottom: 2px;
}
.cap-bar {
height: 14px; background: #1c1f2e; border-radius: 3px; overflow: hidden;
}
.cap-bar-fill {
height: 100%; background: #6366f1; border-radius: 3px;
transition: width 0.3s ease;
}
.cap-bar-fill.warn { background: #ff9800; }
.cap-bar-fill.crit { background: #f44336; }
/* Buttons */
button {
background: #1e2030; color: #e0e0e0; border: 1px solid #2a2d3e;
border-radius: 4px; padding: 6px 14px; font-family: inherit;
font-size: 13px; cursor: pointer; min-height: 38px;
transition: border-color 0.15s;
}
button:hover { border-color: #6366f1; color: #fff; }
button:disabled { opacity: 0.4; cursor: default; }
button.primary { background: #6366f1; border-color: #6366f1; color: #fff; font-weight: 600; }
button.primary:hover { background: #5558e6; }
button.danger:hover { border-color: #f44336; }
/* No-pool message */
.no-pool {
display: flex; align-items: center; justify-content: center;
height: calc(100vh - 49px); color: #555; font-size: 16px;
flex-direction: column; gap: 8px;
}
/* Toast */
.toast {
position: fixed; bottom: 20px; right: 20px; padding: 10px 16px;
border-radius: 4px; font-size: 12px; z-index: 100; opacity: 0;
transition: opacity 0.3s; pointer-events: none;
}
.toast.show { opacity: 1; }
.toast.success { background: #4caf50; color: #fff; }
.toast.error { background: #f44336; color: #fff; }
::-webkit-scrollbar { width: 6px; }
::-webkit-scrollbar-track { background: #0f1117; }
::-webkit-scrollbar-thumb { background: #2a2d3e; border-radius: 3px; }
</style>
</head>
<body>
<div class="header">
<div class="header-left">
<h1>
Swactor Runtime Dashboard
<span id="statusDot" class="status-dot disconnected"></span>
</h1>
<nav class="nav-links">
<a href="/" class="nav-link">Overview</a>
<a href="/actors" class="nav-link">Actors</a>
<a href="/distribution" class="nav-link">Distribution</a>
<a href="/datastore" class="nav-link">Datastore</a>
<a href="/pool" class="nav-link active">Pool</a>
</nav>
</div>
<div class="header-right">
<button id="joinBtn" class="primary" onclick="joinPool()" style="display:none">Join Pool</button>
<button id="leaveBtn" class="danger" onclick="leavePool()" style="display:none">Leave Pool</button>
</div>
</div>
<div id="noPool" class="no-pool" style="display:none">
<div>No pool configured</div>
<div style="font-size:12px;color:#444;">Start the node with --pool-name to enable pooled storage</div>
</div>
<div id="poolContent" style="display:none">
<div class="grid">
<!-- Summary cards -->
<div class="panel full-width">
<h2>Pool Summary</h2>
<div class="stats-cards">
<div class="stat-card"><div class="value" id="statPoolName">-</div><div class="label">Pool Name</div></div>
<div class="stat-card"><div class="value" id="statMembers">0</div><div class="label">Members</div></div>
<div class="stat-card"><div class="value" id="statContent">0</div><div class="label">Content Items</div></div>
<div class="stat-card"><div class="value" id="statTotal">0</div><div class="label">Total Capacity</div></div>
<div class="stat-card"><div class="value" id="statUsed">0</div><div class="label">Used</div></div>
</div>
</div>
<!-- Capacity chart -->
<div class="panel">
<h2>Capacity by Node</h2>
<div id="capacityChart"></div>
<div id="capEmpty" style="color:#555;font-size:11px;">No members yet</div>
</div>
<!-- Members table -->
<div class="panel">
<h2>Members <span id="memberCount" style="color:#555;font-weight:400;"></span></h2>
<div class="members-table">
<table>
<thead><tr><th>Node ID</th><th>State</th><th>Total</th><th>Used</th><th>Free</th></tr></thead>
<tbody id="membersBody"></tbody>
</table>
</div>
</div>
<!-- Content locations -->
<div class="panel full-width">
<h2>Content Location Map <span id="contentCount" style="color:#555;font-weight:400;"></span></h2>
<div class="content-table">
<table>
<thead><tr><th>Content Hash</th><th>Replicas</th><th>Nodes</th></tr></thead>
<tbody id="contentBody"></tbody>
</table>
</div>
</div>
<!-- ACL panel -->
<div class="panel full-width">
<h2>Access Control <span id="aclMode" style="color:#555;font-weight:400;"></span></h2>
<div id="aclOpen" style="color:#555;font-size:11px;">Open mode &mdash; any node may join the pool</div>
<div id="aclTable" class="content-table" style="display:none">
<table>
<thead><tr><th>Node ID</th><th>Granted By</th><th>Status</th></tr></thead>
<tbody id="aclBody"></tbody>
</table>
</div>
</div>
</div>
</div>
<div class="toast" id="toast"></div>
<script>
(function() {
var dot = document.getElementById('statusDot');
var poolConfigured = false;
function $(id) { return document.getElementById(id); }
function formatBytes(b) {
if (b === 0) return '0 B';
var units = ['B', 'KB', 'MB', 'GB', 'TB'];
var i = Math.floor(Math.log(b) / Math.log(1024));
if (i >= units.length) i = units.length - 1;
return (b / Math.pow(1024, i)).toFixed(i > 0 ? 1 : 0) + ' ' + units[i];
}
function escapeHtml(s) {
if (!s) return '';
return s.replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;');
}
window.toast = function(msg, type) {
var t = $('toast');
t.textContent = msg;
t.className = 'toast show ' + type;
setTimeout(function() { t.className = 'toast'; }, 2500);
};
function updatePool(snap) {
if (!snap) {
if (!poolConfigured) {
$('noPool').style.display = 'flex';
$('poolContent').style.display = 'none';
$('joinBtn').style.display = 'none';
$('leaveBtn').style.display = 'none';
}
return;
}
poolConfigured = true;
$('noPool').style.display = 'none';
$('poolContent').style.display = 'block';
$('joinBtn').style.display = 'inline-block';
$('leaveBtn').style.display = 'inline-block';
// Summary cards
$('statPoolName').textContent = snap.pool_name || '-';
$('statPoolName').style.fontSize = '14px';
$('statMembers').textContent = snap.member_count;
$('statContent').textContent = snap.content_count;
$('statTotal').textContent = formatBytes(snap.total_bytes);
$('statUsed').textContent = formatBytes(snap.used_bytes);
// Members table
var mbody = $('membersBody');
mbody.innerHTML = '';
$('memberCount').textContent = '(' + snap.members.length + ')';
for (var i = 0; i < snap.members.length; i++) {
var m = snap.members[i];
var free = m.total_bytes - m.used_bytes;
var tr = document.createElement('tr');
tr.innerHTML =
'<td style="color:#6366f1;font-size:11px;" title="' + escapeHtml(m.node_id) + '">' + m.node_id.substring(0, 16) + '\u2026</td>' +
'<td style="color:#4caf50;">' + escapeHtml(m.state) + '</td>' +
'<td style="color:#888;">' + formatBytes(m.total_bytes) + '</td>' +
'<td style="color:#888;">' + formatBytes(m.used_bytes) + '</td>' +
'<td style="color:#888;">' + formatBytes(free) + '</td>';
mbody.appendChild(tr);
}
// Capacity chart
var chart = $('capacityChart');
chart.innerHTML = '';
var capEmpty = $('capEmpty');
if (snap.members.length === 0) {
capEmpty.style.display = 'block';
} else {
capEmpty.style.display = 'none';
for (var i = 0; i < snap.members.length; i++) {
var m = snap.members[i];
var pct = m.total_bytes > 0 ? Math.round((m.used_bytes / m.total_bytes) * 100) : 0;
var fillClass = 'cap-bar-fill';
if (pct > 90) fillClass += ' crit';
else if (pct > 70) fillClass += ' warn';
var wrap = document.createElement('div');
wrap.className = 'cap-bar-wrap';
wrap.innerHTML =
'<div class="cap-bar-label"><span>' + m.node_id.substring(0, 12) + '\u2026</span><span>' + pct + '% (' + formatBytes(m.used_bytes) + ' / ' + formatBytes(m.total_bytes) + ')</span></div>' +
'<div class="cap-bar"><div class="' + fillClass + '" style="width:' + pct + '%"></div></div>';
chart.appendChild(wrap);
}
}
// Content locations
var cbody = $('contentBody');
cbody.innerHTML = '';
$('contentCount').textContent = '(' + snap.content_locations.length + ')';
for (var i = 0; i < snap.content_locations.length; i++) {
var cl = snap.content_locations[i];
var nodeList = cl.nodes.map(function(n) { return n.substring(0, 12) + '\u2026'; }).join(', ');
var tr = document.createElement('tr');
tr.innerHTML =
'<td style="color:#6366f1;font-size:11px;" title="' + escapeHtml(cl.content_hash) + '">' + cl.content_hash.substring(0, 16) + '\u2026</td>' +
'<td>' + cl.replica_count + '</td>' +
'<td style="color:#888;font-size:10px;">' + escapeHtml(nodeList) + '</td>';
cbody.appendChild(tr);
}
// ACL panel
var aclMode = snap.acl_mode || 'open';
var aclEntries = snap.acl || [];
$('aclMode').textContent = '(' + aclMode + ')';
if (aclMode === 'open' || aclEntries.length === 0) {
$('aclOpen').style.display = 'block';
$('aclTable').style.display = 'none';
} else {
$('aclOpen').style.display = 'none';
$('aclTable').style.display = 'block';
var abody = $('aclBody');
abody.innerHTML = '';
for (var i = 0; i < aclEntries.length; i++) {
var a = aclEntries[i];
var status = a.revoked ? 'revoked' : 'granted';
var statusColor = a.revoked ? '#f44336' : '#4caf50';
var tr = document.createElement('tr');
tr.innerHTML =
'<td style="color:#6366f1;font-size:11px;" title="' + escapeHtml(a.node_id) + '">' + a.node_id.substring(0, 16) + '\u2026</td>' +
'<td style="color:#888;font-size:10px;" title="' + escapeHtml(a.granted_by) + '">' + a.granted_by.substring(0, 16) + '\u2026</td>' +
'<td style="color:' + statusColor + ';font-size:10px;">' + status + '</td>';
abody.appendChild(tr);
}
}
}
// Join/Leave actions
window.joinPool = function() {
$('joinBtn').disabled = true;
fetch('/api/pool/join', { method: 'POST' })
.then(function(r) {
if (!r.ok) return r.json().then(function(j) { throw new Error(j.error || r.statusText); });
return r.json();
})
.then(function() { toast('Joined pool', 'success'); })
.catch(function(e) { toast('Join failed: ' + e.message, 'error'); })
.finally(function() { $('joinBtn').disabled = false; });
};
window.leavePool = function() {
$('leaveBtn').disabled = true;
fetch('/api/pool/leave', { method: 'POST' })
.then(function(r) {
if (!r.ok) return r.json().then(function(j) { throw new Error(j.error || r.statusText); });
return r.json();
})
.then(function() { toast('Left pool', 'success'); })
.catch(function(e) { toast('Leave failed: ' + e.message, 'error'); })
.finally(function() { $('leaveBtn').disabled = false; });
};
// SSE connection
var es = new EventSource('/events');
es.addEventListener('pool', function(e) {
try {
var snap = JSON.parse(e.data);
updatePool(snap);
} catch(err) { console.error('pool parse error', err); }
});
es.addEventListener('done', function() {
dot.className = 'status-dot done';
es.close();
});
es.onerror = function() { dot.className = 'status-dot disconnected'; };
es.onopen = function() { dot.className = 'status-dot'; };
})();
</script>
</body>
</html>
"##;

274
crates/datastore/src/pool/coordinator.rs Normal file
View file

@ -0,0 +1,274 @@
//! Pool coordinator actor — placement-aware CRUD facade.
//!
//! Owns an `Arc<Mutex<PoolDisseminator>>` for query access and delegates
//! storage operations to the co-located `DatastoreNode` actor.
use std::collections::BTreeMap;
use std::sync::{Arc, Mutex};
use swactor::actor::{ActorAddress, ActorInterface, Ctx};
use distribution::types::NodeId;
use shared_types::ContentHash;
use shared_types::pool::PoolConfig;
use crate::messages::{DatastoreNodeMsg, DatastoreResponse};
use super::disseminator::PoolDisseminator;
use super::messages::PoolCoordinatorMsg;
/// The pool coordinator actor.
pub struct PoolCoordinator {
node_id: NodeId,
pool_config: PoolConfig,
disseminator: Arc<Mutex<PoolDisseminator>>,
// Co-located actor addresses
datastore_addr: ActorAddress,
tick_count: u64,
}
impl PoolCoordinator {
pub fn new(
node_id: NodeId,
pool_config: PoolConfig,
disseminator: Arc<Mutex<PoolDisseminator>>,
datastore_addr: ActorAddress,
) -> Self {
Self {
node_id,
pool_config,
disseminator,
datastore_addr,
tick_count: 0,
}
}
/// Access the shared disseminator.
pub fn disseminator(&self) -> &Arc<Mutex<PoolDisseminator>> {
&self.disseminator
}
fn cluster_size(&self) -> usize {
let d = self.disseminator.lock().unwrap();
d.member_count().max(1)
}
// ─── Message handlers ──────────────────────────────────────────────
fn handle_pool_put(
&self,
ctx: &Ctx,
data: Vec<u8>,
name: Option<String>,
tags: BTreeMap<String, String>,
reply_to: ActorAddress,
) {
// Delegate to local DatastoreNode for now.
// Future: check capacity and redirect to best node.
let _ = ctx.send(
self.datastore_addr,
DatastoreNodeMsg::Put {
data,
name,
tags,
reply_to,
},
);
// Note: content announcement happens after PutOk is received.
// For now, caller is responsible for announcing content via PoolTick
// or a future PutOk callback.
}
fn handle_pool_get(
&self,
ctx: &Ctx,
content_hash: ContentHash,
reply_to: ActorAddress,
) {
// Try local first via DatastoreNode
let _ = ctx.send(
self.datastore_addr,
DatastoreNodeMsg::Get {
content_hash,
reply_to,
},
);
// Future: if local not found, use disseminator.locate_content()
// to fetch from a specific peer instead of fan-out.
}
fn handle_pool_delete(
&self,
ctx: &Ctx,
content_hash: ContentHash,
reply_to: ActorAddress,
) {
// Delete locally
let _ = ctx.send(
self.datastore_addr,
DatastoreNodeMsg::Delete {
content_hash,
reply_to,
},
);
// Announce tombstone via gossip
let cluster_size = self.cluster_size();
self.disseminator
.lock()
.unwrap()
.remove_content(content_hash, cluster_size);
}
fn handle_pool_list(
&self,
ctx: &Ctx,
name_filter: Option<String>,
reply_to: ActorAddress,
) {
let _ = ctx.send(
self.datastore_addr,
DatastoreNodeMsg::List {
name_filter,
all: false,
reply_to,
},
);
}
fn handle_pool_status(&self, ctx: &Ctx, reply_to: ActorAddress) {
let d = self.disseminator.lock().unwrap();
let (total_bytes, used_bytes) = d.pool_capacity_summary();
let members: Vec<String> = d
.active_members()
.iter()
.map(|id| id.0.iter().map(|b| format!("{b:02x}")).collect())
.collect();
let member_count = d.member_count();
let content_count = d.content_count();
drop(d);
let json = serde_json::json!({
"pool_name": self.pool_config.pool_name,
"pool_id": self.pool_config.pool_id.to_hex(),
"member_count": member_count,
"content_count": content_count,
"total_bytes": total_bytes,
"used_bytes": used_bytes,
"members": members,
});
let _ = ctx.send(
reply_to,
DatastoreResponse::PoolStatus {
json: json.to_string(),
},
);
}
fn handle_join_pool(&mut self, ctx: &Ctx, reply_to: ActorAddress) {
let cluster_size = self.cluster_size();
let mut d = self.disseminator.lock().unwrap();
if !d.is_node_authorized(&self.node_id) {
let _ = ctx.send(
reply_to,
DatastoreResponse::Error {
reason: "not authorized to join pool".into(),
},
);
return;
}
d.join(cluster_size);
d.announce_capacity(self.pool_config.capacity_bytes, 0, cluster_size);
drop(d);
let _ = ctx.send(reply_to, DatastoreResponse::Bool(true));
}
fn handle_leave_pool(&mut self, ctx: &Ctx, reply_to: ActorAddress) {
let cluster_size = self.cluster_size();
self.disseminator.lock().unwrap().leave(cluster_size);
let _ = ctx.send(reply_to, DatastoreResponse::Bool(true));
}
fn handle_grant_access(&mut self, ctx: &Ctx, target: NodeId, reply_to: ActorAddress) {
let cluster_size = self.cluster_size();
self.disseminator
.lock()
.unwrap()
.grant_access(target, cluster_size);
let _ = ctx.send(reply_to, DatastoreResponse::Bool(true));
}
fn handle_revoke_access(&mut self, ctx: &Ctx, target: NodeId, reply_to: ActorAddress) {
let cluster_size = self.cluster_size();
self.disseminator
.lock()
.unwrap()
.revoke_access(target, cluster_size);
let _ = ctx.send(reply_to, DatastoreResponse::Bool(true));
}
fn handle_pool_tick(&mut self) {
self.tick_count += 1;
// Periodic capacity re-announcement (every 100 ticks)
if self.tick_count % 100 == 0 {
let cluster_size = self.cluster_size();
self.disseminator.lock().unwrap().announce_capacity(
self.pool_config.capacity_bytes,
0, // TODO: query actual usage from BlobStore
cluster_size,
);
}
}
}
impl ActorInterface for PoolCoordinator {
type Incoming = PoolCoordinatorMsg;
type Response = DatastoreResponse;
fn handle(&mut self, ctx: &Ctx, msg: PoolCoordinatorMsg) {
match msg {
PoolCoordinatorMsg::PoolPut {
data,
name,
tags,
reply_to,
} => self.handle_pool_put(ctx, data, name, tags, reply_to),
PoolCoordinatorMsg::PoolGet {
content_hash,
reply_to,
} => self.handle_pool_get(ctx, content_hash, reply_to),
PoolCoordinatorMsg::PoolDelete {
content_hash,
reply_to,
} => self.handle_pool_delete(ctx, content_hash, reply_to),
PoolCoordinatorMsg::PoolList {
name_filter,
reply_to,
} => self.handle_pool_list(ctx, name_filter, reply_to),
PoolCoordinatorMsg::PoolStatus { reply_to } => {
self.handle_pool_status(ctx, reply_to)
}
PoolCoordinatorMsg::JoinPool { reply_to } => {
self.handle_join_pool(ctx, reply_to)
}
PoolCoordinatorMsg::LeavePool { reply_to } => {
self.handle_leave_pool(ctx, reply_to)
}
PoolCoordinatorMsg::GrantPoolAccess { target, reply_to } => {
self.handle_grant_access(ctx, target, reply_to)
}
PoolCoordinatorMsg::RevokePoolAccess { target, reply_to } => {
self.handle_revoke_access(ctx, target, reply_to)
}
PoolCoordinatorMsg::PoolTick => self.handle_pool_tick(),
}
}
}

716
crates/datastore/src/pool/disseminator.rs Normal file
View file

@ -0,0 +1,716 @@
//! Pool disseminator — gossip-converged state for pool membership,
//! capacity, content locations, and ACL.
//!
//! Implements `GossipChannel` to plug into the generic gossip system
//! via `DistributedNode::register_channel()`.
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use distribution::gossip_channel::{DisseminationBuffer, GossipChannel, deserialize_each, serialize_each};
use distribution::types::NodeId;
use shared_types::ContentHash;
use shared_types::pool::*;
// ─── PoolDisseminator ──────────────────────────────────────────────────────
/// Configuration for the pool disseminator.
#[derive(Debug, Clone)]
pub struct PoolDisseminatorConfig {
pub tombstone_ttl: u64,
pub gc_interval: u64,
}
impl Default for PoolDisseminatorConfig {
fn default() -> Self {
Self {
tombstone_ttl: 3600,
gc_interval: 1000,
}
}
}
/// Manages converged pool state via gossip dissemination.
#[derive(Debug)]
pub struct PoolDisseminator {
pool_id: PoolId,
pool_name: String,
local_node_id: NodeId,
// Converged state maps
members: HashMap<[u8; 32], PoolMemberEntry>,
capacity: HashMap<[u8; 32], PoolCapacityEntry>,
content_locations: HashMap<(ContentHash, [u8; 32]), ContentLocationEntry>,
acl: HashMap<[u8; 32], PoolACLEntry>,
// Dissemination buffer
buffer: DisseminationBuffer<PoolEntry>,
// Local generation counters
local_member_gen: u64,
local_capacity_gen: u64,
config: PoolDisseminatorConfig,
tick_count: u64,
}
impl PoolDisseminator {
pub fn new(pool_id: PoolId, pool_name: String, local_node_id: NodeId, lambda: usize) -> Self {
Self {
pool_id,
pool_name,
local_node_id,
members: HashMap::new(),
capacity: HashMap::new(),
content_locations: HashMap::new(),
acl: HashMap::new(),
buffer: DisseminationBuffer::new(lambda),
local_member_gen: 0,
local_capacity_gen: 0,
config: PoolDisseminatorConfig::default(),
tick_count: 0,
}
}
pub fn with_config(mut self, config: PoolDisseminatorConfig) -> Self {
self.config = config;
self
}
pub fn pool_id(&self) -> PoolId {
self.pool_id
}
// ─── Lifecycle methods ──────────────────────────────────────────────
/// Join the pool. Announces Active membership.
pub fn join(&mut self, cluster_size: usize) {
self.local_member_gen += 1;
let entry = PoolMemberEntry {
pool_id: self.pool_id,
node_id: self.local_node_id.0,
state: PoolMemberState::Active,
generation: self.local_member_gen,
};
self.merge_membership(entry.clone());
self.buffer.enqueue(PoolEntry::Membership(entry), cluster_size);
}
/// Leave the pool. Announces Left membership.
pub fn leave(&mut self, cluster_size: usize) {
self.local_member_gen += 1;
let entry = PoolMemberEntry {
pool_id: self.pool_id,
node_id: self.local_node_id.0,
state: PoolMemberState::Left,
generation: self.local_member_gen,
};
self.merge_membership(entry.clone());
self.buffer.enqueue(PoolEntry::Membership(entry), cluster_size);
}
/// Announce storage capacity.
pub fn announce_capacity(&mut self, total: u64, used: u64, cluster_size: usize) {
self.local_capacity_gen += 1;
let entry = PoolCapacityEntry {
pool_id: self.pool_id,
node_id: self.local_node_id.0,
total_bytes: total,
used_bytes: used,
generation: self.local_capacity_gen,
};
self.merge_capacity(entry.clone());
self.buffer.enqueue(PoolEntry::Capacity(entry), cluster_size);
}
/// Announce that this node has a piece of content.
pub fn announce_content(&mut self, hash: ContentHash, cluster_size: usize) {
let key = (hash, self.local_node_id.0);
let next_gen = self.content_locations.get(&key).map_or(1, |e| e.generation + 1);
let entry = ContentLocationEntry {
pool_id: self.pool_id,
content_hash: hash,
node_id: self.local_node_id.0,
generation: next_gen,
tombstone: false,
};
self.merge_content_location(entry.clone());
self.buffer.enqueue(PoolEntry::ContentLocation(entry), cluster_size);
}
/// Remove content announcement (tombstone).
pub fn remove_content(&mut self, hash: ContentHash, cluster_size: usize) {
let key = (hash, self.local_node_id.0);
let next_gen = self.content_locations.get(&key).map_or(1, |e| e.generation + 1);
let entry = ContentLocationEntry {
pool_id: self.pool_id,
content_hash: hash,
node_id: self.local_node_id.0,
generation: next_gen,
tombstone: true,
};
self.merge_content_location(entry.clone());
self.buffer.enqueue(PoolEntry::ContentLocation(entry), cluster_size);
}
/// Grant access to a node.
pub fn grant_access(&mut self, target: NodeId, cluster_size: usize) {
let next_gen = self.acl.get(&target.0).map_or(1, |e| e.generation + 1);
let entry = PoolACLEntry {
pool_id: self.pool_id,
node_id: target.0,
granted_by: self.local_node_id.0,
generation: next_gen,
revoked: false,
};
self.merge_acl(entry.clone());
self.buffer.enqueue(PoolEntry::ACL(entry), cluster_size);
}
/// Revoke access from a node.
pub fn revoke_access(&mut self, target: NodeId, cluster_size: usize) {
let next_gen = self.acl.get(&target.0).map_or(1, |e| e.generation + 1);
let entry = PoolACLEntry {
pool_id: self.pool_id,
node_id: target.0,
granted_by: self.local_node_id.0,
generation: next_gen,
revoked: true,
};
self.merge_acl(entry.clone());
self.buffer.enqueue(PoolEntry::ACL(entry), cluster_size);
}
// ─── Query API ──────────────────────────────────────────────────────
/// All active pool members.
pub fn active_members(&self) -> Vec<NodeId> {
self.members
.values()
.filter(|m| m.state == PoolMemberState::Active)
.map(|m| NodeId(m.node_id))
.collect()
}
/// Total and used capacity across the pool.
pub fn pool_capacity_summary(&self) -> (u64, u64) {
let mut total = 0u64;
let mut used = 0u64;
for cap in self.capacity.values() {
// Only count active members
if let Some(m) = self.members.get(&cap.node_id) {
if m.state == PoolMemberState::Active {
total = total.saturating_add(cap.total_bytes);
used = used.saturating_add(cap.used_bytes);
}
}
}
(total, used)
}
/// Find which nodes have a given content hash.
pub fn locate_content(&self, hash: &ContentHash) -> Vec<NodeId> {
self.content_locations
.iter()
.filter(|((h, _), entry)| h == hash && !entry.tombstone)
.map(|((_, node_id), _)| NodeId(*node_id))
.collect()
}
/// Find the node with the most free space.
pub fn node_with_most_free_space(&self) -> Option<NodeId> {
self.capacity
.values()
.filter(|cap| {
self.members
.get(&cap.node_id)
.is_some_and(|m| m.state == PoolMemberState::Active)
})
.max_by_key(|cap| cap.total_bytes.saturating_sub(cap.used_bytes))
.map(|cap| NodeId(cap.node_id))
}
/// Check if a node is authorized to join this pool.
pub fn is_node_authorized(&self, node_id: &NodeId) -> bool {
// If no ACL entries exist, the pool is open
if self.acl.is_empty() {
return true;
}
self.acl
.get(&node_id.0)
.is_some_and(|entry| !entry.revoked)
}
/// Number of active members.
pub fn member_count(&self) -> usize {
self.members
.values()
.filter(|m| m.state == PoolMemberState::Active)
.count()
}
/// Number of live content location entries (non-tombstone).
pub fn content_count(&self) -> usize {
self.content_locations
.values()
.filter(|e| !e.tombstone)
.count()
}
/// Serialize the current pool state to a JSON string for the dashboard.
pub fn snapshot_json(&self) -> String {
fn hex(bytes: &[u8; 32]) -> String {
bytes.iter().map(|b| format!("{b:02x}")).collect()
}
let (total_bytes, used_bytes) = self.pool_capacity_summary();
let members: Vec<serde_json::Value> = self
.members
.values()
.filter(|m| m.state == PoolMemberState::Active)
.map(|m| {
let cap = self.capacity.get(&m.node_id);
serde_json::json!({
"node_id": hex(&m.node_id),
"state": format!("{:?}", m.state),
"generation": m.generation,
"total_bytes": cap.map_or(0, |c| c.total_bytes),
"used_bytes": cap.map_or(0, |c| c.used_bytes),
})
})
.collect();
// Group content locations by hash
let mut by_hash: HashMap<ContentHash, Vec<[u8; 32]>> = HashMap::new();
for ((hash, _), entry) in &self.content_locations {
if !entry.tombstone {
by_hash.entry(*hash).or_default().push(entry.node_id);
}
}
let content_locations: Vec<serde_json::Value> = by_hash
.iter()
.map(|(hash, nodes)| {
serde_json::json!({
"content_hash": hash.to_hex(),
"nodes": nodes.iter().map(hex).collect::<Vec<_>>(),
"replica_count": nodes.len(),
})
})
.collect();
let acl: Vec<serde_json::Value> = self
.acl
.values()
.map(|a| {
serde_json::json!({
"node_id": hex(&a.node_id),
"granted_by": hex(&a.granted_by),
"revoked": a.revoked,
})
})
.collect();
let acl_mode = if self.acl.is_empty() { "open" } else { "allow-list" };
serde_json::json!({
"pool_name": self.pool_name,
"pool_id": self.pool_id.to_hex(),
"member_count": self.member_count(),
"content_count": self.content_count(),
"total_bytes": total_bytes,
"used_bytes": used_bytes,
"members": members,
"content_locations": content_locations,
"acl": acl,
"acl_mode": acl_mode,
})
.to_string()
}
// ─── Internal merge logic ───────────────────────────────────────────
fn merge_membership(&mut self, entry: PoolMemberEntry) -> bool {
let key = entry.node_id;
if let Some(existing) = self.members.get(&key) {
if entry.generation <= existing.generation {
return false;
}
}
self.members.insert(key, entry);
true
}
fn merge_capacity(&mut self, entry: PoolCapacityEntry) -> bool {
let key = entry.node_id;
if let Some(existing) = self.capacity.get(&key) {
if entry.generation <= existing.generation {
return false;
}
}
self.capacity.insert(key, entry);
true
}
fn merge_content_location(&mut self, entry: ContentLocationEntry) -> bool {
let key = (entry.content_hash, entry.node_id);
if let Some(existing) = self.content_locations.get(&key) {
if entry.generation <= existing.generation {
return false;
}
}
self.content_locations.insert(key, entry);
true
}
fn merge_acl(&mut self, entry: PoolACLEntry) -> bool {
let key = entry.node_id;
if let Some(existing) = self.acl.get(&key) {
if entry.generation <= existing.generation {
return false;
}
}
self.acl.insert(key, entry);
true
}
/// Merge a single pool entry and return whether state changed.
fn merge_entry(&mut self, entry: PoolEntry) -> bool {
match entry {
PoolEntry::Membership(m) => self.merge_membership(m),
PoolEntry::Capacity(c) => self.merge_capacity(c),
PoolEntry::ContentLocation(cl) => self.merge_content_location(cl),
PoolEntry::ACL(a) => self.merge_acl(a),
}
}
/// Take pending entries (internal, typed).
fn take_pending_inner(&mut self, max_count: usize) -> Vec<PoolEntry> {
self.buffer.take(max_count)
}
/// Apply incoming entries (internal, typed).
fn apply_incoming_inner(&mut self, entries: Vec<PoolEntry>, cluster_size: usize) {
for entry in entries {
if self.merge_entry(entry.clone()) {
self.buffer.enqueue(entry, cluster_size);
}
}
}
/// Re-enqueue all state (internal).
fn re_disseminate_all_inner(&mut self, cluster_size: usize) {
let mut all_entries: Vec<PoolEntry> = Vec::new();
for m in self.members.values().cloned() {
all_entries.push(PoolEntry::Membership(m));
}
for c in self.capacity.values().cloned() {
all_entries.push(PoolEntry::Capacity(c));
}
for cl in self.content_locations.values().cloned() {
all_entries.push(PoolEntry::ContentLocation(cl));
}
for a in self.acl.values().cloned() {
all_entries.push(PoolEntry::ACL(a));
}
self.buffer.re_enqueue_all(all_entries, cluster_size);
}
/// GC: evict tombstones past TTL.
fn gc_tick_inner(&mut self) {
self.tick_count += 1;
if self.tick_count % self.config.gc_interval != 0 {
return;
}
let ttl = self.config.tombstone_ttl;
let tick = self.tick_count;
// GC left members
self.members.retain(|_, m| {
if m.state == PoolMemberState::Left {
m.generation + ttl > tick
} else {
true
}
});
// GC tombstoned content locations
self.content_locations.retain(|_, cl| {
if cl.tombstone {
cl.generation + ttl > tick
} else {
true
}
});
// GC revoked ACL entries
self.acl.retain(|_, a| {
if a.revoked {
a.generation + ttl > tick
} else {
true
}
});
}
}
// ─── SharedPoolChannel ─────────────────────────────────────────────────────
/// Wrapper around `Arc<Mutex<PoolDisseminator>>` that implements `GossipChannel`.
///
/// This enables shared ownership between the `PoolCoordinator` actor
/// (which needs query/lifecycle access) and `DistributedNode` (which
/// drives gossip piggyback).
pub struct SharedPoolChannel {
inner: Arc<Mutex<PoolDisseminator>>,
}
impl SharedPoolChannel {
pub fn new(disseminator: Arc<Mutex<PoolDisseminator>>) -> Self {
Self { inner: disseminator }
}
/// Consume the channel and return the underlying `Arc<Mutex<PoolDisseminator>>`.
pub fn into_inner(self) -> Arc<Mutex<PoolDisseminator>> {
self.inner
}
}
impl GossipChannel for SharedPoolChannel {
fn topic_tag(&self) -> &'static str {
"pool"
}
fn take_pending_bytes(&mut self, max_entries: usize) -> Vec<Vec<u8>> {
let entries = self.inner.lock().unwrap().take_pending_inner(max_entries);
serialize_each(&entries)
}
fn apply_incoming_bytes(&mut self, entries: &[Vec<u8>], cluster_size: usize) {
let parsed: Vec<PoolEntry> = deserialize_each(entries);
self.inner.lock().unwrap().apply_incoming_inner(parsed, cluster_size);
}
fn re_disseminate_all(&mut self, cluster_size: usize) {
self.inner.lock().unwrap().re_disseminate_all_inner(cluster_size);
}
fn on_node_death(&mut self, _node_id: &NodeId) {
// Pool membership is explicit (join/leave), not auto-removed on node death.
// Capacity becomes unreliable but we don't remove it.
}
fn gc_tick(&mut self) {
self.inner.lock().unwrap().gc_tick_inner();
}
}
#[cfg(test)]
mod tests {
use super::*;
fn node_id(b: u8) -> NodeId {
NodeId([b; 32])
}
fn make_disseminator(b: u8) -> PoolDisseminator {
PoolDisseminator::new(
PoolId::from_name("test-pool"),
"test-pool".into(),
node_id(b),
3,
)
}
#[test]
fn join_and_query_members() {
let mut d = make_disseminator(1);
d.join(2);
assert_eq!(d.member_count(), 1);
assert_eq!(d.active_members(), vec![node_id(1)]);
}
#[test]
fn leave_removes_from_active() {
let mut d = make_disseminator(1);
d.join(2);
d.leave(2);
assert_eq!(d.member_count(), 0);
assert!(d.active_members().is_empty());
}
#[test]
fn announce_and_locate_content() {
let mut d = make_disseminator(1);
d.join(2);
let hash = ContentHash::of(b"test-data");
d.announce_content(hash, 2);
let locations = d.locate_content(&hash);
assert_eq!(locations, vec![node_id(1)]);
}
#[test]
fn remove_content_tombstones() {
let mut d = make_disseminator(1);
d.join(2);
let hash = ContentHash::of(b"test-data");
d.announce_content(hash, 2);
d.remove_content(hash, 2);
assert!(d.locate_content(&hash).is_empty());
}
#[test]
fn capacity_summary() {
let mut d = make_disseminator(1);
d.join(2);
d.announce_capacity(1000, 300, 2);
let (total, used) = d.pool_capacity_summary();
assert_eq!(total, 1000);
assert_eq!(used, 300);
}
#[test]
fn acl_grant_and_check() {
let mut d = make_disseminator(1);
d.grant_access(node_id(2), 2);
assert!(d.is_node_authorized(&node_id(2)));
assert!(!d.is_node_authorized(&node_id(3)));
}
#[test]
fn acl_revoke() {
let mut d = make_disseminator(1);
d.grant_access(node_id(2), 2);
d.revoke_access(node_id(2), 2);
assert!(!d.is_node_authorized(&node_id(2)));
}
#[test]
fn empty_acl_means_open() {
let d = make_disseminator(1);
assert!(d.is_node_authorized(&node_id(99)));
}
#[test]
fn higher_generation_wins_merge() {
let mut d1 = make_disseminator(1);
let mut d2 = make_disseminator(2);
// d1 joins
d1.join(2);
// d1 leaves
d1.leave(2);
// Gossip d1's entries to d2 out of order:
// First send the Active (gen 1), then the Left (gen 2)
let active_entry = PoolEntry::Membership(PoolMemberEntry {
pool_id: PoolId::from_name("test-pool"),
node_id: [1u8; 32],
state: PoolMemberState::Active,
generation: 1,
});
let left_entry = PoolEntry::Membership(PoolMemberEntry {
pool_id: PoolId::from_name("test-pool"),
node_id: [1u8; 32],
state: PoolMemberState::Left,
generation: 2,
});
// Apply Left first (gen 2), then Active (gen 1) — Active should be rejected
d2.merge_entry(left_entry);
let changed = d2.merge_entry(active_entry);
assert!(!changed, "lower generation should not win");
// d2 should see node_id(1) as Left
assert_eq!(d2.member_count(), 0); // Active count is 0
}
#[test]
fn two_disseminators_converge_via_gossip_exchange() {
let mut d1 = make_disseminator(1);
let mut d2 = make_disseminator(2);
// d1 joins and announces content
d1.join(2);
let hash = ContentHash::of(b"shared-file");
d1.announce_content(hash, 2);
// d2 joins
d2.join(2);
// Simulate gossip: d1 → d2
let pending = d1.take_pending_inner(100);
let bytes = serialize_each(&pending);
let parsed: Vec<PoolEntry> = deserialize_each(&bytes);
d2.apply_incoming_inner(parsed, 2);
// d2 should now see d1 as a member and know about the content
assert_eq!(d2.member_count(), 2);
assert_eq!(d2.locate_content(&hash), vec![node_id(1)]);
// Simulate gossip: d2 → d1
let pending = d2.take_pending_inner(100);
let bytes = serialize_each(&pending);
let parsed: Vec<PoolEntry> = deserialize_each(&bytes);
d1.apply_incoming_inner(parsed, 2);
// d1 should now see d2 as a member
assert_eq!(d1.member_count(), 2);
}
#[test]
fn three_node_convergence_loop() {
let pool = PoolId::from_name("test-pool");
let mut nodes: Vec<PoolDisseminator> = (0..3)
.map(|i| PoolDisseminator::new(pool, "test-pool".into(), node_id(i as u8), 3))
.collect();
// Each node joins
for n in &mut nodes {
n.join(3);
}
// Node 0 announces content
let hash = ContentHash::of(b"convergence-test");
nodes[0].announce_content(hash, 3);
// Run 5 gossip rounds where each node exchanges with all others
for _ in 0..5 {
// Collect pending from each node
let pending_bytes: Vec<Vec<Vec<u8>>> = nodes
.iter_mut()
.map(|n| serialize_each(&n.take_pending_inner(100)))
.collect();
// Apply each node's pending to all other nodes
for (sender_idx, bytes) in pending_bytes.iter().enumerate() {
for (receiver_idx, node) in nodes.iter_mut().enumerate() {
if sender_idx != receiver_idx {
let parsed: Vec<PoolEntry> = deserialize_each(bytes);
node.apply_incoming_inner(parsed, 3);
}
}
}
}
// All nodes should agree on membership and content locations
for (i, node) in nodes.iter().enumerate() {
assert_eq!(node.member_count(), 3, "node {i} should see 3 members");
assert_eq!(
node.locate_content(&hash),
vec![node_id(0)],
"node {i} should know content is on node 0"
);
}
}
}

79
crates/datastore/src/pool/messages.rs Normal file
View file

@ -0,0 +1,79 @@
//! Messages for the `PoolCoordinator` actor.
use std::collections::BTreeMap;
use distribution::types::NodeId;
use shared_types::ContentHash;
/// Messages handled by the `PoolCoordinator` actor.
#[derive(Debug, Clone)]
pub enum PoolCoordinatorMsg {
// ── User-facing operations ──────────────────────────────────────────
/// Store data in the pool (placement-aware).
PoolPut {
data: Vec<u8>,
name: Option<String>,
tags: BTreeMap<String, String>,
reply_to: swactor::actor::ActorAddress,
},
/// Retrieve data from the pool (location-aware).
PoolGet {
content_hash: ContentHash,
reply_to: swactor::actor::ActorAddress,
},
/// Delete data from the pool.
PoolDelete {
content_hash: ContentHash,
reply_to: swactor::actor::ActorAddress,
},
/// List objects in the pool.
PoolList {
name_filter: Option<String>,
reply_to: swactor::actor::ActorAddress,
},
/// Pool status (members, capacity, content count).
PoolStatus {
reply_to: swactor::actor::ActorAddress,
},
// ── Pool lifecycle ──────────────────────────────────────────────────
/// Join the pool.
JoinPool {
reply_to: swactor::actor::ActorAddress,
},
/// Leave the pool.
LeavePool {
reply_to: swactor::actor::ActorAddress,
},
// ── Auth management ─────────────────────────────────────────────────
/// Grant a node access to the pool.
GrantPoolAccess {
target: NodeId,
reply_to: swactor::actor::ActorAddress,
},
/// Revoke a node's access to the pool.
RevokePoolAccess {
target: NodeId,
reply_to: swactor::actor::ActorAddress,
},
// ── Periodic ────────────────────────────────────────────────────────
/// Periodic tick: announce capacity, drive dissemination.
PoolTick,
}
/// Pool-specific response variants.
#[derive(Debug, Clone)]
pub enum PoolResponse {
/// Pool status snapshot.
PoolStatus {
pool_name: String,
pool_id_hex: String,
member_count: usize,
content_count: usize,
total_bytes: u64,
used_bytes: u64,
members: Vec<String>,
},
}

8
crates/datastore/src/pool/mod.rs Normal file
View file

@ -0,0 +1,8 @@
//! Pooled datastore protocol.
//!
//! A shared storage pool where multiple nodes contribute storage capacity
//! and converge on a shared view of what content lives where.
pub mod disseminator;
pub mod messages;
pub mod coordinator;

1
crates/datastore/tests/dashboard_integration_test.rs
View file

@ -48,6 +48,7 @@ fn dashboard_reflects_datastore_operations() {
port: dash_port,
..Default::default()
});
dash.start_http_standalone();
let collector = dashboard::collector::StatsCollector::new(2);
let mut rt = Runtime::new(RuntimeConfig {

341
crates/datastore/tests/pool_tests.rs Normal file
View file

@ -0,0 +1,341 @@
//! Pool protocol integration tests.
//!
//! Tests the pool disseminator's convergence behavior, the gossip channel
//! integration, and the coordinator actor's lifecycle.
use std::sync::{Arc, Mutex};
use distribution::gossip_channel::{GossipChannel, serialize_each};
use distribution::types::NodeId;
use shared_types::ContentHash;
use shared_types::pool::*;
use swactor_datastore::pool::disseminator::{PoolDisseminator, SharedPoolChannel};
fn node_id(b: u8) -> NodeId {
NodeId([b; 32])
}
fn make_pool_disseminator(node_byte: u8) -> PoolDisseminator {
PoolDisseminator::new(
PoolId::from_name("test-pool"),
"test-pool".into(),
node_id(node_byte),
3,
)
}
// ─── Disseminator convergence scenarios ────────────────────────────────────
/// Two nodes join a pool and exchange gossip until they converge
/// on identical state.
#[test]
fn two_nodes_converge_on_membership() {
let mut d1 = make_pool_disseminator(1);
let mut d2 = make_pool_disseminator(2);
d1.join(2);
d2.join(2);
// Simulate 3 gossip rounds
for _ in 0..3 {
let mut ch1 = SharedPoolChannel::new(Arc::new(Mutex::new(d1)));
let mut ch2 = SharedPoolChannel::new(Arc::new(Mutex::new(d2)));
let bytes1 = ch1.take_pending_bytes(100);
let bytes2 = ch2.take_pending_bytes(100);
ch1.apply_incoming_bytes(&bytes2, 2);
ch2.apply_incoming_bytes(&bytes1, 2);
d1 = Arc::try_unwrap(ch1.into_inner()).unwrap().into_inner().unwrap();
d2 = Arc::try_unwrap(ch2.into_inner()).unwrap().into_inner().unwrap();
}
assert_eq!(d1.member_count(), 2);
assert_eq!(d2.member_count(), 2);
}
/// Content announced on one node is visible on another after gossip.
#[test]
fn content_location_propagates_via_gossip() {
let mut d1 = make_pool_disseminator(1);
let mut d2 = make_pool_disseminator(2);
let hash = ContentHash::of(b"test-content");
d1.join(2);
d2.join(2);
d1.announce_content(hash, 2);
// Gossip d1 → d2
let mut ch1 = SharedPoolChannel::new(Arc::new(Mutex::new(d1)));
let bytes = ch1.take_pending_bytes(100);
let _d1 = Arc::try_unwrap(ch1.into_inner()).unwrap().into_inner().unwrap();
let mut ch2 = SharedPoolChannel::new(Arc::new(Mutex::new(d2)));
ch2.apply_incoming_bytes(&bytes, 2);
let d2 = Arc::try_unwrap(ch2.into_inner()).unwrap().into_inner().unwrap();
assert_eq!(d2.locate_content(&hash), vec![node_id(1)]);
}
/// A node's leave is propagated and removes it from the active member list
/// on the receiving node.
#[test]
fn leave_propagates_via_gossip() {
let mut d1 = make_pool_disseminator(1);
let mut d2 = make_pool_disseminator(2);
d1.join(2);
d2.join(2);
// Exchange so both see each other
let mut ch1 = SharedPoolChannel::new(Arc::new(Mutex::new(d1)));
let mut ch2 = SharedPoolChannel::new(Arc::new(Mutex::new(d2)));
let bytes1 = ch1.take_pending_bytes(100);
let bytes2 = ch2.take_pending_bytes(100);
ch1.apply_incoming_bytes(&bytes2, 2);
ch2.apply_incoming_bytes(&bytes1, 2);
d1 = Arc::try_unwrap(ch1.into_inner()).unwrap().into_inner().unwrap();
d2 = Arc::try_unwrap(ch2.into_inner()).unwrap().into_inner().unwrap();
assert_eq!(d1.member_count(), 2);
assert_eq!(d2.member_count(), 2);
// d1 leaves
d1.leave(2);
// Gossip leave to d2
let mut ch1 = SharedPoolChannel::new(Arc::new(Mutex::new(d1)));
let bytes = ch1.take_pending_bytes(100);
let _d1 = Arc::try_unwrap(ch1.into_inner()).unwrap().into_inner().unwrap();
let mut ch2 = SharedPoolChannel::new(Arc::new(Mutex::new(d2)));
ch2.apply_incoming_bytes(&bytes, 2);
let d2 = Arc::try_unwrap(ch2.into_inner()).unwrap().into_inner().unwrap();
assert_eq!(d2.member_count(), 1); // only d2 remains active
}
/// Content tombstone propagates and removes the location.
#[test]
fn content_deletion_propagates() {
let mut d1 = make_pool_disseminator(1);
let mut d2 = make_pool_disseminator(2);
let hash = ContentHash::of(b"ephemeral");
d1.join(2);
d2.join(2);
d1.announce_content(hash, 2);
// Propagate content announcement
let mut ch1 = SharedPoolChannel::new(Arc::new(Mutex::new(d1)));
let bytes = ch1.take_pending_bytes(100);
d1 = Arc::try_unwrap(ch1.into_inner()).unwrap().into_inner().unwrap();
let mut ch2 = SharedPoolChannel::new(Arc::new(Mutex::new(d2)));
ch2.apply_incoming_bytes(&bytes, 2);
d2 = Arc::try_unwrap(ch2.into_inner()).unwrap().into_inner().unwrap();
assert_eq!(d2.locate_content(&hash).len(), 1);
// d1 removes content
d1.remove_content(hash, 2);
// Propagate tombstone
let mut ch1 = SharedPoolChannel::new(Arc::new(Mutex::new(d1)));
let bytes = ch1.take_pending_bytes(100);
let _ = Arc::try_unwrap(ch1.into_inner()).unwrap().into_inner().unwrap();
let mut ch2 = SharedPoolChannel::new(Arc::new(Mutex::new(d2)));
ch2.apply_incoming_bytes(&bytes, 2);
d2 = Arc::try_unwrap(ch2.into_inner()).unwrap().into_inner().unwrap();
assert!(d2.locate_content(&hash).is_empty());
}
/// ACL grant propagates to other nodes.
#[test]
fn acl_grant_propagates() {
let mut d1 = make_pool_disseminator(1);
let mut d2 = make_pool_disseminator(2);
d1.grant_access(node_id(3), 2);
let mut ch1 = SharedPoolChannel::new(Arc::new(Mutex::new(d1)));
let bytes = ch1.take_pending_bytes(100);
let _ = Arc::try_unwrap(ch1.into_inner()).unwrap().into_inner().unwrap();
let mut ch2 = SharedPoolChannel::new(Arc::new(Mutex::new(d2)));
ch2.apply_incoming_bytes(&bytes, 2);
d2 = Arc::try_unwrap(ch2.into_inner()).unwrap().into_inner().unwrap();
assert!(d2.is_node_authorized(&node_id(3)));
assert!(!d2.is_node_authorized(&node_id(4)));
}
/// Capacity announcement propagates and is reflected in summary.
#[test]
fn capacity_propagates_and_summarizes() {
let mut d1 = make_pool_disseminator(1);
let mut d2 = make_pool_disseminator(2);
d1.join(2);
d2.join(2);
d1.announce_capacity(1_000_000, 100_000, 2);
d2.announce_capacity(2_000_000, 200_000, 2);
// Exchange
let mut ch1 = SharedPoolChannel::new(Arc::new(Mutex::new(d1)));
let mut ch2 = SharedPoolChannel::new(Arc::new(Mutex::new(d2)));
let bytes1 = ch1.take_pending_bytes(100);
let bytes2 = ch2.take_pending_bytes(100);
ch1.apply_incoming_bytes(&bytes2, 2);
ch2.apply_incoming_bytes(&bytes1, 2);
d1 = Arc::try_unwrap(ch1.into_inner()).unwrap().into_inner().unwrap();
d2 = Arc::try_unwrap(ch2.into_inner()).unwrap().into_inner().unwrap();
let (total1, used1) = d1.pool_capacity_summary();
let (total2, used2) = d2.pool_capacity_summary();
assert_eq!(total1, 3_000_000);
assert_eq!(used1, 300_000);
assert_eq!(total2, 3_000_000);
assert_eq!(used2, 300_000);
}
/// The node_with_most_free_space query returns the correct node.
#[test]
fn placement_query_picks_node_with_most_space() {
let mut d = make_pool_disseminator(1);
d.join(3);
// Simulate node 2 joining and having lots of space
let member2 = PoolEntry::Membership(PoolMemberEntry {
pool_id: PoolId::from_name("test-pool"),
node_id: [2u8; 32],
state: PoolMemberState::Active,
generation: 1,
});
let cap2 = PoolEntry::Capacity(PoolCapacityEntry {
pool_id: PoolId::from_name("test-pool"),
node_id: [2u8; 32],
total_bytes: 10_000_000,
used_bytes: 1_000_000,
generation: 1,
});
// Simulate node 3 with less free space
let member3 = PoolEntry::Membership(PoolMemberEntry {
pool_id: PoolId::from_name("test-pool"),
node_id: [3u8; 32],
state: PoolMemberState::Active,
generation: 1,
});
let cap3 = PoolEntry::Capacity(PoolCapacityEntry {
pool_id: PoolId::from_name("test-pool"),
node_id: [3u8; 32],
total_bytes: 5_000_000,
used_bytes: 4_000_000,
generation: 1,
});
let entries = vec![member2, cap2, member3, cap3];
let bytes = serialize_each(&entries);
let mut ch = SharedPoolChannel::new(Arc::new(Mutex::new(d)));
ch.apply_incoming_bytes(&bytes, 3);
d = Arc::try_unwrap(ch.into_inner()).unwrap().into_inner().unwrap();
// d1 has no capacity announced, node 2 has 9M free, node 3 has 1M free
let best = d.node_with_most_free_space().unwrap();
assert_eq!(best, node_id(2));
}
/// Five-node convergence: all nodes join, one announces content, everyone converges.
#[test]
fn five_node_pool_converges() {
let pool = PoolId::from_name("five-pool");
let mut nodes: Vec<PoolDisseminator> = (0..5)
.map(|i| PoolDisseminator::new(pool, "five-pool".into(), node_id(i as u8), 3))
.collect();
// All join
for n in &mut nodes {
n.join(5);
}
// Node 0 and 2 announce content
let hash_a = ContentHash::of(b"file-a");
let hash_b = ContentHash::of(b"file-b");
nodes[0].announce_content(hash_a, 5);
nodes[2].announce_content(hash_b, 5);
// Run 10 gossip rounds
for _ in 0..10 {
let mut channels: Vec<SharedPoolChannel> = nodes
.into_iter()
.map(|d| SharedPoolChannel::new(Arc::new(Mutex::new(d))))
.collect();
let pending: Vec<Vec<Vec<u8>>> = channels
.iter_mut()
.map(|ch| ch.take_pending_bytes(100))
.collect();
for (recv_idx, ch) in channels.iter_mut().enumerate() {
for (send_idx, bytes) in pending.iter().enumerate() {
if recv_idx != send_idx {
ch.apply_incoming_bytes(bytes, 5);
}
}
}
nodes = channels
.into_iter()
.map(|ch| Arc::try_unwrap(ch.into_inner()).unwrap().into_inner().unwrap())
.collect();
}
for (i, node) in nodes.iter().enumerate() {
assert_eq!(node.member_count(), 5, "node {i} should see 5 members");
assert_eq!(
node.locate_content(&hash_a),
vec![node_id(0)],
"node {i} should locate file-a on node 0"
);
assert_eq!(
node.locate_content(&hash_b),
vec![node_id(2)],
"node {i} should locate file-b on node 2"
);
}
}
// ─── GossipChannel wire format tests ───────────────────────────────────────
/// Verify the GossipChannel topic tag is correct.
#[test]
fn shared_pool_channel_topic_tag() {
let d = make_pool_disseminator(1);
let ch = SharedPoolChannel::new(Arc::new(Mutex::new(d)));
assert_eq!(GossipChannel::topic_tag(&ch), "pool");
}
/// Verify serialization round-trip through GossipChannel bytes interface.
#[test]
fn gossip_channel_bytes_roundtrip() {
let mut d = make_pool_disseminator(1);
d.join(2);
let hash = ContentHash::of(b"roundtrip");
d.announce_content(hash, 2);
let mut ch = SharedPoolChannel::new(Arc::new(Mutex::new(d)));
let bytes = ch.take_pending_bytes(100);
assert!(!bytes.is_empty());
// Apply to a fresh disseminator
let d2 = make_pool_disseminator(2);
let mut ch2 = SharedPoolChannel::new(Arc::new(Mutex::new(d2)));
ch2.apply_incoming_bytes(&bytes, 2);
let d2 = Arc::try_unwrap(ch2.into_inner()).unwrap().into_inner().unwrap();
assert_eq!(d2.member_count(), 1);
assert_eq!(d2.locate_content(&hash), vec![node_id(1)]);
}

268
crates/distribution/src/gossip_channel.rs Normal file
View file

@ -0,0 +1,268 @@
//! Generic gossip channel abstraction.
//!
//! Provides `GossipChannel` — a trait for any topic that wants to piggyback
//! on SWIM protocol messages — and `DisseminationBuffer<T>` — a reusable
//! budget-limited dissemination queue that replaces the 4 independent copies
//! of the `Λ * ceil(log₂(n))` pattern.
use serde::{Serialize, de::DeserializeOwned};
// ─── GossipChannel trait ────────────────────────────────────────────────────
/// A gossip channel that can piggyback serialized entries on SWIM messages.
///
/// Each channel has a unique topic tag and handles its own serialization.
/// The wire transport uses type-erased `Vec<u8>` entries.
pub trait GossipChannel: Send {
/// Unique string tag identifying this channel in the piggyback payload.
fn topic_tag(&self) -> &'static str;
/// Take up to `max_entries` pending entries, serialized as bytes.
fn take_pending_bytes(&mut self, max_entries: usize) -> Vec<Vec<u8>>;
/// Apply incoming entries (deserialized from bytes) received from gossip.
fn apply_incoming_bytes(&mut self, entries: &[Vec<u8>], cluster_size: usize);
/// Re-enqueue all state for dissemination (anti-entropy on membership recovery).
fn re_disseminate_all(&mut self, cluster_size: usize);
/// Handle a node being declared dead.
fn on_node_death(&mut self, node_id: &crate::types::NodeId);
/// Periodic garbage collection tick.
fn gc_tick(&mut self);
}
// ─── DisseminationBuffer<T> ────────────────────────────────────────────────
/// A queued entry with a remaining transmit budget.
#[derive(Debug, Clone)]
struct BufferEntry<T> {
item: T,
remaining: usize,
}
/// Reusable generic dissemination buffer.
///
/// Manages budget-limited gossip dissemination for any entry type.
/// Each entry is transmitted `Λ * ceil(log₂(n))` times before eviction.
#[derive(Debug)]
pub struct DisseminationBuffer<T> {
entries: Vec<BufferEntry<T>>,
lambda: usize,
}
impl<T: Clone> DisseminationBuffer<T> {
/// Create a new buffer with the given dissemination multiplier (Λ).
pub fn new(lambda: usize) -> Self {
Self {
entries: Vec::new(),
lambda,
}
}
/// Compute the transmit budget: `Λ * ceil(log₂(max(n, 2)))`.
pub fn transmit_budget(&self, cluster_size: usize) -> usize {
let n = cluster_size.max(2) as f64;
let log_n = n.log2().ceil() as usize;
self.lambda * log_n.max(1)
}
/// Enqueue an entry for dissemination. Does not check for duplicates.
pub fn enqueue(&mut self, item: T, cluster_size: usize) {
let budget = self.transmit_budget(cluster_size);
self.entries.push(BufferEntry {
item,
remaining: budget,
});
}
/// Enqueue an entry, replacing an existing one if `matcher` returns true.
/// If no match is found, pushes a new entry.
pub fn enqueue_or_replace<F>(&mut self, item: T, cluster_size: usize, matcher: F)
where
F: Fn(&T) -> bool,
{
let budget = self.transmit_budget(cluster_size);
if let Some(existing) = self.entries.iter_mut().find(|e| matcher(&e.item)) {
existing.item = item;
existing.remaining = budget;
return;
}
self.entries.push(BufferEntry {
item,
remaining: budget,
});
}
/// Take up to `max_count` entries for piggyback.
/// Decrements remaining budget and evicts exhausted entries.
pub fn take(&mut self, max_count: usize) -> Vec<T> {
let count = max_count.min(self.entries.len());
let mut result = Vec::with_capacity(count);
for entry in self.entries.iter_mut().take(count) {
result.push(entry.item.clone());
entry.remaining = entry.remaining.saturating_sub(1);
}
self.entries.retain(|e| e.remaining > 0);
result
}
/// Re-enqueue all given items with fresh budgets.
pub fn re_enqueue_all(&mut self, items: impl IntoIterator<Item = T>, cluster_size: usize) {
for item in items {
self.enqueue(item, cluster_size);
}
}
/// Retain only entries matching the predicate.
pub fn retain<F>(&mut self, mut predicate: F)
where
F: FnMut(&T) -> bool,
{
self.entries.retain(|e| predicate(&e.item));
}
/// Number of queued entries.
pub fn len(&self) -> usize {
self.entries.len()
}
/// Whether the buffer is empty.
pub fn is_empty(&self) -> bool {
self.entries.is_empty()
}
}
// ─── Serialization helpers ─────────────────────────────────────────────────
/// Serialize a slice of items to a Vec of byte vectors.
pub fn serialize_each<T: Serialize>(items: &[T]) -> Vec<Vec<u8>> {
items
.iter()
.filter_map(|item| serde_json::to_vec(item).ok())
.collect()
}
/// Deserialize a slice of byte vectors into items, skipping failures.
pub fn deserialize_each<T: DeserializeOwned>(entries: &[Vec<u8>]) -> Vec<T> {
entries
.iter()
.filter_map(|bytes| serde_json::from_slice(bytes).ok())
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn budget_math_two_nodes() {
let buf: DisseminationBuffer<u32> = DisseminationBuffer::new(3);
// log2(2) = 1, ceil = 1, 3 * 1 = 3
assert_eq!(buf.transmit_budget(2), 3);
}
#[test]
fn budget_math_single_node_floors_to_two() {
let buf: DisseminationBuffer<u32> = DisseminationBuffer::new(3);
// cluster_size=1 → max(1,2)=2, log2(2)=1, 3*1=3
assert_eq!(buf.transmit_budget(1), 3);
}
#[test]
fn budget_math_32_nodes() {
let buf: DisseminationBuffer<u32> = DisseminationBuffer::new(3);
// log2(32) = 5, 3 * 5 = 15
assert_eq!(buf.transmit_budget(32), 15);
}
#[test]
fn budget_math_33_nodes() {
let buf: DisseminationBuffer<u32> = DisseminationBuffer::new(3);
// log2(33) ≈ 5.04, ceil = 6, 3 * 6 = 18
assert_eq!(buf.transmit_budget(33), 18);
}
#[test]
fn enqueue_take_evicts_after_budget() {
let mut buf = DisseminationBuffer::new(3);
buf.enqueue(42u32, 2); // budget = 3
// Take 3 times — each take decrements once
let r1 = buf.take(1);
assert_eq!(r1, vec![42]);
assert_eq!(buf.len(), 1);
let r2 = buf.take(1);
assert_eq!(r2, vec![42]);
assert_eq!(buf.len(), 1);
let r3 = buf.take(1);
assert_eq!(r3, vec![42]);
// After 3 takes, budget exhausted → evicted
assert_eq!(buf.len(), 0);
}
#[test]
fn enqueue_or_replace_replaces_matching_entry() {
let mut buf = DisseminationBuffer::new(3);
buf.enqueue_or_replace(("key", 1), 2, |e| e.0 == "key");
buf.enqueue_or_replace(("key", 2), 2, |e| e.0 == "key");
assert_eq!(buf.len(), 1);
let taken = buf.take(1);
assert_eq!(taken, vec![("key", 2)]);
}
#[test]
fn enqueue_or_replace_adds_when_no_match() {
let mut buf = DisseminationBuffer::new(3);
buf.enqueue_or_replace(("a", 1), 2, |e| e.0 == "a");
buf.enqueue_or_replace(("b", 2), 2, |e| e.0 == "b");
assert_eq!(buf.len(), 2);
}
#[test]
fn re_enqueue_all_refreshes_budgets() {
let mut buf = DisseminationBuffer::new(3);
buf.enqueue(1u32, 2);
buf.enqueue(2u32, 2);
// Drain them
for _ in 0..3 {
buf.take(2);
}
assert!(buf.is_empty());
// Re-enqueue
buf.re_enqueue_all(vec![1, 2, 3], 2);
assert_eq!(buf.len(), 3);
}
#[test]
fn retain_removes_non_matching() {
let mut buf = DisseminationBuffer::new(3);
buf.enqueue(1u32, 2);
buf.enqueue(2u32, 2);
buf.enqueue(3u32, 2);
buf.retain(|item| *item != 2);
assert_eq!(buf.len(), 2);
let taken = buf.take(3);
assert_eq!(taken, vec![1, 3]);
}
#[test]
fn serialize_deserialize_roundtrip() {
let items = vec![1u32, 2, 3];
let bytes = serialize_each(&items);
let recovered: Vec<u32> = deserialize_each(&bytes);
assert_eq!(recovered, items);
}
}

24
crates/distribution/src/iroh_driver.rs
View file

@ -507,15 +507,31 @@ impl IrohDriver {
self.connections.remove(&node_id);
}
// Resolve relay URL: explicit cache → SWIM metadata gossip → own home relay
let relay = self.peer_relay_urls.get(&node_id).cloned()
.or_else(|| {
self.node
.relay_url(&node_id)
.and_then(|s| s.parse::<iroh::RelayUrl>().ok())
})
.or_else(|| self.endpoint.addr().relay_urls().next().cloned());
let endpoint = self.endpoint.clone();
let conn = if let Some(relay) = self.peer_relay_urls.get(&node_id) {
let addr = EndpointAddr::new(key).with_relay_url(relay.clone());
let connect_timeout = Duration::from_secs(2);
let conn = if let Some(relay) = relay {
let addr = EndpointAddr::new(key).with_relay_url(relay);
self.rt.block_on(async {
endpoint.connect(addr, ALPN).await
match tokio::time::timeout(connect_timeout, endpoint.connect(addr, ALPN)).await {
Ok(result) => result.map_err(|e| -> Box<dyn std::error::Error> { Box::new(e) }),
Err(_) => Err("connect timeout".into()),
}
})?
} else {
self.rt.block_on(async {
endpoint.connect(key, ALPN).await
match tokio::time::timeout(connect_timeout, endpoint.connect(key, ALPN)).await {
Ok(result) => result.map_err(|e| -> Box<dyn std::error::Error> { Box::new(e) }),
Err(_) => Err("connect timeout".into()),
}
})?
};

9
crates/python/src/lib.rs
View file

@ -4,7 +4,7 @@ use std::cell::RefCell;
use pyo3::prelude::*;
use pyo3::types::PyModule;
use ::swactor::actor::{Actor, ActorAddress, ActorInterface, AnyActor, Ctx};
use ::swactor::actor::{Actor, ActorAddress, ActorInterface, AnyActor, Ctx, Environment, SpawnRequest};
use ::swactor::config::{BackoffPolicy, RuntimeConfig};
use ::swactor::runtime::{Inbox, Runtime, RuntimeHandle};
@ -182,7 +182,12 @@ impl ActorInterface for PyActor {
Effect::Spawn { addr, handler } => {
let actor = PyActor::new(handler);
let boxed: Box<dyn AnyActor> = Box::new(Actor::new(actor));
let _ = ctx.raw_inner().spawn_any(addr, boxed);
ctx.raw_inner().spawn_any(SpawnRequest {
addr,
actor: boxed,
parent: Some(ctx.self_addr()),
env: Environment::new(),
});
}
}
}

252
crates/shared-types/src/pool.rs Normal file
View file

@ -0,0 +1,252 @@
//! Shared types for the pooled datastore protocol.
//!
//! Types live in `shared-types` because both `distribution` and `datastore`
//! depend on them.
use std::fmt;
use serde::{Deserialize, Serialize};
use crate::ContentHash;
// Re-export NodeId-shaped bytes — pool uses the same 32-byte node identifier.
// The actual NodeId type lives in `distribution::types`, but we use raw [u8; 32]
// here to avoid a circular dependency. Callers convert as needed.
// ─── PoolId ────────────────────────────────────────────────────────────────
/// Unique pool identifier: blake3(name_bytes).
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct PoolId(pub [u8; 32]);
impl PoolId {
/// Create a pool ID from a human-readable name.
pub fn from_name(name: &str) -> Self {
PoolId(*blake3::hash(name.as_bytes()).as_bytes())
}
/// Encode as lowercase hex string.
pub fn to_hex(&self) -> String {
let mut s = String::with_capacity(64);
for b in &self.0 {
use fmt::Write;
write!(s, "{:02x}", b).unwrap();
}
s
}
/// Parse a 64-character hex string into a PoolId.
pub fn from_hex(hex: &str) -> Option<Self> {
if hex.len() != 64 {
return None;
}
let mut bytes = [0u8; 32];
for (i, chunk) in hex.as_bytes().chunks(2).enumerate() {
let hi = hex_digit(chunk[0])?;
let lo = hex_digit(chunk[1])?;
bytes[i] = (hi << 4) | lo;
}
Some(PoolId(bytes))
}
}
impl fmt::Display for PoolId {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
for b in &self.0[..8] {
write!(f, "{:02x}", b)?;
}
write!(f, "\u{2026}")
}
}
// ─── Pool Membership ───────────────────────────────────────────────────────
/// Whether a node is actively participating in a pool.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum PoolMemberState {
Active,
Left,
}
/// A node's membership in a pool. Higher generation always wins.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct PoolMemberEntry {
pub pool_id: PoolId,
pub node_id: [u8; 32],
pub state: PoolMemberState,
pub generation: u64,
}
// ─── Pool Capacity ─────────────────────────────────────────────────────────
/// A node's storage capacity announcement. Higher generation always wins.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct PoolCapacityEntry {
pub pool_id: PoolId,
pub node_id: [u8; 32],
pub total_bytes: u64,
pub used_bytes: u64,
pub generation: u64,
}
// ─── Content Location ──────────────────────────────────────────────────────
/// Where a content hash is stored. Key: (pool_id, content_hash, node_id).
/// Higher generation wins. Tombstones indicate deletion.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct ContentLocationEntry {
pub pool_id: PoolId,
pub content_hash: ContentHash,
pub node_id: [u8; 32],
pub generation: u64,
pub tombstone: bool,
}
// ─── Pool ACL ──────────────────────────────────────────────────────────────
/// Authorization for a node to join a pool. Higher generation wins.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct PoolACLEntry {
pub pool_id: PoolId,
pub node_id: [u8; 32],
pub granted_by: [u8; 32],
pub generation: u64,
pub revoked: bool,
}
// ─── Tagged Union ──────────────────────────────────────────────────────────
/// All pool entry variants, used for gossip serialization.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum PoolEntry {
Membership(PoolMemberEntry),
Capacity(PoolCapacityEntry),
ContentLocation(ContentLocationEntry),
ACL(PoolACLEntry),
}
// ─── Pool Config ───────────────────────────────────────────────────────────
/// Configuration for a pool.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PoolConfig {
pub pool_name: String,
pub pool_id: PoolId,
/// Total bytes this node pledges to the pool.
pub capacity_bytes: u64,
/// Tombstone TTL in ticks before GC.
pub tombstone_ttl: u64,
/// GC interval in ticks.
pub gc_interval: u64,
/// Dissemination multiplier (Λ).
pub dissemination_lambda: usize,
}
impl PoolConfig {
pub fn new(pool_name: &str, capacity_bytes: u64) -> Self {
Self {
pool_name: pool_name.to_string(),
pool_id: PoolId::from_name(pool_name),
capacity_bytes,
tombstone_ttl: 3600,
gc_interval: 1000,
dissemination_lambda: 3,
}
}
}
// ─── Helpers ───────────────────────────────────────────────────────────────
fn hex_digit(b: u8) -> Option<u8> {
match b {
b'0'..=b'9' => Some(b - b'0'),
b'a'..=b'f' => Some(b - b'a' + 10),
b'A'..=b'F' => Some(b - b'A' + 10),
_ => None,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn pool_id_from_name_is_deterministic() {
let a = PoolId::from_name("test-pool");
let b = PoolId::from_name("test-pool");
assert_eq!(a, b);
}
#[test]
fn pool_id_different_names_differ() {
let a = PoolId::from_name("pool-a");
let b = PoolId::from_name("pool-b");
assert_ne!(a, b);
}
#[test]
fn pool_id_hex_roundtrip() {
let id = PoolId::from_name("roundtrip-test");
let hex = id.to_hex();
let recovered = PoolId::from_hex(&hex).unwrap();
assert_eq!(id, recovered);
}
#[test]
fn pool_entry_serde_roundtrip() {
let entry = PoolEntry::Membership(PoolMemberEntry {
pool_id: PoolId::from_name("test"),
node_id: [1u8; 32],
state: PoolMemberState::Active,
generation: 1,
});
let bytes = serde_json::to_vec(&entry).unwrap();
let recovered: PoolEntry = serde_json::from_slice(&bytes).unwrap();
match recovered {
PoolEntry::Membership(m) => {
assert_eq!(m.node_id, [1u8; 32]);
assert_eq!(m.state, PoolMemberState::Active);
assert_eq!(m.generation, 1);
}
_ => panic!("wrong variant"),
}
}
#[test]
fn higher_generation_wins_for_membership() {
let old = PoolMemberEntry {
pool_id: PoolId::from_name("test"),
node_id: [1u8; 32],
state: PoolMemberState::Active,
generation: 1,
};
let new = PoolMemberEntry {
pool_id: PoolId::from_name("test"),
node_id: [1u8; 32],
state: PoolMemberState::Left,
generation: 2,
};
assert!(new.generation > old.generation);
}
#[test]
fn content_location_tombstone_semantics() {
let live = ContentLocationEntry {
pool_id: PoolId::from_name("test"),
content_hash: ContentHash::of(b"hello"),
node_id: [1u8; 32],
generation: 1,
tombstone: false,
};
let dead = ContentLocationEntry {
pool_id: PoolId::from_name("test"),
content_hash: ContentHash::of(b"hello"),
node_id: [1u8; 32],
generation: 2,
tombstone: true,
};
assert!(!live.tombstone);
assert!(dead.tombstone);
assert!(dead.generation > live.generation);
}
}

56
crates/std/src/children_registry.rs Normal file
View file

@ -0,0 +1,56 @@
use std::sync::RwLock;
use swactor::actor::ActorAddress;
use swactor::{AddrMap, AddrSet};
/// Tracks parent → children relationships for orphan cleanup.
///
/// When a parent dies, unsupervised children are stopped automatically.
/// Entries are added in `on_spawn` and cleaned up on actor death.
pub struct ChildrenRegistry {
/// parent_addr → set of child addresses
children: RwLock<AddrMap<AddrSet>>,
}
impl ChildrenRegistry {
pub fn new() -> Self {
Self {
children: RwLock::new(AddrMap::default()),
}
}
/// Register a parent → child relationship.
pub fn register(&self, parent: ActorAddress, child: ActorAddress) {
self.children
.write()
.unwrap()
.entry(parent)
.or_default()
.insert(child);
}
/// Remove and return all children of a parent (for orphan handling).
pub fn take_children(&self, parent: &ActorAddress) -> Vec<ActorAddress> {
self.children
.write()
.unwrap()
.remove(parent)
.map(|set| set.into_iter().collect())
.unwrap_or_default()
}
/// Clean up entries for dead actors (as both parent and child).
pub fn cleanup(&self, dead: &[ActorAddress]) {
let mut map = self.children.write().unwrap();
for addr in dead {
// Remove as parent
map.remove(addr);
// Remove as child from any parent's set
for set in map.values_mut() {
set.remove(addr);
}
}
// Remove empty parent entries
map.retain(|_, set| !set.is_empty());
}
}

230
crates/std/src/ctx_ext.rs
View file

@ -1,10 +1,11 @@
use swactor::actor::{ActorAddress, ActorInterface, Ctx, Message, MonitorRef};
use swactor::actor::{ActorAddress, ActorInterface, Ctx, Environment, LogicalName, Message, MonitorRef, SystemInfo};
use swactor::Error;
use crate::resource_handle::ResourceHandle;
use crate::StdExtension;
use crate::timer_wheel::{CloneMsg, TimerRequest};
fn get_ext<'a>(ctx: &'a Ctx) -> &'a StdExtension {
pub(crate) fn get_ext<'a>(ctx: &'a Ctx) -> &'a StdExtension {
ctx.extension()
.expect("StdExtension not installed — use Runtime::with_extension()")
.as_any()
@ -18,15 +19,18 @@ fn get_ext<'a>(ctx: &'a Ctx) -> &'a StdExtension {
pub trait CtxMonitoring {
/// Subscribe to death notifications from `target`. Returns a [`MonitorRef`]
/// that can be used to cancel the subscription.
fn monitor(&self, target: ActorAddress) -> MonitorRef;
fn monitor(&self, target: ActorAddress) -> Result<MonitorRef, Error>;
/// Cancel a monitor subscription.
fn demonitor(&self, mref: MonitorRef);
}
impl CtxMonitoring for Ctx<'_> {
fn monitor(&self, target: ActorAddress) -> MonitorRef {
get_ext(self).monitor_registry.register(self.self_addr(), target)
fn monitor(&self, target: ActorAddress) -> Result<MonitorRef, Error> {
if let Some(caps) = self.env::<swactor::CapabilitySet>() {
caps.check_monitor(target)?;
}
Ok(get_ext(self).monitor_registry.register(self.self_addr(), target))
}
fn demonitor(&self, mref: MonitorRef) {
@ -60,7 +64,7 @@ impl CtxNaming for Ctx<'_> {
fn spawn_named<A: ActorInterface>(&self, name: impl Into<String>, actor: A) -> Result<ActorAddress, Error> {
let name = name.into();
let addr = self.spawn(actor)?;
let addr = self.spawn_builder(actor).env(LogicalName(name.clone())).finish()?;
if let Err(e) = get_ext(self).name_registry.register(name, addr) {
let _ = self.stop_actor(addr);
return Err(e);
@ -177,3 +181,217 @@ impl CtxGroups for Ctx<'_> {
get_ext(self).group_registry.members(group)
}
}
/// System introspection extension for [`Ctx`].
///
/// Provides convenience accessors for system-level information. Does NOT
/// require [`StdExtension`] — the data comes from the core runtime.
pub trait CtxSystem {
/// Returns the full [`SystemInfo`] snapshot.
fn system_info(&self) -> SystemInfo;
/// Index of the worker thread this actor is running on.
fn worker_id(&self) -> usize;
/// Total number of worker threads in the runtime.
fn num_workers(&self) -> usize;
/// Total number of live actors across all workers.
fn total_actors(&self) -> usize;
/// Milliseconds since the runtime was created.
fn uptime_ms(&self) -> u64;
}
impl CtxSystem for Ctx<'_> {
fn system_info(&self) -> SystemInfo {
Ctx::system_info(self)
}
fn worker_id(&self) -> usize {
Ctx::system_info(self).worker_id
}
fn num_workers(&self) -> usize {
Ctx::system_info(self).num_workers
}
fn total_actors(&self) -> usize {
Ctx::system_info(self).total_actors
}
fn uptime_ms(&self) -> u64 {
Ctx::system_info(self).uptime_ms
}
}
/// Lineage extension for [`Ctx`].
///
/// Exposes the actor's parent and supervisor. `parent()` does NOT require
/// [`StdExtension`] — the data is stored in core per-actor state.
/// `supervisor()` returns `None` gracefully when StdExtension is absent.
pub trait CtxLineage {
/// Returns the address of the actor that spawned this one, or `None`
/// if this actor was spawned externally via `Runtime::spawn`.
fn parent(&self) -> Option<ActorAddress>;
/// Returns the address of this actor's supervisor, or `None` if
/// unsupervised or StdExtension is not installed.
fn supervisor(&self) -> Option<ActorAddress>;
}
impl CtxLineage for Ctx<'_> {
fn parent(&self) -> Option<ActorAddress> {
Ctx::parent(self)
}
fn supervisor(&self) -> Option<ActorAddress> {
let ext = self.extension()?
.as_any()
.downcast_ref::<StdExtension>()?;
ext.supervisor_registry.lookup(&self.self_addr())
}
}
/// Per-actor self-introspection extension for [`Ctx`].
///
/// Exposes the actor's own operational metrics. Does NOT require
/// [`StdExtension`] — the data is snapshotted from core before each tick.
pub trait CtxSelfStats {
/// Total messages this actor has successfully processed (before the current tick).
fn messages_processed(&self) -> u64;
/// Number of messages in this actor's mailbox at the start of the current tick.
fn mailbox_depth(&self) -> usize;
/// Per-message-type counts for this actor, sorted descending by count.
fn message_type_counts(&self) -> &[(&'static str, u64)];
}
impl CtxSelfStats for Ctx<'_> {
fn messages_processed(&self) -> u64 {
Ctx::messages_processed(self)
}
fn mailbox_depth(&self) -> usize {
Ctx::mailbox_depth(self)
}
fn message_type_counts(&self) -> &[(&'static str, u64)] {
Ctx::message_type_counts(self)
}
}
/// Service resource extension for [`Ctx`].
///
/// Provides typed service discovery via the environment. Does NOT require
/// [`StdExtension`] — reads from the core environment (same as [`CtxEnvironment`]).
pub trait CtxResources {
/// Look up a service address by marker type `S`.
///
/// Returns `None` if no `ServiceBinding<S>` is present in the environment.
fn resource<S: 'static + Send + Sync>(&self) -> Option<ActorAddress>;
}
impl CtxResources for Ctx<'_> {
fn resource<S: 'static + Send + Sync>(&self) -> Option<ActorAddress> {
if let Some(caps) = self.env::<swactor::CapabilitySet>() {
if caps.check_service::<S>().is_err() {
return None;
}
}
self.env::<swactor::ServiceBinding<S>>().map(|b| b.addr)
}
}
/// Environment extension for [`Ctx`].
///
/// Provides access to the actor's inherited typed key-value environment.
/// Does NOT require [`StdExtension`] — the data is stored in core per-actor state.
pub trait CtxEnvironment {
/// Read a typed value from this actor's environment.
fn env<T: std::any::Any + Send + Sync>(&self) -> Option<&T>;
/// Access this actor's full environment.
fn environment(&self) -> &Environment;
}
impl CtxEnvironment for Ctx<'_> {
fn env<T: std::any::Any + Send + Sync>(&self) -> Option<&T> {
Ctx::env(self)
}
fn environment(&self) -> &Environment {
Ctx::environment(self)
}
}
/// Resource handle extension for [`Ctx`].
///
/// Provides `handle::<H>()` to construct typed proxy structs wrapping service
/// addresses for ergonomic domain-specific APIs. See [`ResourceHandle`] for
/// how to define a handle type.
pub trait CtxHandles {
/// Construct a typed resource handle from the service registry.
///
/// Returns `None` if no `ServiceBinding<H::Service>` is present in the
/// actor's environment (consistent with `ctx.resource()`, `ctx.where_is()`, etc).
fn handle<H: ResourceHandle>(&self) -> Option<H>;
}
impl CtxHandles for Ctx<'_> {
fn handle<H: ResourceHandle>(&self) -> Option<H> {
let binding = self.env::<swactor::ServiceBinding<H::Service>>()?;
Some(H::from_parts(binding.addr, self.self_addr()))
}
}
/// Lifecycle extension for [`Ctx`].
///
/// Provides suspend/resume capabilities with authorization:
/// only the actor itself or its supervisor can resume it.
pub trait CtxLifecycle {
/// Suspend this actor. Messages continue to queue but are not processed
/// until resumed by self or supervisor.
fn suspend_self(&self);
/// Resume a suspended actor. Only the actor itself or its supervisor
/// may call this. Returns `Err` if the caller is not authorized.
fn resume(&self, target: ActorAddress) -> Result<(), Error>;
}
impl CtxLifecycle for Ctx<'_> {
fn suspend_self(&self) {
Ctx::suspend_self(self);
}
fn resume(&self, target: ActorAddress) -> Result<(), Error> {
// Self-resume is always allowed
if target == self.self_addr() {
self.raw_inner().request_resume(target);
return Ok(());
}
// Supervisor can resume its child
let ext = get_ext(self);
if ext.supervisor_registry.lookup(&target) == Some(self.self_addr()) {
self.raw_inner().request_resume(target);
return Ok(());
}
Err(Error::from("resume denied: caller is not self or supervisor"))
}
}
/// Capability introspection extension for [`Ctx`].
pub trait CtxCapabilities {
fn capabilities(&self) -> Option<&swactor::CapabilitySet>;
fn is_restricted(&self) -> bool;
}
impl CtxCapabilities for Ctx<'_> {
fn capabilities(&self) -> Option<&swactor::CapabilitySet> {
Ctx::env(self)
}
fn is_restricted(&self) -> bool {
self.env::<swactor::CapabilitySet>().is_some()
}
}

52
crates/std/src/extension.rs
View file

@ -1,11 +1,14 @@
use std::any::Any;
use swactor::actor::{ActorAddress, Down, ExitReason, StopReason};
use swactor::actor::{ActorAddress, Down, Environment, EnvironmentBuilder, ExitReason, ExitValue, SpawnTimestamp, StopReason, StopSignal};
use swactor::extension::{RuntimeExtension, WorkerExtension};
use crate::children_registry::ChildrenRegistry;
use crate::group_registry::GroupRegistry;
use crate::monitor_registry::MonitorRegistry;
use crate::name_registry::NameRegistry;
use crate::service_registry::ServiceRegistry;
use crate::supervisor_registry::SupervisorRegistry;
use crate::timer_wheel::TimerWheel;
use crate::watch_registry::WatchRegistry;
@ -17,6 +20,9 @@ pub struct StdExtension {
pub(crate) monitor_registry: MonitorRegistry,
pub(crate) watch_registry: WatchRegistry,
pub(crate) group_registry: GroupRegistry,
pub(crate) supervisor_registry: SupervisorRegistry,
pub(crate) service_registry: ServiceRegistry,
pub(crate) children_registry: ChildrenRegistry,
}
impl StdExtension {
@ -26,6 +32,9 @@ impl StdExtension {
monitor_registry: MonitorRegistry::new(),
watch_registry: WatchRegistry::new(),
group_registry: GroupRegistry::new(),
supervisor_registry: SupervisorRegistry::new(),
service_registry: ServiceRegistry::new(),
children_registry: ChildrenRegistry::new(),
}
}
@ -33,6 +42,14 @@ impl StdExtension {
pub fn resolve_name(&self, addr: &ActorAddress) -> Option<String> {
self.name_registry.lookup_by_addr(addr)
}
/// Register a supervisor → child relationship.
///
/// This is used by the built-in [`Supervisor`](crate::Supervisor) and can
/// also be called by custom supervisor implementations.
pub fn register_supervisor(&self, supervisor: ActorAddress, child: ActorAddress) {
self.supervisor_registry.register(supervisor, child);
}
}
impl Default for StdExtension {
@ -46,40 +63,54 @@ fn stop_to_exit(reason: StopReason) -> ExitReason {
match reason {
StopReason::Normal => ExitReason::Stopped,
StopReason::Panicked => ExitReason::Panicked,
StopReason::Completed => ExitReason::Completed,
}
}
impl RuntimeExtension for StdExtension {
fn on_actor_death(
&self,
dead: &[(ActorAddress, StopReason)],
dead: &[(ActorAddress, StopReason, Option<ExitValue>)],
) -> Vec<(ActorAddress, Box<dyn Any + Send>)> {
let mut notifications = Vec::new();
for &(addr, reason) in dead {
for (addr, reason, exit_value) in dead {
let addr = *addr;
let reason = *reason;
// Monitor notifications (Down)
let watchers = self.monitor_registry.take_monitors(&addr);
for (_mref, watcher) in watchers {
let down = Down { addr, reason };
let down = Down { addr, reason, exit_value: exit_value.clone() };
notifications.push((watcher, Box::new(down) as Box<dyn Any + Send>));
}
// Watch notifications (ActorExited)
let watch_notifications = self.watch_registry.notify_death(addr, stop_to_exit(reason));
let watch_notifications = self.watch_registry.notify_death(addr, stop_to_exit(reason), exit_value.clone());
for (watcher, exited) in watch_notifications {
notifications.push((watcher, Box::new(exited) as Box<dyn Any + Send>));
}
// Orphan handling: kill unsupervised children
let children = self.children_registry.take_children(&addr);
for child in children {
if self.supervisor_registry.lookup(&child).is_none() {
notifications.push((child, Box::new(StopSignal) as Box<dyn Any + Send>));
}
}
}
notifications
}
fn cleanup_dead(&self, dead: &[ActorAddress]) {
self.children_registry.cleanup(dead);
for addr in dead {
self.name_registry.unregister_by_addr(addr);
self.group_registry.cleanup(addr);
self.monitor_registry.remove_watcher(addr);
self.watch_registry.cleanup_watcher(addr);
self.supervisor_registry.cleanup(addr);
}
}
@ -87,6 +118,17 @@ impl RuntimeExtension for StdExtension {
self
}
fn on_spawn(&self, child: ActorAddress, parent: Option<ActorAddress>, env: Environment, uptime_ms: u64) -> Environment {
// Register parent → child relationship for orphan cleanup
if let Some(parent_addr) = parent {
self.children_registry.register(parent_addr, child);
}
let env = self.service_registry.inject_into(env);
EnvironmentBuilder::from_env(&env)
.set(SpawnTimestamp(uptime_ms))
.build()
}
fn create_worker_extension(&self) -> Option<Box<dyn WorkerExtension>> {
Some(Box::new(TimerWheel::new()))
}

9
crates/std/src/lib.rs
View file

@ -4,6 +4,10 @@ pub mod name_registry;
pub mod monitor_registry;
pub mod watch_registry;
pub mod group_registry;
pub mod supervisor_registry;
pub mod service_registry;
pub mod resource_handle;
pub mod children_registry;
pub(crate) mod timer_wheel;
mod extension;
mod ctx_ext;
@ -12,5 +16,6 @@ mod runtime_ext;
pub use supervisor::{ChildSpec, RestartPolicy, Supervisor, SupervisorStrategy};
pub use router::{Router, RoutingStrategy};
pub use extension::StdExtension;
pub use ctx_ext::{CtxMonitoring, CtxNaming, CtxGroups, CtxWatching, CtxTimers};
pub use runtime_ext::{RuntimeNaming, RuntimeGroups, RuntimeWatching};
pub use ctx_ext::{CtxMonitoring, CtxNaming, CtxGroups, CtxWatching, CtxTimers, CtxSystem, CtxSelfStats, CtxLineage, CtxEnvironment, CtxResources, CtxHandles, CtxLifecycle, CtxCapabilities};
pub use resource_handle::ResourceHandle;
pub use runtime_ext::{RuntimeNaming, RuntimeGroups, RuntimeWatching, RuntimeResources};

44
crates/std/src/resource_handle.rs Normal file
View file

@ -0,0 +1,44 @@
use swactor::actor::ActorAddress;
/// Typed proxy wrapping a service address for ergonomic domain-specific APIs.
///
/// Implement this trait on a struct that wraps a service address and provides
/// domain-specific methods. Methods take `&self` + `&Ctx` (not stored `&Ctx` —
/// avoids lifetime issues with `&mut self` in handlers).
///
/// # Example
///
/// ```ignore
/// struct CounterHandle {
/// service: ActorAddress,
/// self_addr: ActorAddress,
/// }
///
/// impl ResourceHandle for CounterHandle {
/// type Service = CounterService;
/// fn from_parts(service_addr: ActorAddress, self_addr: ActorAddress) -> Self {
/// Self { service: service_addr, self_addr }
/// }
/// fn service_addr(&self) -> ActorAddress { self.service }
/// fn self_addr(&self) -> ActorAddress { self.self_addr }
/// }
///
/// impl CounterHandle {
/// pub fn increment(&self, ctx: &Ctx) -> Result<(), Error> {
/// ctx.send(self.service_addr(), Increment { reply_to: self.self_addr() })
/// }
/// }
/// ```
pub trait ResourceHandle: Sized {
/// Marker type identifying the service (same `S` used with `ServiceRegistry`).
type Service: 'static + Send + Sync;
/// Construct a handle from a service address and the calling actor's address.
fn from_parts(service_addr: ActorAddress, self_addr: ActorAddress) -> Self;
/// The address of the underlying service actor.
fn service_addr(&self) -> ActorAddress;
/// The address of the actor holding this handle (for reply_to patterns).
fn self_addr(&self) -> ActorAddress;
}

2
crates/std/src/router.rs
View file

@ -69,7 +69,7 @@ impl<M: Message> Router<M> {
fn start_worker(&mut self, ctx: &Ctx, idx: usize) -> Result<(), Error> {
let addr = (self.factory)(ctx)?;
let mref = ctx.monitor(addr);
let mref = ctx.monitor(addr)?;
self.workers[idx] = Some(ActiveChild {
addr,
_monitor_ref: mref,

25
crates/std/src/runtime_ext.rs
View file

@ -1,4 +1,4 @@
use swactor::actor::{ActorAddress, ActorInterface, Message};
use swactor::actor::{ActorAddress, ActorInterface, EnvironmentBuilder, LogicalName, Message};
use swactor::runtime::Runtime;
use swactor::Error;
@ -40,7 +40,10 @@ impl RuntimeNaming for Runtime {
fn spawn_named<A: ActorInterface>(&self, name: impl Into<String>, actor: A) -> Result<ActorAddress, Error> {
let name = name.into();
let addr = self.spawn(actor)?;
let env = EnvironmentBuilder::new()
.set(LogicalName(name.clone()))
.build();
let addr = self.spawn_with_env(actor, env)?;
if let Err(e) = get_ext(self).name_registry.register(name, addr) {
let _ = self.stop_actor(addr);
return Err(e);
@ -132,3 +135,21 @@ impl RuntimeGroups for Runtime {
get_ext(self).group_registry.group_names()
}
}
/// Service registry extension for [`Runtime`].
///
/// Allows registering typed service bindings that are automatically injected
/// into every actor's environment at spawn time.
pub trait RuntimeResources {
/// Register a service address under marker type `S`.
///
/// All actors spawned after this call will have `ServiceBinding<S>` in
/// their environment (unless overridden via `spawn_builder`).
fn register_service<S: 'static + Send + Sync>(&self, addr: ActorAddress);
}
impl RuntimeResources for Runtime {
fn register_service<S: 'static + Send + Sync>(&self, addr: ActorAddress) {
get_ext(self).service_registry.register::<S>(addr);
}
}

52
crates/std/src/service_registry.rs Normal file
View file

@ -0,0 +1,52 @@
use std::any::{Any, TypeId};
use std::collections::HashMap;
use std::sync::{Arc, RwLock};
use swactor::actor::{Environment, EnvironmentBuilder};
/// Stores typed service bindings for injection into actor environments.
///
/// Bindings are registered at the runtime level (e.g., during startup) and
/// automatically injected into every actor's environment via the `on_spawn`
/// hook. Existing environment keys are **not** overwritten — this preserves
/// per-subtree overrides set via `spawn_builder`.
pub struct ServiceRegistry {
bindings: RwLock<HashMap<TypeId, Arc<dyn Any + Send + Sync>>>,
}
impl ServiceRegistry {
pub fn new() -> Self {
Self {
bindings: RwLock::new(HashMap::new()),
}
}
/// Register a service binding by marker type `S`.
///
/// Overwrites any previous binding for the same marker type.
pub fn register<S: 'static + Send + Sync>(&self, addr: swactor::actor::ActorAddress) {
let binding = swactor::actor::ServiceBinding::<S>::new(addr);
let type_id = TypeId::of::<swactor::actor::ServiceBinding<S>>();
self.bindings
.write()
.unwrap()
.insert(type_id, Arc::new(binding));
}
/// Merge all registered bindings into an environment, skipping keys
/// that are already present (preserves spawn_builder overrides).
pub fn inject_into(&self, env: Environment) -> Environment {
let bindings = self.bindings.read().unwrap();
if bindings.is_empty() {
return env;
}
let mut builder = EnvironmentBuilder::from_env(&env);
for (&type_id, value) in bindings.iter() {
if !env.contains_type_id(type_id) {
builder.set_raw(type_id, Arc::clone(value));
}
}
builder.build()
}
}

4
crates/std/src/supervisor.rs
View file

@ -3,6 +3,7 @@ use std::sync::Arc;
use swactor::actor::{ActorAddress, ActorInterface, Ctx, Down, MonitorRef, StopReason};
use swactor::Error;
use crate::ctx_ext::get_ext;
use crate::CtxMonitoring;
/// How a child should be restarted when it dies.
@ -140,7 +141,8 @@ impl Supervisor {
fn start_child(&mut self, ctx: &Ctx, idx: usize) -> Result<(), Error> {
let addr = (self.specs[idx].start)(ctx)?;
let mref = ctx.monitor(addr);
let mref = ctx.monitor(addr)?;
get_ext(ctx).supervisor_registry.register(ctx.self_addr(), addr);
self.children[idx] = Some(ActiveChild {
addr,
_monitor_ref: mref,

40
crates/std/src/supervisor_registry.rs Normal file
View file

@ -0,0 +1,40 @@
use std::sync::RwLock;
use swactor::actor::ActorAddress;
use swactor::AddrMap;
/// Maps supervised children to their supervisor.
///
/// Follows the same pattern as `MonitorRegistry`, `GroupRegistry`, etc.
/// Entries are added in `Supervisor::start_child` and cleaned up on actor death.
pub struct SupervisorRegistry {
/// child_addr → supervisor_addr
children: RwLock<AddrMap<ActorAddress>>,
}
impl SupervisorRegistry {
pub fn new() -> Self {
Self {
children: RwLock::new(AddrMap::default()),
}
}
/// Register a supervisor → child relationship.
pub fn register(&self, supervisor: ActorAddress, child: ActorAddress) {
self.children.write().unwrap().insert(child, supervisor);
}
/// Look up the supervisor of a child actor.
pub fn lookup(&self, child: &ActorAddress) -> Option<ActorAddress> {
self.children.read().unwrap().get(child).copied()
}
/// Remove entries where `dead_addr` is either a child or a supervisor.
pub fn cleanup(&self, dead_addr: &ActorAddress) {
let mut map = self.children.write().unwrap();
// Remove the dead actor as a child
map.remove(dead_addr);
// Remove all children supervised by the dead actor
map.retain(|_, supervisor| supervisor != dead_addr);
}
}

4
crates/std/src/watch_registry.rs
View file

@ -1,7 +1,7 @@
use std::collections::{HashMap, HashSet};
use std::sync::Mutex;
use swactor::actor::{ActorAddress, ActorExited, ExitReason};
use swactor::actor::{ActorAddress, ActorExited, ExitReason, ExitValue};
/// Tracks watch relationships between actors.
///
@ -55,11 +55,13 @@ impl WatchRegistry {
&self,
target: ActorAddress,
reason: ExitReason,
exit_value: Option<ExitValue>,
) -> Vec<(ActorAddress, ActorExited)> {
let mut state = self.inner.lock().unwrap();
let notification = ActorExited {
addr: target,
reason,
exit_value,
};
let mut result = Vec::new();

10
crates/swactor-node/src/main.rs
View file

@ -789,8 +789,16 @@ fn run_iroh(
let seed_bytes = parse_node_id_str(&seed_str).expect("invalid seed node ID (expected hex or base58)");
let seed_key =
iroh::PublicKey::from_bytes(&seed_bytes).expect("invalid seed public key");
let mut seed_addr = iroh::EndpointAddr::from(seed_key);
// Include relay URLs so iroh can locate the seed through the relay
for host in &relay_hosts {
let url_str = format!("http://{host}:{relay_port}/");
if let Ok(url) = url_str.parse::<iroh::RelayUrl>() {
seed_addr = seed_addr.with_relay_url(url);
}
}
eprintln!("Joining cluster via seed {}", base58_encode(&seed_bytes));
driver.join(&[iroh::EndpointAddr::from(seed_key)]);
driver.join(&[seed_addr]);
}
// Spawn and register actors

542
docs/development_history/PROCESS_PRIMITIVES.md Normal file
View file

@ -0,0 +1,542 @@
# Process Abstraction for Swactor — Development History
> Design and implementation record for the "process" abstraction layer built
> on top of swactor's actor primitives. This work ran across items 1–9 and
> added 9 extension traits, 3 registries, and ~70 scenario tests.
## Context
Swactor is a distributed actor runtime with local primitives (spawn, send, stop, monitor,
supervise) and distributed primitives (SWIM membership, Kademlia directory, cluster-wide naming,
content-addressed datastore). The goal was to design a "process" abstraction that sits on top of
these primitives, making the experience of running code on a swactor network feel closer to what
an OS process feels like -- with access to an API for requesting resources and querying system
state.
---
## Part 1: OS Process Mapping
### Already strong (direct OS equivalents exist)
OS Concept: PID
Swactor Equivalent: ActorAddress (32-byte random)
Where: src/actor.rs
────────────────────────────────────────
OS Concept: fork+exec
Swactor Equivalent: ctx.spawn(), Runtime::spawn()
Where: src/actor.rs, src/runtime.rs
────────────────────────────────────────
OS Concept: exit(0)
Swactor Equivalent: ctx.stop_self()
Where: src/actor.rs
────────────────────────────────────────
OS Concept: kill(pid, SIGTERM)
Swactor Equivalent: ctx.stop_actor(addr)
Where: src/actor.rs
────────────────────────────────────────
OS Concept: SIGCHLD / waitpid
Swactor Equivalent: ctx.monitor() -> Down, ctx.watch() -> ActorExited
Where: crates/std/src/ctx_ext.rs
────────────────────────────────────────
OS Concept: IPC (message queues)
Swactor Equivalent: Typed message passing (local + cross-worker + cross-runtime)
Where: src/actor.rs, src/transport.rs
────────────────────────────────────────
OS Concept: Service names
Swactor Equivalent: NameRegistry (local), ClusterRegistry (cluster CRDT)
Where: crates/std/src/name_registry.rs, crates/distribution/src/registry.rs
────────────────────────────────────────
OS Concept: Process groups
Swactor Equivalent: GroupRegistry (join/leave/publish/members)
Where: crates/std/src/ctx_ext.rs
────────────────────────────────────────
OS Concept: init/systemd
Swactor Equivalent: Supervisor with restart strategies
Where: crates/std/src/supervisor.rs
────────────────────────────────────────
OS Concept: Scheduler
Swactor Equivalent: Worker pool with load-aware placement + per-actor message budgets
Where: src/worker.rs, src/delivery.rs
────────────────────────────────────────
OS Concept: Machine identity
Swactor Equivalent: NodeId (ed25519 public key)
Where: crates/distribution/src/types.rs
────────────────────────────────────────
OS Concept: Cluster membership
Swactor Equivalent: SWIM protocol
Where: crates/distribution/src/swim/
────────────────────────────────────────
OS Concept: /proc, top, ps
Swactor Equivalent: RuntimeStats, StatsHook, Dashboard, Investigate protocol
Where: src/stats.rs, crates/dashboard/
### Implemented during this work
OS Concept: System introspection from inside
Swactor Equivalent: CtxSystem (worker_id, num_workers, total_actors, uptime_ms) + SystemInfo
Where: src/actor.rs, crates/std/src/ctx_ext.rs
────────────────────────────────────────
OS Concept: Per-actor introspection
Swactor Equivalent: CtxSelfStats (messages_processed, mailbox_depth, message_type_counts)
Where: src/actor.rs, src/worker.rs, crates/std/src/ctx_ext.rs
────────────────────────────────────────
OS Concept: Process lineage (getppid)
Swactor Equivalent: CtxLineage (ctx.parent(), ctx.supervisor())
Where: src/actor.rs, src/worker.rs, crates/std/src/ctx_ext.rs, crates/std/src/supervisor_registry.rs
────────────────────────────────────────
OS Concept: Process environment (environ/getenv)
Swactor Equivalent: CtxEnvironment (ctx.env::<T>(), ctx.environment(), SpawnBuilder for overrides)
Where: src/actor.rs, src/worker.rs, crates/std/src/ctx_ext.rs
────────────────────────────────────────
OS Concept: Well-known environment keys (spawn metadata)
Swactor Equivalent: SpawnTimestamp(u64) injected by StdExtension on_spawn hook;
LogicalName(String) injected by spawn_named (ctx and runtime level)
Where: src/actor.rs, src/extension.rs, src/worker.rs, crates/std/src/extension.rs,
crates/std/src/ctx_ext.rs, crates/std/src/runtime_ext.rs, src/runtime.rs
────────────────────────────────────────
OS Concept: Service discovery
Swactor Equivalent: ServiceRegistry + CtxResources (ctx.resource::<S>() -> Option<ActorAddress>)
Where: src/actor.rs, crates/std/src/service_registry.rs, crates/std/src/ctx_ext.rs,
crates/std/src/runtime_ext.rs, crates/std/src/extension.rs
────────────────────────────────────────
OS Concept: Resource request API (typed handles)
Swactor Equivalent: ResourceHandle trait + CtxHandles (ctx.handle::<H>() -> Option<H>)
Where: crates/std/src/resource_handle.rs, crates/std/src/ctx_ext.rs
────────────────────────────────────────
OS Concept: Exit codes / rich exit values
Swactor Equivalent: ExitValue(Arc<dyn Any + Send + Sync>), ctx.stop_with(value),
StopReason::Completed, ExitReason::Completed. Exit values propagated via Down/ActorExited.
Where: src/actor.rs, src/worker.rs, crates/std/src/extension.rs, crates/std/src/watch_registry.rs
────────────────────────────────────────
OS Concept: Parent-child hierarchy + orphan handling
Swactor Equivalent: ChildrenRegistry tracks parent->children. On parent death, unsupervised
children are killed (StopSignal). Supervised children are left to their supervisor. Cascades
naturally across generations via tick-based cleanup.
Where: crates/std/src/children_registry.rs, crates/std/src/extension.rs
────────────────────────────────────────
OS Concept: Suspend/resume (SIGSTOP/SIGCONT)
Swactor Equivalent: ctx.suspend_self(), ctx.resume(target) with auth (self or supervisor only).
Suspended actors queue messages but don't process them. ResumeSignal via transfer queue for
cross-worker resume.
Where: src/actor.rs, src/worker.rs, src/runtime.rs, crates/std/src/ctx_ext.rs
────────────────────────────────────────
OS Concept: Capability model / sandboxing
Swactor Equivalent: CapabilitySet stored in actor's Environment. Enforced at Ctx level (send,
spawn, stop_actor, monitor, resource). Opt-in: actors without a CapabilitySet are unrestricted.
Where: src/actor.rs, crates/std/src/ctx_ext.rs
### Still partially there
OS Concept: Resource limits
What Exists: Mailbox capacity + message budget
What's Missing: No per-actor memory/CPU/fd limits
────────────────────────────────────────
OS Concept: Auth/permissions
What Exists: Datastore ACL + node-level peer auth + actor-level CapabilitySet
What's Missing: Cluster-level capability propagation (local-only today)
---
## Part 2: Design Primitives
The design followed the existing extension pattern: new capabilities were added as extension traits
on Ctx<'_>, backed by registries in the extension system. This preserved backwards compatibility
and kept the core minimal.
### 2.1 System Queries (CtxSystem)
What it enables: An actor can ask about the system it's running in.
Implemented queries (available via ctx.system_info() or the CtxSystem extension trait):
- ctx.worker_id() -> usize -- which worker thread am I on?
- ctx.num_workers() -> usize -- how many worker threads exist?
- ctx.total_actors() -> usize -- live actors across all workers
- ctx.uptime_ms() -> u64 -- milliseconds since runtime creation
Implementation: SystemInfo struct in src/actor.rs. ContextInner::system_info() implemented on
both Runtime (for spawn-time context) and WorkerContext (for handler context). Data flows through
TickContext (worker_stats + created_at fields in src/delivery.rs). The CtxSystem extension trait
in crates/std/src/ctx_ext.rs provides ergonomic per-field accessors.
Future cluster-level queries (not yet implemented):
- What is my node's identity (NodeId)?
- How many cluster nodes are alive?
- Who are the cluster members?
These require the distribution crate's DistributedNode state to be exposed through the extension
system. The CtxSystem trait can be extended with these when the distribution integration is ready.
### 2.2 Process Environment (CtxEnvironment)
What it enables: Typed configuration that flows from parent to child at spawn time.
Properties:
- Inherited: When actor A spawns actor B via ctx.spawn(), B gets A's environment (Arc clone)
- Overridable: ctx.spawn_builder(actor).env(Key(val)).finish() lazily clones the parent's map
on first override (copy-on-write), leaving the common case (no overrides) allocation-free
- Immutable after spawn: Set at creation, read-only thereafter. Mutable config goes through
messages.
- Typed values: TypeId-keyed (like http::Extensions), not string-to-string
- Runtime-spawned actors start with an empty environment
Implementation: Environment is Arc<HashMap<TypeId, Arc<dyn Any + Send + Sync>>> -- clone is an
Arc bump (zero allocation). EnvironmentBuilder provides from_env() for copy-on-write overrides
(cloning individual entries is cheap since values are also Arc-wrapped). The spawn channel was
replaced with a SpawnRequest struct (addr, actor, parent, env) to avoid further tuple growth.
ActorSlot stores env, and Ctx receives it at both construction sites (tick_all and cleanup_dead).
SpawnBuilder provides the ergonomic override API. The CtxEnvironment extension trait in
crates/std/src/ctx_ext.rs provides the import path, following the same pattern as CtxLineage
(no StdExtension dependency required). Python crate spawns with Environment::new(). 6 scenario
tests in tests/std_extension.rs cover: inheritance, empty for runtime-spawned, grandchild chain,
override-one-inherit-others, readable in on_stop, and sibling independence.
Well-known keys:
- SpawnTimestamp(u64): Injected by StdExtension's on_spawn hook. Milliseconds since runtime
creation, same time base as SystemInfo::uptime_ms. Opt-in at runtime level (present when
StdExtension is installed). Read via ctx.env::<SpawnTimestamp>().
- LogicalName(String): Injected by spawn_named() at both ctx and Runtime levels. Inherited by
children via normal environment inheritance. Read via ctx.env::<LogicalName>().
- ServiceBinding<S>(ActorAddress): Injected by ServiceRegistry's inject_into() hook during
on_spawn. Registered at runtime level via rt.register_service::<S>(addr). Read via
ctx.resource::<S>() (CtxResources trait). Overridable per-subtree via spawn_builder.
- CapabilitySet: Granted at spawn time (via environment or spawn_builder). Inherited by children.
Enforced at Ctx level. See section 2.7.
Analogy: Unix environ -- inherited by default, augmented at fork/exec time, readable via getenv().
### 2.3 Service Discovery (CtxResources)
What it enables: Actors can discover system services by type, not by knowing raw addresses.
How it differs from NameRegistry: NameRegistry maps strings to addresses. CtxResources maps
service marker types to addresses. Looking up "datastore" by name gives you a raw ActorAddress and
you must know what messages it accepts. ctx.resource::<Datastore>() gives you the address of the
service registered under that marker type.
Implementation: Three layers compose the feature:
1. Core type: ServiceBinding<S>(ActorAddress) in src/actor.rs -- a generic environment key
parameterized by a zero-sized marker type. Any struct satisfying 'static + Send + Sync works
as a marker (no special Service trait required, consistent with Environment's existing API).
2. Registry + injection: ServiceRegistry in crates/std/src/service_registry.rs stores registered
bindings as RwLock<HashMap<TypeId, Arc<dyn Any + Send + Sync>>> (same thread-safety pattern
as SupervisorRegistry). StdExtension's on_spawn hook calls inject_into() before adding
SpawnTimestamp -- this merges all registered bindings into the actor's environment, skipping
keys already present (preserves per-subtree overrides set via spawn_builder). Helper methods
on Environment (contains_type_id) and EnvironmentBuilder (set_raw) support type-erased
injection without knowing concrete types at compile time.
3. Read API: CtxResources trait in crates/std/src/ctx_ext.rs provides ctx.resource::<S>() ->
Option<ActorAddress>, a thin wrapper around ctx.env::<ServiceBinding<S>>().map(|b| b.addr).
Does NOT require StdExtension -- reads from core environment (same pattern as CtxEnvironment).
When a CapabilitySet is present, resource() checks check_service::<S>() and returns None if
denied. RuntimeResources trait in crates/std/src/runtime_ext.rs provides
rt.register_service::<S>(addr) for startup-time registration.
Key design decisions:
- No Service marker trait: S: 'static + Send + Sync is sufficient. Any zero-size struct works.
- "Skip if present" injection: The registry doesn't overwrite env keys set by spawn_builder,
enabling per-subtree service overrides (e.g., test doubles, staging vs production services).
- No cleanup on service actor death: A dead service's binding stays in the registry (stale
address). Sends to it will fail. Service lifecycle management is a higher-level concern.
6 scenario tests in tests/std_extension.rs cover: discovery by marker type, child inherits
binding from parent, multiple services each accessible by marker, unregistered returns None,
overridable via spawn_builder, accessible in on_start and on_stop lifecycle hooks.
Well-known services that could be registered (when swactor-node is updated):
- Storage -- content-addressed datastore (currently wired manually in swactor-node)
- Directory -- actor location resolution (currently locked inside DistributedNode)
- Cluster -- membership/topology info (currently snapshot-only for dashboard)
- Metrics -- runtime stats (currently StatsHook push-only)
### 2.4 Resource Handles (CtxHandles)
What it enables: Domain-specific typed proxies that wrap service addresses and provide ergonomic
APIs.
The pattern: A handle wraps (service_address, self_address) and provides methods that construct
and send the right messages, embedding self_addr as reply_to. Responses arrive as normal messages
in the actor's handle().
Implementation: The ResourceHandle trait in crates/std/src/resource_handle.rs defines the contract:
- type Service: 'static + Send + Sync -- the marker type used for service discovery
- from_parts(service_addr, self_addr) -> Self -- construct from addresses
- service_addr() -> ActorAddress -- the underlying service address
- self_addr() -> ActorAddress -- the actor's own address (for reply_to)
The CtxHandles extension trait in crates/std/src/ctx_ext.rs provides ctx.handle::<H>() -> Option<H>,
which looks up ServiceBinding<H::Service> from the actor's environment and constructs the handle.
Returns None if the service is not registered (consistent with ctx.resource(), ctx.where_is()).
Handle methods take &self + &Ctx (not stored &Ctx -- avoids lifetime issues with &mut self in
handlers). Example:
impl MyHandle {
pub fn do_work(&self, ctx: &Ctx, data: Vec<u8>) -> Result<(), Error> {
ctx.send(self.service_addr(), MyMsg::DoWork { data, reply_to: self.self_addr() })
}
}
Key design tension: Handles can't block (no await in swactor). The response arrives asynchronously
as a message. This is inherent to the actor model and not something to "fix" -- the handle just
makes the send side ergonomic.
5 scenario tests: handle wraps service and sends ergonomically, returns None when service not
registered, inherits service binding from parent, constructible in on_start, two actors with same
handle type each get responses at their own address.
### 2.5 Process Lineage (CtxLineage)
What it enables: Actors know their ancestry.
Implemented queries:
- ctx.parent() -> Option<ActorAddress> (who spawned me?)
Returns Some(spawner_addr) for actor-spawned children, None for Runtime::spawn().
Available in handle(), on_start(), and on_stop().
- ctx.supervisor() -> Option<ActorAddress> (who supervises me, if anyone?)
Returns Some(supervisor_addr) for supervised children, None for unsupervised actors.
Gracefully returns None when StdExtension is absent (no panic).
Implementation (parent): The spawn channel uses a SpawnRequest struct (addr, actor, parent, env) --
the original 3-tuple was replaced when CtxEnvironment was added. When Ctx::spawn is called, the
spawning actor's self_addr is passed as Some(parent). Runtime::spawn passes None. The parent is
stored in ActorSlot::parent_addr and threaded into Ctx::self_parent_addr at both construction sites
(tick_all and cleanup_dead). 4 scenario tests cover: child knows parent, runtime-spawned has no
parent, grandchild sees immediate parent (not grandparent), and parent is visible in on_stop.
Implementation (supervisor): SupervisorRegistry in crates/std/src/supervisor_registry.rs stores a
child_addr -> supervisor_addr map (RwLock<AddrMap<ActorAddress>>). Supervisor::start_child calls
register(self_addr, child_addr) after spawning and monitoring. cleanup() removes entries where the
dead address is either child or supervisor (O(n) scan for supervisor death, acceptable since
supervisor death is rare and the map is small). CtxLineage::supervisor() downcasts the extension
gracefully (returns None if StdExtension is absent). 5 scenario tests cover: supervised child knows
supervisor, unsupervised actor returns None, supervisor survives child restart, grandchild not
supervised but parent is, OneForAll restart re-registers all children.
The CtxLineage extension trait in crates/std/src/ctx_ext.rs provides the ergonomic import path.
Orphan handling was implemented as part of item 8 (Lifecycle Enrichment) -- see section 2.8.
### 2.6 Self-Introspection (CtxSelfStats)
What it enables: Actors can see their own operational metrics.
Implemented queries (available directly on Ctx or via the CtxSelfStats extension trait):
- ctx.messages_processed() -> u64 -- total successfully processed before current tick
- ctx.mailbox_depth() -> usize -- messages queued at start of current tick (pre-dequeue)
- ctx.message_type_counts() -> &[(&str, u64)] -- per-type counts, sorted descending
Implementation: Stats are snapshotted from ActorSlot fields into Ctx before each tick_all
iteration (src/worker.rs). The snapshot captures the state before any messages are dequeued
in the current tick, giving actors a consistent view. The same snapshot is provided during
on_stop callbacks in cleanup_dead. The CtxSelfStats extension trait in crates/std/src/ctx_ext.rs
provides the ergonomic import path.
The Vec allocation for type counts is bounded (max 32 entries from ActorSlot's msg_type_counts
cap) and negligible relative to handle_any cost.
### 2.7 Capability Model (CapabilitySet + CtxCapabilities)
What it enables: Controlled access to system resources and other actors. Primarily important for
sandboxing untrusted code (wasm actors in crates/bin-runner/).
Approach: A single CapabilitySet stored in the actor's Environment. When present, enforcement is
active -- the actor can only perform operations granted by the set. When absent, the actor is
unrestricted (backward compatible). Capabilities inherit from parent to child via normal
environment inheritance.
Capability grants (all in CapabilitySet):
- with_send(addr) -- send any message type to a specific address
- with_send_typed::<M>(addr) -- send only messages of type M to a specific address
- with_spawn() -- permission to spawn new actors
- with_service::<S>() -- permission to access system service S via ctx.resource::<S>()
- with_monitor(addr) -- permission to monitor a specific actor
Enforcement points (all in src/actor.rs Ctx methods or crates/std/src/ctx_ext.rs):
- ctx.send::<M>(addr, msg) -- checks check_send::<M>(addr); self-send always allowed
- ctx.spawn() / SpawnBuilder::finish() -- checks check_spawn()
- ctx.stop_actor(addr) -- checks check_send_addr(addr) (stop is a send of StopSignal)
- ctx.monitor(addr) -- checks check_monitor(addr); returns Result<MonitorRef, Error>
- ctx.resource::<S>() -- checks check_service::<S>(); returns None if denied
Key design decisions:
- Opt-in: No CapabilitySet in environment means unrestricted. Zero behavioral change for existing
actors. The only cost is an Option check (env.get::<CapabilitySet>()) at each enforcement point.
- Enforcement at Ctx level only: The core ContextInner::send_any is not gated. This means
extension code (supervisors, timers, etc.) that calls send_any directly bypasses capability
checks, which is intentional -- system infrastructure is trusted.
- Dual send granularity: with_send(addr) grants all message types to an address.
with_send_typed::<M>(addr) grants only type M. The check tries address-only first, then typed.
This allows coarse grants for trusted peers and fine-grained grants for untrusted actors.
- Self-send always allowed: A restricted actor can always send to its own address. This prevents
capabilities from breaking actors that use self-messaging patterns (timers, state machines).
- monitor() returns Result: Changed from -> MonitorRef to -> Result<MonitorRef, Error>. This was
a breaking change to all callers (supervisor.rs, router.rs, test files), fixed mechanically by
adding ? or .unwrap().
Builder API: Fluent (CapabilitySet::new().with_send(addr).with_spawn()) and mutable
(caps.grant_send(addr)) variants. Mutable methods return &mut Self for chaining.
Introspection: CtxCapabilities extension trait in crates/std/src/ctx_ext.rs provides:
- ctx.capabilities() -> Option<&CapabilitySet> -- access the raw set
- ctx.is_restricted() -> bool -- quick check
Implementation locations:
- src/actor.rs: CapabilitySet struct, builder methods, check methods, Ctx::capabilities() helper,
enforcement in send/spawn/stop_actor/SpawnBuilder::finish
- src/lib.rs: CapabilitySet re-export
- crates/std/src/ctx_ext.rs: CtxCapabilities trait, monitor() enforcement, resource() enforcement
- crates/std/src/lib.rs: CtxCapabilities re-export
11 scenario tests in tests/std_extension.rs cover: unrestricted actor sends freely (backward
compat), restricted actor denied send, restricted actor allowed send, typed send grant (Ping
allowed / Pong denied), spawn denied, spawn allowed, capability inheritance (child inherits
parent's CapabilitySet), monitor denied, service access denied, self-send always allowed, stop
requires send permission.
### 2.8 Lifecycle Enrichment
Rich exit values: ExitValue(Arc<dyn Any + Send + Sync>) is an opaque typed wrapper. Actors stop
with ctx.stop_with(value) which stores the value and triggers StopReason::Completed. The value
is propagated through Down (monitors) and ActorExited (watchers) via the exit_value: Option<ExitValue>
field. Manual PartialEq/Eq on ExitValue (always false -- opaque blob), so Down/ActorExited compare
by addr+reason only.
Implementation: StopWithSignal(ExitValue) is a sentinel message intercepted in tick_all (like
StopSignal). ActorSlot gains exit_value: Option<ExitValue>. cleanup_dead returns
Vec<(ActorAddress, StopReason, Option<ExitValue>)> with StopReason::Completed when exit_value is
present. The on_actor_death extension hook receives and propagates exit values to monitors/watchers.
7 scenario tests: stop_with value received in Down, received in ActorExited, normal stop has None,
panic has None, multiple monitors receive cloned value, stop_with from on_start, supervisor receives
rich exit in handle_down (graceful handoff pattern).
Orphan handling: ChildrenRegistry tracks parent -> set of children. Populated in on_spawn when a
parent is present. On parent death (on_actor_death), unsupervised children receive StopSignal.
Supervised children are left to their supervisor. Cascades naturally: parent dies -> children killed
next tick -> grandchildren killed the tick after that. StopSignal made pub (was pub(crate)) to
enable this -- it's not Message (not Clone) so can't be sent via ctx.send().
4 scenario tests: unsupervised children killed on parent death, supervised children not killed,
cascading cleanup across generations, runtime-spawned actors unaffected.
Suspend/resume: ActorSlot gains a suspended: bool flag. Suspended actors queue messages but don't
process them (tick_all skips them). ctx.suspend_self() sets the flag via a suspend_requests buffer.
ResumeSignal is intercepted in deliver() to clear the flag. StopSignal/StopWithSignal are also
intercepted for suspended actors (so stop_actor works on them). Cross-worker resume sends
ResumeSignal via the transfer queue.
Authorization: CtxLifecycle extension trait provides ctx.suspend_self() (always allowed) and
ctx.resume(target) which checks: target == self (self-resume) OR caller is the target's supervisor
via SupervisorRegistry. Returns Err if unauthorized.
5 scenario tests: suspended actor queues then resume processes, supervisor can resume, non-supervisor
cannot resume, suspended actor can be stopped, cross-worker resume via runtime.
Graceful handoff: Built on rich exit values. An outgoing actor stops with its state via
ctx.stop_with(state); the supervisor receives it in handle_down's Down message and can pass it
to the replacement's constructor. Enables zero-downtime upgrades. No additional mechanism needed --
the pattern composes from existing primitives.
---
## Part 3: How These Compose
The primitives form a layered system:
Layer 3: Integration (swactor-node wires services at startup)
Layer 2: Process (CapabilitySet, ProcessBuilder)
Layer 1: Std (CtxSystem, CtxEnvironment, CtxLineage, CtxSelfStats, Well-known env keys,
SupervisorRegistry, CtxResources, CtxHandles, CtxLifecycle, ChildrenRegistry,
CtxCapabilities)
Layer 0: Core (SystemInfo, Ctx self-stats, parent tracking, Environment + SpawnRequest,
on_spawn hook, spawn_with_env, ServiceBinding, suspend flag, rich exit, orphan
handling, CapabilitySet)
A "process" in swactor is an actor that has:
1. An identity (ActorAddress) and a name (NameRegistry)
2. A parent and supervisor it can query (CtxLineage)
3. An environment inherited from its spawner, with well-known keys (CtxEnvironment)
4. Access to system services through discovery (CtxResources)
5. The ability to query the system it lives in (CtxSystem)
6. Awareness of its own operational state (CtxSelfStats)
7. Typed resource handles for ergonomic service interaction (CtxHandles)
8. Rich lifecycle support including typed exit values, orphan handling, and suspend/resume
9. Controlled permissions for what it can access (CapabilitySet)
What stayed the same: The core actor model (message passing, mailboxes, workers, tick-based
execution) was unchanged. ActorInterface, Ctx, Runtime remained the foundation. The process
abstraction was additive -- existing actors continued to work exactly as before.
---
## Part 4: Implementation Sequence
Each item was implemented and merged in dependency order. Earlier items established the
infrastructure (Environment, extension hooks) that later items built on.
1. **CtxSystem + CtxSelfStats** -- Exposed existing internal data to actors. SystemInfo struct,
ContextInner::system_info(), Ctx self-stats snapshot fields. Extension traits CtxSystem and
CtxSelfStats in swactor-std. Covered by 3 scenario tests.
2. **CtxLineage (parent tracking)** -- Option<ActorAddress> threaded through the spawn path.
ContextInner::spawn_any gained a parent parameter. ActorSlot stores parent_addr. Ctx exposes
parent(). CtxLineage extension trait in swactor-std. 4 scenario tests.
Python crate updated to pass parent on spawn.
3. **CtxEnvironment (process environment)** -- Typed key-value map inherited from parent to
child at spawn time. Environment is Arc<HashMap<TypeId, Arc<dyn Any + Send + Sync>>> -- clone
is an Arc bump. EnvironmentBuilder supports copy-on-write overrides via from_env(). The spawn
channel 3-tuple was replaced with a SpawnRequest struct (addr, actor, parent, env) to stop
tuple growth. ActorSlot stores env. Ctx gains env::<T>(), environment(), and spawn_builder().
SpawnBuilder lazily clones the parent's map on first .env() call. CtxEnvironment extension trait
in swactor-std (no StdExtension dependency). Python crate spawns with Environment::new().
6 scenario tests: inheritance, empty for runtime-spawned, grandchild chain,
override-one-inherit-others, readable in on_stop, sibling independence.
4. **Well-known environment keys** -- SpawnTimestamp(u64) and LogicalName(String) types in
src/actor.rs, exported from src/lib.rs. SpawnTimestamp is opt-in at runtime level: injected by
StdExtension's on_spawn hook (new RuntimeExtension::on_spawn hook with default no-op in
src/extension.rs). Worker::drain_spawns now takes &TickContext and calls on_spawn for each
spawn request, passing uptime_ms to avoid exposing the pub(crate) Instant type. LogicalName is
injected by spawn_named at both ctx level (via spawn_builder + env override) and runtime level
(via new Runtime::spawn_with_env method). LogicalName inherits to children automatically via
normal environment inheritance. 7 scenario tests.
5. **Supervisor lineage (ctx.supervisor())** -- SupervisorRegistry in
crates/std/src/supervisor_registry.rs stores child_addr -> supervisor_addr as
RwLock<AddrMap<ActorAddress>>. Supervisor::start_child calls register() after spawning and
monitoring. cleanup() removes entries for dead actors (both as child and as supervisor).
CtxLineage::supervisor() gracefully returns None when StdExtension is absent (downcasts via
as_any, no panic). Distinct from parent() because not every parent is a supervisor. get_ext
made pub(crate) so supervisor.rs can access it. 5 scenario tests.
6. **Service Registry + CtxResources** -- Actors discover system services by type
(ctx.resource::<Datastore>()) rather than by raw address. ServiceBinding<S>(ActorAddress)
is a generic environment key parameterized by a marker type. ServiceRegistry in StdExtension
stores bindings and injects them into every actor's environment via on_spawn (skipping keys
already present to preserve spawn_builder overrides). CtxResources trait provides
ctx.resource::<S>() sugar. RuntimeResources trait provides rt.register_service::<S>(addr).
6 scenario tests.
7. **Resource Handles (CtxHandles)** -- ResourceHandle trait + CtxHandles extension trait.
ctx.handle::<H>() -> Option<H> constructs typed proxies from ServiceBinding<H::Service> in the
actor's environment. Handle methods take &self + &Ctx for ergonomic domain-specific APIs.
5 scenario tests.
8. **Lifecycle enrichment** -- Three sub-features:
a) Rich exit values: ExitValue(Arc<dyn Any + Send + Sync>), ctx.stop_with(value),
StopReason::Completed, ExitReason::Completed. Propagated through Down/ActorExited.
7 scenario tests.
b) Orphan handling: ChildrenRegistry tracks parent->children. Unsupervised children killed
on parent death. Supervised children left to their supervisor. Natural cascade.
4 scenario tests.
c) Suspend/resume: ActorSlot::suspended flag, ctx.suspend_self(), ctx.resume(target) with
auth (self or supervisor only). ResumeSignal for cross-worker resume.
5 scenario tests.
9. **Capability model (CapabilitySet)** -- Per-actor permission set stored in the Environment.
Grants: with_send(addr), with_send_typed::<M>(addr), with_spawn(), with_service::<S>(),
with_monitor(addr). Enforced at Ctx level in send, spawn, stop_actor, monitor, and resource.
Opt-in: actors without a CapabilitySet are unrestricted (zero behavioral change). Self-send
always allowed. monitor() changed from -> MonitorRef to -> Result<MonitorRef, Error> (breaking
change, fixed mechanically in supervisor.rs, router.rs, and all test files). CtxCapabilities
extension trait for introspection. 11 scenario tests.

423
docs/development_history/pooled-datastore/POOLED_DATASTORE.md Normal file
View file

@ -0,0 +1,423 @@
# Pooled Datastore & Sim-Cluster — Development History
> Adds a gossip-converged pooled storage protocol, a generic gossip channel
> abstraction, a pool dashboard page, shared pool types, iroh connection
> hardening, and a Docker-free multi-process sim-cluster test harness.
>
> ~18 new/modified files · ~2,400 insertions
>
> *Branch: `pooled-datastore`*
---
## Table of Contents
1. [Overview & Motivation](#1-overview--motivation)
2. [What Was Built](#2-what-was-built)
3. [Pooled Storage Protocol](#3-pooled-storage-protocol)
4. [Generic Gossip Channel Abstraction](#4-generic-gossip-channel-abstraction)
5. [Pool Disseminator](#5-pool-disseminator)
6. [Pool Coordinator Actor](#6-pool-coordinator-actor)
7. [Shared Pool Types](#7-shared-pool-types)
8. [Dashboard Pool Page](#8-dashboard-pool-page)
9. [Iroh Connection Hardening](#9-iroh-connection-hardening)
10. [Sim-Cluster Test Harness](#10-sim-cluster-test-harness)
11. [Design Decisions & Tradeoffs](#11-design-decisions--tradeoffs)
12. [Test Coverage](#12-test-coverage)
13. [Known Gaps & Future Work](#13-known-gaps--future-work)
---
## 1. Overview & Motivation
The existing datastore provides content-addressed storage on individual nodes,
but there is no mechanism for multiple nodes to form a shared storage pool —
knowing who has what content, how much capacity each node offers, or where to
place new data.
This branch introduces a **pooled datastore protocol** that layers on top of
the existing content-addressed datastore. Multiple nodes join a named pool,
gossip their membership/capacity/content-locations via SWIM piggyback, and
converge on a shared view of the pool's state. This enables:
- **Content location**: find which node(s) hold a given content hash without
fan-out queries.
- **Capacity-aware placement**: route new writes to the node with the most free
space.
- **Pool ACL**: optional allow-list to restrict which nodes can join a pool.
- **Live observability**: a new dashboard page shows pool membership, capacity
bars, content location map, and ACL state in real time via SSE.
Separately, the branch also introduces:
- A **generic gossip channel abstraction** (`GossipChannel` trait +
`DisseminationBuffer<T>`) that replaces the 4 duplicated dissemination
patterns in the distribution crate.
- A **sim-cluster test harness** (`cargo xtask test sim-cluster` / `cargo
xtask sim-cluster`) that spawns real multi-process clusters with a local iroh
relay — no Docker required.
- **Iroh connection hardening**: relay URL resolution cascade and connect
timeouts to prevent indefinite hangs during peer connection.
---
## 2. What Was Built
| Component | Crate / Location | Lines |
|-----------|-----------------|-------|
| Pool types (PoolId, entries, config) | `crates/shared-types/src/pool.rs` | ~170 |
| Gossip channel trait + DisseminationBuffer | `crates/distribution/src/gossip_channel.rs` | ~270 |
| Pool disseminator (CRDT state + gossip) | `crates/datastore/src/pool/disseminator.rs` | ~720 |
| Pool coordinator actor | `crates/datastore/src/pool/coordinator.rs` | ~275 |
| Pool messages | `crates/datastore/src/pool/messages.rs` | ~80 |
| Pool dashboard HTML/JS | `crates/dashboard/src/pool_html.rs` | ~390 |
| Sim-cluster harness | `xtask/src/sim_cluster.rs` | ~700 |
| Pool integration tests | `crates/datastore/tests/pool_tests.rs` | ~340 |
| Docker compose (dev cluster) | `tests/docker/docker-compose.dev-cluster.yml` | ~75 |
Modified files:
| File | Change |
|------|--------|
| `crates/distribution/src/iroh_driver.rs` | Relay URL cascade + connect timeout |
| `crates/swactor-node/src/main.rs` | Seed node relay URL hints for iroh |
| `crates/datastore/tests/dashboard_integration_test.rs` | Start HTTP standalone |
| `xtask/src/main.rs` | `sim-cluster` subcommand + test group |
| `xtask/Cargo.toml` | reqwest, tokio, iroh-relay deps |
---
## 3. Pooled Storage Protocol
The pool protocol is a set of four CRDT entry types that converge via gossip:
```
┌────────────────────────────────────────────────────┐
│ Pool State (per node) │
├──────────────┬─────────────┬───────────┬───────────┤
│ Membership │ Capacity │ Content │ ACL │
│ │ │ Location │ │
│ node→state │ node→bytes │ (hash, │ node→ │
│ (Active/Left)│ (total/used)│ node)→ │ grant/ │
│ │ │ tombstone│ revoke │
├──────────────┴─────────────┴───────────┴───────────┤
│ Higher generation always wins │
│ (last-writer-wins register per key) │
└────────────────────────────────────────────────────┘
```
**Convergence rule**: For each entry type, the key is derived from the entry
(e.g. `node_id` for membership, `(content_hash, node_id)` for content
locations). When two entries share a key, the one with the higher `generation`
wins. This makes all merges commutative, associative, and idempotent — a CRDT.
**Deletion**: Content locations and ACL entries use tombstones (`tombstone:
true` / `revoked: true`) with a generation bump. Tombstones are garbage
collected after a configurable TTL.
**Dissemination**: All entries go through a shared `DisseminationBuffer<PoolEntry>`
which transmits each entry `Λ * ceil(log₂(n))` times before eviction, matching
the standard SWIM protocol budget.
---
## 4. Generic Gossip Channel Abstraction
**File**: `crates/distribution/src/gossip_channel.rs`
Before this branch, SWIM piggyback dissemination was hardcoded for membership
updates, directory entries, and dead-letter notifications — each with its own
copy of the `Λ * ceil(log₂(n))` budget logic.
The new abstraction provides:
- **`GossipChannel` trait**: A topic-tagged channel that produces/consumes
`Vec<u8>` entries for piggyback. Methods: `topic_tag()`,
`take_pending_bytes()`, `apply_incoming_bytes()`, `re_disseminate_all()`,
`on_node_death()`, `gc_tick()`.
- **`DisseminationBuffer<T>`**: A generic budget-limited queue. Entries are
enqueued with a transmit budget of `Λ * ceil(log₂(n))` and evicted after
exhaustion. Supports `enqueue`, `enqueue_or_replace` (idempotent upsert),
`take`, `re_enqueue_all`, and `retain`.
- **Serialization helpers**: `serialize_each()` and `deserialize_each()` for
converting between typed entries and `Vec<u8>`.
The pool disseminator is the first consumer, plugging into the distribution
layer via `SharedPoolChannel` which wraps `Arc<Mutex<PoolDisseminator>>`.
---
## 5. Pool Disseminator
**File**: `crates/datastore/src/pool/disseminator.rs`
The core state machine. Manages four `HashMap` tables (membership, capacity,
content locations, ACL) and a single `DisseminationBuffer<PoolEntry>`.
Key methods:
- **Lifecycle**: `join()`, `leave()` — announce membership state changes.
- **Storage**: `announce_content()`, `remove_content()`, `announce_capacity()`.
- **ACL**: `grant_access()`, `revoke_access()`, `is_node_authorized()`.
- **Queries**: `active_members()`, `member_count()`, `content_count()`,
`locate_content()`, `node_with_most_free_space()`, `pool_capacity_summary()`.
- **Dashboard**: `snapshot_json()` — full JSON snapshot for SSE.
- **Internal**: `merge_entry()` applies the higher-generation-wins rule.
`take_pending_inner()` / `apply_incoming_inner()` drive gossip exchange.
`SharedPoolChannel` wraps this in an `Arc<Mutex<>>` and implements
`GossipChannel`, bridging ownership between the `PoolCoordinator` actor
(lifecycle/queries) and `DistributedNode` (gossip transport).
---
## 6. Pool Coordinator Actor
**File**: `crates/datastore/src/pool/coordinator.rs`
An actor implementing `ActorInterface` for `PoolCoordinatorMsg`. It acts as a
placement-aware CRUD facade:
- **PoolPut/Get/Delete/List**: Delegates to the co-located `DatastoreNode`
actor. Future: redirect to best node based on capacity.
- **PoolStatus**: Queries the disseminator and returns a JSON status snapshot.
- **JoinPool/LeavePool**: Checks ACL authorization, then calls the
disseminator.
- **GrantPoolAccess/RevokePoolAccess**: Manages the allow-list.
- **PoolTick**: Periodic capacity re-announcement (every 100 ticks).
---
## 7. Shared Pool Types
**File**: `crates/shared-types/src/pool.rs`
Types live in `shared-types` to avoid circular dependencies between
`distribution` and `datastore`:
- **`PoolId`**: `blake3(name_bytes)` — 32-byte deterministic pool identifier.
Supports hex encoding/decoding and truncated display.
- **`PoolMemberEntry`**: Node membership with `Active`/`Left` state.
- **`PoolCapacityEntry`**: Storage capacity announcement (total/used bytes).
- **`ContentLocationEntry`**: Where a content hash is stored, with tombstone
support.
- **`PoolACLEntry`**: Authorization grant/revoke with `granted_by` provenance.
- **`PoolEntry`**: Tagged enum wrapping all four entry types for gossip
serialization.
- **`PoolConfig`**: Pool configuration (name, capacity, TTL, GC interval, Λ).
---
## 8. Dashboard Pool Page
**File**: `crates/dashboard/src/pool_html.rs`
A new `/pool` page in the dashboard with:
- **Summary cards**: pool name, member count, content count, total/used
capacity.
- **Capacity bars**: per-node usage with color thresholds (green < 70%, orange
< 90%, red >= 90%).
- **Members table**: node ID (truncated with tooltip), state, total/used/free.
- **Content location map**: content hash → replica count → node list.
- **ACL panel**: open mode indicator or allow-list table.
- **Join/Leave buttons**: POST to `/api/pool/join` and `/api/pool/leave`.
- **Live updates**: SSE `pool` events drive real-time state refresh.
---
## 9. Iroh Connection Hardening
**File**: `crates/distribution/src/iroh_driver.rs`
Two problems fixed:
1. **Relay URL resolution cascade**: When connecting to a peer, the driver now
tries three sources in order: (a) explicit relay URL cache from prior
connections, (b) SWIM metadata gossip (via `node.relay_url()`), (c) the
local node's own home relay. Previously only the explicit cache was checked,
causing connections to fail when the cache was empty.
2. **Connect timeout**: All `endpoint.connect()` calls now have a 2-second
`tokio::time::timeout` wrapper. Previously, connections could hang
indefinitely if a peer was unreachable.
**File**: `crates/swactor-node/src/main.rs`
Seed node addresses now include relay URLs so iroh can locate the seed through
the relay server, rather than relying solely on direct addressing.
---
## 10. Sim-Cluster Test Harness
**File**: `xtask/src/sim_cluster.rs`
A new test stage and development tool that spawns real multi-process swactor
clusters without Docker:
### Test mode: `cargo xtask test sim-cluster`
Runs 4 scenarios sequentially, each with a fresh 5-node cluster:
| # | Scenario | Validates |
|---|----------|-----------|
| 1 | Cluster convergence | All 5 nodes see >= 4 alive peers, routing table >= 4 |
| 2 | Node death detection | Kill node 2, survivors detect alive drop, dead count >= 1 |
| 3 | Killed node rejoins | Kill node 2, restart it, rejoined node sees alive >= 1 |
| 4 | Actors resolvable | Each node has >= 2 directory entries, total >= 10 |
### Interactive mode: `cargo xtask sim-cluster --nodes N`
Spawns a persistent cluster for development. Prints dashboard URLs and blocks
until Ctrl-C.
### Infrastructure
- **Local relay server**: Embedded `iroh-relay` server on an ephemeral port.
Nodes connect through the relay rather than requiring direct connectivity.
- **RAII lifecycle**: `SimCluster` owns child processes and SIGTERM's them on
drop. `RelayServer` owns its tokio runtime.
- **Config generation**: Each node gets a `node.toml` with dashboard port,
actor count, relay host/port, and optional seed node ID.
- **Seed key discovery**: Polls the seed node's key file on disk to extract the
public key before spawning joiner nodes.
- **HTTP observation**: Polls `/api/distribution` on each node's dashboard.
Uses `serde_json::Value` to avoid compile-time coupling to protocol types.
- **Node lifecycle**: `kill_node()` sends SIGTERM, `restart_node()` re-spawns
with the same config (non-seed nodes get the seed's public key).
### Dev cluster compose
**File**: `tests/docker/docker-compose.dev-cluster.yml`
A 3-node Docker Compose file for development with pool configuration
(`--pool-name dev-pool --pool-capacity 104857600`). Uses a bridge network with
static IPs.
---
## 11. Design Decisions & Tradeoffs
**Higher-generation-wins CRDT over vector clocks**: Pool entries use a simple
monotonic generation counter per entry key. This is sufficient because each
entry has a single writer (the node that owns it). Vector clocks would add
complexity without benefit since there are no concurrent writers for the same
key.
**Tombstones with TTL over immediate deletion**: Content locations and ACL
revocations use tombstones that propagate via gossip before being GC'd. Without
tombstones, a deleted entry could be re-introduced by a node that hasn't yet
received the deletion.
**Shared `Arc<Mutex<>>` over message-passing for disseminator**: The pool
disseminator needs to be accessed by both the coordinator actor (for
lifecycle/queries) and the distribution layer (for gossip). Rather than adding
an actor-to-actor message protocol, the disseminator is wrapped in
`Arc<Mutex<PoolDisseminator>>`. The lock is held only briefly for individual
operations.
**Sim-cluster over Docker for testing**: Docker adds build time, image
management, and network configuration complexity. The sim-cluster spawns bare
processes on localhost, uses an embedded iroh relay, and tears down in
milliseconds. Scenarios that previously required Docker Compose now run with
`cargo xtask test sim-cluster`.
**HTTP polling over direct protocol observation**: The sim-cluster observes
node state via HTTP (`/api/distribution`) rather than linking against protocol
types. This makes the test harness resilient to protocol changes and mirrors
how an operator would observe a real cluster.
**Pool types in `shared-types`**: Pool entry types live in `shared-types`
rather than `datastore` to avoid a circular dependency — `distribution` needs
to know about pool entries for gossip serialization, and `datastore` depends on
`distribution`.
---
## 12. Test Coverage
### Unit tests (disseminator internals)
In `crates/datastore/src/pool/disseminator.rs`:
- `join_and_query_members` — join lifecycle
- `leave_removes_from_active` — leave lifecycle
- `announce_and_locate_content` — content announcement + query
- `remove_content_tombstones` — tombstone semantics
- `capacity_summary` — capacity aggregation
- `acl_grant_and_check` / `acl_revoke` / `empty_acl_means_open` — ACL logic
- `higher_generation_wins_merge` — CRDT merge rule
- `two_disseminators_converge_via_gossip_exchange` — two-node gossip
- `three_node_convergence_loop` — multi-round gossip convergence
### Unit tests (gossip channel)
In `crates/distribution/src/gossip_channel.rs`:
- `budget_math_*` (4 tests) — transmit budget calculation
- `enqueue_take_evicts_after_budget` — budget exhaustion
- `enqueue_or_replace_*` (2 tests) — idempotent upsert
- `re_enqueue_all_refreshes_budgets` — anti-entropy
- `retain_removes_non_matching` — predicate-based eviction
- `serialize_deserialize_roundtrip` — wire format
### Unit tests (shared types)
In `crates/shared-types/src/pool.rs`:
- `pool_id_from_name_is_deterministic` / `pool_id_different_names_differ`
- `pool_id_hex_roundtrip`
- `pool_entry_serde_roundtrip`
- `higher_generation_wins_for_membership`
- `content_location_tombstone_semantics`
### Integration tests (pool protocol)
In `crates/datastore/tests/pool_tests.rs`:
- `two_nodes_converge_on_membership` — two-node gossip convergence
- `content_location_propagates_via_gossip` — cross-node content discovery
- `leave_propagates_via_gossip` — membership leave propagation
- `content_deletion_propagates` — tombstone propagation
- `acl_grant_propagates` — ACL gossip
- `capacity_propagates_and_summarizes` — capacity gossip + aggregation
- `placement_query_picks_node_with_most_space` — capacity-aware placement
- `five_node_pool_converges` — 5-node full convergence
- `shared_pool_channel_topic_tag` — GossipChannel interface
- `gossip_channel_bytes_roundtrip` — wire format through GossipChannel
### Sim-cluster scenarios (multi-process)
In `xtask/src/sim_cluster.rs`:
- Cluster convergence (5 nodes)
- Node death detection (kill + observe)
- Killed node rejoins (kill + restart + observe)
- Actors resolvable (directory entry propagation)
---
## 13. Known Gaps & Future Work
- **Remote content fetch**: `PoolGet` currently only checks the local
datastore. It should use `locate_content()` to fetch from the node that
actually has the content.
- **Capacity-aware placement**: `PoolPut` delegates to the local datastore.
It should use `node_with_most_free_space()` to route writes to the best node.
- **Actual usage tracking**: `PoolTick` re-announces capacity with `used: 0`.
It should query the `BlobStore` for actual disk usage.
- **GossipChannel integration**: The `GossipChannel` trait and
`SharedPoolChannel` are built but not yet wired into `DistributedNode`'s
piggyback system. The existing hardcoded dissemination channels need to be
migrated to the new trait.
- **Dashboard SSE integration**: The pool dashboard HTML is built, but the
server-side SSE event source for `pool` events needs to be wired to the
`PoolDisseminator::snapshot_json()` method.
- **Sim-cluster namespace isolation**: The harness uses high ephemeral ports
for isolation. Full Linux network namespace isolation (as designed in
`TEST_ISOLATION.md`) is a future enhancement.
- **Sim-cluster in CI**: The sim-cluster test group is opt-in and excluded
from `essential`/`all`. Once proven stable, it should be added to CI.

505
src/actor.rs
View file

@ -1,7 +1,60 @@
use std::any::Any;
use std::any::TypeId;
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use crate::Error;
// ─── ExitValue ──────────────────────────────────────────────────────────────
/// Opaque typed value attached to a completed actor's exit.
///
/// Created via [`Ctx::stop_with`]. Delivered to monitors/watchers in
/// [`Down::exit_value`] and [`ActorExited::exit_value`].
///
/// Clone is an `Arc` bump (zero allocation).
#[derive(Clone)]
pub struct ExitValue(Arc<dyn Any + Send + Sync>);
impl ExitValue {
/// Wrap a typed value as an opaque exit value.
pub fn new<T: Any + Send + Sync>(value: T) -> Self {
Self(Arc::new(value))
}
/// Attempt to downcast to a concrete type by reference.
pub fn downcast_ref<T: Any>(&self) -> Option<&T> {
self.0.downcast_ref::<T>()
}
}
impl PartialEq for ExitValue {
fn eq(&self, _other: &Self) -> bool {
false // opaque blob — always not equal
}
}
impl Eq for ExitValue {}
impl std::fmt::Debug for ExitValue {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("ExitValue(..)")
}
}
/// System-level information visible to actors.
#[derive(Debug, Clone)]
pub struct SystemInfo {
/// Index of the worker thread this actor is running on.
pub worker_id: usize,
/// Total number of worker threads in the runtime.
pub num_workers: usize,
/// Total number of live actors across all workers.
pub total_actors: usize,
/// Milliseconds since the runtime was created.
pub uptime_ms: u64,
}
/// Why an actor exited.
#[derive(Debug, Clone, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
@ -12,6 +65,8 @@ pub enum ExitReason {
Panicked,
/// The node hosting the actor left the cluster (SWIM Dead).
NodeDown,
/// Actor stopped with a typed exit value (via [`Ctx::stop_with`]).
Completed,
}
/// Delivered to watchers when a watched actor exits.
@ -25,8 +80,19 @@ pub struct ActorExited {
pub addr: ActorAddress,
/// Why it exited.
pub reason: ExitReason,
/// Typed exit value if the actor called [`Ctx::stop_with`].
#[cfg_attr(feature = "serde", serde(skip))]
pub exit_value: Option<ExitValue>,
}
impl PartialEq for ActorExited {
fn eq(&self, other: &Self) -> bool {
self.addr == other.addr && self.reason == other.reason
}
}
impl Eq for ActorExited {}
/// The primary trait defining data that can be passed to and from actor processes
pub trait Message: 'static + Sized + Clone + Send + Sync {}
impl<T: 'static + Sized + Clone + Send + Sync> Message for T {}
@ -100,6 +166,250 @@ impl ActorAddress {
}
}
// ─── Environment ─────────────────────────────────────────────────────────────
/// A typed key-value map that flows from parent to child at spawn time.
///
/// Analogous to Unix `environ` — provides inherited configuration without
/// threading values through every constructor. Clone is an Arc bump (zero allocation).
///
/// Values are stored as `Arc<dyn Any>` so that [`EnvironmentBuilder::from_env`]
/// can clone individual entries cheaply (Arc bump) for copy-on-write overrides.
#[derive(Clone, Default)]
pub struct Environment {
inner: Arc<HashMap<TypeId, Arc<dyn Any + Send + Sync>>>,
}
impl Environment {
/// Create an empty environment.
pub fn new() -> Self {
Self {
inner: Arc::new(HashMap::new()),
}
}
/// Read a typed value from the environment.
pub fn get<T: Any + Send + Sync>(&self) -> Option<&T> {
self.inner
.get(&TypeId::of::<T>())
.and_then(|v| v.downcast_ref::<T>())
}
/// Check if the environment contains a value of type `T`.
pub fn contains<T: Any + Send + Sync>(&self) -> bool {
self.inner.contains_key(&TypeId::of::<T>())
}
/// Returns `true` if the environment has no values.
pub fn is_empty(&self) -> bool {
self.inner.is_empty()
}
/// Number of typed values in the environment.
pub fn len(&self) -> usize {
self.inner.len()
}
/// Check if the environment contains a value with the given `TypeId`.
///
/// Type-erased version of [`contains`](Self::contains) — used by
/// `ServiceRegistry::inject_into` to skip keys already present.
pub fn contains_type_id(&self, type_id: TypeId) -> bool {
self.inner.contains_key(&type_id)
}
}
/// Builder for constructing an [`Environment`].
///
/// Allows inserting/replacing typed values before freezing into an immutable `Environment`.
pub struct EnvironmentBuilder {
map: HashMap<TypeId, Arc<dyn Any + Send + Sync>>,
}
impl EnvironmentBuilder {
/// Create an empty builder.
pub fn new() -> Self {
Self {
map: HashMap::new(),
}
}
/// Create a builder pre-populated with values from an existing environment.
///
/// This enables copy-on-write overrides: clone the parent's map, modify, then freeze.
/// Cloning entries is cheap — each value is `Arc`-wrapped.
pub fn from_env(env: &Environment) -> Self {
let map = env.inner.as_ref().clone();
Self { map }
}
/// Insert or replace a typed value.
pub fn set<T: Any + Send + Sync>(mut self, value: T) -> Self {
self.map.insert(TypeId::of::<T>(), Arc::new(value));
self
}
/// Insert or replace a typed value (mutable reference version).
pub fn set_mut<T: Any + Send + Sync>(&mut self, value: T) -> &mut Self {
self.map.insert(TypeId::of::<T>(), Arc::new(value));
self
}
/// Insert a type-erased value by `TypeId`.
///
/// Used by `ServiceRegistry::inject_into` to merge pre-built bindings
/// without knowing concrete types at compile time.
pub fn set_raw(&mut self, type_id: TypeId, value: Arc<dyn Any + Send + Sync>) -> &mut Self {
self.map.insert(type_id, value);
self
}
/// Freeze the builder into an immutable `Environment`.
pub fn build(self) -> Environment {
Environment {
inner: Arc::new(self.map),
}
}
}
impl Default for EnvironmentBuilder {
fn default() -> Self {
Self::new()
}
}
// ─── Well-Known Environment Keys ─────────────────────────────────────────────
/// Milliseconds since runtime creation when this actor was spawned.
/// Injected by StdExtension (opt-in at runtime level). Read via `ctx.env::<SpawnTimestamp>()`.
/// Uses the same time base as `SystemInfo::uptime_ms`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct SpawnTimestamp(pub u64);
/// Logical name assigned via `spawn_named()`. Read via `ctx.env::<LogicalName>()`.
/// None for unnamed actors.
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct LogicalName(pub String);
impl LogicalName {
pub fn as_str(&self) -> &str {
&self.0
}
}
/// A typed service binding stored in the environment.
///
/// `S` is a zero-sized marker type that identifies the service (e.g., `struct Datastore;`).
/// Stored via `ServiceRegistry` and read via `ctx.resource::<S>()`.
#[derive(Clone, Debug)]
pub struct ServiceBinding<S: 'static + Send + Sync> {
pub addr: ActorAddress,
_marker: std::marker::PhantomData<S>,
}
impl<S: 'static + Send + Sync> ServiceBinding<S> {
pub fn new(addr: ActorAddress) -> Self {
Self {
addr,
_marker: std::marker::PhantomData,
}
}
}
// ─── Capabilities ────────────────────────────────────────────────────────────
/// Per-actor capability set controlling what operations the actor can perform.
///
/// When present in an actor's [`Environment`], enforcement is active — the actor
/// can only perform operations granted by the set. When absent, the actor is
/// unrestricted (backward compatible). Inherits from parent to child via normal
/// environment inheritance.
///
/// Built via fluent API: `CapabilitySet::new().with_send(addr).with_spawn()`.
#[derive(Clone, Default)]
pub struct CapabilitySet {
send_any: HashSet<ActorAddress, crate::delivery::AddrBuildHasher>,
send_typed: HashSet<(TypeId, ActorAddress)>,
can_spawn: bool,
service_types: HashSet<TypeId>,
monitor_targets: HashSet<ActorAddress, crate::delivery::AddrBuildHasher>,
}
impl CapabilitySet {
pub fn new() -> Self { Self::default() }
// ── Builder methods (fluent) ──
pub fn with_send(mut self, addr: ActorAddress) -> Self {
self.send_any.insert(addr); self
}
pub fn with_send_typed<M: Message>(mut self, addr: ActorAddress) -> Self {
self.send_typed.insert((TypeId::of::<M>(), addr)); self
}
pub fn with_spawn(mut self) -> Self {
self.can_spawn = true; self
}
pub fn with_service<S: 'static + Send + Sync>(mut self) -> Self {
self.service_types.insert(TypeId::of::<S>()); self
}
pub fn with_monitor(mut self, addr: ActorAddress) -> Self {
self.monitor_targets.insert(addr); self
}
// ── Mutable builder methods ──
pub fn grant_send(&mut self, addr: ActorAddress) -> &mut Self {
self.send_any.insert(addr); self
}
pub fn grant_send_typed<M: Message>(&mut self, addr: ActorAddress) -> &mut Self {
self.send_typed.insert((TypeId::of::<M>(), addr)); self
}
// ── Check methods ──
pub fn check_send<M: Message>(&self, addr: ActorAddress) -> Result<(), crate::Error> {
if self.send_any.contains(&addr) { return Ok(()); }
if self.send_typed.contains(&(TypeId::of::<M>(), addr)) { return Ok(()); }
Err(crate::Error::from("capability denied: send"))
}
pub fn check_send_addr(&self, addr: ActorAddress) -> Result<(), crate::Error> {
if self.send_any.contains(&addr) { return Ok(()); }
Err(crate::Error::from("capability denied: send"))
}
pub fn check_spawn(&self) -> Result<(), crate::Error> {
if self.can_spawn { Ok(()) } else { Err(crate::Error::from("capability denied: spawn")) }
}
pub fn check_service<S: 'static + Send + Sync>(&self) -> Result<(), crate::Error> {
if self.service_types.contains(&TypeId::of::<S>()) { Ok(()) }
else { Err(crate::Error::from("capability denied: service")) }
}
pub fn check_monitor(&self, addr: ActorAddress) -> Result<(), crate::Error> {
if self.monitor_targets.contains(&addr) { Ok(()) }
else { Err(crate::Error::from("capability denied: monitor")) }
}
}
impl std::fmt::Debug for CapabilitySet {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("CapabilitySet")
.field("send_any", &self.send_any.len())
.field("send_typed", &self.send_typed.len())
.field("can_spawn", &self.can_spawn)
.field("services", &self.service_types.len())
.field("monitors", &self.monitor_targets.len())
.finish()
}
}
// ─── SpawnRequest ────────────────────────────────────────────────────────────
/// Bundled arguments for spawning an actor.
///
/// Replaces the spawn channel 3-tuple to stop tuple growth as new fields are added.
pub struct SpawnRequest {
pub addr: ActorAddress,
pub actor: Box<dyn AnyActor>,
pub parent: Option<ActorAddress>,
pub env: Environment,
}
/// The actor process as represented in the Runtime — thin wrapper around user state.
pub struct Actor<A: ActorInterface> {
inner: A,
@ -181,23 +491,45 @@ pub enum StopReason {
Normal,
/// Actor panicked and could not be restarted.
Panicked,
/// Actor stopped with a typed exit value (via [`Ctx::stop_with`]).
Completed,
}
/// Death notification delivered as a normal message when a monitored actor dies.
///
/// Subscribe via [`Ctx::monitor`]. The `Down` message arrives in the watcher's
/// regular `handle()` method — no special callback needed.
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Debug, Clone)]
pub struct Down {
/// Address of the dead actor.
pub addr: ActorAddress,
/// Why it died.
pub reason: StopReason,
/// Typed exit value if the actor called [`Ctx::stop_with`].
pub exit_value: Option<ExitValue>,
}
impl PartialEq for Down {
fn eq(&self, other: &Self) -> bool {
self.addr == other.addr && self.reason == other.reason
}
}
impl Eq for Down {}
/// Internal sentinel message for graceful actor stop.
/// Not a `Message` — intercepted in `tick_all` before reaching `handle_any`.
pub(crate) struct StopSignal;
/// Not a `Message` (not Clone) — intercepted in `tick_all` / `deliver` before
/// reaching `handle_any`. Public so extension crates can construct it for
/// orphan cleanup, but users cannot send it via `ctx.send()`.
pub struct StopSignal;
/// Like [`StopSignal`] but carries a typed exit value.
/// Intercepted in `tick_all` / `deliver`.
pub struct StopWithSignal(pub ExitValue);
/// Internal sentinel message for resuming a suspended actor.
/// Intercepted in [`ActorPool::deliver`] — not delivered to user code.
pub(crate) struct ResumeSignal;
/// Object-safe inner trait for sending type-erased messages.
///
@ -207,13 +539,21 @@ pub(crate) struct StopSignal;
#[allow(private_interfaces)]
pub trait ContextInner {
fn send_any(&self, addr: ActorAddress, msg: Box<dyn Any + Send>) -> Result<(), Error>;
fn spawn_any(&self, addr: ActorAddress, actor: Box<dyn AnyActor>);
fn spawn_any(&self, request: SpawnRequest);
/// Request graceful stop for an actor. Takes effect after the current message.
fn request_stop(&self, addr: ActorAddress);
/// Request graceful stop with a typed exit value. Takes effect after the current message.
fn request_stop_with(&self, addr: ActorAddress, value: ExitValue);
/// Request suspension for an actor. Takes effect after the current message.
fn request_suspend(&self, addr: ActorAddress);
/// Request resumption for a suspended actor. Sends a [`ResumeSignal`].
fn request_resume(&self, addr: ActorAddress);
/// Post a request to the per-worker extension (e.g., timer scheduling).
fn post_worker_request(&self, request: Box<dyn Any + Send>);
/// Access the runtime extension (if installed).
fn extension(&self) -> Option<&dyn crate::extension::RuntimeExtension>;
/// Return system-level information (worker count, actor count, uptime).
fn system_info(&self) -> SystemInfo;
}
/// Actor syscall interface — passed to `ActorInterface::handle()`.
@ -224,11 +564,32 @@ pub trait ContextInner {
pub struct Ctx<'a> {
inner: &'a dyn ContextInner,
self_addr: ActorAddress,
self_parent_addr: Option<ActorAddress>,
self_env: Environment,
self_messages_processed: u64,
self_mailbox_depth: usize,
self_msg_type_counts: Vec<(&'static str, u64)>,
}
impl<'a> Ctx<'a> {
pub(crate) fn new(inner: &'a dyn ContextInner, self_addr: ActorAddress) -> Self {
Self { inner, self_addr }
pub(crate) fn new(
inner: &'a dyn ContextInner,
self_addr: ActorAddress,
self_parent_addr: Option<ActorAddress>,
self_env: Environment,
self_messages_processed: u64,
self_mailbox_depth: usize,
self_msg_type_counts: Vec<(&'static str, u64)>,
) -> Self {
Self {
inner,
self_addr,
self_parent_addr,
self_env,
self_messages_processed,
self_mailbox_depth,
self_msg_type_counts,
}
}
pub fn raw_inner(&self) -> &dyn ContextInner {
@ -240,24 +601,92 @@ impl<'a> Ctx<'a> {
self.self_addr
}
/// Returns the address of the actor that spawned this one, or `None`
/// if this actor was spawned externally via `Runtime::spawn`.
pub fn parent(&self) -> Option<ActorAddress> {
self.self_parent_addr
}
/// Access the runtime extension (if installed).
pub fn extension(&self) -> Option<&dyn crate::extension::RuntimeExtension> {
self.inner.extension()
}
/// Return system-level information (worker count, actor count, uptime).
pub fn system_info(&self) -> SystemInfo {
self.inner.system_info()
}
/// Total messages this actor has successfully processed (before the current tick).
pub fn messages_processed(&self) -> u64 {
self.self_messages_processed
}
/// Number of messages in this actor's mailbox at the start of the current tick.
pub fn mailbox_depth(&self) -> usize {
self.self_mailbox_depth
}
/// Per-message-type counts for this actor, sorted descending by count.
pub fn message_type_counts(&self) -> &[(&'static str, u64)] {
&self.self_msg_type_counts
}
/// Check if this actor has a capability set (i.e., is restricted).
fn capabilities(&self) -> Option<&CapabilitySet> {
self.self_env.get::<CapabilitySet>()
}
/// Send a typed message to an actor address.
pub fn send<M: Message>(&self, addr: ActorAddress, msg: M) -> Result<(), Error> {
if let Some(caps) = self.capabilities() {
if addr != self.self_addr {
caps.check_send::<M>(addr)?;
}
}
self.inner.send_any(addr, Box::new(msg))
}
/// Read a typed value from this actor's environment.
pub fn env<T: Any + Send + Sync>(&self) -> Option<&T> {
self.self_env.get::<T>()
}
/// Access this actor's full environment.
pub fn environment(&self) -> &Environment {
&self.self_env
}
/// Spawn a new actor, returning its address.
///
/// The child inherits this actor's environment (Arc clone — zero allocation).
pub fn spawn<A: ActorInterface>(&self, actor: A) -> Result<ActorAddress, Error> {
if let Some(caps) = self.capabilities() {
caps.check_spawn()?;
}
let addr = ActorAddress::new_random();
let boxed: Box<dyn AnyActor> = Box::new(Actor::new(actor));
self.inner.spawn_any(addr, boxed);
self.inner.spawn_any(SpawnRequest {
addr,
actor: boxed,
parent: Some(self.self_addr),
env: self.self_env.clone(),
});
Ok(addr)
}
/// Create a [`SpawnBuilder`] to spawn an actor with environment overrides.
///
/// Common case (`ctx.spawn(actor)`) is unchanged — this is for when you
/// need to add or replace environment values for the child.
pub fn spawn_builder<A: ActorInterface>(&self, actor: A) -> SpawnBuilder<'_, A> {
SpawnBuilder {
ctx: self,
actor,
env_builder: None,
}
}
/// Request graceful stop for this actor after the current message completes.
///
/// The actor's `on_stop()` hook is called and the actor is removed from the
@ -272,7 +701,69 @@ impl<'a> Ctx<'a> {
/// the stop signal, then its `on_stop()` hook is called and it is removed.
/// Uses PoisonPill semantics — queued after existing messages.
pub fn stop_actor(&self, addr: ActorAddress) -> Result<(), Error> {
if let Some(caps) = self.capabilities() {
caps.check_send_addr(addr)?;
}
self.inner.send_any(addr, Box::new(StopSignal))
}
/// Stop this actor with a typed exit value.
///
/// Like [`stop_self`](Self::stop_self), but the value is delivered to
/// monitors (in [`Down::exit_value`]) and watchers (in [`ActorExited::exit_value`]).
/// The stop reason is [`StopReason::Completed`].
pub fn stop_with<T: Any + Send + Sync + 'static>(&self, value: T) {
self.inner.request_stop_with(self.self_addr, ExitValue::new(value));
}
/// Suspend this actor. Messages continue to queue but are not processed
/// until a supervisor (or self) calls resume.
pub fn suspend_self(&self) {
self.inner.request_suspend(self.self_addr);
}
}
// ─── SpawnBuilder ────────────────────────────────────────────────────────────
/// Builder for spawning an actor with environment overrides.
///
/// Created via [`Ctx::spawn_builder`]. Lazily clones the parent environment
/// on the first `.env()` call to avoid allocation when no overrides are needed.
pub struct SpawnBuilder<'a, A: ActorInterface> {
ctx: &'a Ctx<'a>,
actor: A,
env_builder: Option<EnvironmentBuilder>,
}
impl<'a, A: ActorInterface> SpawnBuilder<'a, A> {
/// Add or replace a typed environment value for the child.
///
/// On the first call, lazily clones the parent's environment map.
pub fn env<T: Any + Send + Sync>(mut self, value: T) -> Self {
let builder = self.env_builder.get_or_insert_with(|| {
EnvironmentBuilder::from_env(self.ctx.environment())
});
builder.set_mut(value);
self
}
/// Spawn the actor, returning its address.
pub fn finish(self) -> Result<ActorAddress, Error> {
if let Some(caps) = self.ctx.capabilities() {
caps.check_spawn()?;
}
let addr = ActorAddress::new_random();
let boxed: Box<dyn AnyActor> = Box::new(Actor::new(self.actor));
let env = match self.env_builder {
Some(builder) => builder.build(),
None => self.ctx.self_env.clone(),
};
self.ctx.inner.spawn_any(SpawnRequest {
addr,
actor: boxed,
parent: Some(self.ctx.self_addr),
env,
});
Ok(addr)
}
}

8
src/delivery.rs
View file

@ -5,7 +5,7 @@ use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, OnceLock, RwLock};
use std::thread::Thread;
use crate::actor::{ActorAddress, AnyActor, Message};
use crate::actor::{ActorAddress, Message, SpawnRequest};
use crate::channel::Sender;
use crate::config::RuntimeConfig;
use crate::stats::WorkerStats;
@ -235,7 +235,7 @@ impl InboxRegistry {
pub(crate) struct TickContext<'a> {
pub(crate) address_map: &'a AddressMap,
pub(crate) transfer_txs: &'a [Sender<Envelope>],
pub(crate) spawn_txs: &'a [Sender<(ActorAddress, Box<dyn AnyActor>)>],
pub(crate) spawn_txs: &'a [Sender<SpawnRequest>],
pub(crate) placement: &'a Placement,
pub(crate) inbox_registry: &'a InboxRegistry,
pub(crate) config: &'a RuntimeConfig,
@ -243,6 +243,10 @@ pub(crate) struct TickContext<'a> {
pub(crate) stats_hook: Option<&'a dyn crate::stats::StatsHook>,
/// Thread handles for waking parked workers on cross-worker sends.
pub(crate) worker_threads: &'a [OnceLock<Thread>],
/// Per-worker stats for summing total_actors across workers.
pub(crate) worker_stats: &'a [Arc<WorkerStats>],
/// Runtime creation time for computing uptime_ms.
pub(crate) created_at: crate::Instant,
#[cfg(feature = "transport")]
pub(crate) codec_registry: Option<&'a crate::transport::CodecRegistry>,
#[cfg(feature = "transport")]

15
src/extension.rs
View file

@ -1,6 +1,6 @@
use std::any::Any;
use crate::actor::{ActorAddress, StopReason};
use crate::actor::{ActorAddress, Environment, ExitValue, StopReason};
/// Extension hook for runtime lifecycle events.
///
@ -17,12 +17,23 @@ pub trait RuntimeExtension: Send + Sync {
/// The core delivers these through normal routing (pending_local or transfer queue).
fn on_actor_death(
&self,
dead: &[(ActorAddress, StopReason)],
dead: &[(ActorAddress, StopReason, Option<ExitValue>)],
) -> Vec<(ActorAddress, Box<dyn Any + Send>)>;
/// Clean up extension state for dead actors (names, groups, monitors).
fn cleanup_dead(&self, dead: &[ActorAddress]);
/// Called for each newly spawned actor, before it enters the pool.
/// Extensions can enrich the actor's environment (e.g., inject SpawnTimestamp).
/// `child` is the address of the newly spawned actor.
/// `parent` is the address of the spawning actor, or `None` for runtime-spawned actors.
/// `uptime_ms` is milliseconds since runtime creation.
/// Default: no-op (returns env unchanged).
fn on_spawn(&self, child: ActorAddress, parent: Option<ActorAddress>, env: Environment, uptime_ms: u64) -> Environment {
let _ = (child, parent, uptime_ms);
env
}
/// Downcast support for Ctx extension traits.
fn as_any(&self) -> &dyn Any;

3
src/lib.rs
View file

@ -2,6 +2,9 @@ pub mod actor;
pub mod extension;
pub mod worker;
// Re-export well-known environment key types for convenient access.
pub use actor::{SpawnTimestamp, LogicalName, ServiceBinding, ExitValue, CapabilitySet};
pub(crate) mod channel;
pub(crate) mod error;
pub use error::Error;

69
src/runtime.rs
View file

@ -7,7 +7,7 @@ use std::thread::{self, JoinHandle};
use std::thread::Thread;
use crate::Instant;
use crate::actor::{Actor, ActorAddress, ActorInterface, AnyActor, Message, StopSignal};
use crate::actor::{Actor, ActorAddress, ActorInterface, AnyActor, Environment, ExitValue, Message, ResumeSignal, SpawnRequest, StopSignal, StopWithSignal, SystemInfo};
use crate::channel::{Receiver, Sender};
// Re-export config types so existing code using `runtime::RuntimeConfig` still works
pub use crate::config::{BackoffPolicy, MailboxOverflow, RuntimeConfig};
@ -102,7 +102,7 @@ pub struct Runtime {
inbox_registry: Arc<InboxRegistry>,
extension: Option<Arc<dyn RuntimeExtension>>,
transfer_txs: Vec<Sender<Envelope>>,
spawn_txs: Vec<Sender<(ActorAddress, Box<dyn AnyActor>)>>,
spawn_txs: Vec<Sender<SpawnRequest>>,
placement: Placement,
is_running: AtomicBool,
worker_stats: Vec<Arc<WorkerStats>>,
@ -170,7 +170,7 @@ impl Runtime {
transfer_txs.push(transfer_tx);
let spawn_rx =
Receiver::<(ActorAddress, Box<dyn AnyActor>)>::new(config.max_actors);
Receiver::<SpawnRequest>::new(config.max_actors);
let spawn_tx = spawn_rx.new_sender();
spawn_txs.push(spawn_tx);
@ -228,7 +228,26 @@ impl Runtime {
self.address_map.insert(addr, worker_id);
let boxed: Box<dyn AnyActor> = Box::new(Actor::new(actor));
self.spawn_txs[worker_id.as_usize()]
.send((addr, boxed));
.send(SpawnRequest { addr, actor: boxed, parent: None, env: Environment::new() });
#[cfg(feature = "tracing")]
tracing::info!(
actor_addr = %addr,
worker_id = worker_id.as_usize(),
"actor.spawned"
);
Ok(addr)
}
/// Spawn an actor with a pre-built environment, returns its address.
pub fn spawn_with_env<A: ActorInterface>(&self, actor: A, env: Environment) -> Result<ActorAddress, Error> {
let addr = ActorAddress::new_random();
let worker_id = self.placement.next_worker();
self.address_map.insert(addr, worker_id);
let boxed: Box<dyn AnyActor> = Box::new(Actor::new(actor));
self.spawn_txs[worker_id.as_usize()]
.send(SpawnRequest { addr, actor: boxed, parent: None, env });
#[cfg(feature = "tracing")]
tracing::info!(
@ -309,6 +328,8 @@ impl Runtime {
extension: self.extension.as_deref(),
stats_hook: self.stats_hook.as_deref(),
worker_threads: &self.worker_threads,
worker_stats: &self.worker_stats,
created_at: self.created_at,
#[cfg(feature = "transport")]
codec_registry: self.codec_registry.as_deref(),
#[cfg(feature = "transport")]
@ -487,11 +508,11 @@ impl ContextInner for Runtime {
}
}
fn spawn_any(&self, addr: ActorAddress, actor: Box<dyn AnyActor>) {
fn spawn_any(&self, request: SpawnRequest) {
let worker_id = self.placement.next_worker();
self.address_map.insert(addr, worker_id);
self.address_map.insert(request.addr, worker_id);
self.spawn_txs[worker_id.as_usize()]
.send((addr, actor));
.send(request);
notify_worker(&self.worker_threads, worker_id.as_usize());
}
@ -504,6 +525,27 @@ impl ContextInner for Runtime {
}
}
fn request_stop_with(&self, addr: ActorAddress, value: ExitValue) {
if let Some(wid) = self.address_map.lookup(&addr) {
self.transfer_txs[wid.as_usize()]
.send(Envelope::new(addr, Box::new(StopWithSignal(value))));
notify_worker(&self.worker_threads, wid.as_usize());
}
}
fn request_suspend(&self, addr: ActorAddress) {
// Outside worker context — not supported (suspend is per-actor, from handler)
eprintln!("swactor: request_suspend called outside worker context for {addr} — ignored");
}
fn request_resume(&self, addr: ActorAddress) {
if let Some(wid) = self.address_map.lookup(&addr) {
self.transfer_txs[wid.as_usize()]
.send(Envelope::new(addr, Box::new(ResumeSignal)));
notify_worker(&self.worker_threads, wid.as_usize());
}
}
fn post_worker_request(&self, _request: Box<dyn Any + Send>) {
// Worker requests (e.g., timers) are per-worker; posting from outside
// a worker context (e.g., rt.spawn() callback) is not supported.
@ -513,4 +555,17 @@ impl ContextInner for Runtime {
fn extension(&self) -> Option<&dyn RuntimeExtension> {
self.extension.as_deref()
}
fn system_info(&self) -> SystemInfo {
let num_workers = self.config.num_threads.max(1);
let total_actors: usize = self.worker_stats.iter()
.map(|ws| ws.num_actors.load(Ordering::Relaxed))
.sum();
SystemInfo {
worker_id: 0,
num_workers,
total_actors,
uptime_ms: self.created_at.elapsed().as_millis() as u64,
}
}
}

227
src/worker.rs
View file

@ -6,7 +6,7 @@ use std::sync::Arc;
use std::thread;
use crate::Instant;
use crate::actor::{ActorAddress, AnyActor, ContextInner, Ctx, StopReason, StopSignal};
use crate::actor::{ActorAddress, AnyActor, ContextInner, Ctx, Environment, ExitValue, ResumeSignal, SpawnRequest, StopReason, StopSignal, StopWithSignal, SystemInfo};
use crate::channel::Receiver;
use crate::config::MailboxOverflow;
use crate::delivery::{AddrBuildHasher, AddrMap, Envelope, TickContext, WorkerId};
@ -45,7 +45,7 @@ pub(crate) struct Worker {
pub(crate) id: WorkerId,
pub(crate) pool: ActorPool,
transfer_rx: Receiver<Envelope>,
spawn_rx: Receiver<(ActorAddress, Box<dyn AnyActor>)>,
spawn_rx: Receiver<SpawnRequest>,
stats: Arc<WorkerStats>,
/// Reusable scratch buffer for building per-actor snapshots.
snapshot_buf: Vec<ActorSnapshot>,
@ -57,7 +57,7 @@ impl Worker {
pub(crate) fn new(
id: WorkerId,
transfer_rx: Receiver<Envelope>,
spawn_rx: Receiver<(ActorAddress, Box<dyn AnyActor>)>,
spawn_rx: Receiver<SpawnRequest>,
stats: Arc<WorkerStats>,
default_mailbox_capacity: usize,
default_overflow_policy: MailboxOverflow,
@ -76,12 +76,15 @@ impl Worker {
/// Run one iteration of the worker loop. Returns `true` if any work was done.
/// Drain the spawn queue, inserting new actors into the pool.
/// Used in phases 1 and 4 of tick_once.
fn drain_spawns(&mut self) -> bool {
fn drain_spawns(&mut self, tc: &TickContext) -> bool {
let mut did_work = false;
#[cfg(feature = "tracing")]
let mut spawn_count: usize = 0;
while let Some((addr, actor)) = self.spawn_rx.try_recv() {
self.pool.insert(addr, actor);
while let Some(mut req) = self.spawn_rx.try_recv() {
if let Some(ext) = tc.extension {
req.env = ext.on_spawn(req.addr, req.parent, req.env, tc.created_at.elapsed().as_millis() as u64);
}
self.pool.insert(req);
#[cfg(feature = "tracing")]
{ spawn_count += 1; }
did_work = true;
@ -98,6 +101,8 @@ impl Worker {
let cleanup_pending: RefCell<Vec<(ActorAddress, Box<dyn Any + Send>)>> =
RefCell::new(Vec::new());
let cleanup_stops: RefCell<Vec<ActorAddress>> = RefCell::new(Vec::new());
let cleanup_stop_withs: RefCell<Vec<(ActorAddress, ExitValue)>> = RefCell::new(Vec::new());
let cleanup_suspends: RefCell<Vec<ActorAddress>> = RefCell::new(Vec::new());
let cleanup_requests: RefCell<Vec<Box<dyn Any + Send>>> = RefCell::new(Vec::new());
let dead = {
let cleanup_ctx = WorkerContext {
@ -105,6 +110,8 @@ impl Worker {
tc,
pending_local: &cleanup_pending,
stop_requests: &cleanup_stops,
stop_with_values: &cleanup_stop_withs,
suspend_requests: &cleanup_suspends,
worker_requests: &cleanup_requests,
stats: &self.stats,
};
@ -113,13 +120,13 @@ impl Worker {
let had_dead = !dead.is_empty();
if had_dead {
for &(addr, _) in &dead {
tc.address_map.remove(&addr);
for (addr, _, _) in &dead {
tc.address_map.remove(addr);
}
if let Some(ext) = tc.extension {
let notifications = ext.on_actor_death(&dead);
let dead_addrs: Vec<_> = dead.iter().map(|(a, _)| *a).collect();
let dead_addrs: Vec<_> = dead.iter().map(|(a, _, _)| *a).collect();
ext.cleanup_dead(&dead_addrs);
for (dest, msg) in notifications {
route_to_pool_or_remote(&mut self.pool, tc, dest, msg);
@ -136,7 +143,7 @@ impl Worker {
// GC per-worker extension state for dead actors
if let Some(ext) = &mut self.worker_ext {
let dead_addrs: Vec<ActorAddress> = dead.iter().map(|(a, _)| *a).collect();
let dead_addrs: Vec<ActorAddress> = dead.iter().map(|(a, _, _)| *a).collect();
ext.gc_dead(&dead_addrs);
}
@ -151,7 +158,7 @@ impl Worker {
let t0 = Instant::now();
// 1. Drain spawn queue → add actors to pool
did_work |= self.drain_spawns();
did_work |= self.drain_spawns(tc);
let t1 = Instant::now();
// 2. Drain transfer queue → deliver envelopes to actors
@ -176,6 +183,8 @@ impl Worker {
let pending_local: RefCell<Vec<(ActorAddress, Box<dyn Any + Send>)>> =
RefCell::new(Vec::new());
let stop_requests: RefCell<Vec<ActorAddress>> = RefCell::new(Vec::new());
let stop_with_values: RefCell<Vec<(ActorAddress, ExitValue)>> = RefCell::new(Vec::new());
let suspend_requests: RefCell<Vec<ActorAddress>> = RefCell::new(Vec::new());
let worker_requests: RefCell<Vec<Box<dyn Any + Send>>> = RefCell::new(Vec::new());
let processed;
@ -185,10 +194,12 @@ impl Worker {
tc,
pending_local: &pending_local,
stop_requests: &stop_requests,
stop_with_values: &stop_with_values,
suspend_requests: &suspend_requests,
worker_requests: &worker_requests,
stats: &self.stats,
};
processed = self.pool.tick_all(&worker_ctx, &self.stats, tc.config.actor_message_budget, &stop_requests);
processed = self.pool.tick_all(&worker_ctx, &self.stats, tc.config.actor_message_budget, &stop_requests, &stop_with_values, &suspend_requests);
if processed > 0 {
did_work = true;
}
@ -206,7 +217,7 @@ impl Worker {
// 4. Drain spawn queue again — actors spawned during step 3
// must be in the pool before pending_local delivery.
did_work |= self.drain_spawns();
did_work |= self.drain_spawns(tc);
let t4 = Instant::now();
// 5. Drain pending_local buffer → deliver to local actors
@ -316,6 +327,8 @@ struct WorkerContext<'a> {
tc: &'a TickContext<'a>,
pending_local: &'a RefCell<Vec<(ActorAddress, Box<dyn Any + Send>)>>,
stop_requests: &'a RefCell<Vec<ActorAddress>>,
stop_with_values: &'a RefCell<Vec<(ActorAddress, ExitValue)>>,
suspend_requests: &'a RefCell<Vec<ActorAddress>>,
worker_requests: &'a RefCell<Vec<Box<dyn Any + Send>>>,
stats: &'a WorkerStats,
}
@ -341,11 +354,11 @@ impl ContextInner for WorkerContext<'_> {
}
}
fn spawn_any(&self, addr: ActorAddress, actor: Box<dyn AnyActor>) {
fn spawn_any(&self, request: SpawnRequest) {
let worker_id = self.tc.placement.next_worker();
self.tc.address_map.insert(addr, worker_id);
self.tc.address_map.insert(request.addr, worker_id);
self.tc.spawn_txs[worker_id.as_usize()]
.send((addr, actor));
.send(request);
crate::runtime::notify_worker(self.tc.worker_threads, worker_id.as_usize());
}
@ -353,6 +366,20 @@ impl ContextInner for WorkerContext<'_> {
self.stop_requests.borrow_mut().push(addr);
}
fn request_stop_with(&self, addr: ActorAddress, value: ExitValue) {
self.stop_with_values.borrow_mut().push((addr, value));
}
fn request_suspend(&self, addr: ActorAddress) {
self.suspend_requests.borrow_mut().push(addr);
}
fn request_resume(&self, addr: ActorAddress) {
// Same-worker: buffer as pending_local ResumeSignal
// Cross-worker: would go through transfer queue (handled by Runtime impl)
self.pending_local.borrow_mut().push((addr, Box::new(ResumeSignal)));
}
fn post_worker_request(&self, request: Box<dyn Any + Send>) {
self.worker_requests.borrow_mut().push(request);
}
@ -360,6 +387,19 @@ impl ContextInner for WorkerContext<'_> {
fn extension(&self) -> Option<&dyn crate::extension::RuntimeExtension> {
self.tc.extension
}
fn system_info(&self) -> SystemInfo {
let num_workers = self.tc.config.num_threads.max(1);
let total_actors: usize = self.tc.worker_stats.iter()
.map(|ws| ws.num_actors.load(Ordering::Relaxed))
.sum();
SystemInfo {
worker_id: self.worker_id.0,
num_workers,
total_actors,
uptime_ms: self.tc.created_at.elapsed().as_millis() as u64,
}
}
}
struct ActorSlot {
@ -370,6 +410,8 @@ struct ActorSlot {
stopping: bool,
/// Whether on_start has been called for this actor.
started: bool,
/// Actor is suspended — messages queue but are not processed.
suspended: bool,
last_msg_type: Option<&'static str>,
messages_processed: u64,
/// Per-message-type counters (bounded to 32 entries).
@ -377,6 +419,12 @@ struct ActorSlot {
/// Per-actor mailbox capacity. 0 = unbounded.
mailbox_capacity: usize,
overflow_policy: MailboxOverflow,
/// Address of the actor that spawned this one, or `None` for externally-spawned actors.
parent_addr: Option<ActorAddress>,
/// Inherited environment from parent (or empty for runtime-spawned actors).
env: Environment,
/// Typed exit value set by `ctx.stop_with()`.
exit_value: Option<ExitValue>,
}
/// Per-worker actor storage. Owns per-actor mailboxes.
@ -398,20 +446,24 @@ impl ActorPool {
}
}
pub fn insert(&mut self, addr: ActorAddress, actor: Box<dyn AnyActor>) {
pub fn insert(&mut self, req: SpawnRequest) {
let cap = self.default_mailbox_capacity;
let prealloc = if cap > 0 { cap.min(64) } else { 16 };
self.actors.insert(addr, ActorSlot {
self.actors.insert(req.addr, ActorSlot {
mailbox: VecDeque::with_capacity(prealloc),
actor,
actor: req.actor,
poisoned: false,
stopping: false,
started: false,
suspended: false,
last_msg_type: None,
messages_processed: 0,
msg_type_counts: HashMap::new(),
mailbox_capacity: self.default_mailbox_capacity,
overflow_policy: self.default_overflow_policy,
parent_addr: req.parent,
env: req.env,
exit_value: None,
});
}
@ -419,6 +471,27 @@ impl ActorPool {
/// Returns `true` if the actor exists (message handled or dropped; type check deferred to tick).
pub fn deliver(&mut self, addr: &ActorAddress, msg: Box<dyn Any + Send>) -> bool {
if let Some(slot) = self.actors.get_mut(addr) {
// Intercept control signals for suspended actors: they skip tick_all
// so we must handle resume/stop at delivery time.
if slot.suspended {
if msg.is::<ResumeSignal>() {
slot.suspended = false;
return true;
}
if msg.is::<StopSignal>() {
slot.stopping = true;
slot.mailbox.clear();
return true;
}
if msg.is::<StopWithSignal>() {
if let Ok(sig) = msg.downcast::<StopWithSignal>() {
slot.exit_value = Some(sig.0);
}
slot.stopping = true;
slot.mailbox.clear();
return true;
}
}
if slot.mailbox_capacity > 0 && slot.mailbox.len() >= slot.mailbox_capacity {
match slot.overflow_policy {
MailboxOverflow::DropNewest => {
@ -453,6 +526,8 @@ impl ActorPool {
stats: &WorkerStats,
budget: usize,
stop_requests: &RefCell<Vec<ActorAddress>>,
stop_with_values: &RefCell<Vec<(ActorAddress, ExitValue)>>,
suspend_requests: &RefCell<Vec<ActorAddress>>,
) -> usize {
let mut count = 0;
for (&addr, slot) in self.actors.iter_mut() {
@ -462,10 +537,22 @@ impl ActorPool {
continue;
}
// Skip suspended actors — messages keep queueing
if slot.suspended {
continue;
}
#[cfg(feature = "tracing")]
let _actor_span = tracing::trace_span!("actor.tick", actor_addr = %addr).entered();
let ctx = Ctx::new(inner, addr);
// Snapshot self-stats before creating Ctx
let snap_processed = slot.messages_processed;
let snap_depth = slot.mailbox.len();
let mut snap_type_counts: Vec<(&'static str, u64)> =
slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect();
snap_type_counts.sort_by(|a, b| b.1.cmp(&a.1));
let ctx = Ctx::new(inner, addr, slot.parent_addr, slot.env.clone(), snap_processed, snap_depth, snap_type_counts);
// Call on_start once, before first message
if !slot.started {
@ -482,7 +569,7 @@ impl ActorPool {
slot.mailbox.clear();
continue;
}
// Check if on_start requested stop
// Check if on_start requested stop or stop_with
{
let stops = stop_requests.borrow();
if !stops.is_empty() && stops.contains(&addr) {
@ -490,6 +577,33 @@ impl ActorPool {
slot.stopping = true;
stats.stops.fetch_add(1, Ordering::Relaxed);
slot.mailbox.clear();
// Check for stop_with value
let mut sws = stop_with_values.borrow_mut();
if let Some(pos) = sws.iter().position(|(a, _)| *a == addr) {
let (_, val) = sws.swap_remove(pos);
slot.exit_value = Some(val);
}
continue;
}
}
// Check if on_start requested stop_with (without plain stop)
{
let mut sws = stop_with_values.borrow_mut();
if let Some(pos) = sws.iter().position(|(a, _)| *a == addr) {
let (_, val) = sws.swap_remove(pos);
slot.exit_value = Some(val);
slot.stopping = true;
stats.stops.fetch_add(1, Ordering::Relaxed);
slot.mailbox.clear();
continue;
}
}
// Check if on_start requested suspend
{
let suspends = suspend_requests.borrow();
if !suspends.is_empty() && suspends.contains(&addr) {
drop(suspends);
slot.suspended = true;
continue;
}
}
@ -507,6 +621,17 @@ impl ActorPool {
break;
}
// Intercept StopWithSignal (from external runtime)
if msg.is::<StopWithSignal>() {
if let Ok(sig) = msg.downcast::<StopWithSignal>() {
slot.exit_value = Some(sig.0);
}
slot.stopping = true;
stats.stops.fetch_add(1, Ordering::Relaxed);
slot.mailbox.clear();
break;
}
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
slot.actor.handle_any(&ctx, msg)
}));
@ -536,16 +661,37 @@ impl ActorPool {
count += 1;
actor_count += 1;
// Check if handler requested self-stop (via ctx.stop_self())
// Check if handler requested self-stop or stop_with
{
let stops = stop_requests.borrow();
if !stops.is_empty() && stops.contains(&addr) {
drop(stops);
let has_stop = !stops.is_empty() && stops.contains(&addr);
drop(stops);
let mut sws = stop_with_values.borrow_mut();
let sw_pos = sws.iter().position(|(a, _)| *a == addr);
if has_stop || sw_pos.is_some() {
if let Some(pos) = sw_pos {
let (_, val) = sws.swap_remove(pos);
slot.exit_value = Some(val);
}
drop(sws);
slot.stopping = true;
stats.stops.fetch_add(1, Ordering::Relaxed);
slot.mailbox.clear();
break;
}
drop(sws);
}
// Check if handler requested suspend
{
let suspends = suspend_requests.borrow();
if !suspends.is_empty() && suspends.contains(&addr) {
drop(suspends);
slot.suspended = true;
break; // stop processing this actor's messages this tick
}
}
if budget > 0 && actor_count >= budget {
@ -568,30 +714,45 @@ impl ActorPool {
self.actors.values().map(|slot| slot.mailbox.len()).sum()
}
/// Remove poisoned and stopping actors, returning their addresses and stop reasons.
/// Remove poisoned and stopping actors, returning their addresses, stop reasons,
/// and optional exit values.
/// Called after tick_all so the caller can clean up the address map.
///
/// For stopping actors: calls `on_stop()` before removal (wrapped in catch_unwind).
/// For poisoned actors: `on_stop()` is NOT called (state may be corrupt).
pub fn cleanup_dead(&mut self, inner: &dyn ContextInner) -> Vec<(ActorAddress, StopReason)> {
let dead: Vec<(ActorAddress, StopReason)> = self
pub fn cleanup_dead(&mut self, inner: &dyn ContextInner) -> Vec<(ActorAddress, StopReason, Option<ExitValue>)> {
let dead_addrs: Vec<ActorAddress> = self
.actors
.iter()
.filter(|(_, slot)| slot.poisoned || slot.stopping)
.map(|(&addr, slot)| {
let reason = if slot.poisoned { StopReason::Panicked } else { StopReason::Normal };
(addr, reason)
})
.map(|(&addr, _)| addr)
.collect();
for &(addr, _) in &dead {
let mut dead = Vec::with_capacity(dead_addrs.len());
for addr in dead_addrs {
if let Some(mut slot) = self.actors.remove(&addr) {
let reason = if slot.poisoned {
StopReason::Panicked
} else if slot.exit_value.is_some() {
StopReason::Completed
} else {
StopReason::Normal
};
// Call on_stop for gracefully stopping actors only
if slot.stopping && !slot.poisoned {
let ctx = Ctx::new(inner, addr);
let mut type_counts: Vec<(&'static str, u64)> =
slot.msg_type_counts.iter().map(|(&k, &v)| (k, v)).collect();
type_counts.sort_by(|a, b| b.1.cmp(&a.1));
let ctx = Ctx::new(
inner, addr, slot.parent_addr, slot.env.clone(),
slot.messages_processed,
slot.mailbox.len(),
type_counts,
);
let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
slot.actor.on_stop(&ctx);
}));
}
dead.push((addr, reason, slot.exit_value.take()));
// slot is dropped here — actor resources freed
}
}

12
tests/actor_lifecycle.rs
View file

@ -216,7 +216,7 @@ impl ActorInterface for MonitorWatcherActor {
type Incoming = Down;
type Response = ();
fn on_start(&mut self, ctx: &Ctx) {
self.mref = Some(ctx.monitor(self.watch_target));
self.mref = Some(ctx.monitor(self.watch_target).unwrap());
}
fn handle(&mut self, ctx: &Ctx, msg: Down) {
ctx.send(self.reply_to, msg).unwrap();
@ -233,7 +233,7 @@ impl ActorInterface for DemonitorActor {
type Incoming = Ping;
type Response = ();
fn on_start(&mut self, ctx: &Ctx) {
self.mref = Some(ctx.monitor(self.watch_target));
self.mref = Some(ctx.monitor(self.watch_target).unwrap());
}
fn handle(&mut self, ctx: &Ctx, _msg: Ping) {
if let Some(mref) = self.mref.take() {
@ -705,8 +705,8 @@ fn monitor_death_notification_contract() {
type Incoming = Down;
type Response = ();
fn on_start(&mut self, ctx: &Ctx) {
ctx.monitor(self.target);
ctx.monitor(self.target);
ctx.monitor(self.target).unwrap();
ctx.monitor(self.target).unwrap();
}
fn handle(&mut self, ctx: &Ctx, msg: Down) {
ctx.send(self.reply_to, msg).unwrap();
@ -734,7 +734,7 @@ fn monitor_death_notification_contract() {
type Incoming = Ping;
type Response = ();
fn on_start(&mut self, ctx: &Ctx) {
ctx.monitor(self.target);
ctx.monitor(self.target).unwrap();
}
fn handle(&mut self, ctx: &Ctx, _msg: Ping) {
let _ = ctx.send(self.inbox, Count(self.downs.len()));
@ -768,7 +768,7 @@ fn monitor_death_notification_contract() {
type Incoming = Down;
type Response = ();
fn on_start(&mut self, ctx: &Ctx) {
ctx.monitor(self.target);
ctx.monitor(self.target).unwrap();
}
fn handle(&mut self, ctx: &Ctx, msg: Down) {
let _ = ctx.send(self.inbox, msg);

11
tests/common/mod.rs
View file

@ -6,13 +6,16 @@ use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
pub use swactor::actor::{
ActorAddress, ActorExited, ActorInterface, Down, ExitReason, MonitorRef, StopReason,
ActorAddress, ActorExited, ActorInterface, CapabilitySet, Down, Environment, EnvironmentBuilder,
ExitReason, ExitValue, LogicalName, MonitorRef, ServiceBinding, SpawnBuilder, SpawnTimestamp,
StopReason,
};
pub use swactor::runtime::{Ctx, Inbox, MailboxOverflow, Runtime, RuntimeConfig};
pub use swactor_std::{
ChildSpec, CtxGroups, CtxMonitoring, CtxNaming, CtxTimers, CtxWatching, RestartPolicy, Router,
RoutingStrategy, RuntimeGroups, RuntimeNaming, RuntimeWatching, StdExtension, Supervisor,
SupervisorStrategy,
ChildSpec, CtxCapabilities, CtxEnvironment, CtxGroups, CtxHandles, CtxLifecycle, CtxLineage,
CtxMonitoring, CtxNaming, CtxResources, CtxSelfStats, CtxSystem, CtxTimers, CtxWatching,
ResourceHandle, RestartPolicy, Router, RoutingStrategy, RuntimeGroups, RuntimeNaming,
RuntimeResources, RuntimeWatching, StdExtension, Supervisor, SupervisorStrategy,
};
// ── Messages ────────────────────────────────────────────────────────────────

77
tests/docker/docker-compose.dev-cluster.yml Normal file
View file

@ -0,0 +1,77 @@
services:
seed:
image: swactor-dev-cluster
command:
- "--identity-dir"
- "/identity"
- "--listen"
- "0.0.0.0:11204"
- "--no-relay"
- "--pool-name"
- "dev-pool"
- "--pool-capacity"
- "104857600"
- "--actors"
- "2"
volumes:
- /home/aaron/swactor-pooled-datastore/.dev-cluster/seed-identity:/identity:ro
networks:
dev-cluster:
ipv4_address: 10.0.2.10
ports:
- "9100:9090"
node-2:
image: swactor-dev-cluster
command:
- "--seed-node-id"
- "d5469553d4408073022fee3e41c99cf9778bf3621e430f75f8c8ae7d6de0153f"
- "--seed-addrs"
- "10.0.2.10:11204"
- "--listen"
- "0.0.0.0:11204"
- "--no-relay"
- "--pool-name"
- "dev-pool"
- "--pool-capacity"
- "104857600"
- "--actors"
- "2"
networks:
dev-cluster:
ipv4_address: 10.0.2.11
ports:
- "9101:9090"
depends_on:
- seed
node-3:
image: swactor-dev-cluster
command:
- "--seed-node-id"
- "d5469553d4408073022fee3e41c99cf9778bf3621e430f75f8c8ae7d6de0153f"
- "--seed-addrs"
- "10.0.2.10:11204"
- "--listen"
- "0.0.0.0:11204"
- "--no-relay"
- "--pool-name"
- "dev-pool"
- "--pool-capacity"
- "104857600"
- "--actors"
- "2"
networks:
dev-cluster:
ipv4_address: 10.0.2.12
ports:
- "9102:9090"
depends_on:
- seed
networks:
dev-cluster:
driver: bridge
ipam:
config:
- subnet: 10.0.2.0/24

2756
tests/std_extension.rs
View file

File diff suppressed because it is too large Load diff

3
xtask/Cargo.toml
View file

@ -9,3 +9,6 @@ toml = "0.8"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
libc = "0.2"
reqwest = { version = "0.12", features = ["blocking", "json"] }
tokio = { version = "1", features = ["rt-multi-thread"] }
iroh-relay = { version = "0.96", features = ["server"] }

16
xtask/src/main.rs
View file

@ -1,4 +1,5 @@
mod deploy;
mod sim_cluster;
use std::path::{Path, PathBuf};
use std::process::Command;
@ -107,6 +108,13 @@ enum Cmd {
dirs: Vec<String>,
},
/// Launch a local sim-cluster (relay + N nodes) for development
SimCluster {
/// Number of nodes (default: 5)
#[arg(long, default_value = "5")]
nodes: usize,
},
/// Deploy swactor to remote machines
Deploy {
/// Deploy via Docker over SSH (build image, push, run containers)
@ -285,6 +293,7 @@ TEST GROUPS:
distribution Distribution protocol + datastore
cluster-sims Deterministic cluster simulations
integrated HTTP API + dashboard end-to-end tests
sim-cluster Multi-process cluster with local iroh relay
essential core + distribution + integrated (merge gate)
all Every test group
@ -317,6 +326,7 @@ fn print_list() {
println!();
}
println!(" {:<14}Multi-process cluster with local iroh relay", "sim-cluster");
println!(" {:<14}core + distribution + integrated (merge gate)", "essential");
println!(" {:<14}Every test group", "all");
}
@ -337,6 +347,11 @@ fn run_test(group: Option<String>, list: bool) {
}
};
if group_name == "sim-cluster" {
sim_cluster::run();
return;
}
let groups = match groups_for(&group_name) {
Some(g) => g,
None => {
@ -930,6 +945,7 @@ fn main() {
} => run_cli(url, key, extra, &config.cli),
Cmd::InitNode { role, dir } => run_init_node(&role, dir.as_deref()),
Cmd::GenPeers { dirs } => run_gen_peers(&dirs),
Cmd::SimCluster { nodes } => sim_cluster::run_interactive(nodes),
Cmd::Deploy { docker, config, skip_build, skip_verify, skip_peers } => {
let config = config.unwrap_or_else(|| {
if docker { ".deploy/docker.toml" } else { ".deploy/deploy.toml" }.into()

701
xtask/src/sim_cluster.rs Normal file
View file

@ -0,0 +1,701 @@
//! sim-cluster: multi-process cluster tests using iroh transport.
//!
//! Spawns 5 swactor nodes with iroh transport connected through a local
//! relay server, then runs the same scenarios as `tests/docker/tests/cluster.rs`
//! without Docker.
use std::fs;
use std::net::Ipv4Addr;
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Stdio};
use std::thread;
use std::time::{Duration, Instant};
use crate::workspace_root;
// ── Constants ───────────────────────────────────────────────────────────
const NODE_COUNT: usize = 5;
const ACTORS_PER_NODE: usize = 2;
const TIMEOUT: Duration = Duration::from_secs(30);
/// Dashboard HTTP ports — high ports to avoid conflicts.
fn dashboard_port(index: usize) -> u16 {
19091 + index as u16
}
fn all_dashboard_ports(node_count: usize) -> Vec<u16> {
(0..node_count).map(dashboard_port).collect()
}
// ── Local relay server ──────────────────────────────────────────────────
/// Owns a tokio runtime + iroh-relay server. RAII cleanup on drop.
struct RelayServer {
_server: iroh_relay::server::Server,
_rt: tokio::runtime::Runtime,
port: u16,
}
impl RelayServer {
fn start() -> Self {
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(1)
.enable_all()
.build()
.expect("failed to create tokio runtime for relay");
let server = rt.block_on(async {
iroh_relay::server::Server::spawn(iroh_relay::server::ServerConfig::<(), ()> {
relay: Some(iroh_relay::server::RelayConfig {
http_bind_addr: (Ipv4Addr::LOCALHOST, 0).into(),
tls: None,
limits: Default::default(),
key_cache_capacity: Some(256),
access: iroh_relay::server::AccessConfig::Everyone,
}),
quic: None,
metrics_addr: None,
})
.await
})
.expect("failed to spawn relay server");
let addr = server.http_addr().expect("relay has no HTTP address");
let port = addr.port();
RelayServer {
_server: server,
_rt: rt,
port,
}
}
}
// ── Build ───────────────────────────────────────────────────────────────
fn build_swactor(root: &Path) -> PathBuf {
let status = Command::new("cargo")
.args(["build", "-p", "swactor-node"])
.current_dir(root)
.status()
.expect("failed to run cargo build");
if !status.success() {
eprintln!("cargo build failed");
std::process::exit(1);
}
let binary = root.join("target/debug/swactor");
if !binary.exists() {
eprintln!("binary not found at {}", binary.display());
std::process::exit(1);
}
binary
}
// ── Config generation ───────────────────────────────────────────────────
fn write_node_config(
dir: &Path,
index: usize,
relay_port: u16,
seed_public_key: Option<&str>,
) -> PathBuf {
let identity_dir = dir.join("identity");
fs::create_dir_all(&identity_dir).expect("failed to create identity dir");
let config_path = dir.join("node.toml");
let dashboard = dashboard_port(index);
let mut config = format!(
r#"dashboard_port = {dashboard}
actors = {ACTORS_PER_NODE}
no_datastore = true
identity_dir = "{identity}"
relay = false
relay_port = {relay_port}
relay_hosts = ["127.0.0.1"]
"#,
identity = identity_dir.display(),
);
if let Some(seed_id) = seed_public_key {
config.push_str(&format!("seed_node_id = \"{seed_id}\"\n"));
}
fs::write(&config_path, &config).expect("failed to write node config");
config_path
}
// ── Seed key discovery ──────────────────────────────────────────────────
fn read_seed_public_key(run_dir: &Path) -> String {
let key_path = run_dir.join("node-0/identity/node.key.json");
let deadline = Instant::now() + Duration::from_secs(10);
loop {
if Instant::now() > deadline {
panic!(
"timed out waiting for seed key file: {}",
key_path.display()
);
}
if key_path.exists() {
if let Ok(data) = fs::read_to_string(&key_path) {
if let Ok(json) = serde_json::from_str::<serde_json::Value>(&data) {
if let Some(pk) = json.get("public_key").and_then(|v| v.as_str()) {
return pk.to_string();
}
}
}
}
thread::sleep(Duration::from_millis(100));
}
}
// ── Dashboard readiness ─────────────────────────────────────────────────
fn wait_for_dashboard(port: u16, timeout: Duration) {
let start = Instant::now();
loop {
if start.elapsed() > timeout {
panic!("timed out waiting for dashboard on port {port}");
}
if poll_distribution(port).is_some() {
return;
}
thread::sleep(Duration::from_millis(200));
}
}
// ── Cluster handle (RAII) ───────────────────────────────────────────────
struct SimCluster {
children: Vec<(usize, Child)>,
run_dir: PathBuf,
binary: PathBuf,
relay: RelayServer,
seed_public_key: String,
_node_count: usize,
}
impl SimCluster {
fn spawn(binary: &Path, run_dir: &Path, node_count: usize) -> Self {
fs::create_dir_all(run_dir).expect("failed to create run dir");
// Phase 0: start relay
let relay = RelayServer::start();
println!(
" relay at http://127.0.0.1:{}/ (port {})",
relay.port, relay.port
);
let mut children = Vec::new();
// Phase 1: spawn seed node (index 0) — no seed_node_id
let child = spawn_node(binary, run_dir, 0, relay.port, None);
children.push((0, child));
// Phase 2: wait for seed's key file
let seed_public_key = read_seed_public_key(run_dir);
println!(" seed key: {seed_public_key}");
// Wait for seed's dashboard to be ready before spawning joiners
wait_for_dashboard(dashboard_port(0), Duration::from_secs(15));
// Phase 3: spawn remaining nodes with seed_node_id
for i in 1..node_count {
let child = spawn_node(binary, run_dir, i, relay.port, Some(&seed_public_key));
children.push((i, child));
}
SimCluster {
children,
run_dir: run_dir.to_path_buf(),
binary: binary.to_path_buf(),
relay,
seed_public_key,
_node_count: node_count,
}
}
fn kill_node(&mut self, index: usize) {
if let Some(pos) = self.children.iter().position(|(i, _)| *i == index) {
let (_, mut child) = self.children.remove(pos);
let _ = signal_term(child.id());
let _ = child.wait();
}
}
fn restart_node(&mut self, index: usize) {
// Non-seed nodes need the seed's public key; the seed itself doesn't
let seed_id = if index != 0 {
Some(self.seed_public_key.as_str())
} else {
None
};
let child = spawn_node(
&self.binary,
&self.run_dir,
index,
self.relay.port,
seed_id,
);
self.children.push((index, child));
}
}
impl Drop for SimCluster {
fn drop(&mut self) {
for (_, child) in &self.children {
let _ = signal_term(child.id());
}
for (_, child) in &mut self.children {
let _ = child.wait();
}
// relay drops automatically via RelayServer Drop
// NOTE: leaving run_dir for debugging; uncomment below for production
// let _ = fs::remove_dir_all(&self.run_dir);
}
}
fn spawn_node(
binary: &Path,
run_dir: &Path,
index: usize,
relay_port: u16,
seed_public_key: Option<&str>,
) -> Child {
let node_dir = run_dir.join(format!("node-{index}"));
let config_path = write_node_config(&node_dir, index, relay_port, seed_public_key);
let log_path = node_dir.join("stderr.log");
let log_file = fs::File::create(&log_path)
.unwrap_or_else(|e| panic!("failed to create log for node {index}: {e}"));
Command::new(binary)
.args(["--config", &config_path.to_string_lossy()])
.stdout(Stdio::null())
.stderr(Stdio::from(log_file))
.spawn()
.unwrap_or_else(|e| panic!("failed to spawn node {index}: {e}"))
}
fn signal_term(pid: u32) -> std::io::Result<()> {
unsafe { libc::kill(pid as i32, libc::SIGTERM) };
Ok(())
}
// ── HTTP polling (decoupled from distribution crate) ────────────────────
fn poll_distribution(port: u16) -> Option<serde_json::Value> {
let url = format!("http://127.0.0.1:{port}/api/distribution");
let client = reqwest::blocking::Client::builder()
.timeout(Duration::from_secs(2))
.build()
.ok()?;
let resp = client.get(&url).send().ok()?;
if !resp.status().is_success() {
return None;
}
let text = resp.text().ok()?;
if text == "{}" {
return None;
}
serde_json::from_str(&text).ok()
}
fn get_usize(val: &serde_json::Value, key: &str) -> Option<usize> {
val.get(key).and_then(|v| v.as_u64()).map(|n| n as usize)
}
fn wait_for_convergence(
ports: &[u16],
expected_alive: usize,
timeout: Duration,
) -> Result<Duration, String> {
let start = Instant::now();
loop {
if start.elapsed() > timeout {
let mut diag = String::from("Convergence timeout. Last seen: ");
for &port in ports {
match poll_distribution(port) {
Some(snap) => {
let alive = get_usize(&snap, "alive_count").unwrap_or(0);
diag.push_str(&format!("port {port}={alive}, "));
}
None => diag.push_str(&format!("port {port}=unreachable, ")),
}
}
return Err(diag);
}
let all_converged = ports.iter().all(|&port| {
poll_distribution(port)
.and_then(|snap| get_usize(&snap, "alive_count"))
.map(|alive| alive >= expected_alive)
.unwrap_or(false)
});
if all_converged {
return Ok(start.elapsed());
}
thread::sleep(Duration::from_secs(1));
}
}
fn wait_for_death_detection(
ports: &[u16],
max_alive: usize,
timeout: Duration,
) -> Result<Duration, String> {
let start = Instant::now();
loop {
if start.elapsed() > timeout {
let mut diag = String::from("Death detection timeout. Last seen: ");
for &port in ports {
match poll_distribution(port) {
Some(snap) => {
let alive = get_usize(&snap, "alive_count").unwrap_or(0);
diag.push_str(&format!("port {port}={alive} alive, "));
}
None => diag.push_str(&format!("port {port}=unreachable, ")),
}
}
return Err(diag);
}
let all_detected = ports.iter().all(|&port| {
poll_distribution(port)
.and_then(|snap| get_usize(&snap, "alive_count"))
.map(|alive| alive <= max_alive)
.unwrap_or(false)
});
if all_detected {
return Ok(start.elapsed());
}
thread::sleep(Duration::from_secs(1));
}
}
fn wait_for_dead_count(ports: &[u16], min_dead: usize, timeout: Duration) -> bool {
let start = Instant::now();
loop {
if start.elapsed() > timeout {
return false;
}
let any_sees_dead = ports.iter().any(|&port| {
poll_distribution(port)
.and_then(|snap| get_usize(&snap, "dead_count"))
.map(|dead| dead >= min_dead)
.unwrap_or(false)
});
if any_sees_dead {
return true;
}
thread::sleep(Duration::from_secs(1));
}
}
// ── Scenarios ───────────────────────────────────────────────────────────
fn scenario_cluster_convergence(binary: &Path, base_dir: &Path) -> Result<(), String> {
let run_dir = base_dir.join("scenario-1");
let cluster = SimCluster::spawn(binary, &run_dir, NODE_COUNT);
let ports = all_dashboard_ports(NODE_COUNT);
// alive_count excludes self, so each node sees NODE_COUNT - 1 peers
let expected_alive = NODE_COUNT - 1;
let elapsed = wait_for_convergence(&ports, expected_alive, TIMEOUT)?;
println!(" converged in {:.1}s", elapsed.as_secs_f64());
// Verify each node's snapshot
for (i, &port) in ports.iter().enumerate() {
let snap = poll_distribution(port)
.ok_or_else(|| format!("node {i} (port {port}) unreachable after convergence"))?;
let alive = get_usize(&snap, "alive_count").unwrap_or(0);
let routing = get_usize(&snap, "routing_table_size").unwrap_or(0);
if alive < expected_alive {
return Err(format!("node {i} sees {alive} alive, expected >= {expected_alive}"));
}
if routing < NODE_COUNT - 1 {
return Err(format!(
"node {i} has routing_table_size {routing}, expected >= {}",
NODE_COUNT - 1
));
}
}
drop(cluster);
Ok(())
}
fn scenario_node_death_detection(binary: &Path, base_dir: &Path) -> Result<(), String> {
let run_dir = base_dir.join("scenario-2");
let mut cluster = SimCluster::spawn(binary, &run_dir, NODE_COUNT);
let ports = all_dashboard_ports(NODE_COUNT);
let expected_alive = NODE_COUNT - 1;
wait_for_convergence(&ports, expected_alive, TIMEOUT)
.map_err(|e| format!("pre-kill convergence failed: {e}"))?;
// Kill node 2
cluster.kill_node(2);
// Survivors: all except index 2
let survivor_ports: Vec<u16> = (0..NODE_COUNT)
.filter(|&i| i != 2)
.map(dashboard_port)
.collect();
// Wait for alive count to drop (alive_count excludes self, so 5-node cluster
// sees 4 alive; after killing 1, survivors should see <= 3)
let elapsed = wait_for_death_detection(&survivor_ports, NODE_COUNT - 2, TIMEOUT)?;
println!(" detected in {:.1}s", elapsed.as_secs_f64());
// Poll for dead_count — SWIM transitions suspect→dead with a delay
let dead_detected = wait_for_dead_count(&survivor_ports, 1, TIMEOUT);
if !dead_detected {
return Err("no survivor detected a dead member".into());
}
drop(cluster);
Ok(())
}
fn scenario_killed_node_rejoins(binary: &Path, base_dir: &Path) -> Result<(), String> {
let run_dir = base_dir.join("scenario-3");
let mut cluster = SimCluster::spawn(binary, &run_dir, NODE_COUNT);
let ports = all_dashboard_ports(NODE_COUNT);
let expected_alive = NODE_COUNT - 1;
wait_for_convergence(&ports, expected_alive, TIMEOUT)
.map_err(|e| format!("pre-kill convergence failed: {e}"))?;
// Kill node 2
cluster.kill_node(2);
let survivor_ports: Vec<u16> = (0..NODE_COUNT)
.filter(|&i| i != 2)
.map(dashboard_port)
.collect();
wait_for_death_detection(&survivor_ports, NODE_COUNT - 2, TIMEOUT)
.map_err(|e| format!("death detection failed: {e}"))?;
// Restart node 2 — identity persists, so cluster recognizes it
cluster.restart_node(2);
let rejoined_port = dashboard_port(2);
let elapsed = wait_for_convergence(&[rejoined_port], 1, TIMEOUT)?;
println!(" rejoined in {:.1}s", elapsed.as_secs_f64());
let snap = poll_distribution(rejoined_port)
.ok_or("rejoined node unreachable")?;
let alive = get_usize(&snap, "alive_count").unwrap_or(0);
if alive < 1 {
return Err(format!("rejoined node sees {alive} alive, expected >= 1"));
}
drop(cluster);
Ok(())
}
fn scenario_actors_resolvable(binary: &Path, base_dir: &Path) -> Result<(), String> {
let run_dir = base_dir.join("scenario-4");
let cluster = SimCluster::spawn(binary, &run_dir, NODE_COUNT);
let ports = all_dashboard_ports(NODE_COUNT);
let expected_alive = NODE_COUNT - 1;
wait_for_convergence(&ports, expected_alive, TIMEOUT)
.map_err(|e| format!("convergence failed: {e}"))?;
let mut total_directory_entries = 0usize;
for (i, &port) in ports.iter().enumerate() {
let snap = poll_distribution(port)
.ok_or_else(|| format!("node {i} unreachable"))?;
let dir_count = get_usize(&snap, "directory_entry_count").unwrap_or(0);
if dir_count < ACTORS_PER_NODE {
return Err(format!(
"node {i} has {dir_count} directory entries, expected >= {ACTORS_PER_NODE}"
));
}
total_directory_entries += dir_count;
}
let expected_total = NODE_COUNT * ACTORS_PER_NODE;
if total_directory_entries < expected_total {
return Err(format!(
"total directory entries {total_directory_entries}, expected >= {expected_total}"
));
}
println!(" total: {total_directory_entries} directory entries");
drop(cluster);
Ok(())
}
// ── Interactive mode ─────────────────────────────────────────────────────
pub fn run_interactive(node_count: usize) {
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Condvar, Mutex};
let root = workspace_root();
println!("=== sim-cluster: building swactor (iroh) ===");
let binary = build_swactor(&root);
let base_dir = root.join(".sim-cluster");
let _ = fs::remove_dir_all(&base_dir);
let run_dir = base_dir.join("interactive");
println!("=== sim-cluster: spawning {node_count} nodes ===");
let cluster = SimCluster::spawn(&binary, &run_dir, node_count);
let ports = all_dashboard_ports(node_count);
// Wait for all dashboards
println!("=== sim-cluster: waiting for dashboards ===");
for &port in &ports {
wait_for_dashboard(port, Duration::from_secs(30));
}
// Wait for convergence
println!("=== sim-cluster: waiting for convergence ===");
let expected_alive = node_count - 1;
let convergence = wait_for_convergence(&ports, expected_alive, TIMEOUT);
let converged_msg = match &convergence {
Ok(elapsed) => format!("{node_count} nodes, all converged in {:.1}s", elapsed.as_secs_f64()),
Err(e) => format!("{node_count} nodes, convergence issue: {e}"),
};
// Print summary
println!();
println!("=== sim-cluster ready ===");
println!(" relay: http://127.0.0.1:{}/", cluster.relay.port);
for i in 0..node_count {
let port = dashboard_port(i);
let label = if i == 0 { " (seed)" } else { "" };
println!(" node-{i}: http://127.0.0.1:{port}/{label}");
}
println!(" cluster: {converged_msg}");
println!();
println!(" Logs: .sim-cluster/interactive/node-N/stderr.log");
println!(" Press Ctrl-C to shut down.");
// Block on Ctrl-C
let shutdown = Arc::new((Mutex::new(false), Condvar::new()));
let shutdown2 = Arc::clone(&shutdown);
let flag = Arc::new(AtomicBool::new(false));
let flag2 = Arc::clone(&flag);
unsafe {
let shutdown_ptr = Arc::into_raw(shutdown2) as usize;
let flag_ptr = Arc::into_raw(flag2) as usize;
libc::signal(libc::SIGINT, handler as *const () as libc::sighandler_t);
SHUTDOWN_PTR.store(shutdown_ptr, Ordering::SeqCst);
FLAG_PTR.store(flag_ptr, Ordering::SeqCst);
}
let (lock, cvar) = &*shutdown;
let mut stopped = lock.lock().unwrap();
while !*stopped {
stopped = cvar.wait(stopped).unwrap();
}
println!("\n=== sim-cluster: shutting down ===");
drop(cluster);
println!("=== sim-cluster: stopped ===");
}
// Signal handler support for run_interactive
use std::sync::atomic::{AtomicUsize, Ordering};
static SHUTDOWN_PTR: AtomicUsize = AtomicUsize::new(0);
static FLAG_PTR: AtomicUsize = AtomicUsize::new(0);
extern "C" fn handler(_sig: libc::c_int) {
use std::sync::atomic::AtomicBool;
use std::sync::{Condvar, Mutex};
let flag_ptr = FLAG_PTR.load(Ordering::SeqCst);
if flag_ptr != 0 {
let flag = unsafe { &*(flag_ptr as *const AtomicBool) };
if flag.swap(true, Ordering::SeqCst) {
// Second Ctrl-C — force exit
std::process::exit(1);
}
}
let ptr = SHUTDOWN_PTR.load(Ordering::SeqCst);
if ptr != 0 {
let pair = unsafe { &*(ptr as *const (Mutex<bool>, Condvar)) };
if let Ok(mut stopped) = pair.0.lock() {
*stopped = true;
pair.1.notify_one();
}
}
}
// ── Entry point (test mode) ─────────────────────────────────────────────
pub fn run() {
let root = workspace_root();
let overall_start = Instant::now();
println!("=== sim-cluster: building swactor (iroh) ===");
let binary = build_swactor(&root);
let base_dir = root.join(".sim-cluster");
// Clean any stale runs
let _ = fs::remove_dir_all(&base_dir);
let scenarios: &[(&str, fn(&Path, &Path) -> Result<(), String>)] = &[
("cluster convergence", scenario_cluster_convergence),
("node death detection", scenario_node_death_detection),
("killed node rejoins", scenario_killed_node_rejoins),
("actors resolvable", scenario_actors_resolvable),
];
let total = scenarios.len();
let mut passed = 0usize;
for (i, (name, func)) in scenarios.iter().enumerate() {
println!(
"=== sim-cluster: scenario {}/{total} \u{2014} {name} ===",
i + 1
);
match func(&binary, &base_dir) {
Ok(()) => passed += 1,
Err(e) => {
let elapsed = overall_start.elapsed();
eprintln!(" FAILED: {e}");
eprintln!(
"\n--- FAILED after {:.1}s ({passed}/{total} passed) ---",
elapsed.as_secs_f64()
);
// Leave .sim-cluster for debugging
std::process::exit(1);
}
}
}
// Clean up base dir
let _ = fs::remove_dir_all(&base_dir);
let elapsed = overall_start.elapsed();
println!(
"\n--- All {total} scenario(s) passed in {:.1}s ---",
elapsed.as_secs_f64()
);
}