feat: datastore complete with manual testing and docs

This commit is contained in:
Zachery Aaron Shores-Chmielewski 2026-02-16 00:01:30 +07:00
parent ebf778fc3f
commit b74d3f7b5f
33 changed files with 2461 additions and 33 deletions

2
.cargo/config.toml Normal file
View file

@ -0,0 +1,2 @@
[alias]
xtask = "run --package xtask --"

5
Cargo.lock generated
View file

@ -4391,6 +4391,7 @@ dependencies = [
"swactor-std",
"tempfile",
"tiny_http",
"toml",
"ureq",
]
@ -6231,6 +6232,10 @@ dependencies = [
"xml-rs",
]
[[package]]
name = "xtask"
version = "0.1.0"
[[package]]
name = "yoke"
version = "0.8.1"

View file

@ -1,5 +1,5 @@
[workspace]
members = [".", "crates/python", "crates/wasm", "crates/bin-runner", "crates/simulation", "crates/runtime-dashboard", "crates/distribution", "crates/std", "crates/datastore", "tests/docker"]
members = [".", "crates/python", "crates/wasm", "crates/bin-runner", "crates/simulation", "crates/runtime-dashboard", "crates/distribution", "crates/std", "crates/datastore", "tests/docker", "xtask"]
exclude = ["tools/depgraph"]
[package]

View file

@ -14,6 +14,7 @@ clap = { version = "4", features = ["derive"], optional = true }
ureq = { version = "2", features = ["json"], optional = true }
ctrlc = { version = "3", optional = true }
runtime-dashboard = { path = "../runtime-dashboard", optional = true }
toml = { version = "0.8", optional = true }
[dev-dependencies]
serde_json = "1"
@ -21,9 +22,12 @@ proptest = "1"
tempfile = "3"
swactor = { path = "../.." }
swactor-std = { path = "../std" }
ureq = { version = "2", features = ["json"] }
tiny_http = "0.12"
runtime-dashboard = { path = "../runtime-dashboard" }
[features]
node = ["dep:tiny_http", "dep:clap", "dep:ctrlc", "dep:runtime-dashboard"]
node = ["dep:tiny_http", "dep:clap", "dep:ctrlc", "dep:runtime-dashboard", "dep:toml"]
cli = ["dep:clap", "dep:ureq"]
[[bin]]

141
crates/datastore/README.md Normal file
View file

@ -0,0 +1,141 @@
# swactor-datastore
Distributed content-addressed datastore built on [swactor](../../README.md). Objects are split into fixed-size chunks, identified by their blake3 hash, and replicated across a peer-to-peer network via epidemic gossip.
## Building
Node binary (HTTP server + actor runtime):
```sh
cargo build -p swactor-datastore --features node
```
CLI client:
```sh
cargo build -p swactor-datastore --features cli
```
Both at once:
```sh
cargo build -p swactor-datastore --features node,cli
```
## Node
Start a datastore node:
```sh
swactor-store-node
```
### Flags
| Flag | Default | Description |
|------|---------|-------------|
| `--port` | `9091` | HTTP API port |
| `--storage-path` | *(in-memory)* | Directory for persistent storage |
| `--dashboard-port` | *(disabled)* | Runtime dashboard port |
| `--chunk-size` | `1048576` | Chunk size in bytes (1 MB) |
| `--gc-interval` | `1000` | GC interval in ticks (~100s) |
| `--disseminate-interval` | `50` | Gossip interval in ticks (~5s) |
| `--config` | *(none)* | Path to a TOML config file |
Example with persistent storage and dashboard:
```sh
swactor-store-node --storage-path ./data --dashboard-port 9090
```
### Config file
Create a `store.toml` and pass it with `--config`:
```toml
port = 9091
storage_path = "./my-data"
dashboard_port = 9090
chunk_size = 1048576
gc_interval = 1000
disseminate_interval = 50
```
CLI flags override config file values. Omitted fields use built-in defaults.
```sh
swactor-store-node --config store.toml --port 8080
```
## CLI
The `swactor-store` command talks to a running node over HTTP.
### Status
```sh
swactor-store status
```
### Put
```sh
swactor-store put photo.jpg --name "vacation"
```
### Get (metadata)
```sh
swactor-store get <hash>
```
### Get (download)
```sh
swactor-store get <hash> --output photo.jpg
```
### Delete
```sh
swactor-store delete <hash>
```
### List (local)
```sh
swactor-store list
```
### List (swarm-wide)
```sh
swactor-store list --all
```
### Filter by name
```sh
swactor-store list --name vacation
```
Use `--url` to point at a different node:
```sh
swactor-store --url http://192.168.1.50:9091 list
```
## Web UI
Visit `http://<host>:<port>/` in a browser. The UI supports uploading, listing, downloading, inspecting, and deleting objects — works on desktop and mobile.
## HTTP API
| Method | Path | Description |
|--------|------|-------------|
| `GET` | `/api/status` | Node identity |
| `POST` | `/api/put?name=...` | Upload (body = raw bytes) |
| `GET` | `/api/list` | List objects (`?all=true` for swarm) |
| `GET` | `/api/get?hash=...` | Object metadata + manifest |
| `GET` | `/api/data?hash=...` | Download reassembled binary |
| `POST` | `/api/delete?hash=...` | Delete object |

View file

@ -14,6 +14,7 @@ use swactor::runtime::{Inbox, Runtime};
use crate::chunking::reassemble_blob;
use crate::messages::{BlobStoreMsg, DatastoreNodeMsg, DatastoreResponse, MetadataMsg};
use crate::metrics::DatastoreMetrics;
use crate::types::ContentHash;
/// Per-peer actor addresses needed for remote operations.
@ -30,6 +31,7 @@ struct ApiState {
metadata_addr: ActorAddress,
blob_store_addr: ActorAddress,
peers: Arc<Mutex<Vec<PeerInfo>>>,
metrics: Arc<DatastoreMetrics>,
}
const POLL_TIMEOUT: Duration = Duration::from_secs(5);
@ -67,6 +69,16 @@ fn respond_bytes(request: tiny_http::Request, data: &[u8]) {
let _ = request.respond(response);
}
fn respond_html(request: tiny_http::Request) {
let response =
tiny_http::Response::from_string(crate::ui_html::DATASTORE_UI_HTML).with_header(
"Content-Type: text/html; charset=utf-8"
.parse::<tiny_http::Header>()
.unwrap(),
);
let _ = request.respond(response);
}
fn respond_error(request: tiny_http::Request, status: u16, msg: &str) {
let json = serde_json::json!({ "error": msg }).to_string();
let response = tiny_http::Response::from_string(json)
@ -186,6 +198,7 @@ fn handle_put(mut request: tiny_http::Request, url: &str, state: &ApiState) {
// Can't respond — request consumed
return;
}
let body_len = body.len();
let inbox = match state.runtime.new_inbox::<DatastoreResponse>() {
Ok(i) => i,
@ -207,7 +220,13 @@ fn handle_put(mut request: tiny_http::Request, url: &str, state: &ApiState) {
match poll_response(&inbox, POLL_TIMEOUT) {
Some(DatastoreResponse::PutOk { content_hash }) => {
let json = serde_json::json!({ "content_hash": content_hash.to_hex() }).to_string();
let hex = content_hash.to_hex();
state.metrics.record_put(
&hex,
params.get("name").map(|s| s.as_str()),
body_len as u64,
);
let json = serde_json::json!({ "content_hash": hex }).to_string();
respond_json(request, &json);
}
Some(DatastoreResponse::Error { reason }) => {
@ -257,6 +276,7 @@ fn handle_get(request: tiny_http::Request, url: &str, state: &ApiState) {
match poll_response(&inbox, POLL_TIMEOUT) {
Some(DatastoreResponse::GetOk { entry, manifest }) => {
state.metrics.record_get(&content_hash.to_hex());
let json = serde_json::json!({
"entry": entry_to_json(&entry),
"manifest": manifest_to_json(&manifest),
@ -313,6 +333,8 @@ fn handle_data(request: tiny_http::Request, url: &str, state: &ApiState) {
},
);
state.metrics.record_get(&content_hash.to_hex());
let (entry, manifest) = match poll_response(&inbox, POLL_TIMEOUT) {
Some(DatastoreResponse::GetOk { entry, manifest }) => (entry, manifest),
Some(DatastoreResponse::NotFound) => {
@ -411,7 +433,9 @@ fn handle_delete(request: tiny_http::Request, url: &str, state: &ApiState) {
match poll_response(&inbox, POLL_TIMEOUT) {
Some(DatastoreResponse::DeleteOk { content_hash }) => {
let json = serde_json::json!({ "content_hash": content_hash.to_hex() }).to_string();
let hex = content_hash.to_hex();
state.metrics.record_delete(&hex, 0);
let json = serde_json::json!({ "content_hash": hex }).to_string();
respond_json(request, &json);
}
Some(DatastoreResponse::NotFound) => {
@ -609,6 +633,8 @@ fn try_remote_get(
};
// Fetch each chunk from peer and store locally
let hash_hex = content_hash.to_hex();
state.metrics.begin_transfer(&hash_hex, manifest.chunks.len());
let mut all_ok = true;
for chunk_ref in &manifest.chunks {
let chunk_inbox = match state.runtime.new_inbox::<DatastoreResponse>() {
@ -648,6 +674,7 @@ fn try_remote_get(
);
// Wait for confirmation
let _ = poll_response(&store_inbox, Duration::from_secs(2));
state.metrics.advance_transfer(&hash_hex);
}
_ => {
all_ok = false;
@ -656,6 +683,8 @@ fn try_remote_get(
}
}
state.metrics.end_transfer(&hash_hex);
if !all_ok {
continue;
}
@ -707,6 +736,7 @@ pub fn start_api_server(
metadata_addr: ActorAddress,
blob_store_addr: ActorAddress,
port: u16,
metrics: Arc<DatastoreMetrics>,
) -> (Arc<AtomicBool>, Arc<Mutex<Vec<PeerInfo>>>) {
let shutdown = Arc::new(AtomicBool::new(false));
let peers: Arc<Mutex<Vec<PeerInfo>>> = Arc::new(Mutex::new(Vec::new()));
@ -717,6 +747,7 @@ pub fn start_api_server(
metadata_addr,
blob_store_addr,
peers: Arc::clone(&peers),
metrics,
});
let addr = format!("0.0.0.0:{port}");
@ -749,6 +780,7 @@ pub fn start_api_server(
("POST", "/api/delete") => handle_delete(request, &url, &state),
("GET", "/api/list") => handle_list(request, &url, &state),
("GET", "/api/status") => handle_status(request, &state),
("GET", "/") => respond_html(request),
_ => {
respond_error(request, 404, "not found");
}

View file

@ -9,6 +9,7 @@ use std::thread;
use std::time::Duration;
use clap::Parser;
use serde::Deserialize;
use swactor::config::RuntimeConfig;
use swactor::runtime::Runtime;
@ -16,6 +17,7 @@ use swactor::runtime::Runtime;
use swactor_datastore::actors::{BlobStoreActor, DatastoreNode, MetadataActor};
use swactor_datastore::api::start_api_server;
use swactor_datastore::messages::MetadataMsg;
use swactor_datastore::metrics::DatastoreMetrics;
use swactor_datastore::storage::{FilesystemBackend, InMemoryBackend};
use swactor_datastore::DatastoreConfig;
@ -24,9 +26,13 @@ use distribution::types::NodeId;
#[derive(Parser)]
#[command(name = "swactor-store-node", about = "Swactor distributed datastore node")]
struct Args {
/// Path to a TOML config file
#[arg(long)]
config: Option<std::path::PathBuf>,
/// HTTP API port
#[arg(long, default_value = "9091")]
port: u16,
#[arg(long)]
port: Option<u16>,
/// Storage directory (omit for in-memory)
#[arg(long)]
@ -37,20 +43,62 @@ struct Args {
dashboard_port: Option<u16>,
/// Chunk size in bytes
#[arg(long, default_value = "1048576")]
chunk_size: u32,
#[arg(long)]
chunk_size: Option<u32>,
/// GC interval in ticks (each tick is ~100ms)
#[arg(long, default_value = "1000")]
gc_interval: u64,
#[arg(long)]
gc_interval: Option<u64>,
/// Dissemination interval in ticks
#[arg(long, default_value = "50")]
#[arg(long)]
disseminate_interval: Option<u64>,
}
#[derive(Deserialize, Default)]
struct NodeConfig {
port: Option<u16>,
storage_path: Option<String>,
dashboard_port: Option<u16>,
chunk_size: Option<u32>,
gc_interval: Option<u64>,
disseminate_interval: Option<u64>,
}
/// Resolved configuration with CLI > config file > defaults applied.
struct ResolvedConfig {
port: u16,
storage_path: Option<String>,
dashboard_port: Option<u16>,
chunk_size: u32,
gc_interval: u64,
disseminate_interval: u64,
}
fn resolve_config(args: &Args) -> ResolvedConfig {
let file_cfg = match &args.config {
Some(path) => {
let contents = std::fs::read_to_string(path)
.unwrap_or_else(|e| panic!("failed to read config file {}: {e}", path.display()));
toml::from_str::<NodeConfig>(&contents)
.unwrap_or_else(|e| panic!("failed to parse config file {}: {e}", path.display()))
}
None => NodeConfig::default(),
};
ResolvedConfig {
port: args.port.or(file_cfg.port).unwrap_or(9091),
storage_path: args.storage_path.clone().or(file_cfg.storage_path),
dashboard_port: args.dashboard_port.or(file_cfg.dashboard_port),
chunk_size: args.chunk_size.or(file_cfg.chunk_size).unwrap_or(1_048_576),
gc_interval: args.gc_interval.or(file_cfg.gc_interval).unwrap_or(1000),
disseminate_interval: args.disseminate_interval.or(file_cfg.disseminate_interval).unwrap_or(50),
}
}
fn main() {
let args = Args::parse();
let cfg = resolve_config(&args);
let stop = Arc::new(AtomicBool::new(false));
// Signal handler
@ -63,7 +111,7 @@ fn main() {
}
// Optionally start dashboard
let dash = args.dashboard_port.map(|port| {
let dash = cfg.dashboard_port.map(|port| {
let d = runtime_dashboard::start_dashboard(runtime_dashboard::DashboardConfig {
port,
..Default::default()
@ -106,18 +154,18 @@ fn main() {
// Datastore config
let config = DatastoreConfig {
chunk_size: args.chunk_size,
storage_path: args
chunk_size: cfg.chunk_size,
storage_path: cfg
.storage_path
.as_ref()
.map(|s| s.into())
.unwrap_or_else(|| "datastore".into()),
gc_interval: args.gc_interval,
gc_interval: cfg.gc_interval,
..Default::default()
};
// Create storage backend
let backend: Box<dyn swactor_datastore::StorageBackend> = match &args.storage_path {
let backend: Box<dyn swactor_datastore::StorageBackend> = match &cfg.storage_path {
Some(path) => {
let p = std::path::PathBuf::from(path);
std::fs::create_dir_all(&p).expect("failed to create storage directory");
@ -145,8 +193,14 @@ fn main() {
// Start runtime
let handle = rt.run().expect("failed to start runtime");
// Create datastore metrics
let node_hex: String = node_id.0.iter().map(|b| format!("{b:02x}")).collect();
let metrics = Arc::new(DatastoreMetrics::new());
metrics.set_node_id(node_hex.clone());
if let Some(ref d) = dash {
d.set_runtime(handle.runtime.clone(), collector);
d.set_datastore(Arc::clone(&metrics) as Arc<dyn runtime_dashboard::datastore_collector::DatastoreStatsProvider>);
}
// Start HTTP API
@ -155,17 +209,17 @@ fn main() {
datastore_addr,
metadata_addr,
blob_store_addr,
args.port,
cfg.port,
Arc::clone(&metrics),
);
let node_hex: String = node_id.0.iter().map(|b| format!("{b:02x}")).collect();
eprintln!("Node {} started", &node_hex[..8]);
eprintln!("API at http://0.0.0.0:{}", args.port);
if let Some(port) = args.dashboard_port {
eprintln!("API at http://0.0.0.0:{}", cfg.port);
if let Some(port) = cfg.dashboard_port {
eprintln!("Dashboard at http://0.0.0.0:{port}");
}
if args.storage_path.is_some() {
eprintln!("Storage: {}", args.storage_path.as_ref().unwrap());
if cfg.storage_path.is_some() {
eprintln!("Storage: {}", cfg.storage_path.as_ref().unwrap());
} else {
eprintln!("Storage: in-memory");
}
@ -175,13 +229,13 @@ fn main() {
while !stop.load(Ordering::Relaxed) {
round += 1;
if round % args.gc_interval == 0 {
if round % cfg.gc_interval == 0 {
let _ = handle
.runtime
.send_to(metadata_addr, MetadataMsg::GcTick);
}
if round % args.disseminate_interval == 0 {
if round % cfg.disseminate_interval == 0 {
let _ = handle
.runtime
.send_to(metadata_addr, MetadataMsg::DisseminateTick);

View file

@ -4,8 +4,11 @@ pub mod chunking;
pub mod storage;
pub mod actors;
pub mod cli;
pub mod metrics;
#[cfg(feature = "node")]
pub mod api;
#[cfg(feature = "node")]
pub mod ui_html;
pub use types::{ChunkRef, ContentHash, DatastoreConfig, ObjectEntry, ObjectManifest};
pub use messages::{BlobStoreMsg, DatastoreNodeMsg, DatastoreResponse, MetadataMsg, TransferMsg};

View file

@ -0,0 +1,197 @@
//! Thread-safe metrics for the datastore, consumed by the runtime dashboard.
//!
//! `DatastoreMetrics` accumulates counters and event history from any thread
//! (API handlers run on `tiny_http` worker threads). The dashboard polls
//! `snapshot()` every ~200ms via the `DatastoreStatsProvider` trait.
use std::collections::VecDeque;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Mutex;
use std::time::{SystemTime, UNIX_EPOCH};
use serde::{Deserialize, Serialize};
// ── Snapshot types (serializable, sent to the dashboard) ────────────────────
/// Point-in-time snapshot of the datastore's state and metrics.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DatastoreSnapshot {
pub node_id: String,
pub object_count: u64,
pub total_bytes: u64,
pub put_ops: u64,
pub get_ops: u64,
pub delete_ops: u64,
pub objects: Vec<ObjectSummary>,
pub recent_events: Vec<DatastoreEvent>,
pub active_transfers: Vec<TransferProgress>,
}
/// Summary of a single stored object (for the dashboard table).
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ObjectSummary {
pub hash: String,
pub name: Option<String>,
pub size_bytes: u64,
}
/// A recorded datastore operation.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DatastoreEvent {
pub timestamp_ms: u64,
pub kind: String,
pub hash: String,
pub name: Option<String>,
pub size_bytes: u64,
}
/// Progress of an in-flight chunk transfer.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TransferProgress {
pub hash: String,
pub chunks_received: usize,
pub chunks_total: usize,
}
// ── Live metrics (thread-safe, mutated from API handlers) ───────────────────
const MAX_EVENTS: usize = 200;
/// Thread-safe metrics accumulator for the datastore.
///
/// Atomic counters for the hot path (put/get/delete counts). A `Mutex`-guarded
/// ring buffer for the event timeline and a small vec for active transfers.
pub struct DatastoreMetrics {
node_id: Mutex<String>,
put_ops: AtomicU64,
get_ops: AtomicU64,
delete_ops: AtomicU64,
objects: Mutex<Vec<ObjectSummary>>,
events: Mutex<VecDeque<DatastoreEvent>>,
transfers: Mutex<Vec<TransferProgress>>,
}
impl DatastoreMetrics {
pub fn new() -> Self {
Self {
node_id: Mutex::new(String::new()),
put_ops: AtomicU64::new(0),
get_ops: AtomicU64::new(0),
delete_ops: AtomicU64::new(0),
objects: Mutex::new(Vec::new()),
events: Mutex::new(VecDeque::with_capacity(MAX_EVENTS + 1)),
transfers: Mutex::new(Vec::new()),
}
}
pub fn set_node_id(&self, hex: String) {
*self.node_id.lock().unwrap() = hex;
}
/// Record a successful PUT operation.
pub fn record_put(&self, hash: &str, name: Option<&str>, size_bytes: u64) {
self.put_ops.fetch_add(1, Ordering::Relaxed);
self.push_event("put", hash, name, size_bytes);
let mut objs = self.objects.lock().unwrap();
objs.push(ObjectSummary {
hash: hash.to_string(),
name: name.map(|s| s.to_string()),
size_bytes,
});
}
/// Record a successful GET operation.
pub fn record_get(&self, hash: &str) {
self.get_ops.fetch_add(1, Ordering::Relaxed);
self.push_event("get", hash, None, 0);
}
/// Record a successful DELETE operation.
pub fn record_delete(&self, hash: &str, size_bytes: u64) {
self.delete_ops.fetch_add(1, Ordering::Relaxed);
self.push_event("delete", hash, None, size_bytes);
let mut objs = self.objects.lock().unwrap();
objs.retain(|o| o.hash != hash);
}
/// Begin tracking a chunk transfer.
pub fn begin_transfer(&self, hash: &str, chunks_total: usize) {
let mut transfers = self.transfers.lock().unwrap();
transfers.push(TransferProgress {
hash: hash.to_string(),
chunks_received: 0,
chunks_total,
});
}
/// Advance a tracked transfer by one chunk.
pub fn advance_transfer(&self, hash: &str) {
let mut transfers = self.transfers.lock().unwrap();
if let Some(t) = transfers.iter_mut().find(|t| t.hash == hash) {
t.chunks_received += 1;
}
}
/// Remove a completed/failed transfer from tracking.
pub fn end_transfer(&self, hash: &str) {
let mut transfers = self.transfers.lock().unwrap();
transfers.retain(|t| t.hash != hash);
}
/// Seed the object list (e.g. from an initial LIST query at startup).
pub fn seed_objects(&self, objects: Vec<ObjectSummary>) {
*self.objects.lock().unwrap() = objects;
}
/// Capture a serializable snapshot of the current metrics.
pub fn snapshot(&self) -> DatastoreSnapshot {
let objs = self.objects.lock().unwrap();
let total_bytes: u64 = objs.iter().map(|o| o.size_bytes).sum();
let events = self.events.lock().unwrap();
let transfers = self.transfers.lock().unwrap();
DatastoreSnapshot {
node_id: self.node_id.lock().unwrap().clone(),
object_count: objs.len() as u64,
total_bytes,
put_ops: self.put_ops.load(Ordering::Relaxed),
get_ops: self.get_ops.load(Ordering::Relaxed),
delete_ops: self.delete_ops.load(Ordering::Relaxed),
objects: objs.clone(),
recent_events: events.iter().cloned().collect(),
active_transfers: transfers.clone(),
}
}
fn push_event(&self, kind: &str, hash: &str, name: Option<&str>, size_bytes: u64) {
let event = DatastoreEvent {
timestamp_ms: now_ms(),
kind: kind.to_string(),
hash: hash.to_string(),
name: name.map(|s| s.to_string()),
size_bytes,
};
let mut events = self.events.lock().unwrap();
if events.len() >= MAX_EVENTS {
events.pop_front();
}
events.push_back(event);
}
}
fn now_ms() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis() as u64
}
// ── Dashboard integration (only when runtime-dashboard is available) ────────
#[cfg(feature = "node")]
impl runtime_dashboard::datastore_collector::DatastoreStatsProvider for DatastoreMetrics {
fn snapshot_json(&self) -> Option<String> {
let snap = self.snapshot();
serde_json::to_string(&snap).ok()
}
}

View file

@ -0,0 +1,326 @@
pub const DATASTORE_UI_HTML: &str = r##"<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>swactor-store</title>
<style>
* { margin: 0; padding: 0; box-sizing: border-box; }
body { font-family: 'Menlo', 'Consolas', 'Monaco', monospace; background: #0f1117; color: #e0e0e0; font-size: 13px; }
.header {
display: flex; align-items: center; justify-content: space-between;
padding: 12px 20px; background: #161822; border-bottom: 1px solid #2a2d3e;
}
.header h1 { font-size: 16px; font-weight: 600; color: #fff; }
.header .node-id { font-size: 11px; color: #888; margin-left: 12px; }
.container { max-width: 960px; margin: 0 auto; padding: 16px; }
.panel {
background: #161822; border: 1px solid #2a2d3e; border-radius: 6px;
padding: 16px; margin-bottom: 16px;
}
.panel h2 {
font-size: 12px; color: #888; text-transform: uppercase;
letter-spacing: 1px; margin-bottom: 12px;
}
.upload-row {
display: flex; gap: 10px; align-items: center; flex-wrap: wrap;
}
input[type="file"] {
background: #1c1f2e; color: #e0e0e0; border: 1px solid #2a2d3e;
border-radius: 4px; padding: 8px; font-family: inherit; font-size: 13px;
min-height: 44px; cursor: pointer;
}
input[type="file"]::file-selector-button {
background: #1e2030; color: #e0e0e0; border: 1px solid #2a2d3e;
border-radius: 4px; padding: 6px 12px; font-family: inherit;
font-size: 12px; cursor: pointer; margin-right: 8px;
}
input[type="file"]::file-selector-button:hover { border-color: #6366f1; }
input[type="text"] {
background: #1c1f2e; color: #e0e0e0; border: 1px solid #2a2d3e;
border-radius: 4px; padding: 8px 12px; font-family: inherit;
font-size: 13px; min-height: 44px; width: 200px;
}
input[type="text"]:focus { outline: none; border-color: #6366f1; }
button {
background: #1e2030; color: #e0e0e0; border: 1px solid #2a2d3e;
border-radius: 4px; padding: 8px 16px; font-family: inherit;
font-size: 13px; cursor: pointer; min-height: 44px;
transition: border-color 0.15s;
}
button:hover { border-color: #6366f1; color: #fff; }
button:disabled { opacity: 0.4; cursor: default; }
button.danger:hover { border-color: #f44336; }
button.primary { background: #6366f1; border-color: #6366f1; color: #fff; font-weight: 600; }
button.primary:hover { background: #5558e6; }
.toast {
position: fixed; bottom: 20px; right: 20px; padding: 10px 16px;
border-radius: 4px; font-size: 12px; z-index: 100; opacity: 0;
transition: opacity 0.3s; pointer-events: none;
}
.toast.show { opacity: 1; }
.toast.success { background: #4caf50; color: #fff; }
.toast.error { background: #f44336; color: #fff; }
table { width: 100%; border-collapse: collapse; }
th {
text-align: left; font-size: 10px; color: #888;
text-transform: uppercase; letter-spacing: 1px;
padding: 6px 8px; border-bottom: 1px solid #2a2d3e;
}
td {
padding: 8px; border-bottom: 1px solid #1c1f2e;
font-size: 13px; vertical-align: middle;
}
tr:hover td { background: #1c1f2e; }
tr { cursor: pointer; }
.hash-cell { color: #6366f1; font-size: 12px; }
.size-cell { color: #888; white-space: nowrap; }
.actions-cell { white-space: nowrap; text-align: right; }
.actions-cell button { min-height: 32px; padding: 4px 10px; font-size: 11px; }
.empty-state {
text-align: center; color: #555; padding: 32px; font-size: 14px;
}
/* Modal overlay */
.modal-overlay {
display: none; position: fixed; inset: 0;
background: rgba(0,0,0,0.6); z-index: 50;
align-items: center; justify-content: center;
}
.modal-overlay.open { display: flex; }
.modal {
background: #161822; border: 1px solid #2a2d3e; border-radius: 6px;
padding: 20px; width: 90%; max-width: 560px; max-height: 80vh;
overflow-y: auto;
}
.modal h2 { font-size: 14px; color: #fff; margin-bottom: 16px; text-transform: none; letter-spacing: 0; }
.modal-close {
float: right; background: none; border: none; color: #888;
font-size: 18px; cursor: pointer; min-height: auto; padding: 0;
}
.modal-close:hover { color: #fff; border: none; }
.detail-row { display: flex; margin-bottom: 8px; }
.detail-label { color: #888; width: 110px; flex-shrink: 0; font-size: 11px; text-transform: uppercase; padding-top: 2px; }
.detail-value { color: #e0e0e0; word-break: break-all; font-size: 13px; }
.chunk-list { margin-top: 8px; }
.chunk-item { color: #888; font-size: 11px; padding: 2px 0; }
@media (max-width: 640px) {
.upload-row { flex-direction: column; align-items: stretch; }
input[type="text"] { width: 100%; }
.header { flex-direction: column; align-items: flex-start; gap: 4px; }
.header .node-id { margin-left: 0; }
.actions-cell { display: flex; gap: 4px; justify-content: flex-end; }
}
</style>
</head>
<body>
<div class="header">
<div style="display:flex;align-items:center;flex-wrap:wrap;">
<h1>swactor-store</h1>
<span class="node-id" id="nodeId">connecting...</span>
</div>
</div>
<div class="container">
<!-- Upload panel -->
<div class="panel">
<h2>Upload</h2>
<div class="upload-row">
<input type="file" id="fileInput" />
<input type="text" id="nameInput" placeholder="name (optional)" />
<button class="primary" id="uploadBtn" onclick="upload()">Upload</button>
</div>
</div>
<!-- Object table -->
<div class="panel">
<h2>Objects</h2>
<div id="tableWrap"></div>
</div>
</div>
<!-- Detail modal -->
<div class="modal-overlay" id="modal" onclick="if(event.target===this)closeModal()">
<div class="modal">
<button class="modal-close" onclick="closeModal()">&times;</button>
<h2>Object Detail</h2>
<div id="modalBody"></div>
</div>
</div>
<div class="toast" id="toast"></div>
<script>
const $ = id => document.getElementById(id);
function toast(msg, type) {
const t = $('toast');
t.textContent = msg;
t.className = 'toast show ' + type;
setTimeout(() => t.className = 'toast', 2500);
}
function fmtSize(b) {
if (b < 1024) return b + ' B';
if (b < 1048576) return (b / 1024).toFixed(1) + ' KB';
if (b < 1073741824) return (b / 1048576).toFixed(1) + ' MB';
return (b / 1073741824).toFixed(1) + ' GB';
}
async function fetchStatus() {
try {
const r = await fetch('/api/status');
const j = await r.json();
$('nodeId').textContent = j.node_id.substring(0, 16) + '...';
$('nodeId').title = j.node_id;
} catch(e) {
$('nodeId').textContent = 'offline';
}
}
async function refreshList() {
try {
const r = await fetch('/api/list');
const j = await r.json();
renderTable(j.entries || []);
} catch(e) {
$('tableWrap').innerHTML = '<div class="empty-state">failed to load</div>';
}
}
function renderTable(entries) {
if (entries.length === 0) {
$('tableWrap').innerHTML = '<div class="empty-state">no objects stored</div>';
return;
}
let html = '<table><thead><tr><th>Hash</th><th>Name</th><th>Size</th><th style="text-align:right">Actions</th></tr></thead><tbody>';
for (const e of entries) {
const h = e.content_hash;
const short = h.substring(0, 16);
const name = e.name || '—';
const size = fmtSize(e.size_bytes);
html += '<tr onclick="showDetail(\'' + h + '\')">';
html += '<td class="hash-cell" title="' + h + '">' + short + '</td>';
html += '<td>' + escHtml(name) + '</td>';
html += '<td class="size-cell">' + size + '</td>';
html += '<td class="actions-cell">';
html += '<button onclick="event.stopPropagation();download(\'' + h + '\',\'' + escAttr(e.name || h.substring(0,12)) + '\')">download</button> ';
html += '<button class="danger" onclick="event.stopPropagation();del(\'' + h + '\')">delete</button>';
html += '</td></tr>';
}
html += '</tbody></table>';
$('tableWrap').innerHTML = html;
}
function escHtml(s) { const d = document.createElement('div'); d.textContent = s; return d.innerHTML; }
function escAttr(s) { return s.replace(/'/g, "\\'").replace(/"/g, '&quot;'); }
async function upload() {
const file = $('fileInput').files[0];
if (!file) { toast('select a file first', 'error'); return; }
const name = $('nameInput').value.trim();
const btn = $('uploadBtn');
btn.disabled = true;
btn.textContent = 'uploading...';
try {
let url = '/api/put';
if (name) url += '?name=' + encodeURIComponent(name);
const r = await fetch(url, { method: 'POST', body: file });
if (!r.ok) throw new Error((await r.json()).error || r.statusText);
const j = await r.json();
toast('uploaded ' + j.content_hash.substring(0, 12), 'success');
$('fileInput').value = '';
$('nameInput').value = '';
refreshList();
} catch(e) {
toast('upload failed: ' + e.message, 'error');
} finally {
btn.disabled = false;
btn.textContent = 'Upload';
}
}
async function download(hash, filename) {
try {
const r = await fetch('/api/data?hash=' + hash);
if (!r.ok) throw new Error('not found');
const blob = await r.blob();
const a = document.createElement('a');
a.href = URL.createObjectURL(blob);
a.download = filename;
a.click();
URL.revokeObjectURL(a.href);
} catch(e) {
toast('download failed: ' + e.message, 'error');
}
}
async function del(hash) {
if (!confirm('Delete ' + hash.substring(0, 16) + '?')) return;
try {
const r = await fetch('/api/delete?hash=' + hash, { method: 'POST' });
if (!r.ok) throw new Error((await r.json()).error || r.statusText);
toast('deleted', 'success');
refreshList();
} catch(e) {
toast('delete failed: ' + e.message, 'error');
}
}
async function showDetail(hash) {
try {
const r = await fetch('/api/get?hash=' + hash);
if (!r.ok) throw new Error('not found');
const j = await r.json();
const e = j.entry;
const m = j.manifest;
let html = '';
html += row('Hash', e.content_hash);
html += row('Name', e.name || '—');
html += row('Size', fmtSize(e.size_bytes));
html += row('Node', e.node_id.substring(0, 16) + '...');
if (e.tags && Object.keys(e.tags).length > 0) {
html += row('Tags', Object.entries(e.tags).map(([k,v]) => k + '=' + v).join(', '));
}
html += row('Chunks', m.chunks.length + ' (' + fmtSize(m.chunk_size) + ' each)');
if (m.chunks.length > 0) {
html += '<div class="chunk-list">';
for (let i = 0; i < m.chunks.length; i++) {
const c = m.chunks[i];
html += '<div class="chunk-item">#' + i + ' ' + c.hash.substring(0, 16) + ' (' + fmtSize(c.size) + ')</div>';
}
html += '</div>';
}
$('modalBody').innerHTML = html;
$('modal').classList.add('open');
} catch(e) {
toast('failed to load detail', 'error');
}
}
function row(label, value) {
return '<div class="detail-row"><div class="detail-label">' + label + '</div><div class="detail-value">' + escHtml(String(value)) + '</div></div>';
}
function closeModal() { $('modal').classList.remove('open'); }
document.addEventListener('keydown', e => { if (e.key === 'Escape') closeModal(); });
// Init
fetchStatus();
refreshList();
</script>
</body>
</html>"##;

View file

@ -0,0 +1,190 @@
#![cfg(feature = "node")]
//! Integration test: spins up a real datastore node with HTTP API and exercises
//! the full CRUD lifecycle over HTTP.
use std::sync::Arc;
use std::thread;
use std::time::Duration;
use swactor::config::RuntimeConfig;
use swactor::runtime::Runtime;
use swactor_datastore::actors::{BlobStoreActor, DatastoreNode, MetadataActor};
use swactor_datastore::api::start_api_server;
use swactor_datastore::metrics::DatastoreMetrics;
use swactor_datastore::storage::InMemoryBackend;
use swactor_datastore::DatastoreConfig;
use distribution::types::NodeId;
fn find_free_port() -> u16 {
std::net::TcpListener::bind("127.0.0.1:0")
.unwrap()
.local_addr()
.unwrap()
.port()
}
/// Full CRUD lifecycle over HTTP:
/// status → put → list → get metadata → get data → delete → list (empty) → get (404)
#[test]
fn http_crud_lifecycle() {
let port = find_free_port();
let base = format!("http://127.0.0.1:{port}");
// Set up runtime with worker threads (needed for HTTP server)
let collector = runtime_dashboard::collector::StatsCollector::new(2);
let mut rt = Runtime::new(RuntimeConfig {
num_threads: 2,
max_actors: 1024,
channel_buffer_size: 2000,
..Default::default()
});
rt.set_stats_hook(collector);
let node_id = NodeId([0xAA; 32]);
let config = DatastoreConfig {
chunk_size: 1_048_576,
gc_interval: 1000,
..Default::default()
};
let backend: Box<dyn swactor_datastore::StorageBackend> = Box::new(InMemoryBackend::new());
let blob_store_addr = rt.spawn(BlobStoreActor::new(backend)).unwrap();
let mut metadata = MetadataActor::new(node_id, &config);
metadata.set_blob_store(blob_store_addr);
let metadata_addr = rt.spawn(metadata).unwrap();
let datastore_node = DatastoreNode::new(node_id, blob_store_addr, metadata_addr, config);
let datastore_addr = rt.spawn(datastore_node).unwrap();
let handle = rt.run().expect("failed to start runtime");
let metrics = Arc::new(DatastoreMetrics::new());
let (api_shutdown, _peers) = start_api_server(
handle.runtime.clone(),
datastore_addr,
metadata_addr,
blob_store_addr,
port,
Arc::clone(&metrics),
);
// Give the HTTP server time to bind
thread::sleep(Duration::from_millis(200));
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
run_crud_scenario(&base, &metrics);
}));
// Cleanup
api_shutdown.store(true, std::sync::atomic::Ordering::Relaxed);
handle.shutdown();
handle.join();
if let Err(e) = result {
std::panic::resume_unwind(e);
}
}
fn run_crud_scenario(base: &str, metrics: &Arc<DatastoreMetrics>) {
// 1. Status — should return a node_id
let status: serde_json::Value = ureq::get(&format!("{base}/api/status"))
.call()
.expect("status request failed")
.into_json()
.unwrap();
let node_id = status["node_id"].as_str().expect("node_id should be a string");
assert_eq!(node_id.len(), 64, "node_id should be 64 hex chars");
// 2. Put — upload some content
let content = b"hello from the integration test!";
let put_resp: serde_json::Value = ureq::post(&format!("{base}/api/put?name=greeting"))
.send_bytes(content)
.expect("put request failed")
.into_json()
.unwrap();
let hash = put_resp["content_hash"]
.as_str()
.expect("put should return content_hash");
assert_eq!(hash.len(), 64, "content_hash should be 64 hex chars");
// 3. List — should contain exactly one entry matching our upload
let list_resp: serde_json::Value = ureq::get(&format!("{base}/api/list"))
.call()
.expect("list request failed")
.into_json()
.unwrap();
let entries = list_resp["entries"].as_array().expect("entries should be an array");
assert_eq!(entries.len(), 1, "should have exactly 1 entry after put");
assert_eq!(entries[0]["content_hash"].as_str().unwrap(), hash);
assert_eq!(entries[0]["name"].as_str().unwrap(), "greeting");
// 4. Get metadata — entry + manifest for the uploaded object
let get_resp: serde_json::Value = ureq::get(&format!("{base}/api/get?hash={hash}"))
.call()
.expect("get request failed")
.into_json()
.unwrap();
let entry = &get_resp["entry"];
assert_eq!(entry["content_hash"].as_str().unwrap(), hash);
assert_eq!(entry["name"].as_str().unwrap(), "greeting");
assert_eq!(entry["size_bytes"].as_u64().unwrap(), content.len() as u64);
let manifest = &get_resp["manifest"];
let chunks = manifest["chunks"].as_array().expect("manifest should have chunks");
assert!(!chunks.is_empty(), "manifest should have at least one chunk");
// 5. Get data — download the raw bytes and verify content matches
let data_resp = ureq::get(&format!("{base}/api/data?hash={hash}"))
.call()
.expect("data request failed");
let mut downloaded = Vec::new();
data_resp
.into_reader()
.read_to_end(&mut downloaded)
.unwrap();
assert_eq!(downloaded, content, "downloaded bytes should match uploaded content");
// 6. Delete — remove the object
let del_resp: serde_json::Value = ureq::post(&format!("{base}/api/delete?hash={hash}"))
.call()
.expect("delete request failed")
.into_json()
.unwrap();
assert_eq!(del_resp["content_hash"].as_str().unwrap(), hash);
// 7. List after delete — should be empty
let list_resp2: serde_json::Value = ureq::get(&format!("{base}/api/list"))
.call()
.expect("list request failed")
.into_json()
.unwrap();
let entries2 = list_resp2["entries"]
.as_array()
.expect("entries should be an array");
assert!(entries2.is_empty(), "list should be empty after delete");
// 8. Get after delete — should 404
let get_err = ureq::get(&format!("{base}/api/get?hash={hash}")).call();
match get_err {
Err(ureq::Error::Status(404, _)) => {} // expected
Err(e) => panic!("expected 404, got error: {e}"),
Ok(_) => panic!("expected 404, got 200"),
}
// 9. Verify metrics snapshot reflects the full lifecycle
let snap = metrics.snapshot();
assert_eq!(snap.put_ops, 1, "one put recorded");
// handle_get + handle_data = 2 get operations
assert_eq!(snap.get_ops, 2, "metadata-get + data-get recorded");
assert_eq!(snap.delete_ops, 1, "one delete recorded");
assert!(snap.objects.is_empty(), "no objects after delete");
assert!(
snap.recent_events.len() >= 4,
"at least 4 events (put + get + get + delete), got {}",
snap.recent_events.len()
);
}

View file

@ -0,0 +1,205 @@
#![cfg(feature = "node")]
//! End-to-end integration test: spins up a datastore node with an HTTP API
//! **and** a runtime dashboard, performs CRUD over HTTP, then verifies:
//!
//! - `DatastoreMetrics::snapshot()` reflects the operations
//! - Dashboard `/datastore` serves HTML
//! - Dashboard `/api/datastore` returns a JSON snapshot matching the metrics
use std::sync::Arc;
use std::thread;
use std::time::Duration;
use swactor::config::RuntimeConfig;
use swactor::runtime::Runtime;
use swactor_datastore::actors::{BlobStoreActor, DatastoreNode, MetadataActor};
use swactor_datastore::api::start_api_server;
use swactor_datastore::metrics::DatastoreMetrics;
use swactor_datastore::storage::InMemoryBackend;
use swactor_datastore::DatastoreConfig;
use distribution::types::NodeId;
fn find_free_port() -> u16 {
std::net::TcpListener::bind("127.0.0.1:0")
.unwrap()
.local_addr()
.unwrap()
.port()
}
/// Scenario: datastore CRUD → dashboard reflects live operation metrics.
///
/// Story:
/// Two files are uploaded. One is fetched (metadata + data). The other
/// is deleted. Afterwards we check that the metrics snapshot, the
/// dashboard HTML page, and the dashboard JSON API all agree on what
/// happened.
#[test]
fn dashboard_reflects_datastore_operations() {
let api_port = find_free_port();
let dash_port = find_free_port();
let api_base = format!("http://127.0.0.1:{api_port}");
let dash_base = format!("http://127.0.0.1:{dash_port}");
// ── Infrastructure: runtime + actors + dashboard + API ──────────────
let dash = runtime_dashboard::start_dashboard(runtime_dashboard::DashboardConfig {
port: dash_port,
..Default::default()
});
let collector = runtime_dashboard::collector::StatsCollector::new(2);
let mut rt = Runtime::new(RuntimeConfig {
num_threads: 2,
max_actors: 1024,
channel_buffer_size: 2000,
..Default::default()
});
rt.set_stats_hook(collector.clone());
let node_id = NodeId([0xBB; 32]);
let config = DatastoreConfig {
chunk_size: 1_048_576,
gc_interval: 1000,
..Default::default()
};
let backend: Box<dyn swactor_datastore::StorageBackend> = Box::new(InMemoryBackend::new());
let blob_store_addr = rt.spawn(BlobStoreActor::new(backend)).unwrap();
let mut metadata = MetadataActor::new(node_id, &config);
metadata.set_blob_store(blob_store_addr);
let metadata_addr = rt.spawn(metadata).unwrap();
let datastore_node = DatastoreNode::new(node_id, blob_store_addr, metadata_addr, config);
let datastore_addr = rt.spawn(datastore_node).unwrap();
let handle = rt.run().expect("failed to start runtime");
let metrics = Arc::new(DatastoreMetrics::new());
let node_hex: String = node_id.0.iter().map(|b| format!("{b:02x}")).collect();
metrics.set_node_id(node_hex);
dash.set_runtime(handle.runtime.clone(), collector);
dash.set_datastore(
Arc::clone(&metrics)
as Arc<dyn runtime_dashboard::datastore_collector::DatastoreStatsProvider>,
);
let (api_shutdown, _peers) = start_api_server(
handle.runtime.clone(),
datastore_addr,
metadata_addr,
blob_store_addr,
api_port,
Arc::clone(&metrics),
);
thread::sleep(Duration::from_millis(300));
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
run_dashboard_scenario(&api_base, &dash_base, &metrics);
}));
// Cleanup
api_shutdown.store(true, std::sync::atomic::Ordering::Relaxed);
dash.shutdown();
handle.shutdown();
handle.join();
if let Err(e) = result {
std::panic::resume_unwind(e);
}
}
fn run_dashboard_scenario(api: &str, dash: &str, metrics: &Arc<DatastoreMetrics>) {
// ── 1. Upload two objects ───────────────────────────────────────────
let put_a: serde_json::Value = ureq::post(&format!("{api}/api/put?name=alpha"))
.send_bytes(b"payload-alpha")
.expect("put A failed")
.into_json()
.unwrap();
let hash_a = put_a["content_hash"].as_str().unwrap().to_string();
let _put_b: serde_json::Value = ureq::post(&format!("{api}/api/put?name=bravo"))
.send_bytes(b"payload-bravo")
.expect("put B failed")
.into_json()
.unwrap();
let hash_b = _put_b["content_hash"].as_str().unwrap().to_string();
// ── 2. GET object A (metadata + raw data → 2 get ops) ──────────────
let _: serde_json::Value = ureq::get(&format!("{api}/api/get?hash={hash_a}"))
.call()
.unwrap()
.into_json()
.unwrap();
let data_resp = ureq::get(&format!("{api}/api/data?hash={hash_a}"))
.call()
.unwrap();
let mut body = Vec::new();
data_resp.into_reader().read_to_end(&mut body).unwrap();
assert_eq!(body, b"payload-alpha", "downloaded data should match");
// ── 3. DELETE object B ──────────────────────────────────────────────
let _: serde_json::Value = ureq::post(&format!("{api}/api/delete?hash={hash_b}"))
.call()
.unwrap()
.into_json()
.unwrap();
// ── 4. Assert: in-process metrics snapshot ──────────────────────────
let snap = metrics.snapshot();
assert_eq!(snap.put_ops, 2, "two puts recorded");
assert_eq!(snap.get_ops, 2, "metadata-get + data-get recorded");
assert_eq!(snap.delete_ops, 1, "one delete recorded");
assert_eq!(snap.objects.len(), 1, "only alpha remains after deleting bravo");
assert_eq!(snap.objects[0].hash, hash_a);
assert!(
snap.recent_events.len() >= 5,
"at least 5 events (2 put + 2 get + 1 delete), got {}",
snap.recent_events.len()
);
// ── 5. Assert: dashboard /datastore serves HTML ─────────────────────
let page = ureq::get(&format!("{dash}/datastore")).call().unwrap();
assert_eq!(page.status(), 200);
assert!(
page.header("Content-Type")
.unwrap_or("")
.contains("text/html"),
);
let html = page.into_string().unwrap();
assert!(html.contains("Datastore"), "page should mention Datastore");
// ── 6. Assert: /api/datastore JSON matches metrics ──────────────────
let ds: serde_json::Value = ureq::get(&format!("{dash}/api/datastore"))
.call()
.unwrap()
.into_json()
.unwrap();
assert_eq!(ds["put_ops"].as_u64().unwrap(), 2);
assert_eq!(ds["get_ops"].as_u64().unwrap(), 2);
assert_eq!(ds["delete_ops"].as_u64().unwrap(), 1);
let objects = ds["objects"].as_array().expect("objects should be array");
assert_eq!(objects.len(), 1);
assert_eq!(objects[0]["hash"].as_str().unwrap(), hash_a);
let events = ds["recent_events"]
.as_array()
.expect("recent_events should be array");
assert!(events.len() >= 5);
}

View file

@ -96,7 +96,6 @@ impl KBucket {
pub struct RoutingTable {
self_id: NodeId,
buckets: Vec<KBucket>,
k: usize,
}
impl RoutingTable {
@ -109,7 +108,7 @@ impl RoutingTable {
for _ in 0..NUM_BUCKETS {
buckets.push(KBucket::new(k));
}
Self { self_id, buckets, k }
Self { self_id, buckets }
}
pub fn self_id(&self) -> NodeId {

View file

@ -195,6 +195,7 @@ pub fn encode_wire_envelope(envelope: &WireEnvelope) -> Vec<u8> {
enum ReadError {
WouldBlock,
Disconnected,
#[allow(dead_code)]
Other(std::io::Error),
}

View file

@ -3,6 +3,7 @@
//! `TestCluster` makes sender-misattribution structurally impossible by
//! tagging every response with the responder's index, mirroring the
//! simulation crate's `deliver_actions_tagged_with_net`.
#![allow(dead_code)]
use distribution::node::{DistributedNode, DistributedNodeConfig};
use distribution::registry::RegistryConfig;

View file

@ -3,7 +3,7 @@ use distribution::crypto::Keypair;
use distribution::kademlia::directory::{
actor_addr_as_node_id, resolve_quorum, DirectoryShard, QuorumResult,
};
use distribution::types::{DirectoryEntry, NodeId, Signature};
use distribution::types::{NodeId, Signature};
// ─── DirectoryShard ─────────────────────────────────────────────────────────

View file

@ -123,6 +123,8 @@ pub const ACTOR_DETAIL_HTML: &str = r##"<!DOCTYPE html>
<nav class="nav-links">
<a href="/" class="nav-link">Overview</a>
<a href="/actors" class="nav-link">Actors</a>
<a href="/distribution" class="nav-link">Distribution</a>
<a href="/datastore" class="nav-link">Datastore</a>
</nav>
</div>
</div>

View file

@ -152,6 +152,7 @@ pub const ACTORS_HTML: &str = r##"<!DOCTYPE html>
<a href="/" class="nav-link">Overview</a>
<a href="/actors" class="nav-link active">Actors</a>
<a href="/distribution" class="nav-link">Distribution</a>
<a href="/datastore" class="nav-link">Datastore</a>
</nav>
</div>
<div class="header-right">

View file

@ -153,6 +153,7 @@ pub const DASHBOARD_HTML: &str = r##"<!DOCTYPE html>
<a href="/" class="nav-link active">Overview</a>
<a href="/actors" class="nav-link">Actors</a>
<a href="/distribution" class="nav-link">Distribution</a>
<a href="/datastore" class="nav-link">Datastore</a>
</nav>
</div>
<div class="header-right">

View file

@ -0,0 +1,16 @@
//! Datastore stats provider for the runtime dashboard.
//!
//! The trait returns a pre-serialized JSON string so that `runtime-dashboard`
//! has no compile-time dependency on `swactor-datastore` (which would create a
//! circular dependency since `swactor-datastore[node]` depends on us).
//!
//! The `swactor-datastore` crate implements this trait in its `node` feature.
/// Trait for providing datastore stats to the dashboard.
///
/// Implementations capture a point-in-time snapshot as serialized JSON.
/// The dashboard polls this every ~200ms via SSE.
pub trait DatastoreStatsProvider: Send + Sync {
/// Return a JSON-serialized datastore snapshot, or `None` if unavailable.
fn snapshot_json(&self) -> Option<String>;
}

View file

@ -0,0 +1,286 @@
pub const DATASTORE_HTML: &str = r##"<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Swactor Runtime – Datastore</title>
<style>
* { margin: 0; padding: 0; box-sizing: border-box; }
body { font-family: 'Menlo', 'Consolas', 'Monaco', monospace; background: #0f1117; color: #e0e0e0; font-size: 13px; }
.header {
display: flex; align-items: center; justify-content: space-between;
padding: 12px 20px; background: #161822; border-bottom: 1px solid #2a2d3e;
}
.header-left { display: flex; align-items: center; }
.header h1 { font-size: 16px; font-weight: 600; color: #fff; }
.status-dot {
width: 10px; height: 10px; border-radius: 50%; background: #4caf50;
display: inline-block; margin-left: 8px; vertical-align: middle;
}
.status-dot.disconnected { background: #f44336; }
.status-dot.done { background: #ff9800; }
.nav-links { display: flex; gap: 4px; margin-left: 20px; }
.nav-link {
color: #888; text-decoration: none; font-size: 12px;
padding: 4px 10px; border-radius: 3px; transition: color 0.2s;
}
.nav-link:hover { color: #e0e0e0; }
.nav-link.active { color: #fff; background: #2a2d3e; }
.header-right { display: flex; align-items: center; gap: 12px; }
.grid {
display: grid;
grid-template-columns: 1fr 1fr;
gap: 12px; padding: 12px;
}
.panel {
background: #161822; border: 1px solid #2a2d3e; border-radius: 6px;
padding: 14px; overflow: hidden;
}
.panel h2 { font-size: 12px; color: #888; text-transform: uppercase; letter-spacing: 1px; margin-bottom: 10px; }
.full-width { grid-column: 1 / -1; }
.stats-cards {
display: grid; grid-template-columns: repeat(5, 1fr); gap: 10px;
}
.stat-card {
background: #1c1f2e; border-radius: 4px; padding: 10px; text-align: center;
}
.stat-card .value { font-size: 22px; font-weight: 700; color: #fff; }
.stat-card .label { font-size: 10px; color: #888; text-transform: uppercase; margin-top: 2px; }
.event-timeline {
max-height: 400px; overflow-y: auto;
}
.event-row {
display: flex; gap: 8px; padding: 4px 0; border-bottom: 1px solid #1c1f2e;
font-size: 11px; align-items: center;
}
.event-time { color: #555; min-width: 70px; }
.event-kind {
min-width: 48px; font-weight: 700; text-transform: uppercase; font-size: 10px;
padding: 1px 6px; border-radius: 3px; text-align: center;
}
.event-kind.put { background: #1b3a2a; color: #4caf50; }
.event-kind.get { background: #1a2a3e; color: #2196f3; }
.event-kind.delete { background: #3a1a1a; color: #f44336; }
.event-hash { color: #aaa; font-size: 10px; min-width: 120px; }
.event-name { color: #e0e0e0; flex: 1; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
.event-size { color: #888; min-width: 70px; text-align: right; }
.objects-table { max-height: 400px; overflow-y: auto; }
.objects-table table { width: 100%; border-collapse: collapse; }
.objects-table th, .objects-table td {
padding: 4px 8px; text-align: left; border-bottom: 1px solid #1c1f2e; font-size: 11px;
white-space: nowrap;
}
.objects-table th { color: #888; font-weight: 500; position: sticky; top: 0; background: #161822; }
.transfers-section { display: none; }
.transfers-section.visible { display: block; }
.transfer-row {
display: flex; align-items: center; gap: 10px; margin-bottom: 8px;
}
.transfer-hash { color: #aaa; font-size: 10px; min-width: 100px; }
.transfer-bar {
flex: 1; height: 16px; background: #1c1f2e; border-radius: 3px; overflow: hidden;
}
.transfer-fill {
height: 100%; background: #6366f1; border-radius: 3px;
transition: width 0.3s ease;
}
.transfer-label { color: #888; font-size: 11px; min-width: 80px; text-align: right; }
::-webkit-scrollbar { width: 6px; }
::-webkit-scrollbar-track { background: #0f1117; }
::-webkit-scrollbar-thumb { background: #2a2d3e; border-radius: 3px; }
</style>
</head>
<body>
<div class="header">
<div class="header-left">
<h1>
Swactor Runtime Dashboard
<span id="statusDot" class="status-dot disconnected"></span>
</h1>
<nav class="nav-links">
<a href="/" class="nav-link">Overview</a>
<a href="/actors" class="nav-link">Actors</a>
<a href="/distribution" class="nav-link">Distribution</a>
<a href="/datastore" class="nav-link active">Datastore</a>
</nav>
</div>
<div class="header-right">
<span id="nodeLabel" style="color:#888;font-size:12px;">Waiting for data...</span>
</div>
</div>
<div class="grid">
<!-- Stat cards -->
<div class="panel full-width">
<h2>Datastore Stats</h2>
<div class="stats-cards">
<div class="stat-card"><div class="value" id="statObjects">0</div><div class="label">Objects</div></div>
<div class="stat-card"><div class="value" id="statSize">0</div><div class="label">Total Size</div></div>
<div class="stat-card"><div class="value" id="statPuts">0</div><div class="label">Puts</div></div>
<div class="stat-card"><div class="value" id="statGets">0</div><div class="label">Gets</div></div>
<div class="stat-card"><div class="value" id="statDeletes">0</div><div class="label">Deletes</div></div>
</div>
</div>
<!-- Event timeline -->
<div class="panel">
<h2>Event Timeline <span id="eventCount" style="color:#555;font-weight:400;"></span></h2>
<div class="event-timeline" id="eventTimeline"></div>
</div>
<!-- Objects table -->
<div class="panel">
<h2>Objects <span id="objectCount" style="color:#555;font-weight:400;"></span></h2>
<div class="objects-table">
<table>
<thead><tr><th>Hash</th><th>Name</th><th>Size</th></tr></thead>
<tbody id="objectsBody"></tbody>
</table>
</div>
</div>
<!-- Transfers -->
<div class="panel full-width transfers-section" id="transfersPanel">
<h2>Active Transfers</h2>
<div id="transfersList"></div>
</div>
</div>
<script>
(function() {
var DASHBOARD_MODE = '__DASHBOARD_MODE__';
var dot = document.getElementById('statusDot');
function formatBytes(b) {
if (b === 0) return '0 B';
var units = ['B', 'KB', 'MB', 'GB', 'TB'];
var i = Math.floor(Math.log(b) / Math.log(1024));
if (i >= units.length) i = units.length - 1;
return (b / Math.pow(1024, i)).toFixed(i > 0 ? 1 : 0) + ' ' + units[i];
}
function formatTime(ms) {
var d = new Date(ms);
return ('0' + d.getHours()).slice(-2) + ':' +
('0' + d.getMinutes()).slice(-2) + ':' +
('0' + d.getSeconds()).slice(-2);
}
function escapeHtml(s) {
if (!s) return '';
return s.replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;');
}
function updateFromSnapshot(snap) {
// Node label
if (snap.node_id) {
document.getElementById('nodeLabel').textContent = 'Node: ' + snap.node_id.substring(0, 16) + '\u2026';
}
// Stat cards
document.getElementById('statObjects').textContent = snap.object_count;
document.getElementById('statSize').textContent = formatBytes(snap.total_bytes);
document.getElementById('statPuts').textContent = snap.put_ops;
document.getElementById('statGets').textContent = snap.get_ops;
document.getElementById('statDeletes').textContent = snap.delete_ops;
// Event timeline
var timeline = document.getElementById('eventTimeline');
var wasAtBottom = timeline.scrollTop + timeline.clientHeight >= timeline.scrollHeight - 20;
timeline.innerHTML = '';
document.getElementById('eventCount').textContent = '(' + snap.recent_events.length + ')';
for (var i = snap.recent_events.length - 1; i >= 0; i--) {
var ev = snap.recent_events[i];
var row = document.createElement('div');
row.className = 'event-row';
row.innerHTML =
'<span class="event-time">' + formatTime(ev.timestamp_ms) + '</span>' +
'<span class="event-kind ' + ev.kind + '">' + ev.kind + '</span>' +
'<span class="event-hash">' + ev.hash.substring(0, 16) + '\u2026</span>' +
'<span class="event-name">' + escapeHtml(ev.name || '') + '</span>' +
'<span class="event-size">' + (ev.size_bytes > 0 ? formatBytes(ev.size_bytes) : '') + '</span>';
timeline.appendChild(row);
}
if (wasAtBottom) {
timeline.scrollTop = timeline.scrollHeight;
}
// Objects table
var tbody = document.getElementById('objectsBody');
tbody.innerHTML = '';
document.getElementById('objectCount').textContent = '(' + snap.objects.length + ')';
for (var i = 0; i < snap.objects.length; i++) {
var obj = snap.objects[i];
var tr = document.createElement('tr');
tr.innerHTML =
'<td style="color:#aaa;font-size:10px;">' + obj.hash.substring(0, 16) + '\u2026</td>' +
'<td>' + escapeHtml(obj.name || '\u2014') + '</td>' +
'<td style="color:#888;">' + formatBytes(obj.size_bytes) + '</td>';
tbody.appendChild(tr);
}
// Transfers
var panel = document.getElementById('transfersPanel');
var list = document.getElementById('transfersList');
if (snap.active_transfers.length === 0) {
panel.className = 'panel full-width transfers-section';
} else {
panel.className = 'panel full-width transfers-section visible';
list.innerHTML = '';
for (var i = 0; i < snap.active_transfers.length; i++) {
var t = snap.active_transfers[i];
var pct = t.chunks_total > 0 ? Math.round((t.chunks_received / t.chunks_total) * 100) : 0;
var row = document.createElement('div');
row.className = 'transfer-row';
row.innerHTML =
'<span class="transfer-hash">' + t.hash.substring(0, 16) + '\u2026</span>' +
'<div class="transfer-bar"><div class="transfer-fill" style="width:' + pct + '%"></div></div>' +
'<span class="transfer-label">' + t.chunks_received + ' / ' + t.chunks_total + '</span>';
list.appendChild(row);
}
}
}
// SSE connection
var es = new EventSource('/events');
es.addEventListener('datastore', function(e) {
try {
var snap = JSON.parse(e.data);
updateFromSnapshot(snap);
} catch(err) { console.error('datastore parse error', err); }
});
es.addEventListener('done', function() {
dot.className = 'status-dot done';
es.close();
});
es.onerror = function() {
dot.className = 'status-dot disconnected';
};
es.onopen = function() {
dot.className = 'status-dot';
};
})();
</script>
</body>
</html>
"##;

View file

@ -121,6 +121,7 @@ pub const DISTRIBUTION_HTML: &str = r##"<!DOCTYPE html>
<a href="/" class="nav-link">Overview</a>
<a href="/actors" class="nav-link">Actors</a>
<a href="/distribution" class="nav-link active">Distribution</a>
<a href="/datastore" class="nav-link">Datastore</a>
</nav>
</div>
<div class="header-right">

View file

@ -20,6 +20,9 @@ mod distribution_html;
#[cfg(feature = "distribution")]
pub mod distribution_collector;
mod datastore_html;
pub mod datastore_collector;
use std::io;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
@ -91,6 +94,7 @@ pub struct DashboardHandle {
recording: bool,
#[cfg(feature = "distribution")]
distribution: Arc<Mutex<Option<Arc<dyn distribution_collector::DistributionStatsProvider>>>>,
datastore: Arc<Mutex<Option<Arc<dyn datastore_collector::DatastoreStatsProvider>>>>,
}
impl DashboardHandle {
@ -123,6 +127,11 @@ impl DashboardHandle {
*self.distribution.lock().unwrap() = Some(provider);
}
/// Attach a datastore stats provider, enabling the `/datastore` page.
pub fn set_datastore(&self, provider: Arc<dyn datastore_collector::DatastoreStatsProvider>) {
*self.datastore.lock().unwrap() = Some(provider);
}
/// Access the time-series history store (for TUI sparklines, etc.).
pub fn history(&self) -> &Arc<DashboardHistory> {
&self.history
@ -178,6 +187,9 @@ pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle {
let distribution: Arc<Mutex<Option<Arc<dyn distribution_collector::DistributionStatsProvider>>>> =
Arc::new(Mutex::new(None));
let datastore: Arc<Mutex<Option<Arc<dyn datastore_collector::DatastoreStatsProvider>>>> =
Arc::new(Mutex::new(None));
server::spawn_http_server(
Arc::clone(&store),
Arc::clone(&runtime),
@ -187,6 +199,7 @@ pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle {
config.port,
#[cfg(feature = "distribution")]
Arc::clone(&distribution),
Arc::clone(&datastore),
);
// Start stats recorder thread when recording is enabled
@ -229,6 +242,7 @@ pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle {
recording: config.record,
#[cfg(feature = "distribution")]
distribution,
datastore,
}
}

View file

@ -24,6 +24,9 @@ use crate::distribution_collector::DistributionStatsProvider;
#[cfg(feature = "distribution")]
use crate::distribution_html::DISTRIBUTION_HTML;
use crate::datastore_collector::DatastoreStatsProvider;
use crate::datastore_html::DATASTORE_HTML;
/// Format a server-sent event.
fn format_sse(event: &str, data: &str) -> Vec<u8> {
format!("event: {event}\ndata: {data}\n\n").into_bytes()
@ -128,6 +131,7 @@ pub(crate) fn spawn_http_server(
port: u16,
#[cfg(feature = "distribution")]
distribution: Arc<Mutex<Option<Arc<dyn DistributionStatsProvider>>>>,
datastore: Arc<Mutex<Option<Arc<dyn DatastoreStatsProvider>>>>,
) {
let addr = format!("0.0.0.0:{port}");
let server = tiny_http::Server::http(&addr).expect("failed to bind HTTP server");
@ -144,6 +148,7 @@ pub(crate) fn spawn_http_server(
let cmd_router = Arc::clone(&cmd_router);
#[cfg(feature = "distribution")]
let distribution = Arc::clone(&distribution);
let datastore = Arc::clone(&datastore);
thread::spawn(move || {
loop {
let request = match server.recv() {
@ -159,6 +164,7 @@ pub(crate) fn spawn_http_server(
"/topology" => respond_html(request, TOPOLOGY_HTML, "live"),
#[cfg(feature = "distribution")]
"/distribution" => respond_html(request, DISTRIBUTION_HTML, "live"),
"/datastore" => respond_html(request, DATASTORE_HTML, "live"),
"/events" => {
handle_live_sse(
request,
@ -169,6 +175,7 @@ pub(crate) fn spawn_http_server(
Arc::clone(&history),
#[cfg(feature = "distribution")]
Arc::clone(&distribution),
Arc::clone(&datastore),
);
}
"/api/stats" => {
@ -204,6 +211,12 @@ pub(crate) fn spawn_http_server(
Arc::clone(&distribution),
);
}
"/api/datastore" => {
handle_datastore_api(
request,
Arc::clone(&datastore),
);
}
"/api/logs" => {
handle_logs_api(request, &url, Arc::clone(&store));
}
@ -239,6 +252,7 @@ fn handle_live_sse(
history: Arc<DashboardHistory>,
#[cfg(feature = "distribution")]
distribution: Arc<Mutex<Option<Arc<dyn DistributionStatsProvider>>>>,
datastore: Arc<Mutex<Option<Arc<dyn DatastoreStatsProvider>>>>,
) {
let (tx, rx) = mpsc::channel::<Vec<u8>>();
let response = make_sse_response(rx);
@ -309,6 +323,18 @@ fn handle_live_sse(
}
}
// Send datastore snapshot if provider is attached
{
let maybe_ds = datastore.lock().unwrap().clone();
if let Some(provider) = maybe_ds {
if let Some(json) = provider.snapshot_json() {
if tx.send(format_sse("datastore", &json)).is_err() {
return;
}
}
}
}
// Send new activity events
let (batch, new_cursor) = store.read_from(cursor);
if !batch.is_empty() {
@ -423,6 +449,26 @@ fn handle_distribution_api(
let _ = request.respond(response);
}
fn handle_datastore_api(
request: tiny_http::Request,
datastore: Arc<Mutex<Option<Arc<dyn DatastoreStatsProvider>>>>,
) {
let json = match datastore.lock().unwrap().as_ref() {
Some(provider) => provider.snapshot_json().unwrap_or_else(|| "{}".into()),
None => serde_json::json!({
"error": "datastore provider not attached"
})
.to_string(),
};
let response = tiny_http::Response::from_string(json).with_header(
"Content-Type: application/json"
.parse::<tiny_http::Header>()
.unwrap(),
);
let _ = request.respond(response);
}
fn handle_topology_api(
request: tiny_http::Request,
runtime: Arc<Mutex<Option<Arc<Runtime>>>>,

View file

@ -47,6 +47,7 @@ pub const TOPOLOGY_HTML: &str = r##"<!DOCTYPE html>
<a href="/actors" class="nav-link">Actors</a>
<a href="/topology" class="nav-link active">Topology</a>
<a href="/distribution" class="nav-link">Distribution</a>
<a href="/datastore" class="nav-link">Datastore</a>
</nav>
</div>
</div>

View file

@ -8,7 +8,7 @@ use std::sync::Arc;
use swactor::actor::ActorInterface;
use swactor::runtime::{Ctx, Runtime, RuntimeConfig};
use runtime_dashboard::command::{
from_query_params, parse_line, CommandContext, CommandRequest, CommandResponse, CommandRouter,
from_query_params, parse_line, CommandContext, CommandResponse, CommandRouter,
};
// ── Test Helpers ─────────────────────────────────────────────────────────────

278
docs/datastore/actors.md Normal file
View file

@ -0,0 +1,278 @@
# Datastore Actor Reference
## Overview
The datastore is built from four actors within the swactor runtime. `DatastoreNode` is the public facade — all external requests (HTTP API, network protocol) enter through it and are routed to two long-lived worker actors: `BlobStoreActor` (content-addressed chunk/manifest I/O) and `MetadataActor` (object index, DHT replication, GC). A fourth actor, `TransferActor`, is spawned ephemerally for each remote download and self-terminates on completion or failure.
```
┌─────────────────────────┐
│ store_node (main) │
│ spawns all 3 long-lived │
│ actors, drives ticks │
└────┬──────┬──────┬───────┘
│ │ │
spawn │ │ │ spawn
┌──────────────┘ │ └──────────────┐
▼ │ spawn ▼
┌───────────────────┐ │ ┌───────────────────┐
│ BlobStoreActor │ │ │ MetadataActor │
│ (chunks, manifests)│ │ │ (index, DHT, GC) │
└─────────▲─────────┘ │ └──▲────────┬───────┘
│ ▼ │ │
│ ┌───────────────────┐ │ │
│ │ DatastoreNode │────┘ │
│ │ (facade/router) │─────────────┘
└────────────│ │
└────────┬──────────┘
│ spawns (per download)
▼
┌───────────────────┐
│ TransferActor │
│ (ephemeral) │
└───────────────────┘
Arrows: ──▶ sends messages to
```
## Actors
### DatastoreNode
| | |
|---|---|
| **Role** | Top-level coordinator/facade. Accepts user-facing commands and incoming network protocol messages, delegates all work to `BlobStoreActor` and `MetadataActor`. |
| **Source** | `crates/datastore/src/actors/datastore_node.rs` |
| **Spawned by** | `store_node` binary (`crates/datastore/src/bin/store_node.rs:188`) |
| **Lifecycle** | Long-lived — runs for the lifetime of the process |
**Inbound messages** (`DatastoreNodeMsg` — 11 variants):
User-facing commands:
- `Put { data, name, tags, reply_to }` — chunk a blob, write chunks + manifest to `BlobStoreActor`, register in `MetadataActor`
- `Get { content_hash, reply_to }` — retrieve object metadata + manifest via `MetadataActor`
- `Delete { content_hash, reply_to }` — remove object via `MetadataActor`
- `List { name_filter, all, reply_to }` — list objects (local or swarm-wide) via `MetadataActor`
- `Status { reply_to }` — return this node's `NodeId`
- `ReadChunk { hash, reply_to }` — read a single chunk via `BlobStoreActor`
Protocol routing (incoming network messages):
- `IncomingGetChunk` — forwards to `BlobStoreActor::ReadChunk`
- `IncomingGetManifest` — forwards to `BlobStoreActor::ReadManifest`
- `IncomingStoreObject` — forwards to `MetadataActor::HandleStoreObject`
- `IncomingFindObject` — forwards to `MetadataActor::HandleFindObject`
- `IncomingListObjects` — forwards to `MetadataActor::ListLocal`
**Key outbound messages:**
- `BlobStoreMsg::WriteChunk`, `WriteManifest`, `ReadChunk`, `ReadManifest` — to `BlobStoreActor`
- `MetadataMsg::PutObject`, `GetObject`, `DeleteObject`, `ListLocal`, `ListSwarm`, `HandleStoreObject`, `HandleFindObject` — to `MetadataActor`
- `DatastoreResponse::NodeStatus` — directly to caller for `Status`
---
### BlobStoreActor
| | |
|---|---|
| **Role** | Content-addressed storage for chunks and manifests. All I/O goes through a pluggable `StorageBackend` (filesystem or in-memory). |
| **Source** | `crates/datastore/src/actors/blob_store.rs` |
| **Spawned by** | `store_node` binary (`store_node.rs:178`) |
| **Lifecycle** | Long-lived — runs for the lifetime of the process |
**Inbound messages** (`BlobStoreMsg` — 8 variants):
Chunk operations:
- `WriteChunk { hash, data, reply_to }` — persist a chunk, reply `ChunkStored`
- `ReadChunk { hash, reply_to }` — read a chunk, reply `ChunkOk` or `NotFound`
- `DeleteChunk { hash }` — remove a chunk (fire-and-forget)
- `HasChunk { hash, reply_to }` — existence check, reply `Bool`
- `ListChunks { reply_to }` — list all chunk hashes, reply `ChunkList`
- `GcUnreferenced { referenced }` — delete chunks not in the referenced set (fire-and-forget)
Manifest operations:
- `WriteManifest { manifest, reply_to }` — persist a manifest, reply `ManifestStored`
- `ReadManifest { hash, reply_to }` — read a manifest, reply `ManifestOk` or `NotFound`
**Key outbound messages:**
- `DatastoreResponse` variants (`ChunkStored`, `ChunkOk`, `ManifestStored`, `ManifestOk`, `NotFound`, `Error`, `Bool`, `ChunkList`) — always back to the `reply_to` address
---
### MetadataActor
| | |
|---|---|
| **Role** | Object metadata index. Maintains a `HashMap<ContentHash, ObjectEntry>` and a manifest cache. Handles DHT-style find/store operations, epidemic dissemination of entries to peers, and periodic garbage collection. |
| **Source** | `crates/datastore/src/actors/metadata.rs` |
| **Spawned by** | `store_node` binary (`store_node.rs:185`) |
| **Lifecycle** | Long-lived — runs for the lifetime of the process |
**Inbound messages** (`MetadataMsg` — 11 variants):
Object operations:
- `PutObject { entry, manifest, reply_to }` — store metadata + manifest locally, enqueue for dissemination, reply `PutOk`
- `GetObject { content_hash, reply_to }` — local lookup, reply `GetOk` or `NotFound`
- `DeleteObject { content_hash, reply_to }` — remove from local index, reply `DeleteOk` or `NotFound`
- `ListLocal { name_filter, reply_to }` — list local entries with optional name filter, reply `ListOk`
- `ListSwarm { name_filter, reply_to }` — swarm-wide list (currently delegates to `ListLocal`)
DHT protocol:
- `HandleFindObject { from, content_hash, reply_to }` — answer an incoming FIND_VALUE from a peer
- `HandleStoreObject { entry, manifest }` — accept an incoming STORE from a peer (fire-and-forget)
Peer management:
- `SetPeers { peers }` — update the list of peer `MetadataActor` addresses for dissemination
Periodic ticks (driven by the `store_node` main loop):
- `DisseminateTick` — send pending entries to all known peers
- `GcTick` — collect referenced chunks from all manifests, send `BlobStoreMsg::GcUnreferenced` to `BlobStoreActor`
**Key outbound messages:**
- `DatastoreResponse` variants (`PutOk`, `GetOk`, `DeleteOk`, `ListOk`, `NotFound`, `Error`) — to caller
- `MetadataMsg::HandleStoreObject` — to peer `MetadataActor` addresses during dissemination
- `BlobStoreMsg::GcUnreferenced` — to local `BlobStoreActor` during GC
---
### TransferActor
| | |
|---|---|
| **Role** | Manages a single object download from a remote node. Tracks pending/received chunks, forwards received data to the local `BlobStoreActor`, and reports completion or failure to the original requester. |
| **Source** | `crates/datastore/src/actors/transfer.rs` |
| **Spawned by** | `DatastoreNode` (one per remote download) |
| **Lifecycle** | Ephemeral — self-terminates via `ctx.stop_self()` on completion, failure, or cancel |
**Inbound messages** (`TransferMsg` — 4 variants):
- `StartDownload { manifest, source_node, reply_to }` — initialize the download with a manifest and source
- `ChunkReceived { hash, data }` — a chunk arrived from the remote node
- `ChunkFailed { hash, reason }` — a chunk fetch failed (retries up to `max_retries`, then fails the whole transfer)
- `Cancel` — abort the transfer immediately
**Key outbound messages:**
- `BlobStoreMsg::WriteChunk` — to local `BlobStoreActor` for each received chunk
- `DatastoreResponse::TransferComplete` — to `reply_to` when all chunks received
- `DatastoreResponse::TransferFailed` — to `reply_to` when retries are exhausted
---
## Message Reference
All message types are defined in `crates/datastore/src/messages.rs`.
### Intra-node actor messages
| Enum | Variants | Handled by |
|------|----------|------------|
| `DatastoreNodeMsg` | 11 (6 user-facing + 5 protocol routing) | `DatastoreNode` |
| `BlobStoreMsg` | 8 (5 chunk ops + 1 GC + 2 manifest ops) | `BlobStoreActor` |
| `MetadataMsg` | 11 (5 object ops + 2 DHT + 1 peer mgmt + 2 ticks) | `MetadataActor` |
| `TransferMsg` | 4 (start + chunk received + chunk failed + cancel) | `TransferActor` |
### Shared response enum
`DatastoreResponse` — 15 variants used as the return type for all four actors:
| Variant | Meaning |
|---------|---------|
| `PutOk { content_hash }` | Object stored successfully |
| `GetOk { entry, manifest }` | Object found |
| `DeleteOk { content_hash }` | Object deleted |
| `ListOk { entries }` | List result |
| `ChunkOk { hash, data }` | Chunk data retrieved |
| `ChunkStored { hash }` | Chunk written to storage |
| `ManifestStored { hash }` | Manifest written to storage |
| `ManifestOk { manifest }` | Manifest retrieved |
| `TransferComplete { content_hash }` | All chunks downloaded |
| `TransferFailed { reason }` | Transfer failed |
| `NodeStatus { node_id }` | Node identity |
| `NotFound` | Resource not found |
| `Error { reason }` | Generic error |
| `Bool(bool)` | Boolean result (e.g. `HasChunk`) |
| `ChunkList { hashes }` | List of chunk hashes |
### Inter-node wire messages (NetworkMessage)
| Struct | Direction | Purpose |
|--------|-----------|---------|
| `GetChunkRequest` | requester → holder | Fetch a chunk by hash |
| `GetChunkResponse` | holder → requester | Return chunk data (or `None`) |
| `StoreObjectRequest` | origin → DHT peer | Kademlia STORE for object metadata |
| `FindObjectRequest` | requester → DHT peer | Kademlia FIND_VALUE for object metadata |
| `FindObjectResponse` | DHT peer → requester | Return `Found(entry)` or `Closer(nodes)` |
| `GetManifestRequest` | requester → holder | Fetch a manifest by content hash |
| `GetManifestResponse` | holder → requester | Return manifest (or `None`) |
| `ListObjectsRequest` | requester → peer | List objects with optional name filter |
| `ListObjectsResponse` | peer → requester | Return matching entries |
Wire messages are distinguished from intra-node messages by implementing the `NetworkMessage` trait with a stable `type_tag()` string. They are serialized with serde for transport over iroh/QUIC.
---
## Key Flows
### Put (store a blob)
```
Client → DatastoreNode::Put
→ chunk_blob() splits data into chunks
→ BlobStoreActor::WriteChunk (for each chunk, fire-and-forget)
→ BlobStoreActor::WriteManifest
→ MetadataActor::PutObject
→ stores entry + manifest locally
→ enqueues for dissemination
→ replies DatastoreResponse::PutOk
```
### Get (retrieve metadata)
```
Client → DatastoreNode::Get
→ MetadataActor::GetObject
→ local index lookup
→ replies DatastoreResponse::GetOk (or NotFound)
```
### Data (reassemble from chunks)
See [streaming.md](streaming.md) for the full transfer protocol. In summary:
```
API server → DatastoreNode::Get → MetadataActor (local miss)
→ iterate peers:
→ FindObjectRequest (wire) → peer MetadataActor
→ GetManifestRequest (wire) → peer BlobStoreActor
→ GetChunkRequest (wire) → peer BlobStoreActor (per chunk)
→ BlobStoreActor::WriteChunk (store locally)
→ BlobStoreActor::WriteManifest
→ MetadataActor::PutObject
→ reassemble_blob() → verify blake3 → respond
```
### Dissemination (epidemic replication)
```
store_node main loop (every disseminate_interval ticks)
→ MetadataActor::DisseminateTick
→ take_pending() selects entries with remaining budget
→ for each peer: MetadataActor::HandleStoreObject
→ peer inserts if absent, re-enqueues for further dissemination
```
Budget per entry = `Λ * ceil(log2(cluster_size))` (SWIM-style, Λ=3).
### GC (garbage collection)
```
store_node main loop (every gc_interval ticks)
→ MetadataActor::GcTick
→ scans all manifests → builds referenced chunk set
→ BlobStoreActor::GcUnreferenced { referenced }
→ deletes any chunk not in the referenced set
```
---
## Related
- [streaming.md](streaming.md) — chunking, transfer protocol, reassembly, and progress tracking

103
docs/datastore/streaming.md Normal file
View file

@ -0,0 +1,103 @@
# Datastore Streaming Architecture
## Overview
"Streaming" in the swactor datastore refers to **progressive chunk-based transfer**, not byte-level streaming. When an object is stored, it is split into fixed-size chunks, each content-addressed with blake3. When retrieved from a remote peer, chunks are fetched individually and reassembled — enabling progress tracking and partial recovery.
This design trades a small amount of per-chunk overhead for:
- **Progress visibility**: the dashboard shows `chunks_received / chunks_total` in real time
- **Resumability**: a failed transfer can (in principle) restart from the last chunk
- **Deduplication**: identical chunks across objects are stored once
## Content-Addressed Chunking
The `chunk_blob()` function (`chunking.rs`) splits raw bytes into fixed-size pieces:
1. Compute `ContentHash = blake3(entire_blob)` — this is the object's identity
2. Split the blob into `ceil(total_size / chunk_size)` pieces (default chunk size: 1 MB)
3. For each piece, compute `chunk_hash = blake3(piece_bytes)`
4. Build a `ChunkRef { hash, offset, size }` for each piece
5. Return an `ObjectManifest` containing the full list of `ChunkRef`s
```
Blob (5.2 MB, chunk_size=1MB)
├── Chunk 0: hash=abc1…, offset=0, size=1048576
├── Chunk 1: hash=def2…, offset=1048576, size=1048576
├── Chunk 2: hash=789a…, offset=2097152, size=1048576
├── Chunk 3: hash=bcd3…, offset=3145728, size=1048576
└── Chunk 4: hash=ef45…, offset=4194304, size=1048576 (last: 209920 bytes)
```
The object's identity (`ContentHash`) is the hash of the *entire* blob, not of the manifest. This means the same data always produces the same hash regardless of chunk size.
## Transfer Protocol
When a client requests an object via `GET /api/data?hash=...`, the API server:
1. Sends a `DatastoreNodeMsg::Get` to the local `DatastoreNode` actor
2. If found locally, reads all chunks from the local `BlobStore` and reassembles
3. If **not found locally**, enters `try_remote_get()`:
![Transfer Flow](../diagrams/datastore_transfer.svg)
### Remote GET step-by-step
1. **Iterate peers**: for each known peer node:
2. **FindObject**: send `MetadataMsg::HandleFindObject` to the peer's `MetadataActor`
3. **Read manifest**: send `BlobStoreMsg::ReadManifest` to the peer's `BlobStore`
4. **Fetch chunks**: for each `ChunkRef` in the manifest:
- Send `BlobStoreMsg::ReadChunk` to the peer
- Receive `DatastoreResponse::ChunkOk { hash, data }`
- Store locally via `BlobStoreMsg::WriteChunk`
- Update `DatastoreMetrics::advance_transfer()` for dashboard progress
5. **Store manifest locally**: `BlobStoreMsg::WriteManifest`
6. **Store metadata locally**: `MetadataMsg::PutObject`
7. **Reassemble and respond**: `reassemble_blob()` concatenates chunks and verifies integrity
If a peer doesn't have the object (or any step fails), the loop continues to the next peer.
## Reassembly
`reassemble_blob()` (`chunking.rs`) takes a manifest and a set of `(hash, data)` pairs:
1. For each `ChunkRef` in manifest order, find the matching `(hash, data)` pair
2. Concatenate all chunk data into a single buffer
3. Compute `blake3(result)` and verify it matches `manifest.content_hash`
4. Return the reassembled blob (or a `ChunkingError` on mismatch)
This integrity check ensures that even if individual chunks are corrupted or swapped, the final result is always verified against the original content hash.
## Progress Tracking
The `DatastoreMetrics` struct provides thread-safe transfer tracking:
```
begin_transfer(hash, chunks_total) // called when remote GET starts
advance_transfer(hash) // called after each chunk is stored locally
end_transfer(hash) // called on completion or failure
```
The dashboard SSE stream includes a `datastore` event every ~200ms with a `DatastoreSnapshot` containing `active_transfers: Vec<TransferProgress>`. The web UI renders these as animated progress bars.
```
TransferProgress {
hash: "abc123...",
chunks_received: 3,
chunks_total: 5,
}
```
## GC Integration
When an object is deleted, its `ObjectEntry` and `ObjectManifest` are removed from the `MetadataActor`. However, the underlying chunks are **not immediately deleted** — they may be referenced by other manifests (deduplication).
Instead, garbage collection runs periodically:
1. `MetadataMsg::GcTick` triggers a scan
2. The `MetadataActor` collects all chunk hashes referenced by any live manifest
3. Sends `BlobStoreMsg::GcUnreferenced` with the referenced set
4. The `BlobStore` deletes any chunks **not** in the referenced set
This two-phase approach prevents data loss when chunks are shared between objects.
![Chunk Lifecycle](../diagrams/datastore_chunk_lifecycle.svg)

View file

@ -0,0 +1,170 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1000 600" font-family="system-ui, sans-serif">
<defs>
<filter id="shadow" x="-4%" y="-4%" width="108%" height="108%">
<feDropShadow dx="1" dy="2" stdDeviation="3" flood-color="#000" flood-opacity="0.10"/>
</filter>
<marker id="arrow-gray" viewBox="0 0 10 7" refX="10" refY="3.5"
markerWidth="10" markerHeight="7" orient="auto-start-reverse" fill="#5f6368">
<polygon points="0 0, 10 3.5, 0 7"/>
</marker>
<marker id="arrow-green" viewBox="0 0 10 7" refX="10" refY="3.5"
markerWidth="10" markerHeight="7" orient="auto-start-reverse" fill="#34a853">
<polygon points="0 0, 10 3.5, 0 7"/>
</marker>
<marker id="arrow-red" viewBox="0 0 10 7" refX="10" refY="3.5"
markerWidth="10" markerHeight="7" orient="auto-start-reverse" fill="#ea4335">
<polygon points="0 0, 10 3.5, 0 7"/>
</marker>
</defs>
<!-- Background -->
<rect width="1000" height="600" rx="8" fill="#f8f9fb" stroke="#e0e3e8" stroke-width="1.5"/>
<!-- Title -->
<text x="500" y="42" text-anchor="middle" font-size="20" font-weight="bold" fill="#202124">Chunk Lifecycle</text>
<text x="500" y="62" text-anchor="middle" font-size="12" fill="#5f6368">chunking.rs &#xB7; storage.rs &#xB7; actors/blob_store.rs</text>
<!-- ═══════════════════════════════════════════════════════════ -->
<!-- Row 1: Creation -->
<!-- ═══════════════════════════════════════════════════════════ -->
<!-- Blob Data -->
<rect x="40" y="100" width="120" height="50" rx="12" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.8" filter="url(#shadow)"/>
<text x="100" y="125" text-anchor="middle" font-size="13" font-weight="600" fill="#1a73e8">Blob Data</text>
<text x="100" y="140" text-anchor="middle" font-size="10" fill="#5f6368">raw bytes</text>
<!-- Arrow -->
<line x1="160" y1="125" x2="210" y2="125" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
<!-- blake3 hash -->
<rect x="222" y="100" width="130" height="50" rx="12" fill="#fef7e0" stroke="#f9ab00" stroke-width="1.8" filter="url(#shadow)"/>
<text x="287" y="125" text-anchor="middle" font-size="13" font-weight="600" fill="#e37400">blake3 hash</text>
<text x="287" y="140" text-anchor="middle" font-size="10" fill="#5f6368">ContentHash</text>
<!-- Arrow -->
<line x1="352" y1="125" x2="405" y2="125" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
<!-- chunk_blob() -->
<rect x="417" y="100" width="140" height="50" rx="12" fill="#e6f4ea" stroke="#34a853" stroke-width="1.8" filter="url(#shadow)"/>
<text x="487" y="125" text-anchor="middle" font-size="13" font-weight="600" fill="#137333">chunk_blob()</text>
<text x="487" y="140" text-anchor="middle" font-size="10" fill="#5f6368">split by chunk_size</text>
<!-- Arrow fan-out -->
<line x1="557" y1="115" x2="620" y2="105" stroke="#34a853" stroke-width="1.5" marker-end="url(#arrow-green)"/>
<line x1="557" y1="125" x2="620" y2="125" stroke="#34a853" stroke-width="1.5" marker-end="url(#arrow-green)"/>
<line x1="557" y1="135" x2="620" y2="145" stroke="#34a853" stroke-width="1.5" marker-end="url(#arrow-green)"/>
<!-- ChunkRef boxes -->
<rect x="630" y="88" width="110" height="30" rx="6" fill="#e6f4ea" stroke="#34a853" stroke-width="1.2"/>
<text x="685" y="108" text-anchor="middle" font-size="11" font-weight="600" fill="#137333">ChunkRef[0]</text>
<rect x="630" y="122" width="110" height="30" rx="6" fill="#e6f4ea" stroke="#34a853" stroke-width="1.2"/>
<text x="685" y="142" text-anchor="middle" font-size="11" font-weight="600" fill="#137333">ChunkRef[1]</text>
<rect x="630" y="156" width="110" height="30" rx="6" fill="#e6f4ea" stroke="#34a853" stroke-width="1.2"/>
<text x="685" y="176" text-anchor="middle" font-size="11" font-weight="600" fill="#137333">ChunkRef[N]</text>
<!-- Brace → ObjectManifest -->
<line x1="740" y1="103" x2="770" y2="137" stroke="#5f6368" stroke-width="1"/>
<line x1="740" y1="137" x2="770" y2="137" stroke="#5f6368" stroke-width="1"/>
<line x1="740" y1="171" x2="770" y2="137" stroke="#5f6368" stroke-width="1"/>
<line x1="770" y1="137" x2="800" y2="137" stroke="#5f6368" stroke-width="1.5" marker-end="url(#arrow-gray)"/>
<rect x="812" y="112" width="155" height="50" rx="12" fill="#fef7e0" stroke="#f9ab00" stroke-width="1.8" filter="url(#shadow)"/>
<text x="889" y="137" text-anchor="middle" font-size="13" font-weight="600" fill="#e37400">ObjectManifest</text>
<text x="889" y="152" text-anchor="middle" font-size="10" fill="#5f6368">chunks + total_size</text>
<!-- ═══════════════════════════════════════════════════════════ -->
<!-- Row 2: Storage -->
<!-- ═══════════════════════════════════════════════════════════ -->
<text x="500" y="220" text-anchor="middle" font-size="14" font-weight="600" fill="#202124">Storage</text>
<!-- WriteChunk -->
<rect x="100" y="240" width="200" height="50" rx="12" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.8" filter="url(#shadow)"/>
<text x="200" y="260" text-anchor="middle" font-size="13" font-weight="600" fill="#1a73e8">BlobStoreMsg</text>
<text x="200" y="278" text-anchor="middle" font-size="10" fill="#5f6368">::WriteChunk { hash, data }</text>
<line x1="300" y1="265" x2="380" y2="265" stroke="#34a853" stroke-width="1.8" marker-end="url(#arrow-green)"/>
<!-- StorageBackend -->
<rect x="392" y="240" width="220" height="50" rx="12" fill="#e6f4ea" stroke="#34a853" stroke-width="1.8" filter="url(#shadow)"/>
<text x="502" y="260" text-anchor="middle" font-size="13" font-weight="600" fill="#137333">StorageBackend</text>
<text x="502" y="278" text-anchor="middle" font-size="10" fill="#5f6368">::write_chunk(hash, data)</text>
<line x1="612" y1="265" x2="690" y2="265" stroke="#34a853" stroke-width="1.8" marker-end="url(#arrow-green)"/>
<!-- Disk/Memory -->
<rect x="702" y="240" width="180" height="50" rx="12" fill="#f1f3f4" stroke="#9aa0a6" stroke-width="1.8" filter="url(#shadow)"/>
<text x="792" y="260" text-anchor="middle" font-size="13" font-weight="600" fill="#5f6368">Filesystem /</text>
<text x="792" y="278" text-anchor="middle" font-size="10" fill="#5f6368">InMemoryBackend</text>
<!-- ═══════════════════════════════════════════════════════════ -->
<!-- Row 3: Reference tracking -->
<!-- ═══════════════════════════════════════════════════════════ -->
<text x="500" y="340" text-anchor="middle" font-size="14" font-weight="600" fill="#202124">Reference Tracking</text>
<rect x="100" y="360" width="200" height="50" rx="12" fill="#fef7e0" stroke="#f9ab00" stroke-width="1.8" filter="url(#shadow)"/>
<text x="200" y="380" text-anchor="middle" font-size="13" font-weight="600" fill="#e37400">ObjectManifest</text>
<text x="200" y="398" text-anchor="middle" font-size="10" fill="#5f6368">references chunk hashes</text>
<line x1="300" y1="385" x2="380" y2="385" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
<text x="340" y="377" text-anchor="middle" font-size="10" fill="#5f6368">owns</text>
<rect x="392" y="360" width="220" height="50" rx="12" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.8" filter="url(#shadow)"/>
<text x="502" y="385" text-anchor="middle" font-size="13" font-weight="600" fill="#1a73e8">MetadataActor</text>
<text x="502" y="398" text-anchor="middle" font-size="10" fill="#5f6368">tracks all manifests</text>
<!-- ═══════════════════════════════════════════════════════════ -->
<!-- Row 4: GC -->
<!-- ═══════════════════════════════════════════════════════════ -->
<text x="500" y="460" text-anchor="middle" font-size="14" font-weight="600" fill="#202124">Garbage Collection</text>
<rect x="60" y="480" width="150" height="50" rx="12" fill="#fce8e6" stroke="#ea4335" stroke-width="1.8" filter="url(#shadow)"/>
<text x="135" y="505" text-anchor="middle" font-size="13" font-weight="600" fill="#c5221f">GcTick</text>
<text x="135" y="520" text-anchor="middle" font-size="10" fill="#5f6368">periodic trigger</text>
<line x1="210" y1="505" x2="280" y2="505" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
<rect x="292" y="480" width="200" height="50" rx="12" fill="#fef7e0" stroke="#f9ab00" stroke-width="1.8" filter="url(#shadow)"/>
<text x="392" y="500" text-anchor="middle" font-size="12" font-weight="600" fill="#e37400">Collect referenced</text>
<text x="392" y="518" text-anchor="middle" font-size="10" fill="#5f6368">hashes from manifests</text>
<line x1="492" y1="505" x2="560" y2="505" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
<rect x="572" y="480" width="200" height="50" rx="12" fill="#fce8e6" stroke="#ea4335" stroke-width="1.8" filter="url(#shadow)"/>
<text x="672" y="500" text-anchor="middle" font-size="12" font-weight="600" fill="#c5221f">GcUnreferenced</text>
<text x="672" y="518" text-anchor="middle" font-size="10" fill="#5f6368">delete orphan chunks</text>
<line x1="772" y1="505" x2="830" y2="505" stroke="#ea4335" stroke-width="1.8" marker-end="url(#arrow-red)"/>
<!-- Deleted -->
<rect x="842" y="488" width="100" height="34" rx="8" fill="#f1f3f4" stroke="#9aa0a6" stroke-width="1.5"/>
<text x="892" y="510" text-anchor="middle" font-size="12" font-weight="600" fill="#5f6368">Deleted</text>
<!-- ============================================ -->
<!-- Legend -->
<rect x="50" y="555" width="900" height="30" rx="6" fill="none" stroke="#e0e3e8" stroke-width="1"/>
<text x="70" y="575" font-size="11" font-weight="600" fill="#202124">Legend</text>
<line x1="130" y1="572" x2="160" y2="572" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
<text x="168" y="576" font-size="10" fill="#5f6368">Transition</text>
<line x1="260" y1="572" x2="290" y2="572" stroke="#34a853" stroke-width="1.8" marker-end="url(#arrow-green)"/>
<text x="298" y="576" font-size="10" fill="#34a853">Data flow</text>
<line x1="380" y1="572" x2="410" y2="572" stroke="#ea4335" stroke-width="1.8" marker-end="url(#arrow-red)"/>
<text x="418" y="576" font-size="10" fill="#ea4335">Deletion</text>
<rect x="510" y="564" width="40" height="16" rx="4" fill="#e8f0fe" stroke="#4285f4" stroke-width="1"/>
<text x="560" y="576" font-size="10" fill="#4285f4">Actor / Component</text>
<rect x="680" y="564" width="40" height="16" rx="4" fill="#fef7e0" stroke="#f9ab00" stroke-width="1"/>
<text x="730" y="576" font-size="10" fill="#e37400">Data type</text>
<rect x="810" y="564" width="40" height="16" rx="4" fill="#e6f4ea" stroke="#34a853" stroke-width="1"/>
<text x="860" y="576" font-size="10" fill="#34a853">Success state</text>
</svg>

After

Width:  |  Height:  |  Size: 11 KiB

View file

@ -0,0 +1,153 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1000 700" font-family="system-ui, sans-serif">
<defs>
<filter id="shadow" x="-4%" y="-4%" width="108%" height="108%">
<feDropShadow dx="1" dy="2" stdDeviation="3" flood-color="#000" flood-opacity="0.10"/>
</filter>
<marker id="arrow-gray" viewBox="0 0 10 7" refX="10" refY="3.5"
markerWidth="10" markerHeight="7" orient="auto-start-reverse" fill="#5f6368">
<polygon points="0 0, 10 3.5, 0 7"/>
</marker>
<marker id="arrow-green" viewBox="0 0 10 7" refX="10" refY="3.5"
markerWidth="10" markerHeight="7" orient="auto-start-reverse" fill="#34a853">
<polygon points="0 0, 10 3.5, 0 7"/>
</marker>
<marker id="arrow-red" viewBox="0 0 10 7" refX="10" refY="3.5"
markerWidth="10" markerHeight="7" orient="auto-start-reverse" fill="#ea4335">
<polygon points="0 0, 10 3.5, 0 7"/>
</marker>
<marker id="arrow-blue" viewBox="0 0 10 7" refX="10" refY="3.5"
markerWidth="10" markerHeight="7" orient="auto-start-reverse" fill="#4285f4">
<polygon points="0 0, 10 3.5, 0 7"/>
</marker>
</defs>
<!-- Background -->
<rect width="1000" height="700" rx="8" fill="#f8f9fb" stroke="#e0e3e8" stroke-width="1.5"/>
<!-- Title -->
<text x="500" y="42" text-anchor="middle" font-size="20" font-weight="bold" fill="#202124">Datastore Remote GET Transfer Flow</text>
<text x="500" y="62" text-anchor="middle" font-size="12" fill="#5f6368">api.rs:583-706 &#xB7; types.rs</text>
<!-- ── Client ──────────────────────────────────────────── -->
<rect x="40" y="100" width="130" height="50" rx="12" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.8" filter="url(#shadow)"/>
<text x="105" y="130" text-anchor="middle" font-size="14" font-weight="600" fill="#1a73e8">Client</text>
<!-- Arrow: Client → API Server -->
<line x1="170" y1="125" x2="240" y2="125" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
<text x="205" y="117" text-anchor="middle" font-size="10" fill="#5f6368">GET /api/data</text>
<!-- ── API Server ──────────────────────────────────────── -->
<rect x="252" y="100" width="150" height="50" rx="12" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.8" filter="url(#shadow)"/>
<text x="327" y="130" text-anchor="middle" font-size="14" font-weight="600" fill="#1a73e8">API Server</text>
<!-- Arrow: API Server → DatastoreNode -->
<line x1="402" y1="125" x2="470" y2="125" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
<text x="436" y="117" text-anchor="middle" font-size="10" fill="#5f6368">Get msg</text>
<!-- ── DatastoreNode ───────────────────────────────────── -->
<rect x="482" y="100" width="160" height="50" rx="12" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.8" filter="url(#shadow)"/>
<text x="562" y="130" text-anchor="middle" font-size="14" font-weight="600" fill="#1a73e8">DatastoreNode</text>
<!-- Arrow: DatastoreNode → MetadataActor -->
<line x1="562" y1="150" x2="562" y2="200" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
<text x="580" y="180" font-size="10" fill="#5f6368">lookup</text>
<!-- ── MetadataActor (local) ───────────────────────────── -->
<rect x="482" y="210" width="160" height="50" rx="12" fill="#fef7e0" stroke="#f9ab00" stroke-width="1.8" filter="url(#shadow)"/>
<text x="562" y="235" text-anchor="middle" font-size="13" font-weight="600" fill="#e37400">MetadataActor</text>
<text x="562" y="250" text-anchor="middle" font-size="10" fill="#5f6368">(local)</text>
<!-- Arrow: MetadataActor → Not Found (red, left) -->
<line x1="482" y1="235" x2="420" y2="235" stroke="#ea4335" stroke-width="1.8" marker-end="url(#arrow-red)"/>
<text x="450" y="227" text-anchor="middle" font-size="10" fill="#ea4335">Not Found</text>
<!-- ── try_remote_get box ──────────────────────────────── -->
<rect x="210" y="210" width="200" height="50" rx="12" fill="#fce8e6" stroke="#ea4335" stroke-width="1.5" filter="url(#shadow)" stroke-dasharray="5,3"/>
<text x="310" y="235" text-anchor="middle" font-size="12" font-weight="600" fill="#c5221f">try_remote_get()</text>
<text x="310" y="250" text-anchor="middle" font-size="10" fill="#5f6368">iterate peers</text>
<!-- Arrow: try_remote_get → Peer Metadata -->
<line x1="310" y1="260" x2="310" y2="320" stroke="#34a853" stroke-width="1.8" marker-end="url(#arrow-green)"/>
<text x="340" y="295" font-size="10" fill="#34a853">FindObject</text>
<!-- ── Peer MetadataActor ──────────────────────────────── -->
<rect x="210" y="330" width="200" height="50" rx="12" fill="#e6f4ea" stroke="#34a853" stroke-width="1.8" filter="url(#shadow)"/>
<text x="310" y="355" text-anchor="middle" font-size="13" font-weight="600" fill="#137333">Peer MetadataActor</text>
<text x="310" y="370" text-anchor="middle" font-size="10" fill="#5f6368">(remote node)</text>
<!-- Arrow: Peer Metadata → Peer BlobStore -->
<line x1="310" y1="380" x2="310" y2="430" stroke="#34a853" stroke-width="1.8" marker-end="url(#arrow-green)"/>
<text x="340" y="410" font-size="10" fill="#34a853">Found!</text>
<!-- ── Peer BlobStore ──────────────────────────────────── -->
<rect x="210" y="440" width="200" height="50" rx="12" fill="#e6f4ea" stroke="#34a853" stroke-width="1.8" filter="url(#shadow)"/>
<text x="310" y="465" text-anchor="middle" font-size="13" font-weight="600" fill="#137333">Peer BlobStore</text>
<text x="310" y="480" text-anchor="middle" font-size="10" fill="#5f6368">ReadChunk &#xD7; N</text>
<!-- Arrow fan-out: Peer BlobStore → Chunks -->
<line x1="230" y1="490" x2="150" y2="540" stroke="#4285f4" stroke-width="1.5" marker-end="url(#arrow-blue)"/>
<line x1="310" y1="490" x2="310" y2="540" stroke="#4285f4" stroke-width="1.5" marker-end="url(#arrow-blue)"/>
<line x1="390" y1="490" x2="470" y2="540" stroke="#4285f4" stroke-width="1.5" marker-end="url(#arrow-blue)"/>
<!-- Chunk boxes -->
<rect x="100" y="545" width="100" height="36" rx="8" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.2" filter="url(#shadow)"/>
<text x="150" y="568" text-anchor="middle" font-size="12" font-weight="600" fill="#1a73e8">Chunk 1</text>
<rect x="260" y="545" width="100" height="36" rx="8" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.2" filter="url(#shadow)"/>
<text x="310" y="568" text-anchor="middle" font-size="12" font-weight="600" fill="#1a73e8">Chunk 2</text>
<rect x="420" y="545" width="100" height="36" rx="8" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.2" filter="url(#shadow)"/>
<text x="470" y="568" text-anchor="middle" font-size="12" font-weight="600" fill="#1a73e8">Chunk N</text>
<!-- Arrows: Chunks → Local BlobStore -->
<line x1="150" y1="581" x2="560" y2="610" stroke="#4285f4" stroke-width="1.2" marker-end="url(#arrow-blue)"/>
<line x1="310" y1="581" x2="560" y2="610" stroke="#4285f4" stroke-width="1.2" marker-end="url(#arrow-blue)"/>
<line x1="470" y1="581" x2="560" y2="610" stroke="#4285f4" stroke-width="1.2" marker-end="url(#arrow-blue)"/>
<!-- ── Local BlobStore ─────────────────────────────────── -->
<rect x="510" y="600" width="180" height="50" rx="12" fill="#e6f4ea" stroke="#34a853" stroke-width="1.8" filter="url(#shadow)"/>
<text x="600" y="625" text-anchor="middle" font-size="13" font-weight="600" fill="#137333">Local BlobStore</text>
<text x="600" y="640" text-anchor="middle" font-size="10" fill="#5f6368">WriteChunk &#xD7; N</text>
<!-- Arrow: Local BlobStore → Reassemble -->
<line x1="690" y1="625" x2="740" y2="625" stroke="#34a853" stroke-width="1.8" marker-end="url(#arrow-green)"/>
<!-- ── Reassemble ──────────────────────────────────────── -->
<rect x="752" y="600" width="150" height="50" rx="12" fill="#e6f4ea" stroke="#34a853" stroke-width="1.8" filter="url(#shadow)"/>
<text x="827" y="625" text-anchor="middle" font-size="13" font-weight="600" fill="#137333">reassemble_blob</text>
<text x="827" y="640" text-anchor="middle" font-size="10" fill="#5f6368">verify integrity</text>
<!-- Arrow: Reassemble → Client (curved up) -->
<path d="M 902 620 Q 950 600 960 400 Q 960 200 180 130"
fill="none" stroke="#34a853" stroke-width="1.8" marker-end="url(#arrow-green)"/>
<text x="960" y="370" font-size="11" fill="#34a853" font-weight="600">200 OK</text>
<text x="960" y="385" font-size="10" fill="#34a853">blob data</text>
<!-- ── Dashboard integration ────────────────────────────── -->
<rect x="700" y="330" width="180" height="50" rx="12" fill="#f3e8fd" stroke="#9c27b0" stroke-width="1.5" filter="url(#shadow)"/>
<text x="790" y="350" text-anchor="middle" font-size="12" font-weight="600" fill="#7b1fa2">DatastoreMetrics</text>
<text x="790" y="367" text-anchor="middle" font-size="10" fill="#5f6368">begin/advance/end</text>
<line x1="410" y1="460" x2="700" y2="355" stroke="#9c27b0" stroke-width="1.2" stroke-dasharray="4,3" marker-end="url(#arrow-gray)"/>
<text x="570" y="395" font-size="9" fill="#9c27b0">transfer progress</text>
<!-- ============================================ -->
<!-- Legend -->
<rect x="50" y="660" width="900" height="30" rx="6" fill="none" stroke="#e0e3e8" stroke-width="1"/>
<text x="70" y="680" font-size="11" font-weight="600" fill="#202124">Legend</text>
<line x1="130" y1="677" x2="160" y2="677" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
<text x="168" y="681" font-size="10" fill="#5f6368">Transition</text>
<line x1="260" y1="677" x2="290" y2="677" stroke="#34a853" stroke-width="1.8" marker-end="url(#arrow-green)"/>
<text x="298" y="681" font-size="10" fill="#34a853">Success path</text>
<line x1="400" y1="677" x2="430" y2="677" stroke="#ea4335" stroke-width="1.8" marker-end="url(#arrow-red)"/>
<text x="438" y="681" font-size="10" fill="#ea4335">Not found / fallback</text>
<line x1="580" y1="677" x2="610" y2="677" stroke="#4285f4" stroke-width="1.5" marker-end="url(#arrow-blue)"/>
<text x="618" y="681" font-size="10" fill="#4285f4">Chunk transfer</text>
<line x1="720" y1="677" x2="750" y2="677" stroke="#9c27b0" stroke-width="1.2" stroke-dasharray="4,3"/>
<text x="758" y="681" font-size="10" fill="#9c27b0">Metrics tracking</text>
</svg>

After

Width:  |  Height:  |  Size: 11 KiB

View file

@ -265,11 +265,6 @@ impl Runtime {
/// Creates a temporary inbox, calls `msg_builder` with the inbox's address
/// (so you can embed it as `reply_to`), sends the message, and returns an
/// [`Ask`] handle for receiving the response.
///
/// ```ignore
/// let ask = rt.ask(actor, |reply_to| GetValue { reply_to })?;
/// let value = ask.recv_ticking(&rt, 10)?;
/// ```
pub fn ask<Req: Message, Resp: Message>(
&self,
addr: ActorAddress,

4
xtask/Cargo.toml Normal file
View file

@ -0,0 +1,4 @@
[package]
name = "xtask"
version = "0.1.0"
edition = "2024"

197
xtask/src/main.rs Normal file
View file

@ -0,0 +1,197 @@
use std::process::Command;
use std::time::Instant;
struct TestStep {
label: &'static str,
args: &'static [&'static str],
}
struct Group {
name: &'static str,
description: &'static str,
steps: &'static [TestStep],
}
const CORE: Group = Group {
name: "core",
description: "Actor runtime, message delivery, property tests",
steps: &[TestStep {
label: "actor runtime",
args: &["test", "-p", "swactor", "--features", "transport"],
}],
};
const DISTRIBUTION: Group = Group {
name: "distribution",
description: "Distribution protocol + datastore",
steps: &[
TestStep {
label: "distribution protocol",
args: &["test", "-p", "distribution"],
},
TestStep {
label: "datastore",
args: &["test", "-p", "swactor-datastore"],
},
],
};
const CLUSTER_SIMS: Group = Group {
name: "cluster-sims",
description: "Deterministic cluster simulations",
steps: &[TestStep {
label: "cluster simulations",
args: &["test", "-p", "simulation"],
}],
};
const INTEGRATED: Group = Group {
name: "integrated",
description: "HTTP API + dashboard end-to-end tests",
steps: &[
TestStep {
label: "datastore integration (node features)",
args: &[
"test",
"-p",
"swactor-datastore",
"--features",
"node",
"--test",
"api_integration_test",
"--test",
"dashboard_integration_test",
],
},
TestStep {
label: "runtime dashboard",
args: &["test", "-p", "runtime-dashboard"],
},
],
};
fn groups_for(name: &str) -> Option<Vec<&'static Group>> {
match name {
"core" => Some(vec![&CORE]),
"distribution" => Some(vec![&DISTRIBUTION]),
"cluster-sims" => Some(vec![&CLUSTER_SIMS]),
"integrated" => Some(vec![&INTEGRATED]),
"essential" => Some(vec![&CORE, &DISTRIBUTION, &INTEGRATED]),
"all" => Some(vec![&CORE, &DISTRIBUTION, &CLUSTER_SIMS, &INTEGRATED]),
_ => None,
}
}
fn run_step(group_name: &str, step: &TestStep) -> bool {
println!("\n=== {group_name}: {} ===", step.label);
println!(" cargo {}", step.args.join(" "));
println!();
let status = Command::new("cargo")
.args(step.args)
.status();
match status {
Ok(s) => s.success(),
Err(e) => {
eprintln!("Failed to execute cargo: {e}");
false
}
}
}
fn print_usage() {
println!(
"\
USAGE: cargo xtask test <GROUP>
GROUPS:
core Actor runtime, message delivery, property tests
distribution Distribution protocol + datastore
cluster-sims Deterministic cluster simulations
integrated HTTP API + dashboard end-to-end tests
essential core + distribution + integrated (merge gate)
all Every test group
FLAGS:
--list Show all groups and the cargo commands they run"
);
}
fn print_list() {
let all_groups: &[(&[&str], &Group)] = &[
(&[], &CORE),
(&[], &DISTRIBUTION),
(&[], &CLUSTER_SIMS),
(&[], &INTEGRATED),
];
println!("Available test groups:\n");
for &(_, group) in all_groups {
println!(" {:<14}{}", group.name, group.description);
for step in group.steps {
println!(" → cargo {}", step.args.join(" "));
}
println!();
}
println!(" {:<14}core + distribution + integrated (merge gate)", "essential");
println!(" {:<14}Every test group", "all");
}
fn main() {
let args: Vec<String> = std::env::args().collect();
if args.len() < 2 || args[1] != "test" {
print_usage();
std::process::exit(if args.len() < 2 { 1 } else { 1 });
}
if args.len() < 3 {
print_usage();
std::process::exit(1);
}
let target = &args[2];
if target == "--list" {
print_list();
return;
}
let groups = match groups_for(target) {
Some(g) => g,
None => {
eprintln!("Unknown test group: {target}\n");
print_usage();
std::process::exit(1);
}
};
let start = Instant::now();
let mut passed = 0usize;
let mut failed = 0usize;
for group in &groups {
for step in group.steps {
if run_step(group.name, step) {
passed += 1;
} else {
failed += 1;
let elapsed = start.elapsed();
println!(
"\n--- FAILED after {:.1}s ({passed} passed, {failed} failed) ---",
elapsed.as_secs_f64()
);
std::process::exit(1);
}
}
}
let elapsed = start.elapsed();
println!(
"\n--- All {passed} step(s) passed in {:.1}s ---",
elapsed.as_secs_f64()
);
}