feat: datastore complete with manual testing and docs
This commit is contained in:
parent
ebf778fc3f
commit
b74d3f7b5f
33 changed files with 2461 additions and 33 deletions
2
.cargo/config.toml
Normal file
2
.cargo/config.toml
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
[alias]
|
||||||
|
xtask = "run --package xtask --"
|
||||||
5
Cargo.lock
generated
5
Cargo.lock
generated
|
|
@ -4391,6 +4391,7 @@ dependencies = [
|
||||||
"swactor-std",
|
"swactor-std",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
"tiny_http",
|
"tiny_http",
|
||||||
|
"toml",
|
||||||
"ureq",
|
"ureq",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
@ -6231,6 +6232,10 @@ dependencies = [
|
||||||
"xml-rs",
|
"xml-rs",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "xtask"
|
||||||
|
version = "0.1.0"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "yoke"
|
name = "yoke"
|
||||||
version = "0.8.1"
|
version = "0.8.1"
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
[workspace]
|
[workspace]
|
||||||
members = [".", "crates/python", "crates/wasm", "crates/bin-runner", "crates/simulation", "crates/runtime-dashboard", "crates/distribution", "crates/std", "crates/datastore", "tests/docker"]
|
members = [".", "crates/python", "crates/wasm", "crates/bin-runner", "crates/simulation", "crates/runtime-dashboard", "crates/distribution", "crates/std", "crates/datastore", "tests/docker", "xtask"]
|
||||||
exclude = ["tools/depgraph"]
|
exclude = ["tools/depgraph"]
|
||||||
|
|
||||||
[package]
|
[package]
|
||||||
|
|
|
||||||
|
|
@ -14,6 +14,7 @@ clap = { version = "4", features = ["derive"], optional = true }
|
||||||
ureq = { version = "2", features = ["json"], optional = true }
|
ureq = { version = "2", features = ["json"], optional = true }
|
||||||
ctrlc = { version = "3", optional = true }
|
ctrlc = { version = "3", optional = true }
|
||||||
runtime-dashboard = { path = "../runtime-dashboard", optional = true }
|
runtime-dashboard = { path = "../runtime-dashboard", optional = true }
|
||||||
|
toml = { version = "0.8", optional = true }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
|
|
@ -21,9 +22,12 @@ proptest = "1"
|
||||||
tempfile = "3"
|
tempfile = "3"
|
||||||
swactor = { path = "../.." }
|
swactor = { path = "../.." }
|
||||||
swactor-std = { path = "../std" }
|
swactor-std = { path = "../std" }
|
||||||
|
ureq = { version = "2", features = ["json"] }
|
||||||
|
tiny_http = "0.12"
|
||||||
|
runtime-dashboard = { path = "../runtime-dashboard" }
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
node = ["dep:tiny_http", "dep:clap", "dep:ctrlc", "dep:runtime-dashboard"]
|
node = ["dep:tiny_http", "dep:clap", "dep:ctrlc", "dep:runtime-dashboard", "dep:toml"]
|
||||||
cli = ["dep:clap", "dep:ureq"]
|
cli = ["dep:clap", "dep:ureq"]
|
||||||
|
|
||||||
[[bin]]
|
[[bin]]
|
||||||
|
|
|
||||||
141
crates/datastore/README.md
Normal file
141
crates/datastore/README.md
Normal file
|
|
@ -0,0 +1,141 @@
|
||||||
|
# swactor-datastore
|
||||||
|
|
||||||
|
Distributed content-addressed datastore built on [swactor](../../README.md). Objects are split into fixed-size chunks, identified by their blake3 hash, and replicated across a peer-to-peer network via epidemic gossip.
|
||||||
|
|
||||||
|
## Building
|
||||||
|
|
||||||
|
Node binary (HTTP server + actor runtime):
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cargo build -p swactor-datastore --features node
|
||||||
|
```
|
||||||
|
|
||||||
|
CLI client:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cargo build -p swactor-datastore --features cli
|
||||||
|
```
|
||||||
|
|
||||||
|
Both at once:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cargo build -p swactor-datastore --features node,cli
|
||||||
|
```
|
||||||
|
|
||||||
|
## Node
|
||||||
|
|
||||||
|
Start a datastore node:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
swactor-store-node
|
||||||
|
```
|
||||||
|
|
||||||
|
### Flags
|
||||||
|
|
||||||
|
| Flag | Default | Description |
|
||||||
|
|------|---------|-------------|
|
||||||
|
| `--port` | `9091` | HTTP API port |
|
||||||
|
| `--storage-path` | *(in-memory)* | Directory for persistent storage |
|
||||||
|
| `--dashboard-port` | *(disabled)* | Runtime dashboard port |
|
||||||
|
| `--chunk-size` | `1048576` | Chunk size in bytes (1 MB) |
|
||||||
|
| `--gc-interval` | `1000` | GC interval in ticks (~100s) |
|
||||||
|
| `--disseminate-interval` | `50` | Gossip interval in ticks (~5s) |
|
||||||
|
| `--config` | *(none)* | Path to a TOML config file |
|
||||||
|
|
||||||
|
Example with persistent storage and dashboard:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
swactor-store-node --storage-path ./data --dashboard-port 9090
|
||||||
|
```
|
||||||
|
|
||||||
|
### Config file
|
||||||
|
|
||||||
|
Create a `store.toml` and pass it with `--config`:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
port = 9091
|
||||||
|
storage_path = "./my-data"
|
||||||
|
dashboard_port = 9090
|
||||||
|
chunk_size = 1048576
|
||||||
|
gc_interval = 1000
|
||||||
|
disseminate_interval = 50
|
||||||
|
```
|
||||||
|
|
||||||
|
CLI flags override config file values. Omitted fields use built-in defaults.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
swactor-store-node --config store.toml --port 8080
|
||||||
|
```
|
||||||
|
|
||||||
|
## CLI
|
||||||
|
|
||||||
|
The `swactor-store` command talks to a running node over HTTP.
|
||||||
|
|
||||||
|
### Status
|
||||||
|
|
||||||
|
```sh
|
||||||
|
swactor-store status
|
||||||
|
```
|
||||||
|
|
||||||
|
### Put
|
||||||
|
|
||||||
|
```sh
|
||||||
|
swactor-store put photo.jpg --name "vacation"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Get (metadata)
|
||||||
|
|
||||||
|
```sh
|
||||||
|
swactor-store get <hash>
|
||||||
|
```
|
||||||
|
|
||||||
|
### Get (download)
|
||||||
|
|
||||||
|
```sh
|
||||||
|
swactor-store get <hash> --output photo.jpg
|
||||||
|
```
|
||||||
|
|
||||||
|
### Delete
|
||||||
|
|
||||||
|
```sh
|
||||||
|
swactor-store delete <hash>
|
||||||
|
```
|
||||||
|
|
||||||
|
### List (local)
|
||||||
|
|
||||||
|
```sh
|
||||||
|
swactor-store list
|
||||||
|
```
|
||||||
|
|
||||||
|
### List (swarm-wide)
|
||||||
|
|
||||||
|
```sh
|
||||||
|
swactor-store list --all
|
||||||
|
```
|
||||||
|
|
||||||
|
### Filter by name
|
||||||
|
|
||||||
|
```sh
|
||||||
|
swactor-store list --name vacation
|
||||||
|
```
|
||||||
|
|
||||||
|
Use `--url` to point at a different node:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
swactor-store --url http://192.168.1.50:9091 list
|
||||||
|
```
|
||||||
|
|
||||||
|
## Web UI
|
||||||
|
|
||||||
|
Visit `http://<host>:<port>/` in a browser. The UI supports uploading, listing, downloading, inspecting, and deleting objects — works on desktop and mobile.
|
||||||
|
|
||||||
|
## HTTP API
|
||||||
|
|
||||||
|
| Method | Path | Description |
|
||||||
|
|--------|------|-------------|
|
||||||
|
| `GET` | `/api/status` | Node identity |
|
||||||
|
| `POST` | `/api/put?name=...` | Upload (body = raw bytes) |
|
||||||
|
| `GET` | `/api/list` | List objects (`?all=true` for swarm) |
|
||||||
|
| `GET` | `/api/get?hash=...` | Object metadata + manifest |
|
||||||
|
| `GET` | `/api/data?hash=...` | Download reassembled binary |
|
||||||
|
| `POST` | `/api/delete?hash=...` | Delete object |
|
||||||
|
|
@ -14,6 +14,7 @@ use swactor::runtime::{Inbox, Runtime};
|
||||||
|
|
||||||
use crate::chunking::reassemble_blob;
|
use crate::chunking::reassemble_blob;
|
||||||
use crate::messages::{BlobStoreMsg, DatastoreNodeMsg, DatastoreResponse, MetadataMsg};
|
use crate::messages::{BlobStoreMsg, DatastoreNodeMsg, DatastoreResponse, MetadataMsg};
|
||||||
|
use crate::metrics::DatastoreMetrics;
|
||||||
use crate::types::ContentHash;
|
use crate::types::ContentHash;
|
||||||
|
|
||||||
/// Per-peer actor addresses needed for remote operations.
|
/// Per-peer actor addresses needed for remote operations.
|
||||||
|
|
@ -30,6 +31,7 @@ struct ApiState {
|
||||||
metadata_addr: ActorAddress,
|
metadata_addr: ActorAddress,
|
||||||
blob_store_addr: ActorAddress,
|
blob_store_addr: ActorAddress,
|
||||||
peers: Arc<Mutex<Vec<PeerInfo>>>,
|
peers: Arc<Mutex<Vec<PeerInfo>>>,
|
||||||
|
metrics: Arc<DatastoreMetrics>,
|
||||||
}
|
}
|
||||||
|
|
||||||
const POLL_TIMEOUT: Duration = Duration::from_secs(5);
|
const POLL_TIMEOUT: Duration = Duration::from_secs(5);
|
||||||
|
|
@ -67,6 +69,16 @@ fn respond_bytes(request: tiny_http::Request, data: &[u8]) {
|
||||||
let _ = request.respond(response);
|
let _ = request.respond(response);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn respond_html(request: tiny_http::Request) {
|
||||||
|
let response =
|
||||||
|
tiny_http::Response::from_string(crate::ui_html::DATASTORE_UI_HTML).with_header(
|
||||||
|
"Content-Type: text/html; charset=utf-8"
|
||||||
|
.parse::<tiny_http::Header>()
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
let _ = request.respond(response);
|
||||||
|
}
|
||||||
|
|
||||||
fn respond_error(request: tiny_http::Request, status: u16, msg: &str) {
|
fn respond_error(request: tiny_http::Request, status: u16, msg: &str) {
|
||||||
let json = serde_json::json!({ "error": msg }).to_string();
|
let json = serde_json::json!({ "error": msg }).to_string();
|
||||||
let response = tiny_http::Response::from_string(json)
|
let response = tiny_http::Response::from_string(json)
|
||||||
|
|
@ -186,6 +198,7 @@ fn handle_put(mut request: tiny_http::Request, url: &str, state: &ApiState) {
|
||||||
// Can't respond — request consumed
|
// Can't respond — request consumed
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
let body_len = body.len();
|
||||||
|
|
||||||
let inbox = match state.runtime.new_inbox::<DatastoreResponse>() {
|
let inbox = match state.runtime.new_inbox::<DatastoreResponse>() {
|
||||||
Ok(i) => i,
|
Ok(i) => i,
|
||||||
|
|
@ -207,7 +220,13 @@ fn handle_put(mut request: tiny_http::Request, url: &str, state: &ApiState) {
|
||||||
|
|
||||||
match poll_response(&inbox, POLL_TIMEOUT) {
|
match poll_response(&inbox, POLL_TIMEOUT) {
|
||||||
Some(DatastoreResponse::PutOk { content_hash }) => {
|
Some(DatastoreResponse::PutOk { content_hash }) => {
|
||||||
let json = serde_json::json!({ "content_hash": content_hash.to_hex() }).to_string();
|
let hex = content_hash.to_hex();
|
||||||
|
state.metrics.record_put(
|
||||||
|
&hex,
|
||||||
|
params.get("name").map(|s| s.as_str()),
|
||||||
|
body_len as u64,
|
||||||
|
);
|
||||||
|
let json = serde_json::json!({ "content_hash": hex }).to_string();
|
||||||
respond_json(request, &json);
|
respond_json(request, &json);
|
||||||
}
|
}
|
||||||
Some(DatastoreResponse::Error { reason }) => {
|
Some(DatastoreResponse::Error { reason }) => {
|
||||||
|
|
@ -257,6 +276,7 @@ fn handle_get(request: tiny_http::Request, url: &str, state: &ApiState) {
|
||||||
|
|
||||||
match poll_response(&inbox, POLL_TIMEOUT) {
|
match poll_response(&inbox, POLL_TIMEOUT) {
|
||||||
Some(DatastoreResponse::GetOk { entry, manifest }) => {
|
Some(DatastoreResponse::GetOk { entry, manifest }) => {
|
||||||
|
state.metrics.record_get(&content_hash.to_hex());
|
||||||
let json = serde_json::json!({
|
let json = serde_json::json!({
|
||||||
"entry": entry_to_json(&entry),
|
"entry": entry_to_json(&entry),
|
||||||
"manifest": manifest_to_json(&manifest),
|
"manifest": manifest_to_json(&manifest),
|
||||||
|
|
@ -313,6 +333,8 @@ fn handle_data(request: tiny_http::Request, url: &str, state: &ApiState) {
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
|
state.metrics.record_get(&content_hash.to_hex());
|
||||||
|
|
||||||
let (entry, manifest) = match poll_response(&inbox, POLL_TIMEOUT) {
|
let (entry, manifest) = match poll_response(&inbox, POLL_TIMEOUT) {
|
||||||
Some(DatastoreResponse::GetOk { entry, manifest }) => (entry, manifest),
|
Some(DatastoreResponse::GetOk { entry, manifest }) => (entry, manifest),
|
||||||
Some(DatastoreResponse::NotFound) => {
|
Some(DatastoreResponse::NotFound) => {
|
||||||
|
|
@ -411,7 +433,9 @@ fn handle_delete(request: tiny_http::Request, url: &str, state: &ApiState) {
|
||||||
|
|
||||||
match poll_response(&inbox, POLL_TIMEOUT) {
|
match poll_response(&inbox, POLL_TIMEOUT) {
|
||||||
Some(DatastoreResponse::DeleteOk { content_hash }) => {
|
Some(DatastoreResponse::DeleteOk { content_hash }) => {
|
||||||
let json = serde_json::json!({ "content_hash": content_hash.to_hex() }).to_string();
|
let hex = content_hash.to_hex();
|
||||||
|
state.metrics.record_delete(&hex, 0);
|
||||||
|
let json = serde_json::json!({ "content_hash": hex }).to_string();
|
||||||
respond_json(request, &json);
|
respond_json(request, &json);
|
||||||
}
|
}
|
||||||
Some(DatastoreResponse::NotFound) => {
|
Some(DatastoreResponse::NotFound) => {
|
||||||
|
|
@ -609,6 +633,8 @@ fn try_remote_get(
|
||||||
};
|
};
|
||||||
|
|
||||||
// Fetch each chunk from peer and store locally
|
// Fetch each chunk from peer and store locally
|
||||||
|
let hash_hex = content_hash.to_hex();
|
||||||
|
state.metrics.begin_transfer(&hash_hex, manifest.chunks.len());
|
||||||
let mut all_ok = true;
|
let mut all_ok = true;
|
||||||
for chunk_ref in &manifest.chunks {
|
for chunk_ref in &manifest.chunks {
|
||||||
let chunk_inbox = match state.runtime.new_inbox::<DatastoreResponse>() {
|
let chunk_inbox = match state.runtime.new_inbox::<DatastoreResponse>() {
|
||||||
|
|
@ -648,6 +674,7 @@ fn try_remote_get(
|
||||||
);
|
);
|
||||||
// Wait for confirmation
|
// Wait for confirmation
|
||||||
let _ = poll_response(&store_inbox, Duration::from_secs(2));
|
let _ = poll_response(&store_inbox, Duration::from_secs(2));
|
||||||
|
state.metrics.advance_transfer(&hash_hex);
|
||||||
}
|
}
|
||||||
_ => {
|
_ => {
|
||||||
all_ok = false;
|
all_ok = false;
|
||||||
|
|
@ -656,6 +683,8 @@ fn try_remote_get(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
state.metrics.end_transfer(&hash_hex);
|
||||||
|
|
||||||
if !all_ok {
|
if !all_ok {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
@ -707,6 +736,7 @@ pub fn start_api_server(
|
||||||
metadata_addr: ActorAddress,
|
metadata_addr: ActorAddress,
|
||||||
blob_store_addr: ActorAddress,
|
blob_store_addr: ActorAddress,
|
||||||
port: u16,
|
port: u16,
|
||||||
|
metrics: Arc<DatastoreMetrics>,
|
||||||
) -> (Arc<AtomicBool>, Arc<Mutex<Vec<PeerInfo>>>) {
|
) -> (Arc<AtomicBool>, Arc<Mutex<Vec<PeerInfo>>>) {
|
||||||
let shutdown = Arc::new(AtomicBool::new(false));
|
let shutdown = Arc::new(AtomicBool::new(false));
|
||||||
let peers: Arc<Mutex<Vec<PeerInfo>>> = Arc::new(Mutex::new(Vec::new()));
|
let peers: Arc<Mutex<Vec<PeerInfo>>> = Arc::new(Mutex::new(Vec::new()));
|
||||||
|
|
@ -717,6 +747,7 @@ pub fn start_api_server(
|
||||||
metadata_addr,
|
metadata_addr,
|
||||||
blob_store_addr,
|
blob_store_addr,
|
||||||
peers: Arc::clone(&peers),
|
peers: Arc::clone(&peers),
|
||||||
|
metrics,
|
||||||
});
|
});
|
||||||
|
|
||||||
let addr = format!("0.0.0.0:{port}");
|
let addr = format!("0.0.0.0:{port}");
|
||||||
|
|
@ -749,6 +780,7 @@ pub fn start_api_server(
|
||||||
("POST", "/api/delete") => handle_delete(request, &url, &state),
|
("POST", "/api/delete") => handle_delete(request, &url, &state),
|
||||||
("GET", "/api/list") => handle_list(request, &url, &state),
|
("GET", "/api/list") => handle_list(request, &url, &state),
|
||||||
("GET", "/api/status") => handle_status(request, &state),
|
("GET", "/api/status") => handle_status(request, &state),
|
||||||
|
("GET", "/") => respond_html(request),
|
||||||
_ => {
|
_ => {
|
||||||
respond_error(request, 404, "not found");
|
respond_error(request, 404, "not found");
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,7 @@ use std::thread;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
|
use serde::Deserialize;
|
||||||
|
|
||||||
use swactor::config::RuntimeConfig;
|
use swactor::config::RuntimeConfig;
|
||||||
use swactor::runtime::Runtime;
|
use swactor::runtime::Runtime;
|
||||||
|
|
@ -16,6 +17,7 @@ use swactor::runtime::Runtime;
|
||||||
use swactor_datastore::actors::{BlobStoreActor, DatastoreNode, MetadataActor};
|
use swactor_datastore::actors::{BlobStoreActor, DatastoreNode, MetadataActor};
|
||||||
use swactor_datastore::api::start_api_server;
|
use swactor_datastore::api::start_api_server;
|
||||||
use swactor_datastore::messages::MetadataMsg;
|
use swactor_datastore::messages::MetadataMsg;
|
||||||
|
use swactor_datastore::metrics::DatastoreMetrics;
|
||||||
use swactor_datastore::storage::{FilesystemBackend, InMemoryBackend};
|
use swactor_datastore::storage::{FilesystemBackend, InMemoryBackend};
|
||||||
use swactor_datastore::DatastoreConfig;
|
use swactor_datastore::DatastoreConfig;
|
||||||
|
|
||||||
|
|
@ -24,9 +26,13 @@ use distribution::types::NodeId;
|
||||||
#[derive(Parser)]
|
#[derive(Parser)]
|
||||||
#[command(name = "swactor-store-node", about = "Swactor distributed datastore node")]
|
#[command(name = "swactor-store-node", about = "Swactor distributed datastore node")]
|
||||||
struct Args {
|
struct Args {
|
||||||
|
/// Path to a TOML config file
|
||||||
|
#[arg(long)]
|
||||||
|
config: Option<std::path::PathBuf>,
|
||||||
|
|
||||||
/// HTTP API port
|
/// HTTP API port
|
||||||
#[arg(long, default_value = "9091")]
|
#[arg(long)]
|
||||||
port: u16,
|
port: Option<u16>,
|
||||||
|
|
||||||
/// Storage directory (omit for in-memory)
|
/// Storage directory (omit for in-memory)
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
|
|
@ -37,20 +43,62 @@ struct Args {
|
||||||
dashboard_port: Option<u16>,
|
dashboard_port: Option<u16>,
|
||||||
|
|
||||||
/// Chunk size in bytes
|
/// Chunk size in bytes
|
||||||
#[arg(long, default_value = "1048576")]
|
#[arg(long)]
|
||||||
chunk_size: u32,
|
chunk_size: Option<u32>,
|
||||||
|
|
||||||
/// GC interval in ticks (each tick is ~100ms)
|
/// GC interval in ticks (each tick is ~100ms)
|
||||||
#[arg(long, default_value = "1000")]
|
#[arg(long)]
|
||||||
gc_interval: u64,
|
gc_interval: Option<u64>,
|
||||||
|
|
||||||
/// Dissemination interval in ticks
|
/// Dissemination interval in ticks
|
||||||
#[arg(long, default_value = "50")]
|
#[arg(long)]
|
||||||
|
disseminate_interval: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Default)]
|
||||||
|
struct NodeConfig {
|
||||||
|
port: Option<u16>,
|
||||||
|
storage_path: Option<String>,
|
||||||
|
dashboard_port: Option<u16>,
|
||||||
|
chunk_size: Option<u32>,
|
||||||
|
gc_interval: Option<u64>,
|
||||||
|
disseminate_interval: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolved configuration with CLI > config file > defaults applied.
|
||||||
|
struct ResolvedConfig {
|
||||||
|
port: u16,
|
||||||
|
storage_path: Option<String>,
|
||||||
|
dashboard_port: Option<u16>,
|
||||||
|
chunk_size: u32,
|
||||||
|
gc_interval: u64,
|
||||||
disseminate_interval: u64,
|
disseminate_interval: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn resolve_config(args: &Args) -> ResolvedConfig {
|
||||||
|
let file_cfg = match &args.config {
|
||||||
|
Some(path) => {
|
||||||
|
let contents = std::fs::read_to_string(path)
|
||||||
|
.unwrap_or_else(|e| panic!("failed to read config file {}: {e}", path.display()));
|
||||||
|
toml::from_str::<NodeConfig>(&contents)
|
||||||
|
.unwrap_or_else(|e| panic!("failed to parse config file {}: {e}", path.display()))
|
||||||
|
}
|
||||||
|
None => NodeConfig::default(),
|
||||||
|
};
|
||||||
|
|
||||||
|
ResolvedConfig {
|
||||||
|
port: args.port.or(file_cfg.port).unwrap_or(9091),
|
||||||
|
storage_path: args.storage_path.clone().or(file_cfg.storage_path),
|
||||||
|
dashboard_port: args.dashboard_port.or(file_cfg.dashboard_port),
|
||||||
|
chunk_size: args.chunk_size.or(file_cfg.chunk_size).unwrap_or(1_048_576),
|
||||||
|
gc_interval: args.gc_interval.or(file_cfg.gc_interval).unwrap_or(1000),
|
||||||
|
disseminate_interval: args.disseminate_interval.or(file_cfg.disseminate_interval).unwrap_or(50),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn main() {
|
fn main() {
|
||||||
let args = Args::parse();
|
let args = Args::parse();
|
||||||
|
let cfg = resolve_config(&args);
|
||||||
let stop = Arc::new(AtomicBool::new(false));
|
let stop = Arc::new(AtomicBool::new(false));
|
||||||
|
|
||||||
// Signal handler
|
// Signal handler
|
||||||
|
|
@ -63,7 +111,7 @@ fn main() {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Optionally start dashboard
|
// Optionally start dashboard
|
||||||
let dash = args.dashboard_port.map(|port| {
|
let dash = cfg.dashboard_port.map(|port| {
|
||||||
let d = runtime_dashboard::start_dashboard(runtime_dashboard::DashboardConfig {
|
let d = runtime_dashboard::start_dashboard(runtime_dashboard::DashboardConfig {
|
||||||
port,
|
port,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
|
|
@ -106,18 +154,18 @@ fn main() {
|
||||||
|
|
||||||
// Datastore config
|
// Datastore config
|
||||||
let config = DatastoreConfig {
|
let config = DatastoreConfig {
|
||||||
chunk_size: args.chunk_size,
|
chunk_size: cfg.chunk_size,
|
||||||
storage_path: args
|
storage_path: cfg
|
||||||
.storage_path
|
.storage_path
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|s| s.into())
|
.map(|s| s.into())
|
||||||
.unwrap_or_else(|| "datastore".into()),
|
.unwrap_or_else(|| "datastore".into()),
|
||||||
gc_interval: args.gc_interval,
|
gc_interval: cfg.gc_interval,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
// Create storage backend
|
// Create storage backend
|
||||||
let backend: Box<dyn swactor_datastore::StorageBackend> = match &args.storage_path {
|
let backend: Box<dyn swactor_datastore::StorageBackend> = match &cfg.storage_path {
|
||||||
Some(path) => {
|
Some(path) => {
|
||||||
let p = std::path::PathBuf::from(path);
|
let p = std::path::PathBuf::from(path);
|
||||||
std::fs::create_dir_all(&p).expect("failed to create storage directory");
|
std::fs::create_dir_all(&p).expect("failed to create storage directory");
|
||||||
|
|
@ -145,8 +193,14 @@ fn main() {
|
||||||
// Start runtime
|
// Start runtime
|
||||||
let handle = rt.run().expect("failed to start runtime");
|
let handle = rt.run().expect("failed to start runtime");
|
||||||
|
|
||||||
|
// Create datastore metrics
|
||||||
|
let node_hex: String = node_id.0.iter().map(|b| format!("{b:02x}")).collect();
|
||||||
|
let metrics = Arc::new(DatastoreMetrics::new());
|
||||||
|
metrics.set_node_id(node_hex.clone());
|
||||||
|
|
||||||
if let Some(ref d) = dash {
|
if let Some(ref d) = dash {
|
||||||
d.set_runtime(handle.runtime.clone(), collector);
|
d.set_runtime(handle.runtime.clone(), collector);
|
||||||
|
d.set_datastore(Arc::clone(&metrics) as Arc<dyn runtime_dashboard::datastore_collector::DatastoreStatsProvider>);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start HTTP API
|
// Start HTTP API
|
||||||
|
|
@ -155,17 +209,17 @@ fn main() {
|
||||||
datastore_addr,
|
datastore_addr,
|
||||||
metadata_addr,
|
metadata_addr,
|
||||||
blob_store_addr,
|
blob_store_addr,
|
||||||
args.port,
|
cfg.port,
|
||||||
|
Arc::clone(&metrics),
|
||||||
);
|
);
|
||||||
|
|
||||||
let node_hex: String = node_id.0.iter().map(|b| format!("{b:02x}")).collect();
|
|
||||||
eprintln!("Node {} started", &node_hex[..8]);
|
eprintln!("Node {} started", &node_hex[..8]);
|
||||||
eprintln!("API at http://0.0.0.0:{}", args.port);
|
eprintln!("API at http://0.0.0.0:{}", cfg.port);
|
||||||
if let Some(port) = args.dashboard_port {
|
if let Some(port) = cfg.dashboard_port {
|
||||||
eprintln!("Dashboard at http://0.0.0.0:{port}");
|
eprintln!("Dashboard at http://0.0.0.0:{port}");
|
||||||
}
|
}
|
||||||
if args.storage_path.is_some() {
|
if cfg.storage_path.is_some() {
|
||||||
eprintln!("Storage: {}", args.storage_path.as_ref().unwrap());
|
eprintln!("Storage: {}", cfg.storage_path.as_ref().unwrap());
|
||||||
} else {
|
} else {
|
||||||
eprintln!("Storage: in-memory");
|
eprintln!("Storage: in-memory");
|
||||||
}
|
}
|
||||||
|
|
@ -175,13 +229,13 @@ fn main() {
|
||||||
while !stop.load(Ordering::Relaxed) {
|
while !stop.load(Ordering::Relaxed) {
|
||||||
round += 1;
|
round += 1;
|
||||||
|
|
||||||
if round % args.gc_interval == 0 {
|
if round % cfg.gc_interval == 0 {
|
||||||
let _ = handle
|
let _ = handle
|
||||||
.runtime
|
.runtime
|
||||||
.send_to(metadata_addr, MetadataMsg::GcTick);
|
.send_to(metadata_addr, MetadataMsg::GcTick);
|
||||||
}
|
}
|
||||||
|
|
||||||
if round % args.disseminate_interval == 0 {
|
if round % cfg.disseminate_interval == 0 {
|
||||||
let _ = handle
|
let _ = handle
|
||||||
.runtime
|
.runtime
|
||||||
.send_to(metadata_addr, MetadataMsg::DisseminateTick);
|
.send_to(metadata_addr, MetadataMsg::DisseminateTick);
|
||||||
|
|
|
||||||
|
|
@ -4,8 +4,11 @@ pub mod chunking;
|
||||||
pub mod storage;
|
pub mod storage;
|
||||||
pub mod actors;
|
pub mod actors;
|
||||||
pub mod cli;
|
pub mod cli;
|
||||||
|
pub mod metrics;
|
||||||
#[cfg(feature = "node")]
|
#[cfg(feature = "node")]
|
||||||
pub mod api;
|
pub mod api;
|
||||||
|
#[cfg(feature = "node")]
|
||||||
|
pub mod ui_html;
|
||||||
|
|
||||||
pub use types::{ChunkRef, ContentHash, DatastoreConfig, ObjectEntry, ObjectManifest};
|
pub use types::{ChunkRef, ContentHash, DatastoreConfig, ObjectEntry, ObjectManifest};
|
||||||
pub use messages::{BlobStoreMsg, DatastoreNodeMsg, DatastoreResponse, MetadataMsg, TransferMsg};
|
pub use messages::{BlobStoreMsg, DatastoreNodeMsg, DatastoreResponse, MetadataMsg, TransferMsg};
|
||||||
|
|
|
||||||
197
crates/datastore/src/metrics.rs
Normal file
197
crates/datastore/src/metrics.rs
Normal file
|
|
@ -0,0 +1,197 @@
|
||||||
|
//! Thread-safe metrics for the datastore, consumed by the runtime dashboard.
|
||||||
|
//!
|
||||||
|
//! `DatastoreMetrics` accumulates counters and event history from any thread
|
||||||
|
//! (API handlers run on `tiny_http` worker threads). The dashboard polls
|
||||||
|
//! `snapshot()` every ~200ms via the `DatastoreStatsProvider` trait.
|
||||||
|
|
||||||
|
use std::collections::VecDeque;
|
||||||
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
|
use std::sync::Mutex;
|
||||||
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
// ── Snapshot types (serializable, sent to the dashboard) ────────────────────
|
||||||
|
|
||||||
|
/// Point-in-time snapshot of the datastore's state and metrics.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct DatastoreSnapshot {
|
||||||
|
pub node_id: String,
|
||||||
|
pub object_count: u64,
|
||||||
|
pub total_bytes: u64,
|
||||||
|
pub put_ops: u64,
|
||||||
|
pub get_ops: u64,
|
||||||
|
pub delete_ops: u64,
|
||||||
|
pub objects: Vec<ObjectSummary>,
|
||||||
|
pub recent_events: Vec<DatastoreEvent>,
|
||||||
|
pub active_transfers: Vec<TransferProgress>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Summary of a single stored object (for the dashboard table).
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct ObjectSummary {
|
||||||
|
pub hash: String,
|
||||||
|
pub name: Option<String>,
|
||||||
|
pub size_bytes: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A recorded datastore operation.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct DatastoreEvent {
|
||||||
|
pub timestamp_ms: u64,
|
||||||
|
pub kind: String,
|
||||||
|
pub hash: String,
|
||||||
|
pub name: Option<String>,
|
||||||
|
pub size_bytes: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Progress of an in-flight chunk transfer.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct TransferProgress {
|
||||||
|
pub hash: String,
|
||||||
|
pub chunks_received: usize,
|
||||||
|
pub chunks_total: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Live metrics (thread-safe, mutated from API handlers) ───────────────────
|
||||||
|
|
||||||
|
const MAX_EVENTS: usize = 200;
|
||||||
|
|
||||||
|
/// Thread-safe metrics accumulator for the datastore.
|
||||||
|
///
|
||||||
|
/// Atomic counters for the hot path (put/get/delete counts). A `Mutex`-guarded
|
||||||
|
/// ring buffer for the event timeline and a small vec for active transfers.
|
||||||
|
pub struct DatastoreMetrics {
|
||||||
|
node_id: Mutex<String>,
|
||||||
|
put_ops: AtomicU64,
|
||||||
|
get_ops: AtomicU64,
|
||||||
|
delete_ops: AtomicU64,
|
||||||
|
objects: Mutex<Vec<ObjectSummary>>,
|
||||||
|
events: Mutex<VecDeque<DatastoreEvent>>,
|
||||||
|
transfers: Mutex<Vec<TransferProgress>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl DatastoreMetrics {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
node_id: Mutex::new(String::new()),
|
||||||
|
put_ops: AtomicU64::new(0),
|
||||||
|
get_ops: AtomicU64::new(0),
|
||||||
|
delete_ops: AtomicU64::new(0),
|
||||||
|
objects: Mutex::new(Vec::new()),
|
||||||
|
events: Mutex::new(VecDeque::with_capacity(MAX_EVENTS + 1)),
|
||||||
|
transfers: Mutex::new(Vec::new()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn set_node_id(&self, hex: String) {
|
||||||
|
*self.node_id.lock().unwrap() = hex;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record a successful PUT operation.
|
||||||
|
pub fn record_put(&self, hash: &str, name: Option<&str>, size_bytes: u64) {
|
||||||
|
self.put_ops.fetch_add(1, Ordering::Relaxed);
|
||||||
|
self.push_event("put", hash, name, size_bytes);
|
||||||
|
let mut objs = self.objects.lock().unwrap();
|
||||||
|
objs.push(ObjectSummary {
|
||||||
|
hash: hash.to_string(),
|
||||||
|
name: name.map(|s| s.to_string()),
|
||||||
|
size_bytes,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record a successful GET operation.
|
||||||
|
pub fn record_get(&self, hash: &str) {
|
||||||
|
self.get_ops.fetch_add(1, Ordering::Relaxed);
|
||||||
|
self.push_event("get", hash, None, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record a successful DELETE operation.
|
||||||
|
pub fn record_delete(&self, hash: &str, size_bytes: u64) {
|
||||||
|
self.delete_ops.fetch_add(1, Ordering::Relaxed);
|
||||||
|
self.push_event("delete", hash, None, size_bytes);
|
||||||
|
let mut objs = self.objects.lock().unwrap();
|
||||||
|
objs.retain(|o| o.hash != hash);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Begin tracking a chunk transfer.
|
||||||
|
pub fn begin_transfer(&self, hash: &str, chunks_total: usize) {
|
||||||
|
let mut transfers = self.transfers.lock().unwrap();
|
||||||
|
transfers.push(TransferProgress {
|
||||||
|
hash: hash.to_string(),
|
||||||
|
chunks_received: 0,
|
||||||
|
chunks_total,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Advance a tracked transfer by one chunk.
|
||||||
|
pub fn advance_transfer(&self, hash: &str) {
|
||||||
|
let mut transfers = self.transfers.lock().unwrap();
|
||||||
|
if let Some(t) = transfers.iter_mut().find(|t| t.hash == hash) {
|
||||||
|
t.chunks_received += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Remove a completed/failed transfer from tracking.
|
||||||
|
pub fn end_transfer(&self, hash: &str) {
|
||||||
|
let mut transfers = self.transfers.lock().unwrap();
|
||||||
|
transfers.retain(|t| t.hash != hash);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Seed the object list (e.g. from an initial LIST query at startup).
|
||||||
|
pub fn seed_objects(&self, objects: Vec<ObjectSummary>) {
|
||||||
|
*self.objects.lock().unwrap() = objects;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Capture a serializable snapshot of the current metrics.
|
||||||
|
pub fn snapshot(&self) -> DatastoreSnapshot {
|
||||||
|
let objs = self.objects.lock().unwrap();
|
||||||
|
let total_bytes: u64 = objs.iter().map(|o| o.size_bytes).sum();
|
||||||
|
let events = self.events.lock().unwrap();
|
||||||
|
let transfers = self.transfers.lock().unwrap();
|
||||||
|
|
||||||
|
DatastoreSnapshot {
|
||||||
|
node_id: self.node_id.lock().unwrap().clone(),
|
||||||
|
object_count: objs.len() as u64,
|
||||||
|
total_bytes,
|
||||||
|
put_ops: self.put_ops.load(Ordering::Relaxed),
|
||||||
|
get_ops: self.get_ops.load(Ordering::Relaxed),
|
||||||
|
delete_ops: self.delete_ops.load(Ordering::Relaxed),
|
||||||
|
objects: objs.clone(),
|
||||||
|
recent_events: events.iter().cloned().collect(),
|
||||||
|
active_transfers: transfers.clone(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_event(&self, kind: &str, hash: &str, name: Option<&str>, size_bytes: u64) {
|
||||||
|
let event = DatastoreEvent {
|
||||||
|
timestamp_ms: now_ms(),
|
||||||
|
kind: kind.to_string(),
|
||||||
|
hash: hash.to_string(),
|
||||||
|
name: name.map(|s| s.to_string()),
|
||||||
|
size_bytes,
|
||||||
|
};
|
||||||
|
let mut events = self.events.lock().unwrap();
|
||||||
|
if events.len() >= MAX_EVENTS {
|
||||||
|
events.pop_front();
|
||||||
|
}
|
||||||
|
events.push_back(event);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn now_ms() -> u64 {
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_millis() as u64
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Dashboard integration (only when runtime-dashboard is available) ────────
|
||||||
|
|
||||||
|
#[cfg(feature = "node")]
|
||||||
|
impl runtime_dashboard::datastore_collector::DatastoreStatsProvider for DatastoreMetrics {
|
||||||
|
fn snapshot_json(&self) -> Option<String> {
|
||||||
|
let snap = self.snapshot();
|
||||||
|
serde_json::to_string(&snap).ok()
|
||||||
|
}
|
||||||
|
}
|
||||||
326
crates/datastore/src/ui_html.rs
Normal file
326
crates/datastore/src/ui_html.rs
Normal file
|
|
@ -0,0 +1,326 @@
|
||||||
|
pub const DATASTORE_UI_HTML: &str = r##"<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
|
<title>swactor-store</title>
|
||||||
|
<style>
|
||||||
|
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||||
|
body { font-family: 'Menlo', 'Consolas', 'Monaco', monospace; background: #0f1117; color: #e0e0e0; font-size: 13px; }
|
||||||
|
|
||||||
|
.header {
|
||||||
|
display: flex; align-items: center; justify-content: space-between;
|
||||||
|
padding: 12px 20px; background: #161822; border-bottom: 1px solid #2a2d3e;
|
||||||
|
}
|
||||||
|
.header h1 { font-size: 16px; font-weight: 600; color: #fff; }
|
||||||
|
.header .node-id { font-size: 11px; color: #888; margin-left: 12px; }
|
||||||
|
|
||||||
|
.container { max-width: 960px; margin: 0 auto; padding: 16px; }
|
||||||
|
|
||||||
|
.panel {
|
||||||
|
background: #161822; border: 1px solid #2a2d3e; border-radius: 6px;
|
||||||
|
padding: 16px; margin-bottom: 16px;
|
||||||
|
}
|
||||||
|
.panel h2 {
|
||||||
|
font-size: 12px; color: #888; text-transform: uppercase;
|
||||||
|
letter-spacing: 1px; margin-bottom: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.upload-row {
|
||||||
|
display: flex; gap: 10px; align-items: center; flex-wrap: wrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
input[type="file"] {
|
||||||
|
background: #1c1f2e; color: #e0e0e0; border: 1px solid #2a2d3e;
|
||||||
|
border-radius: 4px; padding: 8px; font-family: inherit; font-size: 13px;
|
||||||
|
min-height: 44px; cursor: pointer;
|
||||||
|
}
|
||||||
|
input[type="file"]::file-selector-button {
|
||||||
|
background: #1e2030; color: #e0e0e0; border: 1px solid #2a2d3e;
|
||||||
|
border-radius: 4px; padding: 6px 12px; font-family: inherit;
|
||||||
|
font-size: 12px; cursor: pointer; margin-right: 8px;
|
||||||
|
}
|
||||||
|
input[type="file"]::file-selector-button:hover { border-color: #6366f1; }
|
||||||
|
|
||||||
|
input[type="text"] {
|
||||||
|
background: #1c1f2e; color: #e0e0e0; border: 1px solid #2a2d3e;
|
||||||
|
border-radius: 4px; padding: 8px 12px; font-family: inherit;
|
||||||
|
font-size: 13px; min-height: 44px; width: 200px;
|
||||||
|
}
|
||||||
|
input[type="text"]:focus { outline: none; border-color: #6366f1; }
|
||||||
|
|
||||||
|
button {
|
||||||
|
background: #1e2030; color: #e0e0e0; border: 1px solid #2a2d3e;
|
||||||
|
border-radius: 4px; padding: 8px 16px; font-family: inherit;
|
||||||
|
font-size: 13px; cursor: pointer; min-height: 44px;
|
||||||
|
transition: border-color 0.15s;
|
||||||
|
}
|
||||||
|
button:hover { border-color: #6366f1; color: #fff; }
|
||||||
|
button:disabled { opacity: 0.4; cursor: default; }
|
||||||
|
button.danger:hover { border-color: #f44336; }
|
||||||
|
button.primary { background: #6366f1; border-color: #6366f1; color: #fff; font-weight: 600; }
|
||||||
|
button.primary:hover { background: #5558e6; }
|
||||||
|
|
||||||
|
.toast {
|
||||||
|
position: fixed; bottom: 20px; right: 20px; padding: 10px 16px;
|
||||||
|
border-radius: 4px; font-size: 12px; z-index: 100; opacity: 0;
|
||||||
|
transition: opacity 0.3s; pointer-events: none;
|
||||||
|
}
|
||||||
|
.toast.show { opacity: 1; }
|
||||||
|
.toast.success { background: #4caf50; color: #fff; }
|
||||||
|
.toast.error { background: #f44336; color: #fff; }
|
||||||
|
|
||||||
|
table { width: 100%; border-collapse: collapse; }
|
||||||
|
th {
|
||||||
|
text-align: left; font-size: 10px; color: #888;
|
||||||
|
text-transform: uppercase; letter-spacing: 1px;
|
||||||
|
padding: 6px 8px; border-bottom: 1px solid #2a2d3e;
|
||||||
|
}
|
||||||
|
td {
|
||||||
|
padding: 8px; border-bottom: 1px solid #1c1f2e;
|
||||||
|
font-size: 13px; vertical-align: middle;
|
||||||
|
}
|
||||||
|
tr:hover td { background: #1c1f2e; }
|
||||||
|
tr { cursor: pointer; }
|
||||||
|
|
||||||
|
.hash-cell { color: #6366f1; font-size: 12px; }
|
||||||
|
.size-cell { color: #888; white-space: nowrap; }
|
||||||
|
.actions-cell { white-space: nowrap; text-align: right; }
|
||||||
|
.actions-cell button { min-height: 32px; padding: 4px 10px; font-size: 11px; }
|
||||||
|
|
||||||
|
.empty-state {
|
||||||
|
text-align: center; color: #555; padding: 32px; font-size: 14px;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Modal overlay */
|
||||||
|
.modal-overlay {
|
||||||
|
display: none; position: fixed; inset: 0;
|
||||||
|
background: rgba(0,0,0,0.6); z-index: 50;
|
||||||
|
align-items: center; justify-content: center;
|
||||||
|
}
|
||||||
|
.modal-overlay.open { display: flex; }
|
||||||
|
.modal {
|
||||||
|
background: #161822; border: 1px solid #2a2d3e; border-radius: 6px;
|
||||||
|
padding: 20px; width: 90%; max-width: 560px; max-height: 80vh;
|
||||||
|
overflow-y: auto;
|
||||||
|
}
|
||||||
|
.modal h2 { font-size: 14px; color: #fff; margin-bottom: 16px; text-transform: none; letter-spacing: 0; }
|
||||||
|
.modal-close {
|
||||||
|
float: right; background: none; border: none; color: #888;
|
||||||
|
font-size: 18px; cursor: pointer; min-height: auto; padding: 0;
|
||||||
|
}
|
||||||
|
.modal-close:hover { color: #fff; border: none; }
|
||||||
|
.detail-row { display: flex; margin-bottom: 8px; }
|
||||||
|
.detail-label { color: #888; width: 110px; flex-shrink: 0; font-size: 11px; text-transform: uppercase; padding-top: 2px; }
|
||||||
|
.detail-value { color: #e0e0e0; word-break: break-all; font-size: 13px; }
|
||||||
|
.chunk-list { margin-top: 8px; }
|
||||||
|
.chunk-item { color: #888; font-size: 11px; padding: 2px 0; }
|
||||||
|
|
||||||
|
@media (max-width: 640px) {
|
||||||
|
.upload-row { flex-direction: column; align-items: stretch; }
|
||||||
|
input[type="text"] { width: 100%; }
|
||||||
|
.header { flex-direction: column; align-items: flex-start; gap: 4px; }
|
||||||
|
.header .node-id { margin-left: 0; }
|
||||||
|
.actions-cell { display: flex; gap: 4px; justify-content: flex-end; }
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
|
||||||
|
<div class="header">
|
||||||
|
<div style="display:flex;align-items:center;flex-wrap:wrap;">
|
||||||
|
<h1>swactor-store</h1>
|
||||||
|
<span class="node-id" id="nodeId">connecting...</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="container">
|
||||||
|
<!-- Upload panel -->
|
||||||
|
<div class="panel">
|
||||||
|
<h2>Upload</h2>
|
||||||
|
<div class="upload-row">
|
||||||
|
<input type="file" id="fileInput" />
|
||||||
|
<input type="text" id="nameInput" placeholder="name (optional)" />
|
||||||
|
<button class="primary" id="uploadBtn" onclick="upload()">Upload</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Object table -->
|
||||||
|
<div class="panel">
|
||||||
|
<h2>Objects</h2>
|
||||||
|
<div id="tableWrap"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Detail modal -->
|
||||||
|
<div class="modal-overlay" id="modal" onclick="if(event.target===this)closeModal()">
|
||||||
|
<div class="modal">
|
||||||
|
<button class="modal-close" onclick="closeModal()">×</button>
|
||||||
|
<h2>Object Detail</h2>
|
||||||
|
<div id="modalBody"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="toast" id="toast"></div>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
const $ = id => document.getElementById(id);
|
||||||
|
|
||||||
|
function toast(msg, type) {
|
||||||
|
const t = $('toast');
|
||||||
|
t.textContent = msg;
|
||||||
|
t.className = 'toast show ' + type;
|
||||||
|
setTimeout(() => t.className = 'toast', 2500);
|
||||||
|
}
|
||||||
|
|
||||||
|
function fmtSize(b) {
|
||||||
|
if (b < 1024) return b + ' B';
|
||||||
|
if (b < 1048576) return (b / 1024).toFixed(1) + ' KB';
|
||||||
|
if (b < 1073741824) return (b / 1048576).toFixed(1) + ' MB';
|
||||||
|
return (b / 1073741824).toFixed(1) + ' GB';
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchStatus() {
|
||||||
|
try {
|
||||||
|
const r = await fetch('/api/status');
|
||||||
|
const j = await r.json();
|
||||||
|
$('nodeId').textContent = j.node_id.substring(0, 16) + '...';
|
||||||
|
$('nodeId').title = j.node_id;
|
||||||
|
} catch(e) {
|
||||||
|
$('nodeId').textContent = 'offline';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function refreshList() {
|
||||||
|
try {
|
||||||
|
const r = await fetch('/api/list');
|
||||||
|
const j = await r.json();
|
||||||
|
renderTable(j.entries || []);
|
||||||
|
} catch(e) {
|
||||||
|
$('tableWrap').innerHTML = '<div class="empty-state">failed to load</div>';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderTable(entries) {
|
||||||
|
if (entries.length === 0) {
|
||||||
|
$('tableWrap').innerHTML = '<div class="empty-state">no objects stored</div>';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let html = '<table><thead><tr><th>Hash</th><th>Name</th><th>Size</th><th style="text-align:right">Actions</th></tr></thead><tbody>';
|
||||||
|
for (const e of entries) {
|
||||||
|
const h = e.content_hash;
|
||||||
|
const short = h.substring(0, 16);
|
||||||
|
const name = e.name || '—';
|
||||||
|
const size = fmtSize(e.size_bytes);
|
||||||
|
html += '<tr onclick="showDetail(\'' + h + '\')">';
|
||||||
|
html += '<td class="hash-cell" title="' + h + '">' + short + '</td>';
|
||||||
|
html += '<td>' + escHtml(name) + '</td>';
|
||||||
|
html += '<td class="size-cell">' + size + '</td>';
|
||||||
|
html += '<td class="actions-cell">';
|
||||||
|
html += '<button onclick="event.stopPropagation();download(\'' + h + '\',\'' + escAttr(e.name || h.substring(0,12)) + '\')">download</button> ';
|
||||||
|
html += '<button class="danger" onclick="event.stopPropagation();del(\'' + h + '\')">delete</button>';
|
||||||
|
html += '</td></tr>';
|
||||||
|
}
|
||||||
|
html += '</tbody></table>';
|
||||||
|
$('tableWrap').innerHTML = html;
|
||||||
|
}
|
||||||
|
|
||||||
|
function escHtml(s) { const d = document.createElement('div'); d.textContent = s; return d.innerHTML; }
|
||||||
|
function escAttr(s) { return s.replace(/'/g, "\\'").replace(/"/g, '"'); }
|
||||||
|
|
||||||
|
async function upload() {
|
||||||
|
const file = $('fileInput').files[0];
|
||||||
|
if (!file) { toast('select a file first', 'error'); return; }
|
||||||
|
const name = $('nameInput').value.trim();
|
||||||
|
const btn = $('uploadBtn');
|
||||||
|
btn.disabled = true;
|
||||||
|
btn.textContent = 'uploading...';
|
||||||
|
try {
|
||||||
|
let url = '/api/put';
|
||||||
|
if (name) url += '?name=' + encodeURIComponent(name);
|
||||||
|
const r = await fetch(url, { method: 'POST', body: file });
|
||||||
|
if (!r.ok) throw new Error((await r.json()).error || r.statusText);
|
||||||
|
const j = await r.json();
|
||||||
|
toast('uploaded ' + j.content_hash.substring(0, 12), 'success');
|
||||||
|
$('fileInput').value = '';
|
||||||
|
$('nameInput').value = '';
|
||||||
|
refreshList();
|
||||||
|
} catch(e) {
|
||||||
|
toast('upload failed: ' + e.message, 'error');
|
||||||
|
} finally {
|
||||||
|
btn.disabled = false;
|
||||||
|
btn.textContent = 'Upload';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function download(hash, filename) {
|
||||||
|
try {
|
||||||
|
const r = await fetch('/api/data?hash=' + hash);
|
||||||
|
if (!r.ok) throw new Error('not found');
|
||||||
|
const blob = await r.blob();
|
||||||
|
const a = document.createElement('a');
|
||||||
|
a.href = URL.createObjectURL(blob);
|
||||||
|
a.download = filename;
|
||||||
|
a.click();
|
||||||
|
URL.revokeObjectURL(a.href);
|
||||||
|
} catch(e) {
|
||||||
|
toast('download failed: ' + e.message, 'error');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function del(hash) {
|
||||||
|
if (!confirm('Delete ' + hash.substring(0, 16) + '?')) return;
|
||||||
|
try {
|
||||||
|
const r = await fetch('/api/delete?hash=' + hash, { method: 'POST' });
|
||||||
|
if (!r.ok) throw new Error((await r.json()).error || r.statusText);
|
||||||
|
toast('deleted', 'success');
|
||||||
|
refreshList();
|
||||||
|
} catch(e) {
|
||||||
|
toast('delete failed: ' + e.message, 'error');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function showDetail(hash) {
|
||||||
|
try {
|
||||||
|
const r = await fetch('/api/get?hash=' + hash);
|
||||||
|
if (!r.ok) throw new Error('not found');
|
||||||
|
const j = await r.json();
|
||||||
|
const e = j.entry;
|
||||||
|
const m = j.manifest;
|
||||||
|
let html = '';
|
||||||
|
html += row('Hash', e.content_hash);
|
||||||
|
html += row('Name', e.name || '—');
|
||||||
|
html += row('Size', fmtSize(e.size_bytes));
|
||||||
|
html += row('Node', e.node_id.substring(0, 16) + '...');
|
||||||
|
if (e.tags && Object.keys(e.tags).length > 0) {
|
||||||
|
html += row('Tags', Object.entries(e.tags).map(([k,v]) => k + '=' + v).join(', '));
|
||||||
|
}
|
||||||
|
html += row('Chunks', m.chunks.length + ' (' + fmtSize(m.chunk_size) + ' each)');
|
||||||
|
if (m.chunks.length > 0) {
|
||||||
|
html += '<div class="chunk-list">';
|
||||||
|
for (let i = 0; i < m.chunks.length; i++) {
|
||||||
|
const c = m.chunks[i];
|
||||||
|
html += '<div class="chunk-item">#' + i + ' ' + c.hash.substring(0, 16) + ' (' + fmtSize(c.size) + ')</div>';
|
||||||
|
}
|
||||||
|
html += '</div>';
|
||||||
|
}
|
||||||
|
$('modalBody').innerHTML = html;
|
||||||
|
$('modal').classList.add('open');
|
||||||
|
} catch(e) {
|
||||||
|
toast('failed to load detail', 'error');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function row(label, value) {
|
||||||
|
return '<div class="detail-row"><div class="detail-label">' + label + '</div><div class="detail-value">' + escHtml(String(value)) + '</div></div>';
|
||||||
|
}
|
||||||
|
|
||||||
|
function closeModal() { $('modal').classList.remove('open'); }
|
||||||
|
document.addEventListener('keydown', e => { if (e.key === 'Escape') closeModal(); });
|
||||||
|
|
||||||
|
// Init
|
||||||
|
fetchStatus();
|
||||||
|
refreshList();
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>"##;
|
||||||
190
crates/datastore/tests/api_integration_test.rs
Normal file
190
crates/datastore/tests/api_integration_test.rs
Normal file
|
|
@ -0,0 +1,190 @@
|
||||||
|
#![cfg(feature = "node")]
|
||||||
|
|
||||||
|
//! Integration test: spins up a real datastore node with HTTP API and exercises
|
||||||
|
//! the full CRUD lifecycle over HTTP.
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::thread;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use swactor::config::RuntimeConfig;
|
||||||
|
use swactor::runtime::Runtime;
|
||||||
|
|
||||||
|
use swactor_datastore::actors::{BlobStoreActor, DatastoreNode, MetadataActor};
|
||||||
|
use swactor_datastore::api::start_api_server;
|
||||||
|
use swactor_datastore::metrics::DatastoreMetrics;
|
||||||
|
use swactor_datastore::storage::InMemoryBackend;
|
||||||
|
use swactor_datastore::DatastoreConfig;
|
||||||
|
|
||||||
|
use distribution::types::NodeId;
|
||||||
|
|
||||||
|
fn find_free_port() -> u16 {
|
||||||
|
std::net::TcpListener::bind("127.0.0.1:0")
|
||||||
|
.unwrap()
|
||||||
|
.local_addr()
|
||||||
|
.unwrap()
|
||||||
|
.port()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Full CRUD lifecycle over HTTP:
|
||||||
|
/// status → put → list → get metadata → get data → delete → list (empty) → get (404)
|
||||||
|
#[test]
|
||||||
|
fn http_crud_lifecycle() {
|
||||||
|
let port = find_free_port();
|
||||||
|
let base = format!("http://127.0.0.1:{port}");
|
||||||
|
|
||||||
|
// Set up runtime with worker threads (needed for HTTP server)
|
||||||
|
let collector = runtime_dashboard::collector::StatsCollector::new(2);
|
||||||
|
let mut rt = Runtime::new(RuntimeConfig {
|
||||||
|
num_threads: 2,
|
||||||
|
max_actors: 1024,
|
||||||
|
channel_buffer_size: 2000,
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
rt.set_stats_hook(collector);
|
||||||
|
|
||||||
|
let node_id = NodeId([0xAA; 32]);
|
||||||
|
|
||||||
|
let config = DatastoreConfig {
|
||||||
|
chunk_size: 1_048_576,
|
||||||
|
gc_interval: 1000,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let backend: Box<dyn swactor_datastore::StorageBackend> = Box::new(InMemoryBackend::new());
|
||||||
|
let blob_store_addr = rt.spawn(BlobStoreActor::new(backend)).unwrap();
|
||||||
|
|
||||||
|
let mut metadata = MetadataActor::new(node_id, &config);
|
||||||
|
metadata.set_blob_store(blob_store_addr);
|
||||||
|
let metadata_addr = rt.spawn(metadata).unwrap();
|
||||||
|
|
||||||
|
let datastore_node = DatastoreNode::new(node_id, blob_store_addr, metadata_addr, config);
|
||||||
|
let datastore_addr = rt.spawn(datastore_node).unwrap();
|
||||||
|
|
||||||
|
let handle = rt.run().expect("failed to start runtime");
|
||||||
|
|
||||||
|
let metrics = Arc::new(DatastoreMetrics::new());
|
||||||
|
let (api_shutdown, _peers) = start_api_server(
|
||||||
|
handle.runtime.clone(),
|
||||||
|
datastore_addr,
|
||||||
|
metadata_addr,
|
||||||
|
blob_store_addr,
|
||||||
|
port,
|
||||||
|
Arc::clone(&metrics),
|
||||||
|
);
|
||||||
|
|
||||||
|
// Give the HTTP server time to bind
|
||||||
|
thread::sleep(Duration::from_millis(200));
|
||||||
|
|
||||||
|
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||||
|
run_crud_scenario(&base, &metrics);
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Cleanup
|
||||||
|
api_shutdown.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||||
|
handle.shutdown();
|
||||||
|
handle.join();
|
||||||
|
|
||||||
|
if let Err(e) = result {
|
||||||
|
std::panic::resume_unwind(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run_crud_scenario(base: &str, metrics: &Arc<DatastoreMetrics>) {
|
||||||
|
// 1. Status — should return a node_id
|
||||||
|
let status: serde_json::Value = ureq::get(&format!("{base}/api/status"))
|
||||||
|
.call()
|
||||||
|
.expect("status request failed")
|
||||||
|
.into_json()
|
||||||
|
.unwrap();
|
||||||
|
let node_id = status["node_id"].as_str().expect("node_id should be a string");
|
||||||
|
assert_eq!(node_id.len(), 64, "node_id should be 64 hex chars");
|
||||||
|
|
||||||
|
// 2. Put — upload some content
|
||||||
|
let content = b"hello from the integration test!";
|
||||||
|
let put_resp: serde_json::Value = ureq::post(&format!("{base}/api/put?name=greeting"))
|
||||||
|
.send_bytes(content)
|
||||||
|
.expect("put request failed")
|
||||||
|
.into_json()
|
||||||
|
.unwrap();
|
||||||
|
let hash = put_resp["content_hash"]
|
||||||
|
.as_str()
|
||||||
|
.expect("put should return content_hash");
|
||||||
|
assert_eq!(hash.len(), 64, "content_hash should be 64 hex chars");
|
||||||
|
|
||||||
|
// 3. List — should contain exactly one entry matching our upload
|
||||||
|
let list_resp: serde_json::Value = ureq::get(&format!("{base}/api/list"))
|
||||||
|
.call()
|
||||||
|
.expect("list request failed")
|
||||||
|
.into_json()
|
||||||
|
.unwrap();
|
||||||
|
let entries = list_resp["entries"].as_array().expect("entries should be an array");
|
||||||
|
assert_eq!(entries.len(), 1, "should have exactly 1 entry after put");
|
||||||
|
assert_eq!(entries[0]["content_hash"].as_str().unwrap(), hash);
|
||||||
|
assert_eq!(entries[0]["name"].as_str().unwrap(), "greeting");
|
||||||
|
|
||||||
|
// 4. Get metadata — entry + manifest for the uploaded object
|
||||||
|
let get_resp: serde_json::Value = ureq::get(&format!("{base}/api/get?hash={hash}"))
|
||||||
|
.call()
|
||||||
|
.expect("get request failed")
|
||||||
|
.into_json()
|
||||||
|
.unwrap();
|
||||||
|
let entry = &get_resp["entry"];
|
||||||
|
assert_eq!(entry["content_hash"].as_str().unwrap(), hash);
|
||||||
|
assert_eq!(entry["name"].as_str().unwrap(), "greeting");
|
||||||
|
assert_eq!(entry["size_bytes"].as_u64().unwrap(), content.len() as u64);
|
||||||
|
let manifest = &get_resp["manifest"];
|
||||||
|
let chunks = manifest["chunks"].as_array().expect("manifest should have chunks");
|
||||||
|
assert!(!chunks.is_empty(), "manifest should have at least one chunk");
|
||||||
|
|
||||||
|
// 5. Get data — download the raw bytes and verify content matches
|
||||||
|
let data_resp = ureq::get(&format!("{base}/api/data?hash={hash}"))
|
||||||
|
.call()
|
||||||
|
.expect("data request failed");
|
||||||
|
let mut downloaded = Vec::new();
|
||||||
|
data_resp
|
||||||
|
.into_reader()
|
||||||
|
.read_to_end(&mut downloaded)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(downloaded, content, "downloaded bytes should match uploaded content");
|
||||||
|
|
||||||
|
// 6. Delete — remove the object
|
||||||
|
let del_resp: serde_json::Value = ureq::post(&format!("{base}/api/delete?hash={hash}"))
|
||||||
|
.call()
|
||||||
|
.expect("delete request failed")
|
||||||
|
.into_json()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(del_resp["content_hash"].as_str().unwrap(), hash);
|
||||||
|
|
||||||
|
// 7. List after delete — should be empty
|
||||||
|
let list_resp2: serde_json::Value = ureq::get(&format!("{base}/api/list"))
|
||||||
|
.call()
|
||||||
|
.expect("list request failed")
|
||||||
|
.into_json()
|
||||||
|
.unwrap();
|
||||||
|
let entries2 = list_resp2["entries"]
|
||||||
|
.as_array()
|
||||||
|
.expect("entries should be an array");
|
||||||
|
assert!(entries2.is_empty(), "list should be empty after delete");
|
||||||
|
|
||||||
|
// 8. Get after delete — should 404
|
||||||
|
let get_err = ureq::get(&format!("{base}/api/get?hash={hash}")).call();
|
||||||
|
match get_err {
|
||||||
|
Err(ureq::Error::Status(404, _)) => {} // expected
|
||||||
|
Err(e) => panic!("expected 404, got error: {e}"),
|
||||||
|
Ok(_) => panic!("expected 404, got 200"),
|
||||||
|
}
|
||||||
|
|
||||||
|
// 9. Verify metrics snapshot reflects the full lifecycle
|
||||||
|
let snap = metrics.snapshot();
|
||||||
|
assert_eq!(snap.put_ops, 1, "one put recorded");
|
||||||
|
// handle_get + handle_data = 2 get operations
|
||||||
|
assert_eq!(snap.get_ops, 2, "metadata-get + data-get recorded");
|
||||||
|
assert_eq!(snap.delete_ops, 1, "one delete recorded");
|
||||||
|
assert!(snap.objects.is_empty(), "no objects after delete");
|
||||||
|
assert!(
|
||||||
|
snap.recent_events.len() >= 4,
|
||||||
|
"at least 4 events (put + get + get + delete), got {}",
|
||||||
|
snap.recent_events.len()
|
||||||
|
);
|
||||||
|
}
|
||||||
205
crates/datastore/tests/dashboard_integration_test.rs
Normal file
205
crates/datastore/tests/dashboard_integration_test.rs
Normal file
|
|
@ -0,0 +1,205 @@
|
||||||
|
#![cfg(feature = "node")]
|
||||||
|
|
||||||
|
//! End-to-end integration test: spins up a datastore node with an HTTP API
|
||||||
|
//! **and** a runtime dashboard, performs CRUD over HTTP, then verifies:
|
||||||
|
//!
|
||||||
|
//! - `DatastoreMetrics::snapshot()` reflects the operations
|
||||||
|
//! - Dashboard `/datastore` serves HTML
|
||||||
|
//! - Dashboard `/api/datastore` returns a JSON snapshot matching the metrics
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::thread;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use swactor::config::RuntimeConfig;
|
||||||
|
use swactor::runtime::Runtime;
|
||||||
|
|
||||||
|
use swactor_datastore::actors::{BlobStoreActor, DatastoreNode, MetadataActor};
|
||||||
|
use swactor_datastore::api::start_api_server;
|
||||||
|
use swactor_datastore::metrics::DatastoreMetrics;
|
||||||
|
use swactor_datastore::storage::InMemoryBackend;
|
||||||
|
use swactor_datastore::DatastoreConfig;
|
||||||
|
|
||||||
|
use distribution::types::NodeId;
|
||||||
|
|
||||||
|
fn find_free_port() -> u16 {
|
||||||
|
std::net::TcpListener::bind("127.0.0.1:0")
|
||||||
|
.unwrap()
|
||||||
|
.local_addr()
|
||||||
|
.unwrap()
|
||||||
|
.port()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Scenario: datastore CRUD → dashboard reflects live operation metrics.
|
||||||
|
///
|
||||||
|
/// Story:
|
||||||
|
/// Two files are uploaded. One is fetched (metadata + data). The other
|
||||||
|
/// is deleted. Afterwards we check that the metrics snapshot, the
|
||||||
|
/// dashboard HTML page, and the dashboard JSON API all agree on what
|
||||||
|
/// happened.
|
||||||
|
#[test]
|
||||||
|
fn dashboard_reflects_datastore_operations() {
|
||||||
|
let api_port = find_free_port();
|
||||||
|
let dash_port = find_free_port();
|
||||||
|
let api_base = format!("http://127.0.0.1:{api_port}");
|
||||||
|
let dash_base = format!("http://127.0.0.1:{dash_port}");
|
||||||
|
|
||||||
|
// ── Infrastructure: runtime + actors + dashboard + API ──────────────
|
||||||
|
|
||||||
|
let dash = runtime_dashboard::start_dashboard(runtime_dashboard::DashboardConfig {
|
||||||
|
port: dash_port,
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
|
||||||
|
let collector = runtime_dashboard::collector::StatsCollector::new(2);
|
||||||
|
let mut rt = Runtime::new(RuntimeConfig {
|
||||||
|
num_threads: 2,
|
||||||
|
max_actors: 1024,
|
||||||
|
channel_buffer_size: 2000,
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
rt.set_stats_hook(collector.clone());
|
||||||
|
|
||||||
|
let node_id = NodeId([0xBB; 32]);
|
||||||
|
let config = DatastoreConfig {
|
||||||
|
chunk_size: 1_048_576,
|
||||||
|
gc_interval: 1000,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let backend: Box<dyn swactor_datastore::StorageBackend> = Box::new(InMemoryBackend::new());
|
||||||
|
let blob_store_addr = rt.spawn(BlobStoreActor::new(backend)).unwrap();
|
||||||
|
|
||||||
|
let mut metadata = MetadataActor::new(node_id, &config);
|
||||||
|
metadata.set_blob_store(blob_store_addr);
|
||||||
|
let metadata_addr = rt.spawn(metadata).unwrap();
|
||||||
|
|
||||||
|
let datastore_node = DatastoreNode::new(node_id, blob_store_addr, metadata_addr, config);
|
||||||
|
let datastore_addr = rt.spawn(datastore_node).unwrap();
|
||||||
|
|
||||||
|
let handle = rt.run().expect("failed to start runtime");
|
||||||
|
|
||||||
|
let metrics = Arc::new(DatastoreMetrics::new());
|
||||||
|
let node_hex: String = node_id.0.iter().map(|b| format!("{b:02x}")).collect();
|
||||||
|
metrics.set_node_id(node_hex);
|
||||||
|
|
||||||
|
dash.set_runtime(handle.runtime.clone(), collector);
|
||||||
|
dash.set_datastore(
|
||||||
|
Arc::clone(&metrics)
|
||||||
|
as Arc<dyn runtime_dashboard::datastore_collector::DatastoreStatsProvider>,
|
||||||
|
);
|
||||||
|
|
||||||
|
let (api_shutdown, _peers) = start_api_server(
|
||||||
|
handle.runtime.clone(),
|
||||||
|
datastore_addr,
|
||||||
|
metadata_addr,
|
||||||
|
blob_store_addr,
|
||||||
|
api_port,
|
||||||
|
Arc::clone(&metrics),
|
||||||
|
);
|
||||||
|
|
||||||
|
thread::sleep(Duration::from_millis(300));
|
||||||
|
|
||||||
|
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||||
|
run_dashboard_scenario(&api_base, &dash_base, &metrics);
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Cleanup
|
||||||
|
api_shutdown.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||||
|
dash.shutdown();
|
||||||
|
handle.shutdown();
|
||||||
|
handle.join();
|
||||||
|
|
||||||
|
if let Err(e) = result {
|
||||||
|
std::panic::resume_unwind(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run_dashboard_scenario(api: &str, dash: &str, metrics: &Arc<DatastoreMetrics>) {
|
||||||
|
// ── 1. Upload two objects ───────────────────────────────────────────
|
||||||
|
|
||||||
|
let put_a: serde_json::Value = ureq::post(&format!("{api}/api/put?name=alpha"))
|
||||||
|
.send_bytes(b"payload-alpha")
|
||||||
|
.expect("put A failed")
|
||||||
|
.into_json()
|
||||||
|
.unwrap();
|
||||||
|
let hash_a = put_a["content_hash"].as_str().unwrap().to_string();
|
||||||
|
|
||||||
|
let _put_b: serde_json::Value = ureq::post(&format!("{api}/api/put?name=bravo"))
|
||||||
|
.send_bytes(b"payload-bravo")
|
||||||
|
.expect("put B failed")
|
||||||
|
.into_json()
|
||||||
|
.unwrap();
|
||||||
|
let hash_b = _put_b["content_hash"].as_str().unwrap().to_string();
|
||||||
|
|
||||||
|
// ── 2. GET object A (metadata + raw data → 2 get ops) ──────────────
|
||||||
|
|
||||||
|
let _: serde_json::Value = ureq::get(&format!("{api}/api/get?hash={hash_a}"))
|
||||||
|
.call()
|
||||||
|
.unwrap()
|
||||||
|
.into_json()
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let data_resp = ureq::get(&format!("{api}/api/data?hash={hash_a}"))
|
||||||
|
.call()
|
||||||
|
.unwrap();
|
||||||
|
let mut body = Vec::new();
|
||||||
|
data_resp.into_reader().read_to_end(&mut body).unwrap();
|
||||||
|
assert_eq!(body, b"payload-alpha", "downloaded data should match");
|
||||||
|
|
||||||
|
// ── 3. DELETE object B ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
let _: serde_json::Value = ureq::post(&format!("{api}/api/delete?hash={hash_b}"))
|
||||||
|
.call()
|
||||||
|
.unwrap()
|
||||||
|
.into_json()
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// ── 4. Assert: in-process metrics snapshot ──────────────────────────
|
||||||
|
|
||||||
|
let snap = metrics.snapshot();
|
||||||
|
|
||||||
|
assert_eq!(snap.put_ops, 2, "two puts recorded");
|
||||||
|
assert_eq!(snap.get_ops, 2, "metadata-get + data-get recorded");
|
||||||
|
assert_eq!(snap.delete_ops, 1, "one delete recorded");
|
||||||
|
assert_eq!(snap.objects.len(), 1, "only alpha remains after deleting bravo");
|
||||||
|
assert_eq!(snap.objects[0].hash, hash_a);
|
||||||
|
assert!(
|
||||||
|
snap.recent_events.len() >= 5,
|
||||||
|
"at least 5 events (2 put + 2 get + 1 delete), got {}",
|
||||||
|
snap.recent_events.len()
|
||||||
|
);
|
||||||
|
|
||||||
|
// ── 5. Assert: dashboard /datastore serves HTML ─────────────────────
|
||||||
|
|
||||||
|
let page = ureq::get(&format!("{dash}/datastore")).call().unwrap();
|
||||||
|
assert_eq!(page.status(), 200);
|
||||||
|
assert!(
|
||||||
|
page.header("Content-Type")
|
||||||
|
.unwrap_or("")
|
||||||
|
.contains("text/html"),
|
||||||
|
);
|
||||||
|
let html = page.into_string().unwrap();
|
||||||
|
assert!(html.contains("Datastore"), "page should mention Datastore");
|
||||||
|
|
||||||
|
// ── 6. Assert: /api/datastore JSON matches metrics ──────────────────
|
||||||
|
|
||||||
|
let ds: serde_json::Value = ureq::get(&format!("{dash}/api/datastore"))
|
||||||
|
.call()
|
||||||
|
.unwrap()
|
||||||
|
.into_json()
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(ds["put_ops"].as_u64().unwrap(), 2);
|
||||||
|
assert_eq!(ds["get_ops"].as_u64().unwrap(), 2);
|
||||||
|
assert_eq!(ds["delete_ops"].as_u64().unwrap(), 1);
|
||||||
|
|
||||||
|
let objects = ds["objects"].as_array().expect("objects should be array");
|
||||||
|
assert_eq!(objects.len(), 1);
|
||||||
|
assert_eq!(objects[0]["hash"].as_str().unwrap(), hash_a);
|
||||||
|
|
||||||
|
let events = ds["recent_events"]
|
||||||
|
.as_array()
|
||||||
|
.expect("recent_events should be array");
|
||||||
|
assert!(events.len() >= 5);
|
||||||
|
}
|
||||||
|
|
@ -96,7 +96,6 @@ impl KBucket {
|
||||||
pub struct RoutingTable {
|
pub struct RoutingTable {
|
||||||
self_id: NodeId,
|
self_id: NodeId,
|
||||||
buckets: Vec<KBucket>,
|
buckets: Vec<KBucket>,
|
||||||
k: usize,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RoutingTable {
|
impl RoutingTable {
|
||||||
|
|
@ -109,7 +108,7 @@ impl RoutingTable {
|
||||||
for _ in 0..NUM_BUCKETS {
|
for _ in 0..NUM_BUCKETS {
|
||||||
buckets.push(KBucket::new(k));
|
buckets.push(KBucket::new(k));
|
||||||
}
|
}
|
||||||
Self { self_id, buckets, k }
|
Self { self_id, buckets }
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn self_id(&self) -> NodeId {
|
pub fn self_id(&self) -> NodeId {
|
||||||
|
|
|
||||||
|
|
@ -195,6 +195,7 @@ pub fn encode_wire_envelope(envelope: &WireEnvelope) -> Vec<u8> {
|
||||||
enum ReadError {
|
enum ReadError {
|
||||||
WouldBlock,
|
WouldBlock,
|
||||||
Disconnected,
|
Disconnected,
|
||||||
|
#[allow(dead_code)]
|
||||||
Other(std::io::Error),
|
Other(std::io::Error),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@
|
||||||
//! `TestCluster` makes sender-misattribution structurally impossible by
|
//! `TestCluster` makes sender-misattribution structurally impossible by
|
||||||
//! tagging every response with the responder's index, mirroring the
|
//! tagging every response with the responder's index, mirroring the
|
||||||
//! simulation crate's `deliver_actions_tagged_with_net`.
|
//! simulation crate's `deliver_actions_tagged_with_net`.
|
||||||
|
#![allow(dead_code)]
|
||||||
|
|
||||||
use distribution::node::{DistributedNode, DistributedNodeConfig};
|
use distribution::node::{DistributedNode, DistributedNodeConfig};
|
||||||
use distribution::registry::RegistryConfig;
|
use distribution::registry::RegistryConfig;
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ use distribution::crypto::Keypair;
|
||||||
use distribution::kademlia::directory::{
|
use distribution::kademlia::directory::{
|
||||||
actor_addr_as_node_id, resolve_quorum, DirectoryShard, QuorumResult,
|
actor_addr_as_node_id, resolve_quorum, DirectoryShard, QuorumResult,
|
||||||
};
|
};
|
||||||
use distribution::types::{DirectoryEntry, NodeId, Signature};
|
use distribution::types::{NodeId, Signature};
|
||||||
|
|
||||||
// ─── DirectoryShard ─────────────────────────────────────────────────────────
|
// ─── DirectoryShard ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -123,6 +123,8 @@ pub const ACTOR_DETAIL_HTML: &str = r##"<!DOCTYPE html>
|
||||||
<nav class="nav-links">
|
<nav class="nav-links">
|
||||||
<a href="/" class="nav-link">Overview</a>
|
<a href="/" class="nav-link">Overview</a>
|
||||||
<a href="/actors" class="nav-link">Actors</a>
|
<a href="/actors" class="nav-link">Actors</a>
|
||||||
|
<a href="/distribution" class="nav-link">Distribution</a>
|
||||||
|
<a href="/datastore" class="nav-link">Datastore</a>
|
||||||
</nav>
|
</nav>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
|
|
@ -152,6 +152,7 @@ pub const ACTORS_HTML: &str = r##"<!DOCTYPE html>
|
||||||
<a href="/" class="nav-link">Overview</a>
|
<a href="/" class="nav-link">Overview</a>
|
||||||
<a href="/actors" class="nav-link active">Actors</a>
|
<a href="/actors" class="nav-link active">Actors</a>
|
||||||
<a href="/distribution" class="nav-link">Distribution</a>
|
<a href="/distribution" class="nav-link">Distribution</a>
|
||||||
|
<a href="/datastore" class="nav-link">Datastore</a>
|
||||||
</nav>
|
</nav>
|
||||||
</div>
|
</div>
|
||||||
<div class="header-right">
|
<div class="header-right">
|
||||||
|
|
|
||||||
|
|
@ -153,6 +153,7 @@ pub const DASHBOARD_HTML: &str = r##"<!DOCTYPE html>
|
||||||
<a href="/" class="nav-link active">Overview</a>
|
<a href="/" class="nav-link active">Overview</a>
|
||||||
<a href="/actors" class="nav-link">Actors</a>
|
<a href="/actors" class="nav-link">Actors</a>
|
||||||
<a href="/distribution" class="nav-link">Distribution</a>
|
<a href="/distribution" class="nav-link">Distribution</a>
|
||||||
|
<a href="/datastore" class="nav-link">Datastore</a>
|
||||||
</nav>
|
</nav>
|
||||||
</div>
|
</div>
|
||||||
<div class="header-right">
|
<div class="header-right">
|
||||||
|
|
|
||||||
16
crates/runtime-dashboard/src/datastore_collector.rs
Normal file
16
crates/runtime-dashboard/src/datastore_collector.rs
Normal file
|
|
@ -0,0 +1,16 @@
|
||||||
|
//! Datastore stats provider for the runtime dashboard.
|
||||||
|
//!
|
||||||
|
//! The trait returns a pre-serialized JSON string so that `runtime-dashboard`
|
||||||
|
//! has no compile-time dependency on `swactor-datastore` (which would create a
|
||||||
|
//! circular dependency since `swactor-datastore[node]` depends on us).
|
||||||
|
//!
|
||||||
|
//! The `swactor-datastore` crate implements this trait in its `node` feature.
|
||||||
|
|
||||||
|
/// Trait for providing datastore stats to the dashboard.
|
||||||
|
///
|
||||||
|
/// Implementations capture a point-in-time snapshot as serialized JSON.
|
||||||
|
/// The dashboard polls this every ~200ms via SSE.
|
||||||
|
pub trait DatastoreStatsProvider: Send + Sync {
|
||||||
|
/// Return a JSON-serialized datastore snapshot, or `None` if unavailable.
|
||||||
|
fn snapshot_json(&self) -> Option<String>;
|
||||||
|
}
|
||||||
286
crates/runtime-dashboard/src/datastore_html.rs
Normal file
286
crates/runtime-dashboard/src/datastore_html.rs
Normal file
|
|
@ -0,0 +1,286 @@
|
||||||
|
pub const DATASTORE_HTML: &str = r##"<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
|
<title>Swactor Runtime – Datastore</title>
|
||||||
|
<style>
|
||||||
|
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||||
|
body { font-family: 'Menlo', 'Consolas', 'Monaco', monospace; background: #0f1117; color: #e0e0e0; font-size: 13px; }
|
||||||
|
|
||||||
|
.header {
|
||||||
|
display: flex; align-items: center; justify-content: space-between;
|
||||||
|
padding: 12px 20px; background: #161822; border-bottom: 1px solid #2a2d3e;
|
||||||
|
}
|
||||||
|
.header-left { display: flex; align-items: center; }
|
||||||
|
.header h1 { font-size: 16px; font-weight: 600; color: #fff; }
|
||||||
|
.status-dot {
|
||||||
|
width: 10px; height: 10px; border-radius: 50%; background: #4caf50;
|
||||||
|
display: inline-block; margin-left: 8px; vertical-align: middle;
|
||||||
|
}
|
||||||
|
.status-dot.disconnected { background: #f44336; }
|
||||||
|
.status-dot.done { background: #ff9800; }
|
||||||
|
|
||||||
|
.nav-links { display: flex; gap: 4px; margin-left: 20px; }
|
||||||
|
.nav-link {
|
||||||
|
color: #888; text-decoration: none; font-size: 12px;
|
||||||
|
padding: 4px 10px; border-radius: 3px; transition: color 0.2s;
|
||||||
|
}
|
||||||
|
.nav-link:hover { color: #e0e0e0; }
|
||||||
|
.nav-link.active { color: #fff; background: #2a2d3e; }
|
||||||
|
|
||||||
|
.header-right { display: flex; align-items: center; gap: 12px; }
|
||||||
|
|
||||||
|
.grid {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: 1fr 1fr;
|
||||||
|
gap: 12px; padding: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.panel {
|
||||||
|
background: #161822; border: 1px solid #2a2d3e; border-radius: 6px;
|
||||||
|
padding: 14px; overflow: hidden;
|
||||||
|
}
|
||||||
|
.panel h2 { font-size: 12px; color: #888; text-transform: uppercase; letter-spacing: 1px; margin-bottom: 10px; }
|
||||||
|
|
||||||
|
.full-width { grid-column: 1 / -1; }
|
||||||
|
|
||||||
|
.stats-cards {
|
||||||
|
display: grid; grid-template-columns: repeat(5, 1fr); gap: 10px;
|
||||||
|
}
|
||||||
|
.stat-card {
|
||||||
|
background: #1c1f2e; border-radius: 4px; padding: 10px; text-align: center;
|
||||||
|
}
|
||||||
|
.stat-card .value { font-size: 22px; font-weight: 700; color: #fff; }
|
||||||
|
.stat-card .label { font-size: 10px; color: #888; text-transform: uppercase; margin-top: 2px; }
|
||||||
|
|
||||||
|
.event-timeline {
|
||||||
|
max-height: 400px; overflow-y: auto;
|
||||||
|
}
|
||||||
|
.event-row {
|
||||||
|
display: flex; gap: 8px; padding: 4px 0; border-bottom: 1px solid #1c1f2e;
|
||||||
|
font-size: 11px; align-items: center;
|
||||||
|
}
|
||||||
|
.event-time { color: #555; min-width: 70px; }
|
||||||
|
.event-kind {
|
||||||
|
min-width: 48px; font-weight: 700; text-transform: uppercase; font-size: 10px;
|
||||||
|
padding: 1px 6px; border-radius: 3px; text-align: center;
|
||||||
|
}
|
||||||
|
.event-kind.put { background: #1b3a2a; color: #4caf50; }
|
||||||
|
.event-kind.get { background: #1a2a3e; color: #2196f3; }
|
||||||
|
.event-kind.delete { background: #3a1a1a; color: #f44336; }
|
||||||
|
.event-hash { color: #aaa; font-size: 10px; min-width: 120px; }
|
||||||
|
.event-name { color: #e0e0e0; flex: 1; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||||
|
.event-size { color: #888; min-width: 70px; text-align: right; }
|
||||||
|
|
||||||
|
.objects-table { max-height: 400px; overflow-y: auto; }
|
||||||
|
.objects-table table { width: 100%; border-collapse: collapse; }
|
||||||
|
.objects-table th, .objects-table td {
|
||||||
|
padding: 4px 8px; text-align: left; border-bottom: 1px solid #1c1f2e; font-size: 11px;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
.objects-table th { color: #888; font-weight: 500; position: sticky; top: 0; background: #161822; }
|
||||||
|
|
||||||
|
.transfers-section { display: none; }
|
||||||
|
.transfers-section.visible { display: block; }
|
||||||
|
.transfer-row {
|
||||||
|
display: flex; align-items: center; gap: 10px; margin-bottom: 8px;
|
||||||
|
}
|
||||||
|
.transfer-hash { color: #aaa; font-size: 10px; min-width: 100px; }
|
||||||
|
.transfer-bar {
|
||||||
|
flex: 1; height: 16px; background: #1c1f2e; border-radius: 3px; overflow: hidden;
|
||||||
|
}
|
||||||
|
.transfer-fill {
|
||||||
|
height: 100%; background: #6366f1; border-radius: 3px;
|
||||||
|
transition: width 0.3s ease;
|
||||||
|
}
|
||||||
|
.transfer-label { color: #888; font-size: 11px; min-width: 80px; text-align: right; }
|
||||||
|
|
||||||
|
::-webkit-scrollbar { width: 6px; }
|
||||||
|
::-webkit-scrollbar-track { background: #0f1117; }
|
||||||
|
::-webkit-scrollbar-thumb { background: #2a2d3e; border-radius: 3px; }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div class="header">
|
||||||
|
<div class="header-left">
|
||||||
|
<h1>
|
||||||
|
Swactor Runtime Dashboard
|
||||||
|
<span id="statusDot" class="status-dot disconnected"></span>
|
||||||
|
</h1>
|
||||||
|
<nav class="nav-links">
|
||||||
|
<a href="/" class="nav-link">Overview</a>
|
||||||
|
<a href="/actors" class="nav-link">Actors</a>
|
||||||
|
<a href="/distribution" class="nav-link">Distribution</a>
|
||||||
|
<a href="/datastore" class="nav-link active">Datastore</a>
|
||||||
|
</nav>
|
||||||
|
</div>
|
||||||
|
<div class="header-right">
|
||||||
|
<span id="nodeLabel" style="color:#888;font-size:12px;">Waiting for data...</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="grid">
|
||||||
|
<!-- Stat cards -->
|
||||||
|
<div class="panel full-width">
|
||||||
|
<h2>Datastore Stats</h2>
|
||||||
|
<div class="stats-cards">
|
||||||
|
<div class="stat-card"><div class="value" id="statObjects">0</div><div class="label">Objects</div></div>
|
||||||
|
<div class="stat-card"><div class="value" id="statSize">0</div><div class="label">Total Size</div></div>
|
||||||
|
<div class="stat-card"><div class="value" id="statPuts">0</div><div class="label">Puts</div></div>
|
||||||
|
<div class="stat-card"><div class="value" id="statGets">0</div><div class="label">Gets</div></div>
|
||||||
|
<div class="stat-card"><div class="value" id="statDeletes">0</div><div class="label">Deletes</div></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Event timeline -->
|
||||||
|
<div class="panel">
|
||||||
|
<h2>Event Timeline <span id="eventCount" style="color:#555;font-weight:400;"></span></h2>
|
||||||
|
<div class="event-timeline" id="eventTimeline"></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Objects table -->
|
||||||
|
<div class="panel">
|
||||||
|
<h2>Objects <span id="objectCount" style="color:#555;font-weight:400;"></span></h2>
|
||||||
|
<div class="objects-table">
|
||||||
|
<table>
|
||||||
|
<thead><tr><th>Hash</th><th>Name</th><th>Size</th></tr></thead>
|
||||||
|
<tbody id="objectsBody"></tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Transfers -->
|
||||||
|
<div class="panel full-width transfers-section" id="transfersPanel">
|
||||||
|
<h2>Active Transfers</h2>
|
||||||
|
<div id="transfersList"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
(function() {
|
||||||
|
var DASHBOARD_MODE = '__DASHBOARD_MODE__';
|
||||||
|
var dot = document.getElementById('statusDot');
|
||||||
|
|
||||||
|
function formatBytes(b) {
|
||||||
|
if (b === 0) return '0 B';
|
||||||
|
var units = ['B', 'KB', 'MB', 'GB', 'TB'];
|
||||||
|
var i = Math.floor(Math.log(b) / Math.log(1024));
|
||||||
|
if (i >= units.length) i = units.length - 1;
|
||||||
|
return (b / Math.pow(1024, i)).toFixed(i > 0 ? 1 : 0) + ' ' + units[i];
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatTime(ms) {
|
||||||
|
var d = new Date(ms);
|
||||||
|
return ('0' + d.getHours()).slice(-2) + ':' +
|
||||||
|
('0' + d.getMinutes()).slice(-2) + ':' +
|
||||||
|
('0' + d.getSeconds()).slice(-2);
|
||||||
|
}
|
||||||
|
|
||||||
|
function escapeHtml(s) {
|
||||||
|
if (!s) return '';
|
||||||
|
return s.replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>');
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateFromSnapshot(snap) {
|
||||||
|
// Node label
|
||||||
|
if (snap.node_id) {
|
||||||
|
document.getElementById('nodeLabel').textContent = 'Node: ' + snap.node_id.substring(0, 16) + '\u2026';
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stat cards
|
||||||
|
document.getElementById('statObjects').textContent = snap.object_count;
|
||||||
|
document.getElementById('statSize').textContent = formatBytes(snap.total_bytes);
|
||||||
|
document.getElementById('statPuts').textContent = snap.put_ops;
|
||||||
|
document.getElementById('statGets').textContent = snap.get_ops;
|
||||||
|
document.getElementById('statDeletes').textContent = snap.delete_ops;
|
||||||
|
|
||||||
|
// Event timeline
|
||||||
|
var timeline = document.getElementById('eventTimeline');
|
||||||
|
var wasAtBottom = timeline.scrollTop + timeline.clientHeight >= timeline.scrollHeight - 20;
|
||||||
|
timeline.innerHTML = '';
|
||||||
|
document.getElementById('eventCount').textContent = '(' + snap.recent_events.length + ')';
|
||||||
|
|
||||||
|
for (var i = snap.recent_events.length - 1; i >= 0; i--) {
|
||||||
|
var ev = snap.recent_events[i];
|
||||||
|
var row = document.createElement('div');
|
||||||
|
row.className = 'event-row';
|
||||||
|
row.innerHTML =
|
||||||
|
'<span class="event-time">' + formatTime(ev.timestamp_ms) + '</span>' +
|
||||||
|
'<span class="event-kind ' + ev.kind + '">' + ev.kind + '</span>' +
|
||||||
|
'<span class="event-hash">' + ev.hash.substring(0, 16) + '\u2026</span>' +
|
||||||
|
'<span class="event-name">' + escapeHtml(ev.name || '') + '</span>' +
|
||||||
|
'<span class="event-size">' + (ev.size_bytes > 0 ? formatBytes(ev.size_bytes) : '') + '</span>';
|
||||||
|
timeline.appendChild(row);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (wasAtBottom) {
|
||||||
|
timeline.scrollTop = timeline.scrollHeight;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Objects table
|
||||||
|
var tbody = document.getElementById('objectsBody');
|
||||||
|
tbody.innerHTML = '';
|
||||||
|
document.getElementById('objectCount').textContent = '(' + snap.objects.length + ')';
|
||||||
|
|
||||||
|
for (var i = 0; i < snap.objects.length; i++) {
|
||||||
|
var obj = snap.objects[i];
|
||||||
|
var tr = document.createElement('tr');
|
||||||
|
tr.innerHTML =
|
||||||
|
'<td style="color:#aaa;font-size:10px;">' + obj.hash.substring(0, 16) + '\u2026</td>' +
|
||||||
|
'<td>' + escapeHtml(obj.name || '\u2014') + '</td>' +
|
||||||
|
'<td style="color:#888;">' + formatBytes(obj.size_bytes) + '</td>';
|
||||||
|
tbody.appendChild(tr);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Transfers
|
||||||
|
var panel = document.getElementById('transfersPanel');
|
||||||
|
var list = document.getElementById('transfersList');
|
||||||
|
|
||||||
|
if (snap.active_transfers.length === 0) {
|
||||||
|
panel.className = 'panel full-width transfers-section';
|
||||||
|
} else {
|
||||||
|
panel.className = 'panel full-width transfers-section visible';
|
||||||
|
list.innerHTML = '';
|
||||||
|
|
||||||
|
for (var i = 0; i < snap.active_transfers.length; i++) {
|
||||||
|
var t = snap.active_transfers[i];
|
||||||
|
var pct = t.chunks_total > 0 ? Math.round((t.chunks_received / t.chunks_total) * 100) : 0;
|
||||||
|
var row = document.createElement('div');
|
||||||
|
row.className = 'transfer-row';
|
||||||
|
row.innerHTML =
|
||||||
|
'<span class="transfer-hash">' + t.hash.substring(0, 16) + '\u2026</span>' +
|
||||||
|
'<div class="transfer-bar"><div class="transfer-fill" style="width:' + pct + '%"></div></div>' +
|
||||||
|
'<span class="transfer-label">' + t.chunks_received + ' / ' + t.chunks_total + '</span>';
|
||||||
|
list.appendChild(row);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SSE connection
|
||||||
|
var es = new EventSource('/events');
|
||||||
|
|
||||||
|
es.addEventListener('datastore', function(e) {
|
||||||
|
try {
|
||||||
|
var snap = JSON.parse(e.data);
|
||||||
|
updateFromSnapshot(snap);
|
||||||
|
} catch(err) { console.error('datastore parse error', err); }
|
||||||
|
});
|
||||||
|
|
||||||
|
es.addEventListener('done', function() {
|
||||||
|
dot.className = 'status-dot done';
|
||||||
|
es.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
es.onerror = function() {
|
||||||
|
dot.className = 'status-dot disconnected';
|
||||||
|
};
|
||||||
|
|
||||||
|
es.onopen = function() {
|
||||||
|
dot.className = 'status-dot';
|
||||||
|
};
|
||||||
|
})();
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
"##;
|
||||||
|
|
@ -121,6 +121,7 @@ pub const DISTRIBUTION_HTML: &str = r##"<!DOCTYPE html>
|
||||||
<a href="/" class="nav-link">Overview</a>
|
<a href="/" class="nav-link">Overview</a>
|
||||||
<a href="/actors" class="nav-link">Actors</a>
|
<a href="/actors" class="nav-link">Actors</a>
|
||||||
<a href="/distribution" class="nav-link active">Distribution</a>
|
<a href="/distribution" class="nav-link active">Distribution</a>
|
||||||
|
<a href="/datastore" class="nav-link">Datastore</a>
|
||||||
</nav>
|
</nav>
|
||||||
</div>
|
</div>
|
||||||
<div class="header-right">
|
<div class="header-right">
|
||||||
|
|
|
||||||
|
|
@ -20,6 +20,9 @@ mod distribution_html;
|
||||||
#[cfg(feature = "distribution")]
|
#[cfg(feature = "distribution")]
|
||||||
pub mod distribution_collector;
|
pub mod distribution_collector;
|
||||||
|
|
||||||
|
mod datastore_html;
|
||||||
|
pub mod datastore_collector;
|
||||||
|
|
||||||
use std::io;
|
use std::io;
|
||||||
use std::sync::atomic::{AtomicBool, Ordering};
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
@ -91,6 +94,7 @@ pub struct DashboardHandle {
|
||||||
recording: bool,
|
recording: bool,
|
||||||
#[cfg(feature = "distribution")]
|
#[cfg(feature = "distribution")]
|
||||||
distribution: Arc<Mutex<Option<Arc<dyn distribution_collector::DistributionStatsProvider>>>>,
|
distribution: Arc<Mutex<Option<Arc<dyn distribution_collector::DistributionStatsProvider>>>>,
|
||||||
|
datastore: Arc<Mutex<Option<Arc<dyn datastore_collector::DatastoreStatsProvider>>>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl DashboardHandle {
|
impl DashboardHandle {
|
||||||
|
|
@ -123,6 +127,11 @@ impl DashboardHandle {
|
||||||
*self.distribution.lock().unwrap() = Some(provider);
|
*self.distribution.lock().unwrap() = Some(provider);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Attach a datastore stats provider, enabling the `/datastore` page.
|
||||||
|
pub fn set_datastore(&self, provider: Arc<dyn datastore_collector::DatastoreStatsProvider>) {
|
||||||
|
*self.datastore.lock().unwrap() = Some(provider);
|
||||||
|
}
|
||||||
|
|
||||||
/// Access the time-series history store (for TUI sparklines, etc.).
|
/// Access the time-series history store (for TUI sparklines, etc.).
|
||||||
pub fn history(&self) -> &Arc<DashboardHistory> {
|
pub fn history(&self) -> &Arc<DashboardHistory> {
|
||||||
&self.history
|
&self.history
|
||||||
|
|
@ -178,6 +187,9 @@ pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle {
|
||||||
let distribution: Arc<Mutex<Option<Arc<dyn distribution_collector::DistributionStatsProvider>>>> =
|
let distribution: Arc<Mutex<Option<Arc<dyn distribution_collector::DistributionStatsProvider>>>> =
|
||||||
Arc::new(Mutex::new(None));
|
Arc::new(Mutex::new(None));
|
||||||
|
|
||||||
|
let datastore: Arc<Mutex<Option<Arc<dyn datastore_collector::DatastoreStatsProvider>>>> =
|
||||||
|
Arc::new(Mutex::new(None));
|
||||||
|
|
||||||
server::spawn_http_server(
|
server::spawn_http_server(
|
||||||
Arc::clone(&store),
|
Arc::clone(&store),
|
||||||
Arc::clone(&runtime),
|
Arc::clone(&runtime),
|
||||||
|
|
@ -187,6 +199,7 @@ pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle {
|
||||||
config.port,
|
config.port,
|
||||||
#[cfg(feature = "distribution")]
|
#[cfg(feature = "distribution")]
|
||||||
Arc::clone(&distribution),
|
Arc::clone(&distribution),
|
||||||
|
Arc::clone(&datastore),
|
||||||
);
|
);
|
||||||
|
|
||||||
// Start stats recorder thread when recording is enabled
|
// Start stats recorder thread when recording is enabled
|
||||||
|
|
@ -229,6 +242,7 @@ pub fn start_dashboard(config: DashboardConfig) -> DashboardHandle {
|
||||||
recording: config.record,
|
recording: config.record,
|
||||||
#[cfg(feature = "distribution")]
|
#[cfg(feature = "distribution")]
|
||||||
distribution,
|
distribution,
|
||||||
|
datastore,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -24,6 +24,9 @@ use crate::distribution_collector::DistributionStatsProvider;
|
||||||
#[cfg(feature = "distribution")]
|
#[cfg(feature = "distribution")]
|
||||||
use crate::distribution_html::DISTRIBUTION_HTML;
|
use crate::distribution_html::DISTRIBUTION_HTML;
|
||||||
|
|
||||||
|
use crate::datastore_collector::DatastoreStatsProvider;
|
||||||
|
use crate::datastore_html::DATASTORE_HTML;
|
||||||
|
|
||||||
/// Format a server-sent event.
|
/// Format a server-sent event.
|
||||||
fn format_sse(event: &str, data: &str) -> Vec<u8> {
|
fn format_sse(event: &str, data: &str) -> Vec<u8> {
|
||||||
format!("event: {event}\ndata: {data}\n\n").into_bytes()
|
format!("event: {event}\ndata: {data}\n\n").into_bytes()
|
||||||
|
|
@ -128,6 +131,7 @@ pub(crate) fn spawn_http_server(
|
||||||
port: u16,
|
port: u16,
|
||||||
#[cfg(feature = "distribution")]
|
#[cfg(feature = "distribution")]
|
||||||
distribution: Arc<Mutex<Option<Arc<dyn DistributionStatsProvider>>>>,
|
distribution: Arc<Mutex<Option<Arc<dyn DistributionStatsProvider>>>>,
|
||||||
|
datastore: Arc<Mutex<Option<Arc<dyn DatastoreStatsProvider>>>>,
|
||||||
) {
|
) {
|
||||||
let addr = format!("0.0.0.0:{port}");
|
let addr = format!("0.0.0.0:{port}");
|
||||||
let server = tiny_http::Server::http(&addr).expect("failed to bind HTTP server");
|
let server = tiny_http::Server::http(&addr).expect("failed to bind HTTP server");
|
||||||
|
|
@ -144,6 +148,7 @@ pub(crate) fn spawn_http_server(
|
||||||
let cmd_router = Arc::clone(&cmd_router);
|
let cmd_router = Arc::clone(&cmd_router);
|
||||||
#[cfg(feature = "distribution")]
|
#[cfg(feature = "distribution")]
|
||||||
let distribution = Arc::clone(&distribution);
|
let distribution = Arc::clone(&distribution);
|
||||||
|
let datastore = Arc::clone(&datastore);
|
||||||
thread::spawn(move || {
|
thread::spawn(move || {
|
||||||
loop {
|
loop {
|
||||||
let request = match server.recv() {
|
let request = match server.recv() {
|
||||||
|
|
@ -159,6 +164,7 @@ pub(crate) fn spawn_http_server(
|
||||||
"/topology" => respond_html(request, TOPOLOGY_HTML, "live"),
|
"/topology" => respond_html(request, TOPOLOGY_HTML, "live"),
|
||||||
#[cfg(feature = "distribution")]
|
#[cfg(feature = "distribution")]
|
||||||
"/distribution" => respond_html(request, DISTRIBUTION_HTML, "live"),
|
"/distribution" => respond_html(request, DISTRIBUTION_HTML, "live"),
|
||||||
|
"/datastore" => respond_html(request, DATASTORE_HTML, "live"),
|
||||||
"/events" => {
|
"/events" => {
|
||||||
handle_live_sse(
|
handle_live_sse(
|
||||||
request,
|
request,
|
||||||
|
|
@ -169,6 +175,7 @@ pub(crate) fn spawn_http_server(
|
||||||
Arc::clone(&history),
|
Arc::clone(&history),
|
||||||
#[cfg(feature = "distribution")]
|
#[cfg(feature = "distribution")]
|
||||||
Arc::clone(&distribution),
|
Arc::clone(&distribution),
|
||||||
|
Arc::clone(&datastore),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
"/api/stats" => {
|
"/api/stats" => {
|
||||||
|
|
@ -204,6 +211,12 @@ pub(crate) fn spawn_http_server(
|
||||||
Arc::clone(&distribution),
|
Arc::clone(&distribution),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
"/api/datastore" => {
|
||||||
|
handle_datastore_api(
|
||||||
|
request,
|
||||||
|
Arc::clone(&datastore),
|
||||||
|
);
|
||||||
|
}
|
||||||
"/api/logs" => {
|
"/api/logs" => {
|
||||||
handle_logs_api(request, &url, Arc::clone(&store));
|
handle_logs_api(request, &url, Arc::clone(&store));
|
||||||
}
|
}
|
||||||
|
|
@ -239,6 +252,7 @@ fn handle_live_sse(
|
||||||
history: Arc<DashboardHistory>,
|
history: Arc<DashboardHistory>,
|
||||||
#[cfg(feature = "distribution")]
|
#[cfg(feature = "distribution")]
|
||||||
distribution: Arc<Mutex<Option<Arc<dyn DistributionStatsProvider>>>>,
|
distribution: Arc<Mutex<Option<Arc<dyn DistributionStatsProvider>>>>,
|
||||||
|
datastore: Arc<Mutex<Option<Arc<dyn DatastoreStatsProvider>>>>,
|
||||||
) {
|
) {
|
||||||
let (tx, rx) = mpsc::channel::<Vec<u8>>();
|
let (tx, rx) = mpsc::channel::<Vec<u8>>();
|
||||||
let response = make_sse_response(rx);
|
let response = make_sse_response(rx);
|
||||||
|
|
@ -309,6 +323,18 @@ fn handle_live_sse(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Send datastore snapshot if provider is attached
|
||||||
|
{
|
||||||
|
let maybe_ds = datastore.lock().unwrap().clone();
|
||||||
|
if let Some(provider) = maybe_ds {
|
||||||
|
if let Some(json) = provider.snapshot_json() {
|
||||||
|
if tx.send(format_sse("datastore", &json)).is_err() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Send new activity events
|
// Send new activity events
|
||||||
let (batch, new_cursor) = store.read_from(cursor);
|
let (batch, new_cursor) = store.read_from(cursor);
|
||||||
if !batch.is_empty() {
|
if !batch.is_empty() {
|
||||||
|
|
@ -423,6 +449,26 @@ fn handle_distribution_api(
|
||||||
let _ = request.respond(response);
|
let _ = request.respond(response);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn handle_datastore_api(
|
||||||
|
request: tiny_http::Request,
|
||||||
|
datastore: Arc<Mutex<Option<Arc<dyn DatastoreStatsProvider>>>>,
|
||||||
|
) {
|
||||||
|
let json = match datastore.lock().unwrap().as_ref() {
|
||||||
|
Some(provider) => provider.snapshot_json().unwrap_or_else(|| "{}".into()),
|
||||||
|
None => serde_json::json!({
|
||||||
|
"error": "datastore provider not attached"
|
||||||
|
})
|
||||||
|
.to_string(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let response = tiny_http::Response::from_string(json).with_header(
|
||||||
|
"Content-Type: application/json"
|
||||||
|
.parse::<tiny_http::Header>()
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
let _ = request.respond(response);
|
||||||
|
}
|
||||||
|
|
||||||
fn handle_topology_api(
|
fn handle_topology_api(
|
||||||
request: tiny_http::Request,
|
request: tiny_http::Request,
|
||||||
runtime: Arc<Mutex<Option<Arc<Runtime>>>>,
|
runtime: Arc<Mutex<Option<Arc<Runtime>>>>,
|
||||||
|
|
|
||||||
|
|
@ -47,6 +47,7 @@ pub const TOPOLOGY_HTML: &str = r##"<!DOCTYPE html>
|
||||||
<a href="/actors" class="nav-link">Actors</a>
|
<a href="/actors" class="nav-link">Actors</a>
|
||||||
<a href="/topology" class="nav-link active">Topology</a>
|
<a href="/topology" class="nav-link active">Topology</a>
|
||||||
<a href="/distribution" class="nav-link">Distribution</a>
|
<a href="/distribution" class="nav-link">Distribution</a>
|
||||||
|
<a href="/datastore" class="nav-link">Datastore</a>
|
||||||
</nav>
|
</nav>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@ use std::sync::Arc;
|
||||||
use swactor::actor::ActorInterface;
|
use swactor::actor::ActorInterface;
|
||||||
use swactor::runtime::{Ctx, Runtime, RuntimeConfig};
|
use swactor::runtime::{Ctx, Runtime, RuntimeConfig};
|
||||||
use runtime_dashboard::command::{
|
use runtime_dashboard::command::{
|
||||||
from_query_params, parse_line, CommandContext, CommandRequest, CommandResponse, CommandRouter,
|
from_query_params, parse_line, CommandContext, CommandResponse, CommandRouter,
|
||||||
};
|
};
|
||||||
|
|
||||||
// ── Test Helpers ─────────────────────────────────────────────────────────────
|
// ── Test Helpers ─────────────────────────────────────────────────────────────
|
||||||
|
|
|
||||||
278
docs/datastore/actors.md
Normal file
278
docs/datastore/actors.md
Normal file
|
|
@ -0,0 +1,278 @@
|
||||||
|
# Datastore Actor Reference
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
The datastore is built from four actors within the swactor runtime. `DatastoreNode` is the public facade — all external requests (HTTP API, network protocol) enter through it and are routed to two long-lived worker actors: `BlobStoreActor` (content-addressed chunk/manifest I/O) and `MetadataActor` (object index, DHT replication, GC). A fourth actor, `TransferActor`, is spawned ephemerally for each remote download and self-terminates on completion or failure.
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────────────────────┐
|
||||||
|
│ store_node (main) │
|
||||||
|
│ spawns all 3 long-lived │
|
||||||
|
│ actors, drives ticks │
|
||||||
|
└────┬──────┬──────┬───────┘
|
||||||
|
│ │ │
|
||||||
|
spawn │ │ │ spawn
|
||||||
|
┌──────────────┘ │ └──────────────┐
|
||||||
|
▼ │ spawn ▼
|
||||||
|
┌───────────────────┐ │ ┌───────────────────┐
|
||||||
|
│ BlobStoreActor │ │ │ MetadataActor │
|
||||||
|
│ (chunks, manifests)│ │ │ (index, DHT, GC) │
|
||||||
|
└─────────▲─────────┘ │ └──▲────────┬───────┘
|
||||||
|
│ ▼ │ │
|
||||||
|
│ ┌───────────────────┐ │ │
|
||||||
|
│ │ DatastoreNode │────┘ │
|
||||||
|
│ │ (facade/router) │─────────────┘
|
||||||
|
└────────────│ │
|
||||||
|
└────────┬──────────┘
|
||||||
|
│ spawns (per download)
|
||||||
|
▼
|
||||||
|
┌───────────────────┐
|
||||||
|
│ TransferActor │
|
||||||
|
│ (ephemeral) │
|
||||||
|
└───────────────────┘
|
||||||
|
|
||||||
|
Arrows: ──▶ sends messages to
|
||||||
|
```
|
||||||
|
|
||||||
|
## Actors
|
||||||
|
|
||||||
|
### DatastoreNode
|
||||||
|
|
||||||
|
| | |
|
||||||
|
|---|---|
|
||||||
|
| **Role** | Top-level coordinator/facade. Accepts user-facing commands and incoming network protocol messages, delegates all work to `BlobStoreActor` and `MetadataActor`. |
|
||||||
|
| **Source** | `crates/datastore/src/actors/datastore_node.rs` |
|
||||||
|
| **Spawned by** | `store_node` binary (`crates/datastore/src/bin/store_node.rs:188`) |
|
||||||
|
| **Lifecycle** | Long-lived — runs for the lifetime of the process |
|
||||||
|
|
||||||
|
**Inbound messages** (`DatastoreNodeMsg` — 11 variants):
|
||||||
|
|
||||||
|
User-facing commands:
|
||||||
|
- `Put { data, name, tags, reply_to }` — chunk a blob, write chunks + manifest to `BlobStoreActor`, register in `MetadataActor`
|
||||||
|
- `Get { content_hash, reply_to }` — retrieve object metadata + manifest via `MetadataActor`
|
||||||
|
- `Delete { content_hash, reply_to }` — remove object via `MetadataActor`
|
||||||
|
- `List { name_filter, all, reply_to }` — list objects (local or swarm-wide) via `MetadataActor`
|
||||||
|
- `Status { reply_to }` — return this node's `NodeId`
|
||||||
|
- `ReadChunk { hash, reply_to }` — read a single chunk via `BlobStoreActor`
|
||||||
|
|
||||||
|
Protocol routing (incoming network messages):
|
||||||
|
- `IncomingGetChunk` — forwards to `BlobStoreActor::ReadChunk`
|
||||||
|
- `IncomingGetManifest` — forwards to `BlobStoreActor::ReadManifest`
|
||||||
|
- `IncomingStoreObject` — forwards to `MetadataActor::HandleStoreObject`
|
||||||
|
- `IncomingFindObject` — forwards to `MetadataActor::HandleFindObject`
|
||||||
|
- `IncomingListObjects` — forwards to `MetadataActor::ListLocal`
|
||||||
|
|
||||||
|
**Key outbound messages:**
|
||||||
|
- `BlobStoreMsg::WriteChunk`, `WriteManifest`, `ReadChunk`, `ReadManifest` — to `BlobStoreActor`
|
||||||
|
- `MetadataMsg::PutObject`, `GetObject`, `DeleteObject`, `ListLocal`, `ListSwarm`, `HandleStoreObject`, `HandleFindObject` — to `MetadataActor`
|
||||||
|
- `DatastoreResponse::NodeStatus` — directly to caller for `Status`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### BlobStoreActor
|
||||||
|
|
||||||
|
| | |
|
||||||
|
|---|---|
|
||||||
|
| **Role** | Content-addressed storage for chunks and manifests. All I/O goes through a pluggable `StorageBackend` (filesystem or in-memory). |
|
||||||
|
| **Source** | `crates/datastore/src/actors/blob_store.rs` |
|
||||||
|
| **Spawned by** | `store_node` binary (`store_node.rs:178`) |
|
||||||
|
| **Lifecycle** | Long-lived — runs for the lifetime of the process |
|
||||||
|
|
||||||
|
**Inbound messages** (`BlobStoreMsg` — 8 variants):
|
||||||
|
|
||||||
|
Chunk operations:
|
||||||
|
- `WriteChunk { hash, data, reply_to }` — persist a chunk, reply `ChunkStored`
|
||||||
|
- `ReadChunk { hash, reply_to }` — read a chunk, reply `ChunkOk` or `NotFound`
|
||||||
|
- `DeleteChunk { hash }` — remove a chunk (fire-and-forget)
|
||||||
|
- `HasChunk { hash, reply_to }` — existence check, reply `Bool`
|
||||||
|
- `ListChunks { reply_to }` — list all chunk hashes, reply `ChunkList`
|
||||||
|
- `GcUnreferenced { referenced }` — delete chunks not in the referenced set (fire-and-forget)
|
||||||
|
|
||||||
|
Manifest operations:
|
||||||
|
- `WriteManifest { manifest, reply_to }` — persist a manifest, reply `ManifestStored`
|
||||||
|
- `ReadManifest { hash, reply_to }` — read a manifest, reply `ManifestOk` or `NotFound`
|
||||||
|
|
||||||
|
**Key outbound messages:**
|
||||||
|
- `DatastoreResponse` variants (`ChunkStored`, `ChunkOk`, `ManifestStored`, `ManifestOk`, `NotFound`, `Error`, `Bool`, `ChunkList`) — always back to the `reply_to` address
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### MetadataActor
|
||||||
|
|
||||||
|
| | |
|
||||||
|
|---|---|
|
||||||
|
| **Role** | Object metadata index. Maintains a `HashMap<ContentHash, ObjectEntry>` and a manifest cache. Handles DHT-style find/store operations, epidemic dissemination of entries to peers, and periodic garbage collection. |
|
||||||
|
| **Source** | `crates/datastore/src/actors/metadata.rs` |
|
||||||
|
| **Spawned by** | `store_node` binary (`store_node.rs:185`) |
|
||||||
|
| **Lifecycle** | Long-lived — runs for the lifetime of the process |
|
||||||
|
|
||||||
|
**Inbound messages** (`MetadataMsg` — 11 variants):
|
||||||
|
|
||||||
|
Object operations:
|
||||||
|
- `PutObject { entry, manifest, reply_to }` — store metadata + manifest locally, enqueue for dissemination, reply `PutOk`
|
||||||
|
- `GetObject { content_hash, reply_to }` — local lookup, reply `GetOk` or `NotFound`
|
||||||
|
- `DeleteObject { content_hash, reply_to }` — remove from local index, reply `DeleteOk` or `NotFound`
|
||||||
|
- `ListLocal { name_filter, reply_to }` — list local entries with optional name filter, reply `ListOk`
|
||||||
|
- `ListSwarm { name_filter, reply_to }` — swarm-wide list (currently delegates to `ListLocal`)
|
||||||
|
|
||||||
|
DHT protocol:
|
||||||
|
- `HandleFindObject { from, content_hash, reply_to }` — answer an incoming FIND_VALUE from a peer
|
||||||
|
- `HandleStoreObject { entry, manifest }` — accept an incoming STORE from a peer (fire-and-forget)
|
||||||
|
|
||||||
|
Peer management:
|
||||||
|
- `SetPeers { peers }` — update the list of peer `MetadataActor` addresses for dissemination
|
||||||
|
|
||||||
|
Periodic ticks (driven by the `store_node` main loop):
|
||||||
|
- `DisseminateTick` — send pending entries to all known peers
|
||||||
|
- `GcTick` — collect referenced chunks from all manifests, send `BlobStoreMsg::GcUnreferenced` to `BlobStoreActor`
|
||||||
|
|
||||||
|
**Key outbound messages:**
|
||||||
|
- `DatastoreResponse` variants (`PutOk`, `GetOk`, `DeleteOk`, `ListOk`, `NotFound`, `Error`) — to caller
|
||||||
|
- `MetadataMsg::HandleStoreObject` — to peer `MetadataActor` addresses during dissemination
|
||||||
|
- `BlobStoreMsg::GcUnreferenced` — to local `BlobStoreActor` during GC
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### TransferActor
|
||||||
|
|
||||||
|
| | |
|
||||||
|
|---|---|
|
||||||
|
| **Role** | Manages a single object download from a remote node. Tracks pending/received chunks, forwards received data to the local `BlobStoreActor`, and reports completion or failure to the original requester. |
|
||||||
|
| **Source** | `crates/datastore/src/actors/transfer.rs` |
|
||||||
|
| **Spawned by** | `DatastoreNode` (one per remote download) |
|
||||||
|
| **Lifecycle** | Ephemeral — self-terminates via `ctx.stop_self()` on completion, failure, or cancel |
|
||||||
|
|
||||||
|
**Inbound messages** (`TransferMsg` — 4 variants):
|
||||||
|
|
||||||
|
- `StartDownload { manifest, source_node, reply_to }` — initialize the download with a manifest and source
|
||||||
|
- `ChunkReceived { hash, data }` — a chunk arrived from the remote node
|
||||||
|
- `ChunkFailed { hash, reason }` — a chunk fetch failed (retries up to `max_retries`, then fails the whole transfer)
|
||||||
|
- `Cancel` — abort the transfer immediately
|
||||||
|
|
||||||
|
**Key outbound messages:**
|
||||||
|
- `BlobStoreMsg::WriteChunk` — to local `BlobStoreActor` for each received chunk
|
||||||
|
- `DatastoreResponse::TransferComplete` — to `reply_to` when all chunks received
|
||||||
|
- `DatastoreResponse::TransferFailed` — to `reply_to` when retries are exhausted
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Message Reference
|
||||||
|
|
||||||
|
All message types are defined in `crates/datastore/src/messages.rs`.
|
||||||
|
|
||||||
|
### Intra-node actor messages
|
||||||
|
|
||||||
|
| Enum | Variants | Handled by |
|
||||||
|
|------|----------|------------|
|
||||||
|
| `DatastoreNodeMsg` | 11 (6 user-facing + 5 protocol routing) | `DatastoreNode` |
|
||||||
|
| `BlobStoreMsg` | 8 (5 chunk ops + 1 GC + 2 manifest ops) | `BlobStoreActor` |
|
||||||
|
| `MetadataMsg` | 11 (5 object ops + 2 DHT + 1 peer mgmt + 2 ticks) | `MetadataActor` |
|
||||||
|
| `TransferMsg` | 4 (start + chunk received + chunk failed + cancel) | `TransferActor` |
|
||||||
|
|
||||||
|
### Shared response enum
|
||||||
|
|
||||||
|
`DatastoreResponse` — 15 variants used as the return type for all four actors:
|
||||||
|
|
||||||
|
| Variant | Meaning |
|
||||||
|
|---------|---------|
|
||||||
|
| `PutOk { content_hash }` | Object stored successfully |
|
||||||
|
| `GetOk { entry, manifest }` | Object found |
|
||||||
|
| `DeleteOk { content_hash }` | Object deleted |
|
||||||
|
| `ListOk { entries }` | List result |
|
||||||
|
| `ChunkOk { hash, data }` | Chunk data retrieved |
|
||||||
|
| `ChunkStored { hash }` | Chunk written to storage |
|
||||||
|
| `ManifestStored { hash }` | Manifest written to storage |
|
||||||
|
| `ManifestOk { manifest }` | Manifest retrieved |
|
||||||
|
| `TransferComplete { content_hash }` | All chunks downloaded |
|
||||||
|
| `TransferFailed { reason }` | Transfer failed |
|
||||||
|
| `NodeStatus { node_id }` | Node identity |
|
||||||
|
| `NotFound` | Resource not found |
|
||||||
|
| `Error { reason }` | Generic error |
|
||||||
|
| `Bool(bool)` | Boolean result (e.g. `HasChunk`) |
|
||||||
|
| `ChunkList { hashes }` | List of chunk hashes |
|
||||||
|
|
||||||
|
### Inter-node wire messages (NetworkMessage)
|
||||||
|
|
||||||
|
| Struct | Direction | Purpose |
|
||||||
|
|--------|-----------|---------|
|
||||||
|
| `GetChunkRequest` | requester → holder | Fetch a chunk by hash |
|
||||||
|
| `GetChunkResponse` | holder → requester | Return chunk data (or `None`) |
|
||||||
|
| `StoreObjectRequest` | origin → DHT peer | Kademlia STORE for object metadata |
|
||||||
|
| `FindObjectRequest` | requester → DHT peer | Kademlia FIND_VALUE for object metadata |
|
||||||
|
| `FindObjectResponse` | DHT peer → requester | Return `Found(entry)` or `Closer(nodes)` |
|
||||||
|
| `GetManifestRequest` | requester → holder | Fetch a manifest by content hash |
|
||||||
|
| `GetManifestResponse` | holder → requester | Return manifest (or `None`) |
|
||||||
|
| `ListObjectsRequest` | requester → peer | List objects with optional name filter |
|
||||||
|
| `ListObjectsResponse` | peer → requester | Return matching entries |
|
||||||
|
|
||||||
|
Wire messages are distinguished from intra-node messages by implementing the `NetworkMessage` trait with a stable `type_tag()` string. They are serialized with serde for transport over iroh/QUIC.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Key Flows
|
||||||
|
|
||||||
|
### Put (store a blob)
|
||||||
|
|
||||||
|
```
|
||||||
|
Client → DatastoreNode::Put
|
||||||
|
→ chunk_blob() splits data into chunks
|
||||||
|
→ BlobStoreActor::WriteChunk (for each chunk, fire-and-forget)
|
||||||
|
→ BlobStoreActor::WriteManifest
|
||||||
|
→ MetadataActor::PutObject
|
||||||
|
→ stores entry + manifest locally
|
||||||
|
→ enqueues for dissemination
|
||||||
|
→ replies DatastoreResponse::PutOk
|
||||||
|
```
|
||||||
|
|
||||||
|
### Get (retrieve metadata)
|
||||||
|
|
||||||
|
```
|
||||||
|
Client → DatastoreNode::Get
|
||||||
|
→ MetadataActor::GetObject
|
||||||
|
→ local index lookup
|
||||||
|
→ replies DatastoreResponse::GetOk (or NotFound)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Data (reassemble from chunks)
|
||||||
|
|
||||||
|
See [streaming.md](streaming.md) for the full transfer protocol. In summary:
|
||||||
|
|
||||||
|
```
|
||||||
|
API server → DatastoreNode::Get → MetadataActor (local miss)
|
||||||
|
→ iterate peers:
|
||||||
|
→ FindObjectRequest (wire) → peer MetadataActor
|
||||||
|
→ GetManifestRequest (wire) → peer BlobStoreActor
|
||||||
|
→ GetChunkRequest (wire) → peer BlobStoreActor (per chunk)
|
||||||
|
→ BlobStoreActor::WriteChunk (store locally)
|
||||||
|
→ BlobStoreActor::WriteManifest
|
||||||
|
→ MetadataActor::PutObject
|
||||||
|
→ reassemble_blob() → verify blake3 → respond
|
||||||
|
```
|
||||||
|
|
||||||
|
### Dissemination (epidemic replication)
|
||||||
|
|
||||||
|
```
|
||||||
|
store_node main loop (every disseminate_interval ticks)
|
||||||
|
→ MetadataActor::DisseminateTick
|
||||||
|
→ take_pending() selects entries with remaining budget
|
||||||
|
→ for each peer: MetadataActor::HandleStoreObject
|
||||||
|
→ peer inserts if absent, re-enqueues for further dissemination
|
||||||
|
```
|
||||||
|
|
||||||
|
Budget per entry = `Λ * ceil(log2(cluster_size))` (SWIM-style, Λ=3).
|
||||||
|
|
||||||
|
### GC (garbage collection)
|
||||||
|
|
||||||
|
```
|
||||||
|
store_node main loop (every gc_interval ticks)
|
||||||
|
→ MetadataActor::GcTick
|
||||||
|
→ scans all manifests → builds referenced chunk set
|
||||||
|
→ BlobStoreActor::GcUnreferenced { referenced }
|
||||||
|
→ deletes any chunk not in the referenced set
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Related
|
||||||
|
|
||||||
|
- [streaming.md](streaming.md) — chunking, transfer protocol, reassembly, and progress tracking
|
||||||
103
docs/datastore/streaming.md
Normal file
103
docs/datastore/streaming.md
Normal file
|
|
@ -0,0 +1,103 @@
|
||||||
|
# Datastore Streaming Architecture
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
"Streaming" in the swactor datastore refers to **progressive chunk-based transfer**, not byte-level streaming. When an object is stored, it is split into fixed-size chunks, each content-addressed with blake3. When retrieved from a remote peer, chunks are fetched individually and reassembled — enabling progress tracking and partial recovery.
|
||||||
|
|
||||||
|
This design trades a small amount of per-chunk overhead for:
|
||||||
|
- **Progress visibility**: the dashboard shows `chunks_received / chunks_total` in real time
|
||||||
|
- **Resumability**: a failed transfer can (in principle) restart from the last chunk
|
||||||
|
- **Deduplication**: identical chunks across objects are stored once
|
||||||
|
|
||||||
|
## Content-Addressed Chunking
|
||||||
|
|
||||||
|
The `chunk_blob()` function (`chunking.rs`) splits raw bytes into fixed-size pieces:
|
||||||
|
|
||||||
|
1. Compute `ContentHash = blake3(entire_blob)` — this is the object's identity
|
||||||
|
2. Split the blob into `ceil(total_size / chunk_size)` pieces (default chunk size: 1 MB)
|
||||||
|
3. For each piece, compute `chunk_hash = blake3(piece_bytes)`
|
||||||
|
4. Build a `ChunkRef { hash, offset, size }` for each piece
|
||||||
|
5. Return an `ObjectManifest` containing the full list of `ChunkRef`s
|
||||||
|
|
||||||
|
```
|
||||||
|
Blob (5.2 MB, chunk_size=1MB)
|
||||||
|
├── Chunk 0: hash=abc1…, offset=0, size=1048576
|
||||||
|
├── Chunk 1: hash=def2…, offset=1048576, size=1048576
|
||||||
|
├── Chunk 2: hash=789a…, offset=2097152, size=1048576
|
||||||
|
├── Chunk 3: hash=bcd3…, offset=3145728, size=1048576
|
||||||
|
└── Chunk 4: hash=ef45…, offset=4194304, size=1048576 (last: 209920 bytes)
|
||||||
|
```
|
||||||
|
|
||||||
|
The object's identity (`ContentHash`) is the hash of the *entire* blob, not of the manifest. This means the same data always produces the same hash regardless of chunk size.
|
||||||
|
|
||||||
|
## Transfer Protocol
|
||||||
|
|
||||||
|
When a client requests an object via `GET /api/data?hash=...`, the API server:
|
||||||
|
|
||||||
|
1. Sends a `DatastoreNodeMsg::Get` to the local `DatastoreNode` actor
|
||||||
|
2. If found locally, reads all chunks from the local `BlobStore` and reassembles
|
||||||
|
3. If **not found locally**, enters `try_remote_get()`:
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|
### Remote GET step-by-step
|
||||||
|
|
||||||
|
1. **Iterate peers**: for each known peer node:
|
||||||
|
2. **FindObject**: send `MetadataMsg::HandleFindObject` to the peer's `MetadataActor`
|
||||||
|
3. **Read manifest**: send `BlobStoreMsg::ReadManifest` to the peer's `BlobStore`
|
||||||
|
4. **Fetch chunks**: for each `ChunkRef` in the manifest:
|
||||||
|
- Send `BlobStoreMsg::ReadChunk` to the peer
|
||||||
|
- Receive `DatastoreResponse::ChunkOk { hash, data }`
|
||||||
|
- Store locally via `BlobStoreMsg::WriteChunk`
|
||||||
|
- Update `DatastoreMetrics::advance_transfer()` for dashboard progress
|
||||||
|
5. **Store manifest locally**: `BlobStoreMsg::WriteManifest`
|
||||||
|
6. **Store metadata locally**: `MetadataMsg::PutObject`
|
||||||
|
7. **Reassemble and respond**: `reassemble_blob()` concatenates chunks and verifies integrity
|
||||||
|
|
||||||
|
If a peer doesn't have the object (or any step fails), the loop continues to the next peer.
|
||||||
|
|
||||||
|
## Reassembly
|
||||||
|
|
||||||
|
`reassemble_blob()` (`chunking.rs`) takes a manifest and a set of `(hash, data)` pairs:
|
||||||
|
|
||||||
|
1. For each `ChunkRef` in manifest order, find the matching `(hash, data)` pair
|
||||||
|
2. Concatenate all chunk data into a single buffer
|
||||||
|
3. Compute `blake3(result)` and verify it matches `manifest.content_hash`
|
||||||
|
4. Return the reassembled blob (or a `ChunkingError` on mismatch)
|
||||||
|
|
||||||
|
This integrity check ensures that even if individual chunks are corrupted or swapped, the final result is always verified against the original content hash.
|
||||||
|
|
||||||
|
## Progress Tracking
|
||||||
|
|
||||||
|
The `DatastoreMetrics` struct provides thread-safe transfer tracking:
|
||||||
|
|
||||||
|
```
|
||||||
|
begin_transfer(hash, chunks_total) // called when remote GET starts
|
||||||
|
advance_transfer(hash) // called after each chunk is stored locally
|
||||||
|
end_transfer(hash) // called on completion or failure
|
||||||
|
```
|
||||||
|
|
||||||
|
The dashboard SSE stream includes a `datastore` event every ~200ms with a `DatastoreSnapshot` containing `active_transfers: Vec<TransferProgress>`. The web UI renders these as animated progress bars.
|
||||||
|
|
||||||
|
```
|
||||||
|
TransferProgress {
|
||||||
|
hash: "abc123...",
|
||||||
|
chunks_received: 3,
|
||||||
|
chunks_total: 5,
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## GC Integration
|
||||||
|
|
||||||
|
When an object is deleted, its `ObjectEntry` and `ObjectManifest` are removed from the `MetadataActor`. However, the underlying chunks are **not immediately deleted** — they may be referenced by other manifests (deduplication).
|
||||||
|
|
||||||
|
Instead, garbage collection runs periodically:
|
||||||
|
|
||||||
|
1. `MetadataMsg::GcTick` triggers a scan
|
||||||
|
2. The `MetadataActor` collects all chunk hashes referenced by any live manifest
|
||||||
|
3. Sends `BlobStoreMsg::GcUnreferenced` with the referenced set
|
||||||
|
4. The `BlobStore` deletes any chunks **not** in the referenced set
|
||||||
|
|
||||||
|
This two-phase approach prevents data loss when chunks are shared between objects.
|
||||||
|
|
||||||
|

|
||||||
170
docs/diagrams/datastore_chunk_lifecycle.svg
Normal file
170
docs/diagrams/datastore_chunk_lifecycle.svg
Normal file
|
|
@ -0,0 +1,170 @@
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1000 600" font-family="system-ui, sans-serif">
|
||||||
|
<defs>
|
||||||
|
<filter id="shadow" x="-4%" y="-4%" width="108%" height="108%">
|
||||||
|
<feDropShadow dx="1" dy="2" stdDeviation="3" flood-color="#000" flood-opacity="0.10"/>
|
||||||
|
</filter>
|
||||||
|
<marker id="arrow-gray" viewBox="0 0 10 7" refX="10" refY="3.5"
|
||||||
|
markerWidth="10" markerHeight="7" orient="auto-start-reverse" fill="#5f6368">
|
||||||
|
<polygon points="0 0, 10 3.5, 0 7"/>
|
||||||
|
</marker>
|
||||||
|
<marker id="arrow-green" viewBox="0 0 10 7" refX="10" refY="3.5"
|
||||||
|
markerWidth="10" markerHeight="7" orient="auto-start-reverse" fill="#34a853">
|
||||||
|
<polygon points="0 0, 10 3.5, 0 7"/>
|
||||||
|
</marker>
|
||||||
|
<marker id="arrow-red" viewBox="0 0 10 7" refX="10" refY="3.5"
|
||||||
|
markerWidth="10" markerHeight="7" orient="auto-start-reverse" fill="#ea4335">
|
||||||
|
<polygon points="0 0, 10 3.5, 0 7"/>
|
||||||
|
</marker>
|
||||||
|
</defs>
|
||||||
|
|
||||||
|
<!-- Background -->
|
||||||
|
<rect width="1000" height="600" rx="8" fill="#f8f9fb" stroke="#e0e3e8" stroke-width="1.5"/>
|
||||||
|
|
||||||
|
<!-- Title -->
|
||||||
|
<text x="500" y="42" text-anchor="middle" font-size="20" font-weight="bold" fill="#202124">Chunk Lifecycle</text>
|
||||||
|
<text x="500" y="62" text-anchor="middle" font-size="12" fill="#5f6368">chunking.rs · storage.rs · actors/blob_store.rs</text>
|
||||||
|
|
||||||
|
<!-- ═══════════════════════════════════════════════════════════ -->
|
||||||
|
<!-- Row 1: Creation -->
|
||||||
|
<!-- ═══════════════════════════════════════════════════════════ -->
|
||||||
|
|
||||||
|
<!-- Blob Data -->
|
||||||
|
<rect x="40" y="100" width="120" height="50" rx="12" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.8" filter="url(#shadow)"/>
|
||||||
|
<text x="100" y="125" text-anchor="middle" font-size="13" font-weight="600" fill="#1a73e8">Blob Data</text>
|
||||||
|
<text x="100" y="140" text-anchor="middle" font-size="10" fill="#5f6368">raw bytes</text>
|
||||||
|
|
||||||
|
<!-- Arrow -->
|
||||||
|
<line x1="160" y1="125" x2="210" y2="125" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
|
||||||
|
|
||||||
|
<!-- blake3 hash -->
|
||||||
|
<rect x="222" y="100" width="130" height="50" rx="12" fill="#fef7e0" stroke="#f9ab00" stroke-width="1.8" filter="url(#shadow)"/>
|
||||||
|
<text x="287" y="125" text-anchor="middle" font-size="13" font-weight="600" fill="#e37400">blake3 hash</text>
|
||||||
|
<text x="287" y="140" text-anchor="middle" font-size="10" fill="#5f6368">ContentHash</text>
|
||||||
|
|
||||||
|
<!-- Arrow -->
|
||||||
|
<line x1="352" y1="125" x2="405" y2="125" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
|
||||||
|
|
||||||
|
<!-- chunk_blob() -->
|
||||||
|
<rect x="417" y="100" width="140" height="50" rx="12" fill="#e6f4ea" stroke="#34a853" stroke-width="1.8" filter="url(#shadow)"/>
|
||||||
|
<text x="487" y="125" text-anchor="middle" font-size="13" font-weight="600" fill="#137333">chunk_blob()</text>
|
||||||
|
<text x="487" y="140" text-anchor="middle" font-size="10" fill="#5f6368">split by chunk_size</text>
|
||||||
|
|
||||||
|
<!-- Arrow fan-out -->
|
||||||
|
<line x1="557" y1="115" x2="620" y2="105" stroke="#34a853" stroke-width="1.5" marker-end="url(#arrow-green)"/>
|
||||||
|
<line x1="557" y1="125" x2="620" y2="125" stroke="#34a853" stroke-width="1.5" marker-end="url(#arrow-green)"/>
|
||||||
|
<line x1="557" y1="135" x2="620" y2="145" stroke="#34a853" stroke-width="1.5" marker-end="url(#arrow-green)"/>
|
||||||
|
|
||||||
|
<!-- ChunkRef boxes -->
|
||||||
|
<rect x="630" y="88" width="110" height="30" rx="6" fill="#e6f4ea" stroke="#34a853" stroke-width="1.2"/>
|
||||||
|
<text x="685" y="108" text-anchor="middle" font-size="11" font-weight="600" fill="#137333">ChunkRef[0]</text>
|
||||||
|
|
||||||
|
<rect x="630" y="122" width="110" height="30" rx="6" fill="#e6f4ea" stroke="#34a853" stroke-width="1.2"/>
|
||||||
|
<text x="685" y="142" text-anchor="middle" font-size="11" font-weight="600" fill="#137333">ChunkRef[1]</text>
|
||||||
|
|
||||||
|
<rect x="630" y="156" width="110" height="30" rx="6" fill="#e6f4ea" stroke="#34a853" stroke-width="1.2"/>
|
||||||
|
<text x="685" y="176" text-anchor="middle" font-size="11" font-weight="600" fill="#137333">ChunkRef[N]</text>
|
||||||
|
|
||||||
|
<!-- Brace → ObjectManifest -->
|
||||||
|
<line x1="740" y1="103" x2="770" y2="137" stroke="#5f6368" stroke-width="1"/>
|
||||||
|
<line x1="740" y1="137" x2="770" y2="137" stroke="#5f6368" stroke-width="1"/>
|
||||||
|
<line x1="740" y1="171" x2="770" y2="137" stroke="#5f6368" stroke-width="1"/>
|
||||||
|
<line x1="770" y1="137" x2="800" y2="137" stroke="#5f6368" stroke-width="1.5" marker-end="url(#arrow-gray)"/>
|
||||||
|
|
||||||
|
<rect x="812" y="112" width="155" height="50" rx="12" fill="#fef7e0" stroke="#f9ab00" stroke-width="1.8" filter="url(#shadow)"/>
|
||||||
|
<text x="889" y="137" text-anchor="middle" font-size="13" font-weight="600" fill="#e37400">ObjectManifest</text>
|
||||||
|
<text x="889" y="152" text-anchor="middle" font-size="10" fill="#5f6368">chunks + total_size</text>
|
||||||
|
|
||||||
|
<!-- ═══════════════════════════════════════════════════════════ -->
|
||||||
|
<!-- Row 2: Storage -->
|
||||||
|
<!-- ═══════════════════════════════════════════════════════════ -->
|
||||||
|
|
||||||
|
<text x="500" y="220" text-anchor="middle" font-size="14" font-weight="600" fill="#202124">Storage</text>
|
||||||
|
|
||||||
|
<!-- WriteChunk -->
|
||||||
|
<rect x="100" y="240" width="200" height="50" rx="12" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.8" filter="url(#shadow)"/>
|
||||||
|
<text x="200" y="260" text-anchor="middle" font-size="13" font-weight="600" fill="#1a73e8">BlobStoreMsg</text>
|
||||||
|
<text x="200" y="278" text-anchor="middle" font-size="10" fill="#5f6368">::WriteChunk { hash, data }</text>
|
||||||
|
|
||||||
|
<line x1="300" y1="265" x2="380" y2="265" stroke="#34a853" stroke-width="1.8" marker-end="url(#arrow-green)"/>
|
||||||
|
|
||||||
|
<!-- StorageBackend -->
|
||||||
|
<rect x="392" y="240" width="220" height="50" rx="12" fill="#e6f4ea" stroke="#34a853" stroke-width="1.8" filter="url(#shadow)"/>
|
||||||
|
<text x="502" y="260" text-anchor="middle" font-size="13" font-weight="600" fill="#137333">StorageBackend</text>
|
||||||
|
<text x="502" y="278" text-anchor="middle" font-size="10" fill="#5f6368">::write_chunk(hash, data)</text>
|
||||||
|
|
||||||
|
<line x1="612" y1="265" x2="690" y2="265" stroke="#34a853" stroke-width="1.8" marker-end="url(#arrow-green)"/>
|
||||||
|
|
||||||
|
<!-- Disk/Memory -->
|
||||||
|
<rect x="702" y="240" width="180" height="50" rx="12" fill="#f1f3f4" stroke="#9aa0a6" stroke-width="1.8" filter="url(#shadow)"/>
|
||||||
|
<text x="792" y="260" text-anchor="middle" font-size="13" font-weight="600" fill="#5f6368">Filesystem /</text>
|
||||||
|
<text x="792" y="278" text-anchor="middle" font-size="10" fill="#5f6368">InMemoryBackend</text>
|
||||||
|
|
||||||
|
<!-- ═══════════════════════════════════════════════════════════ -->
|
||||||
|
<!-- Row 3: Reference tracking -->
|
||||||
|
<!-- ═══════════════════════════════════════════════════════════ -->
|
||||||
|
|
||||||
|
<text x="500" y="340" text-anchor="middle" font-size="14" font-weight="600" fill="#202124">Reference Tracking</text>
|
||||||
|
|
||||||
|
<rect x="100" y="360" width="200" height="50" rx="12" fill="#fef7e0" stroke="#f9ab00" stroke-width="1.8" filter="url(#shadow)"/>
|
||||||
|
<text x="200" y="380" text-anchor="middle" font-size="13" font-weight="600" fill="#e37400">ObjectManifest</text>
|
||||||
|
<text x="200" y="398" text-anchor="middle" font-size="10" fill="#5f6368">references chunk hashes</text>
|
||||||
|
|
||||||
|
<line x1="300" y1="385" x2="380" y2="385" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
|
||||||
|
<text x="340" y="377" text-anchor="middle" font-size="10" fill="#5f6368">owns</text>
|
||||||
|
|
||||||
|
<rect x="392" y="360" width="220" height="50" rx="12" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.8" filter="url(#shadow)"/>
|
||||||
|
<text x="502" y="385" text-anchor="middle" font-size="13" font-weight="600" fill="#1a73e8">MetadataActor</text>
|
||||||
|
|
||||||
|
<text x="502" y="398" text-anchor="middle" font-size="10" fill="#5f6368">tracks all manifests</text>
|
||||||
|
|
||||||
|
<!-- ═══════════════════════════════════════════════════════════ -->
|
||||||
|
<!-- Row 4: GC -->
|
||||||
|
<!-- ═══════════════════════════════════════════════════════════ -->
|
||||||
|
|
||||||
|
<text x="500" y="460" text-anchor="middle" font-size="14" font-weight="600" fill="#202124">Garbage Collection</text>
|
||||||
|
|
||||||
|
<rect x="60" y="480" width="150" height="50" rx="12" fill="#fce8e6" stroke="#ea4335" stroke-width="1.8" filter="url(#shadow)"/>
|
||||||
|
<text x="135" y="505" text-anchor="middle" font-size="13" font-weight="600" fill="#c5221f">GcTick</text>
|
||||||
|
<text x="135" y="520" text-anchor="middle" font-size="10" fill="#5f6368">periodic trigger</text>
|
||||||
|
|
||||||
|
<line x1="210" y1="505" x2="280" y2="505" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
|
||||||
|
|
||||||
|
<rect x="292" y="480" width="200" height="50" rx="12" fill="#fef7e0" stroke="#f9ab00" stroke-width="1.8" filter="url(#shadow)"/>
|
||||||
|
<text x="392" y="500" text-anchor="middle" font-size="12" font-weight="600" fill="#e37400">Collect referenced</text>
|
||||||
|
<text x="392" y="518" text-anchor="middle" font-size="10" fill="#5f6368">hashes from manifests</text>
|
||||||
|
|
||||||
|
<line x1="492" y1="505" x2="560" y2="505" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
|
||||||
|
|
||||||
|
<rect x="572" y="480" width="200" height="50" rx="12" fill="#fce8e6" stroke="#ea4335" stroke-width="1.8" filter="url(#shadow)"/>
|
||||||
|
<text x="672" y="500" text-anchor="middle" font-size="12" font-weight="600" fill="#c5221f">GcUnreferenced</text>
|
||||||
|
<text x="672" y="518" text-anchor="middle" font-size="10" fill="#5f6368">delete orphan chunks</text>
|
||||||
|
|
||||||
|
<line x1="772" y1="505" x2="830" y2="505" stroke="#ea4335" stroke-width="1.8" marker-end="url(#arrow-red)"/>
|
||||||
|
|
||||||
|
<!-- Deleted -->
|
||||||
|
<rect x="842" y="488" width="100" height="34" rx="8" fill="#f1f3f4" stroke="#9aa0a6" stroke-width="1.5"/>
|
||||||
|
<text x="892" y="510" text-anchor="middle" font-size="12" font-weight="600" fill="#5f6368">Deleted</text>
|
||||||
|
|
||||||
|
<!-- ============================================ -->
|
||||||
|
<!-- Legend -->
|
||||||
|
<rect x="50" y="555" width="900" height="30" rx="6" fill="none" stroke="#e0e3e8" stroke-width="1"/>
|
||||||
|
<text x="70" y="575" font-size="11" font-weight="600" fill="#202124">Legend</text>
|
||||||
|
|
||||||
|
<line x1="130" y1="572" x2="160" y2="572" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
|
||||||
|
<text x="168" y="576" font-size="10" fill="#5f6368">Transition</text>
|
||||||
|
|
||||||
|
<line x1="260" y1="572" x2="290" y2="572" stroke="#34a853" stroke-width="1.8" marker-end="url(#arrow-green)"/>
|
||||||
|
<text x="298" y="576" font-size="10" fill="#34a853">Data flow</text>
|
||||||
|
|
||||||
|
<line x1="380" y1="572" x2="410" y2="572" stroke="#ea4335" stroke-width="1.8" marker-end="url(#arrow-red)"/>
|
||||||
|
<text x="418" y="576" font-size="10" fill="#ea4335">Deletion</text>
|
||||||
|
|
||||||
|
<rect x="510" y="564" width="40" height="16" rx="4" fill="#e8f0fe" stroke="#4285f4" stroke-width="1"/>
|
||||||
|
<text x="560" y="576" font-size="10" fill="#4285f4">Actor / Component</text>
|
||||||
|
|
||||||
|
<rect x="680" y="564" width="40" height="16" rx="4" fill="#fef7e0" stroke="#f9ab00" stroke-width="1"/>
|
||||||
|
<text x="730" y="576" font-size="10" fill="#e37400">Data type</text>
|
||||||
|
|
||||||
|
<rect x="810" y="564" width="40" height="16" rx="4" fill="#e6f4ea" stroke="#34a853" stroke-width="1"/>
|
||||||
|
<text x="860" y="576" font-size="10" fill="#34a853">Success state</text>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 11 KiB |
153
docs/diagrams/datastore_transfer.svg
Normal file
153
docs/diagrams/datastore_transfer.svg
Normal file
|
|
@ -0,0 +1,153 @@
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1000 700" font-family="system-ui, sans-serif">
|
||||||
|
<defs>
|
||||||
|
<filter id="shadow" x="-4%" y="-4%" width="108%" height="108%">
|
||||||
|
<feDropShadow dx="1" dy="2" stdDeviation="3" flood-color="#000" flood-opacity="0.10"/>
|
||||||
|
</filter>
|
||||||
|
<marker id="arrow-gray" viewBox="0 0 10 7" refX="10" refY="3.5"
|
||||||
|
markerWidth="10" markerHeight="7" orient="auto-start-reverse" fill="#5f6368">
|
||||||
|
<polygon points="0 0, 10 3.5, 0 7"/>
|
||||||
|
</marker>
|
||||||
|
<marker id="arrow-green" viewBox="0 0 10 7" refX="10" refY="3.5"
|
||||||
|
markerWidth="10" markerHeight="7" orient="auto-start-reverse" fill="#34a853">
|
||||||
|
<polygon points="0 0, 10 3.5, 0 7"/>
|
||||||
|
</marker>
|
||||||
|
<marker id="arrow-red" viewBox="0 0 10 7" refX="10" refY="3.5"
|
||||||
|
markerWidth="10" markerHeight="7" orient="auto-start-reverse" fill="#ea4335">
|
||||||
|
<polygon points="0 0, 10 3.5, 0 7"/>
|
||||||
|
</marker>
|
||||||
|
<marker id="arrow-blue" viewBox="0 0 10 7" refX="10" refY="3.5"
|
||||||
|
markerWidth="10" markerHeight="7" orient="auto-start-reverse" fill="#4285f4">
|
||||||
|
<polygon points="0 0, 10 3.5, 0 7"/>
|
||||||
|
</marker>
|
||||||
|
</defs>
|
||||||
|
|
||||||
|
<!-- Background -->
|
||||||
|
<rect width="1000" height="700" rx="8" fill="#f8f9fb" stroke="#e0e3e8" stroke-width="1.5"/>
|
||||||
|
|
||||||
|
<!-- Title -->
|
||||||
|
<text x="500" y="42" text-anchor="middle" font-size="20" font-weight="bold" fill="#202124">Datastore Remote GET Transfer Flow</text>
|
||||||
|
<text x="500" y="62" text-anchor="middle" font-size="12" fill="#5f6368">api.rs:583-706 · types.rs</text>
|
||||||
|
|
||||||
|
<!-- ── Client ──────────────────────────────────────────── -->
|
||||||
|
<rect x="40" y="100" width="130" height="50" rx="12" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.8" filter="url(#shadow)"/>
|
||||||
|
<text x="105" y="130" text-anchor="middle" font-size="14" font-weight="600" fill="#1a73e8">Client</text>
|
||||||
|
|
||||||
|
<!-- Arrow: Client → API Server -->
|
||||||
|
<line x1="170" y1="125" x2="240" y2="125" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
|
||||||
|
<text x="205" y="117" text-anchor="middle" font-size="10" fill="#5f6368">GET /api/data</text>
|
||||||
|
|
||||||
|
<!-- ── API Server ──────────────────────────────────────── -->
|
||||||
|
<rect x="252" y="100" width="150" height="50" rx="12" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.8" filter="url(#shadow)"/>
|
||||||
|
<text x="327" y="130" text-anchor="middle" font-size="14" font-weight="600" fill="#1a73e8">API Server</text>
|
||||||
|
|
||||||
|
<!-- Arrow: API Server → DatastoreNode -->
|
||||||
|
<line x1="402" y1="125" x2="470" y2="125" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
|
||||||
|
<text x="436" y="117" text-anchor="middle" font-size="10" fill="#5f6368">Get msg</text>
|
||||||
|
|
||||||
|
<!-- ── DatastoreNode ───────────────────────────────────── -->
|
||||||
|
<rect x="482" y="100" width="160" height="50" rx="12" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.8" filter="url(#shadow)"/>
|
||||||
|
<text x="562" y="130" text-anchor="middle" font-size="14" font-weight="600" fill="#1a73e8">DatastoreNode</text>
|
||||||
|
|
||||||
|
<!-- Arrow: DatastoreNode → MetadataActor -->
|
||||||
|
<line x1="562" y1="150" x2="562" y2="200" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
|
||||||
|
<text x="580" y="180" font-size="10" fill="#5f6368">lookup</text>
|
||||||
|
|
||||||
|
<!-- ── MetadataActor (local) ───────────────────────────── -->
|
||||||
|
<rect x="482" y="210" width="160" height="50" rx="12" fill="#fef7e0" stroke="#f9ab00" stroke-width="1.8" filter="url(#shadow)"/>
|
||||||
|
<text x="562" y="235" text-anchor="middle" font-size="13" font-weight="600" fill="#e37400">MetadataActor</text>
|
||||||
|
<text x="562" y="250" text-anchor="middle" font-size="10" fill="#5f6368">(local)</text>
|
||||||
|
|
||||||
|
<!-- Arrow: MetadataActor → Not Found (red, left) -->
|
||||||
|
<line x1="482" y1="235" x2="420" y2="235" stroke="#ea4335" stroke-width="1.8" marker-end="url(#arrow-red)"/>
|
||||||
|
<text x="450" y="227" text-anchor="middle" font-size="10" fill="#ea4335">Not Found</text>
|
||||||
|
|
||||||
|
<!-- ── try_remote_get box ──────────────────────────────── -->
|
||||||
|
<rect x="210" y="210" width="200" height="50" rx="12" fill="#fce8e6" stroke="#ea4335" stroke-width="1.5" filter="url(#shadow)" stroke-dasharray="5,3"/>
|
||||||
|
<text x="310" y="235" text-anchor="middle" font-size="12" font-weight="600" fill="#c5221f">try_remote_get()</text>
|
||||||
|
<text x="310" y="250" text-anchor="middle" font-size="10" fill="#5f6368">iterate peers</text>
|
||||||
|
|
||||||
|
<!-- Arrow: try_remote_get → Peer Metadata -->
|
||||||
|
<line x1="310" y1="260" x2="310" y2="320" stroke="#34a853" stroke-width="1.8" marker-end="url(#arrow-green)"/>
|
||||||
|
<text x="340" y="295" font-size="10" fill="#34a853">FindObject</text>
|
||||||
|
|
||||||
|
<!-- ── Peer MetadataActor ──────────────────────────────── -->
|
||||||
|
<rect x="210" y="330" width="200" height="50" rx="12" fill="#e6f4ea" stroke="#34a853" stroke-width="1.8" filter="url(#shadow)"/>
|
||||||
|
<text x="310" y="355" text-anchor="middle" font-size="13" font-weight="600" fill="#137333">Peer MetadataActor</text>
|
||||||
|
<text x="310" y="370" text-anchor="middle" font-size="10" fill="#5f6368">(remote node)</text>
|
||||||
|
|
||||||
|
<!-- Arrow: Peer Metadata → Peer BlobStore -->
|
||||||
|
<line x1="310" y1="380" x2="310" y2="430" stroke="#34a853" stroke-width="1.8" marker-end="url(#arrow-green)"/>
|
||||||
|
<text x="340" y="410" font-size="10" fill="#34a853">Found!</text>
|
||||||
|
|
||||||
|
<!-- ── Peer BlobStore ──────────────────────────────────── -->
|
||||||
|
<rect x="210" y="440" width="200" height="50" rx="12" fill="#e6f4ea" stroke="#34a853" stroke-width="1.8" filter="url(#shadow)"/>
|
||||||
|
<text x="310" y="465" text-anchor="middle" font-size="13" font-weight="600" fill="#137333">Peer BlobStore</text>
|
||||||
|
<text x="310" y="480" text-anchor="middle" font-size="10" fill="#5f6368">ReadChunk × N</text>
|
||||||
|
|
||||||
|
<!-- Arrow fan-out: Peer BlobStore → Chunks -->
|
||||||
|
<line x1="230" y1="490" x2="150" y2="540" stroke="#4285f4" stroke-width="1.5" marker-end="url(#arrow-blue)"/>
|
||||||
|
<line x1="310" y1="490" x2="310" y2="540" stroke="#4285f4" stroke-width="1.5" marker-end="url(#arrow-blue)"/>
|
||||||
|
<line x1="390" y1="490" x2="470" y2="540" stroke="#4285f4" stroke-width="1.5" marker-end="url(#arrow-blue)"/>
|
||||||
|
|
||||||
|
<!-- Chunk boxes -->
|
||||||
|
<rect x="100" y="545" width="100" height="36" rx="8" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.2" filter="url(#shadow)"/>
|
||||||
|
<text x="150" y="568" text-anchor="middle" font-size="12" font-weight="600" fill="#1a73e8">Chunk 1</text>
|
||||||
|
|
||||||
|
<rect x="260" y="545" width="100" height="36" rx="8" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.2" filter="url(#shadow)"/>
|
||||||
|
<text x="310" y="568" text-anchor="middle" font-size="12" font-weight="600" fill="#1a73e8">Chunk 2</text>
|
||||||
|
|
||||||
|
<rect x="420" y="545" width="100" height="36" rx="8" fill="#e8f0fe" stroke="#4285f4" stroke-width="1.2" filter="url(#shadow)"/>
|
||||||
|
<text x="470" y="568" text-anchor="middle" font-size="12" font-weight="600" fill="#1a73e8">Chunk N</text>
|
||||||
|
|
||||||
|
<!-- Arrows: Chunks → Local BlobStore -->
|
||||||
|
<line x1="150" y1="581" x2="560" y2="610" stroke="#4285f4" stroke-width="1.2" marker-end="url(#arrow-blue)"/>
|
||||||
|
<line x1="310" y1="581" x2="560" y2="610" stroke="#4285f4" stroke-width="1.2" marker-end="url(#arrow-blue)"/>
|
||||||
|
<line x1="470" y1="581" x2="560" y2="610" stroke="#4285f4" stroke-width="1.2" marker-end="url(#arrow-blue)"/>
|
||||||
|
|
||||||
|
<!-- ── Local BlobStore ─────────────────────────────────── -->
|
||||||
|
<rect x="510" y="600" width="180" height="50" rx="12" fill="#e6f4ea" stroke="#34a853" stroke-width="1.8" filter="url(#shadow)"/>
|
||||||
|
<text x="600" y="625" text-anchor="middle" font-size="13" font-weight="600" fill="#137333">Local BlobStore</text>
|
||||||
|
<text x="600" y="640" text-anchor="middle" font-size="10" fill="#5f6368">WriteChunk × N</text>
|
||||||
|
|
||||||
|
<!-- Arrow: Local BlobStore → Reassemble -->
|
||||||
|
<line x1="690" y1="625" x2="740" y2="625" stroke="#34a853" stroke-width="1.8" marker-end="url(#arrow-green)"/>
|
||||||
|
|
||||||
|
<!-- ── Reassemble ──────────────────────────────────────── -->
|
||||||
|
<rect x="752" y="600" width="150" height="50" rx="12" fill="#e6f4ea" stroke="#34a853" stroke-width="1.8" filter="url(#shadow)"/>
|
||||||
|
<text x="827" y="625" text-anchor="middle" font-size="13" font-weight="600" fill="#137333">reassemble_blob</text>
|
||||||
|
<text x="827" y="640" text-anchor="middle" font-size="10" fill="#5f6368">verify integrity</text>
|
||||||
|
|
||||||
|
<!-- Arrow: Reassemble → Client (curved up) -->
|
||||||
|
<path d="M 902 620 Q 950 600 960 400 Q 960 200 180 130"
|
||||||
|
fill="none" stroke="#34a853" stroke-width="1.8" marker-end="url(#arrow-green)"/>
|
||||||
|
<text x="960" y="370" font-size="11" fill="#34a853" font-weight="600">200 OK</text>
|
||||||
|
<text x="960" y="385" font-size="10" fill="#34a853">blob data</text>
|
||||||
|
|
||||||
|
<!-- ── Dashboard integration ────────────────────────────── -->
|
||||||
|
<rect x="700" y="330" width="180" height="50" rx="12" fill="#f3e8fd" stroke="#9c27b0" stroke-width="1.5" filter="url(#shadow)"/>
|
||||||
|
<text x="790" y="350" text-anchor="middle" font-size="12" font-weight="600" fill="#7b1fa2">DatastoreMetrics</text>
|
||||||
|
<text x="790" y="367" text-anchor="middle" font-size="10" fill="#5f6368">begin/advance/end</text>
|
||||||
|
|
||||||
|
<line x1="410" y1="460" x2="700" y2="355" stroke="#9c27b0" stroke-width="1.2" stroke-dasharray="4,3" marker-end="url(#arrow-gray)"/>
|
||||||
|
<text x="570" y="395" font-size="9" fill="#9c27b0">transfer progress</text>
|
||||||
|
|
||||||
|
<!-- ============================================ -->
|
||||||
|
<!-- Legend -->
|
||||||
|
<rect x="50" y="660" width="900" height="30" rx="6" fill="none" stroke="#e0e3e8" stroke-width="1"/>
|
||||||
|
<text x="70" y="680" font-size="11" font-weight="600" fill="#202124">Legend</text>
|
||||||
|
|
||||||
|
<line x1="130" y1="677" x2="160" y2="677" stroke="#5f6368" stroke-width="1.8" marker-end="url(#arrow-gray)"/>
|
||||||
|
<text x="168" y="681" font-size="10" fill="#5f6368">Transition</text>
|
||||||
|
|
||||||
|
<line x1="260" y1="677" x2="290" y2="677" stroke="#34a853" stroke-width="1.8" marker-end="url(#arrow-green)"/>
|
||||||
|
<text x="298" y="681" font-size="10" fill="#34a853">Success path</text>
|
||||||
|
|
||||||
|
<line x1="400" y1="677" x2="430" y2="677" stroke="#ea4335" stroke-width="1.8" marker-end="url(#arrow-red)"/>
|
||||||
|
<text x="438" y="681" font-size="10" fill="#ea4335">Not found / fallback</text>
|
||||||
|
|
||||||
|
<line x1="580" y1="677" x2="610" y2="677" stroke="#4285f4" stroke-width="1.5" marker-end="url(#arrow-blue)"/>
|
||||||
|
<text x="618" y="681" font-size="10" fill="#4285f4">Chunk transfer</text>
|
||||||
|
|
||||||
|
<line x1="720" y1="677" x2="750" y2="677" stroke="#9c27b0" stroke-width="1.2" stroke-dasharray="4,3"/>
|
||||||
|
<text x="758" y="681" font-size="10" fill="#9c27b0">Metrics tracking</text>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 11 KiB |
|
|
@ -265,11 +265,6 @@ impl Runtime {
|
||||||
/// Creates a temporary inbox, calls `msg_builder` with the inbox's address
|
/// Creates a temporary inbox, calls `msg_builder` with the inbox's address
|
||||||
/// (so you can embed it as `reply_to`), sends the message, and returns an
|
/// (so you can embed it as `reply_to`), sends the message, and returns an
|
||||||
/// [`Ask`] handle for receiving the response.
|
/// [`Ask`] handle for receiving the response.
|
||||||
///
|
|
||||||
/// ```ignore
|
|
||||||
/// let ask = rt.ask(actor, |reply_to| GetValue { reply_to })?;
|
|
||||||
/// let value = ask.recv_ticking(&rt, 10)?;
|
|
||||||
/// ```
|
|
||||||
pub fn ask<Req: Message, Resp: Message>(
|
pub fn ask<Req: Message, Resp: Message>(
|
||||||
&self,
|
&self,
|
||||||
addr: ActorAddress,
|
addr: ActorAddress,
|
||||||
|
|
|
||||||
4
xtask/Cargo.toml
Normal file
4
xtask/Cargo.toml
Normal file
|
|
@ -0,0 +1,4 @@
|
||||||
|
[package]
|
||||||
|
name = "xtask"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition = "2024"
|
||||||
197
xtask/src/main.rs
Normal file
197
xtask/src/main.rs
Normal file
|
|
@ -0,0 +1,197 @@
|
||||||
|
use std::process::Command;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
struct TestStep {
|
||||||
|
label: &'static str,
|
||||||
|
args: &'static [&'static str],
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Group {
|
||||||
|
name: &'static str,
|
||||||
|
description: &'static str,
|
||||||
|
steps: &'static [TestStep],
|
||||||
|
}
|
||||||
|
|
||||||
|
const CORE: Group = Group {
|
||||||
|
name: "core",
|
||||||
|
description: "Actor runtime, message delivery, property tests",
|
||||||
|
steps: &[TestStep {
|
||||||
|
label: "actor runtime",
|
||||||
|
args: &["test", "-p", "swactor", "--features", "transport"],
|
||||||
|
}],
|
||||||
|
};
|
||||||
|
|
||||||
|
const DISTRIBUTION: Group = Group {
|
||||||
|
name: "distribution",
|
||||||
|
description: "Distribution protocol + datastore",
|
||||||
|
steps: &[
|
||||||
|
TestStep {
|
||||||
|
label: "distribution protocol",
|
||||||
|
args: &["test", "-p", "distribution"],
|
||||||
|
},
|
||||||
|
TestStep {
|
||||||
|
label: "datastore",
|
||||||
|
args: &["test", "-p", "swactor-datastore"],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
const CLUSTER_SIMS: Group = Group {
|
||||||
|
name: "cluster-sims",
|
||||||
|
description: "Deterministic cluster simulations",
|
||||||
|
steps: &[TestStep {
|
||||||
|
label: "cluster simulations",
|
||||||
|
args: &["test", "-p", "simulation"],
|
||||||
|
}],
|
||||||
|
};
|
||||||
|
|
||||||
|
const INTEGRATED: Group = Group {
|
||||||
|
name: "integrated",
|
||||||
|
description: "HTTP API + dashboard end-to-end tests",
|
||||||
|
steps: &[
|
||||||
|
TestStep {
|
||||||
|
label: "datastore integration (node features)",
|
||||||
|
args: &[
|
||||||
|
"test",
|
||||||
|
"-p",
|
||||||
|
"swactor-datastore",
|
||||||
|
"--features",
|
||||||
|
"node",
|
||||||
|
"--test",
|
||||||
|
"api_integration_test",
|
||||||
|
"--test",
|
||||||
|
"dashboard_integration_test",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
TestStep {
|
||||||
|
label: "runtime dashboard",
|
||||||
|
args: &["test", "-p", "runtime-dashboard"],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
fn groups_for(name: &str) -> Option<Vec<&'static Group>> {
|
||||||
|
match name {
|
||||||
|
"core" => Some(vec![&CORE]),
|
||||||
|
"distribution" => Some(vec![&DISTRIBUTION]),
|
||||||
|
"cluster-sims" => Some(vec![&CLUSTER_SIMS]),
|
||||||
|
"integrated" => Some(vec![&INTEGRATED]),
|
||||||
|
"essential" => Some(vec![&CORE, &DISTRIBUTION, &INTEGRATED]),
|
||||||
|
"all" => Some(vec![&CORE, &DISTRIBUTION, &CLUSTER_SIMS, &INTEGRATED]),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run_step(group_name: &str, step: &TestStep) -> bool {
|
||||||
|
println!("\n=== {group_name}: {} ===", step.label);
|
||||||
|
println!(" cargo {}", step.args.join(" "));
|
||||||
|
println!();
|
||||||
|
|
||||||
|
let status = Command::new("cargo")
|
||||||
|
.args(step.args)
|
||||||
|
.status();
|
||||||
|
|
||||||
|
match status {
|
||||||
|
Ok(s) => s.success(),
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("Failed to execute cargo: {e}");
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn print_usage() {
|
||||||
|
println!(
|
||||||
|
"\
|
||||||
|
USAGE: cargo xtask test <GROUP>
|
||||||
|
|
||||||
|
GROUPS:
|
||||||
|
core Actor runtime, message delivery, property tests
|
||||||
|
distribution Distribution protocol + datastore
|
||||||
|
cluster-sims Deterministic cluster simulations
|
||||||
|
integrated HTTP API + dashboard end-to-end tests
|
||||||
|
essential core + distribution + integrated (merge gate)
|
||||||
|
all Every test group
|
||||||
|
|
||||||
|
FLAGS:
|
||||||
|
--list Show all groups and the cargo commands they run"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn print_list() {
|
||||||
|
let all_groups: &[(&[&str], &Group)] = &[
|
||||||
|
(&[], &CORE),
|
||||||
|
(&[], &DISTRIBUTION),
|
||||||
|
(&[], &CLUSTER_SIMS),
|
||||||
|
(&[], &INTEGRATED),
|
||||||
|
];
|
||||||
|
|
||||||
|
println!("Available test groups:\n");
|
||||||
|
|
||||||
|
for &(_, group) in all_groups {
|
||||||
|
println!(" {:<14}{}", group.name, group.description);
|
||||||
|
for step in group.steps {
|
||||||
|
println!(" → cargo {}", step.args.join(" "));
|
||||||
|
}
|
||||||
|
println!();
|
||||||
|
}
|
||||||
|
|
||||||
|
println!(" {:<14}core + distribution + integrated (merge gate)", "essential");
|
||||||
|
println!(" {:<14}Every test group", "all");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn main() {
|
||||||
|
let args: Vec<String> = std::env::args().collect();
|
||||||
|
|
||||||
|
if args.len() < 2 || args[1] != "test" {
|
||||||
|
print_usage();
|
||||||
|
std::process::exit(if args.len() < 2 { 1 } else { 1 });
|
||||||
|
}
|
||||||
|
|
||||||
|
if args.len() < 3 {
|
||||||
|
print_usage();
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
let target = &args[2];
|
||||||
|
|
||||||
|
if target == "--list" {
|
||||||
|
print_list();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let groups = match groups_for(target) {
|
||||||
|
Some(g) => g,
|
||||||
|
None => {
|
||||||
|
eprintln!("Unknown test group: {target}\n");
|
||||||
|
print_usage();
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let start = Instant::now();
|
||||||
|
let mut passed = 0usize;
|
||||||
|
let mut failed = 0usize;
|
||||||
|
|
||||||
|
for group in &groups {
|
||||||
|
for step in group.steps {
|
||||||
|
if run_step(group.name, step) {
|
||||||
|
passed += 1;
|
||||||
|
} else {
|
||||||
|
failed += 1;
|
||||||
|
let elapsed = start.elapsed();
|
||||||
|
println!(
|
||||||
|
"\n--- FAILED after {:.1}s ({passed} passed, {failed} failed) ---",
|
||||||
|
elapsed.as_secs_f64()
|
||||||
|
);
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let elapsed = start.elapsed();
|
||||||
|
println!(
|
||||||
|
"\n--- All {passed} step(s) passed in {:.1}s ---",
|
||||||
|
elapsed.as_secs_f64()
|
||||||
|
);
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue