306 lines
13 KiB
Rust
306 lines
13 KiB
Rust
//! `DirectoryActor` — the actor→host location directory (`DIRECTORY.md`) as a
|
|
//! standalone gossip actor.
|
|
//!
|
|
//! It is the fourth standalone gossip actor, a peer of [`SwimActor`], [`RegistryActor`],
|
|
//! and [`MetadataActor`] — not a layer above them. It owns one signed claim per
|
|
//! actor (a [`DirectoryEntry`], the spec's `Claim`), converges that location map
|
|
//! across the cluster by lazy round-robin gossip on `Tick`, and publishes a
|
|
//! [`RouteView`] read-mirror the egress consults to route an application message to
|
|
//! an actor it knows only by address (`DIRECTORY.md` §5).
|
|
//!
|
|
//! It differs from the registry/metadata template in only three ways:
|
|
//! 1. it stores signed claims directly (no wrapped CRDT engine), one per actor;
|
|
//! 2. it **verifies the signature on every merge**, so a forged claim is dropped;
|
|
//! 3. it publishes a [`RouteView`] (`ActorAddress → NodeId`) instead of a
|
|
//! [`RelayMirror`](crate::transport_bridge::RelayMirror) (`NodeId → relay URL`).
|
|
//!
|
|
//! Routing is **blind, best-effort**: there are no acks, no retries, no awaited
|
|
//! replies. The single wire-crossing edge is [`DirectoryIn::Gossip`]; every other
|
|
//! variant is local control. Like SWIM, the actor reads no ambient clock — time
|
|
//! enters only as `Tick`.
|
|
//!
|
|
//! [`SwimActor`]: crate::swim::actor::SwimActor
|
|
//! [`RegistryActor`]: crate::registry_actor::RegistryActor
|
|
//! [`MetadataActor`]: crate::node_metadata_actor::MetadataActor
|
|
|
|
use std::collections::{BTreeSet, HashMap};
|
|
use std::sync::Arc;
|
|
|
|
use swactor::actor::{ActorAddress, ActorInterface};
|
|
use swactor::runtime::Ctx;
|
|
|
|
use crate::crypto::verify_directory_entry;
|
|
use crate::messages::DirectoryGossip;
|
|
use crate::swim::actor::{MembershipChanged, PeerDirectory};
|
|
use crate::transport_bridge::{RouteBinder, RouteView};
|
|
use crate::types::{DirectoryEntry, MemberState, NodeId};
|
|
|
|
/// Round-robin gossip fan-out per `Tick` (the base dissemination width).
|
|
const FANOUT: u32 = 3;
|
|
/// Budget ceiling, in multiples of [`FANOUT`]: a claim is sent at most
|
|
/// `FANOUT * MAX_ROUNDS` times before it stops being re-pushed.
|
|
const MAX_ROUNDS: u32 = 8;
|
|
/// Claims shipped in a single gossip batch per `Tick`.
|
|
const BATCH: usize = 16;
|
|
/// Grace window (in `Tick`s) before a gone host's claims are reclaimed from the
|
|
/// map. ~5s at the live node's 100 ms cadence. Routing already hides a dead host
|
|
/// (see [`DirectoryActor::republish`]); this is memory hygiene only.
|
|
const GC_GRACE_TICKS: u32 = 50;
|
|
|
|
/// Everything the `DirectoryActor` receives, as one enum. Only [`Gossip`](DirectoryIn::Gossip)
|
|
/// crosses the wire (it carries the registered `DirectoryGossip` tag); the rest
|
|
/// are local control — see [`crate::messages::actor_codec_registry`].
|
|
#[derive(Clone)]
|
|
pub enum DirectoryIn {
|
|
/// Local: a host-signed claim for an actor spawned on this node. The host is
|
|
/// the claim's signer; merging it begins disseminating it.
|
|
Register(DirectoryEntry),
|
|
/// Gossip from a peer: a batch of signed claims to merge.
|
|
Gossip(DirectoryGossip),
|
|
/// Membership delta, adapted from the SwimActor's `MembershipChanged` stream.
|
|
Membership(MembershipChanged),
|
|
/// Local diagnostic read (`DIRECTORY.md` §5): resolve `actor`'s host from the
|
|
/// converged map; the answer is sent to `reply`. Off the hot path — the
|
|
/// load-bearing read is the [`RouteView`], read directly by the egress.
|
|
Resolve {
|
|
actor: ActorAddress,
|
|
reply: ActorAddress,
|
|
},
|
|
/// Clock: disseminate one batch to one peer, then run GC.
|
|
Tick,
|
|
}
|
|
|
|
/// Reply to [`DirectoryIn::Resolve`] (diagnostics/tooling only).
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub struct Located {
|
|
pub actor: ActorAddress,
|
|
pub host: Option<NodeId>,
|
|
}
|
|
|
|
pub struct DirectoryActor {
|
|
self_id: NodeId,
|
|
/// The location map: one signed claim per actor. The only writer is [`Self::merge_one`].
|
|
map: HashMap<ActorAddress, DirectoryEntry>,
|
|
/// Alive peers (excludes self), folded from the membership stream — the
|
|
/// dissemination fan-out set and the `cluster_size` budget basis.
|
|
alive: BTreeSet<NodeId>,
|
|
/// Actors still owing dissemination → remaining sends. A freshly authored or
|
|
/// freshly-superseded claim is armed here; budget decays to zero in steady
|
|
/// state, so a settled cluster goes quiet.
|
|
hot: HashMap<ActorAddress, u32>,
|
|
/// Round-robins the gossip target across `alive`, one per `Tick`.
|
|
cursor: usize,
|
|
/// Per-gone-host GC grace countdown. Internal bookkeeping (not in `DIRECTORY.md`
|
|
/// §4.2's state); a host that returns is cleared in [`Self::on_membership`].
|
|
absent: HashMap<NodeId, u32>,
|
|
peer_directory: Arc<dyn PeerDirectory>,
|
|
/// The §5 read mirror. Single writer = this actor; republished on every change.
|
|
route_view: RouteView,
|
|
/// Registers a route so the runtime can deliver an app message addressed to a
|
|
/// remotely-hosted actor (the §5 egress seam). Called from [`Self::republish`].
|
|
route_binder: Arc<dyn RouteBinder>,
|
|
}
|
|
|
|
impl DirectoryActor {
|
|
pub fn new(
|
|
self_id: NodeId,
|
|
peer_directory: Arc<dyn PeerDirectory>,
|
|
route_view: RouteView,
|
|
route_binder: Arc<dyn RouteBinder>,
|
|
) -> Self {
|
|
Self {
|
|
self_id,
|
|
map: HashMap::new(),
|
|
alive: BTreeSet::new(),
|
|
hot: HashMap::new(),
|
|
cursor: 0,
|
|
absent: HashMap::new(),
|
|
peer_directory,
|
|
route_view,
|
|
route_binder,
|
|
}
|
|
}
|
|
|
|
/// Merge one claim under the supersession rule — the only writer of `map`.
|
|
///
|
|
/// A claim wins iff it is strictly newer by `(generation, node_id)`: a higher
|
|
/// generation supersedes; at equal generation the larger `node_id` is the
|
|
/// deterministic tie-break (so every node converges on the same winner). A
|
|
/// stale, equal, or forged claim is ignored. Merging the same claim twice is a
|
|
/// no-op — it does not re-arm dissemination, which is what lets the cluster go
|
|
/// quiet. Idempotent and commutative.
|
|
fn merge_one(&mut self, claim: DirectoryEntry) {
|
|
if !verify_directory_entry(&claim) {
|
|
return; // not signed by the host it names — drop it
|
|
}
|
|
let supersedes = match self.map.get(&claim.actor_addr) {
|
|
None => true,
|
|
Some(cur) => {
|
|
claim.generation > cur.generation
|
|
|| (claim.generation == cur.generation && claim.node_id > cur.node_id)
|
|
}
|
|
};
|
|
if supersedes {
|
|
let budget = self.budget();
|
|
self.hot.insert(claim.actor_addr, budget); // arm for dissemination
|
|
self.map.insert(claim.actor_addr, claim);
|
|
}
|
|
}
|
|
|
|
fn register(&mut self, claim: DirectoryEntry) {
|
|
self.merge_one(claim);
|
|
self.republish();
|
|
}
|
|
|
|
fn merge_batch(&mut self, claims: Vec<DirectoryEntry>) {
|
|
for c in claims {
|
|
self.merge_one(c);
|
|
}
|
|
self.republish();
|
|
}
|
|
|
|
fn on_membership(&mut self, change: MembershipChanged) {
|
|
match change.state {
|
|
MemberState::Alive if change.node_id != self.self_id => {
|
|
if self.alive.insert(change.node_id) {
|
|
// A (re)joining peer: clear any pending GC and re-arm every held
|
|
// claim so the returning peer is caught up — without a full-cluster
|
|
// reflood (only the actors we hold, and only via the lazy push).
|
|
self.absent.remove(&change.node_id);
|
|
let budget = self.budget();
|
|
let actors: Vec<ActorAddress> = self.map.keys().copied().collect();
|
|
for actor in actors {
|
|
self.hot.insert(actor, budget);
|
|
}
|
|
}
|
|
}
|
|
MemberState::Dead => {
|
|
// Drop from the alive set; republish hides its actors from routing,
|
|
// and gc reclaims the claims after the grace window.
|
|
self.alive.remove(&change.node_id);
|
|
}
|
|
_ => {} // Suspect, or self: ignore (suspicion is SWIM's transient state)
|
|
}
|
|
self.republish();
|
|
}
|
|
|
|
fn tick(&mut self, ctx: &Ctx) {
|
|
self.disseminate(ctx);
|
|
self.gc();
|
|
}
|
|
|
|
/// Send one batch of armed claims to one alive peer (round-robin). Skips the
|
|
/// send when there is no peer or nothing armed, so budget is never burned into
|
|
/// the void — mirrors the registry/metadata disseminators.
|
|
fn disseminate(&mut self, ctx: &Ctx) {
|
|
if self.alive.is_empty() || self.hot.is_empty() {
|
|
return;
|
|
}
|
|
let peers: Vec<NodeId> = self.alive.iter().copied().collect();
|
|
let peer = peers[self.cursor % peers.len()];
|
|
self.cursor = self.cursor.wrapping_add(1);
|
|
let claims = self.take_hot(BATCH);
|
|
if claims.is_empty() {
|
|
return;
|
|
}
|
|
if let Some(addr) = self.peer_directory.resolve(&peer) {
|
|
let _ = ctx.send(addr, DirectoryIn::Gossip(DirectoryGossip { claims }));
|
|
}
|
|
}
|
|
|
|
/// Republish the §5 route view: every actor whose host is reachable right now
|
|
/// (self, or an alive peer). A dead host's actors are omitted, so the egress
|
|
/// never routes to a host SWIM has buried — even before GC reclaims the claim.
|
|
///
|
|
/// Builds the new view locally, swaps it in under the lock, then registers a
|
|
/// route for each remotely-hosted actor *outside* the lock — so the egress
|
|
/// (which takes the view's read lock on every send) never contends with the
|
|
/// route registration.
|
|
fn republish(&self) {
|
|
let mut view = HashMap::with_capacity(self.map.len());
|
|
let mut remote: Vec<ActorAddress> = Vec::new();
|
|
for (actor, claim) in &self.map {
|
|
if claim.node_id == self.self_id {
|
|
view.insert(*actor, claim.node_id);
|
|
} else if self.alive.contains(&claim.node_id) {
|
|
view.insert(*actor, claim.node_id);
|
|
remote.push(*actor);
|
|
}
|
|
}
|
|
*self.route_view.write().expect("route view poisoned") = view;
|
|
for actor in remote {
|
|
self.route_binder.ensure_routable(actor);
|
|
}
|
|
}
|
|
|
|
/// Take up to `limit` armed claims for this tick's batch, spending one unit of
|
|
/// each one's budget. A claim whose budget reaches zero stops being re-pushed.
|
|
fn take_hot(&mut self, limit: usize) -> Vec<DirectoryEntry> {
|
|
let actors: Vec<ActorAddress> = self.hot.keys().copied().take(limit).collect();
|
|
let mut claims = Vec::with_capacity(actors.len());
|
|
for actor in actors {
|
|
if let Some(c) = self.map.get(&actor) {
|
|
claims.push(c.clone());
|
|
}
|
|
if let Some(left) = self.hot.get_mut(&actor) {
|
|
*left = left.saturating_sub(1);
|
|
if *left == 0 {
|
|
self.hot.remove(&actor);
|
|
}
|
|
}
|
|
}
|
|
claims
|
|
}
|
|
|
|
/// Reclaim claims whose host has been gone past the grace window. Memory
|
|
/// hygiene only — [`Self::republish`] already hides a dead host from routing.
|
|
fn gc(&mut self) {
|
|
let gone: Vec<NodeId> = self
|
|
.map
|
|
.values()
|
|
.map(|c| c.node_id)
|
|
.filter(|h| *h != self.self_id && !self.alive.contains(h))
|
|
.collect();
|
|
for h in gone {
|
|
let n = self.absent.entry(h).or_insert(0);
|
|
*n += 1;
|
|
if *n > GC_GRACE_TICKS {
|
|
self.map.retain(|_, c| c.node_id != h);
|
|
self.absent.remove(&h);
|
|
}
|
|
}
|
|
// A host that came back was already cleared in on_membership(Alive).
|
|
}
|
|
|
|
/// Per-claim dissemination budget: `FANOUT · ⌈log2(cluster+1)⌉`, clamped to
|
|
/// `[FANOUT, FANOUT·MAX_ROUNDS]`. Grows gently with cluster size so a claim
|
|
/// reaches enough seeds for epidemic spread to finish the job, but is bounded.
|
|
fn budget(&self) -> u32 {
|
|
let n = (self.alive.len() + 1) as u32; // include self
|
|
let log2 = u32::BITS - n.leading_zeros(); // bit-length of n ≈ ⌈log2(n+1)⌉
|
|
(FANOUT * log2).clamp(FANOUT, FANOUT * MAX_ROUNDS)
|
|
}
|
|
}
|
|
|
|
impl ActorInterface for DirectoryActor {
|
|
type Incoming = DirectoryIn;
|
|
type Response = ();
|
|
|
|
fn handle(&mut self, ctx: &Ctx, msg: DirectoryIn) {
|
|
match msg {
|
|
DirectoryIn::Register(claim) => self.register(claim),
|
|
DirectoryIn::Gossip(batch) => self.merge_batch(batch.claims),
|
|
DirectoryIn::Membership(change) => self.on_membership(change),
|
|
DirectoryIn::Tick => self.tick(ctx),
|
|
DirectoryIn::Resolve { actor, reply } => {
|
|
let host = self
|
|
.route_view
|
|
.read()
|
|
.expect("route view poisoned")
|
|
.get(&actor)
|
|
.copied();
|
|
let _ = ctx.send(reply, Located { actor, host });
|
|
}
|
|
}
|
|
}
|
|
}
|