From 92d62daa3388653baf8d3ed6302d2460f1493469 Mon Sep 17 00:00:00 2001 From: Zachery Aaron Shores-Chmielewski Date: Mon, 25 May 2026 14:33:35 +0400 Subject: [PATCH] feat(sim): add stage host kind and relay scenarios Adds the pipeline-stage host kind wrapping the production stage-supervisor lifecycle (Cold/Registering/Running/Halted per RELAY_SPEC), N3 relay calibration scenarios (own-relay and canary-relay, real-worker and stub), relay invariant and scenario-validation tests, and an adversarial judge test suite hunting spec/code gaps. Expands SIM_SPEC to a three-layer failure decomposition. Signed-off-by: Zachery Aaron Shores-Chmielewski --- crates/simulation/Cargo.toml | 20 + crates/simulation/HOLE_PUNCH_FINDINGS.md | 76 ++ crates/simulation/SIM_SPEC.md | 487 ++++++++++- .../n3_canary_relay_real_worker.toml | 151 ++++ .../calibration/n3_own_relay_real_worker.toml | 143 ++++ .../calibration/n3_own_relay_stub.toml | 133 +++ .../scenarios/parity/reference.toml | 28 +- crates/simulation/src/bundle.rs | 27 +- crates/simulation/src/bundle_file.rs | 33 +- crates/simulation/src/engine.rs | 131 ++- crates/simulation/src/evaluator.rs | 202 +++++ crates/simulation/src/host.rs | 9 + crates/simulation/src/lib.rs | 1 + crates/simulation/src/network.rs | 524 +++++++++++- crates/simulation/src/parity_host.rs | 5 + crates/simulation/src/property.rs | 2 + crates/simulation/src/scenario.rs | 682 ++++++++++++++- crates/simulation/src/stage_host.rs | 220 +++++ crates/simulation/src/swim_host.rs | 13 + crates/simulation/tests/adversarial_judge.rs | 798 ++++++++++++++++++ crates/simulation/tests/cross_arch_parity.rs | 17 +- crates/simulation/tests/engine_invariants.rs | 6 + .../simulation/tests/evaluator_invariants.rs | 1 + .../tests/relay_assertion_invariants.rs | 385 +++++++++ crates/simulation/tests/relay_invariants.rs | 570 +++++++++++++ .../tests/relay_scenario_validation.rs | 290 +++++++ .../simulation/tests/stage_host_invariants.rs | 405 +++++++++ 27 files changed, 5246 insertions(+), 113 deletions(-) create mode 100644 crates/simulation/HOLE_PUNCH_FINDINGS.md create mode 100644 crates/simulation/scenarios/calibration/n3_canary_relay_real_worker.toml create mode 100644 crates/simulation/scenarios/calibration/n3_own_relay_real_worker.toml create mode 100644 crates/simulation/scenarios/calibration/n3_own_relay_stub.toml create mode 100644 crates/simulation/src/stage_host.rs create mode 100644 crates/simulation/tests/adversarial_judge.rs create mode 100644 crates/simulation/tests/relay_assertion_invariants.rs create mode 100644 crates/simulation/tests/relay_invariants.rs create mode 100644 crates/simulation/tests/relay_scenario_validation.rs create mode 100644 crates/simulation/tests/stage_host_invariants.rs diff --git a/crates/simulation/Cargo.toml b/crates/simulation/Cargo.toml index 1f23bcc..7c1bbc9 100644 --- a/crates/simulation/Cargo.toml +++ b/crates/simulation/Cargo.toml @@ -76,3 +76,23 @@ path = "tests/swim_integration.rs" [[test]] name = "property_runner" path = "tests/property_runner.rs" + +[[test]] +name = "relay_invariants" +path = "tests/relay_invariants.rs" + +[[test]] +name = "stage_host_invariants" +path = "tests/stage_host_invariants.rs" + +[[test]] +name = "relay_scenario_validation" +path = "tests/relay_scenario_validation.rs" + +[[test]] +name = "relay_assertion_invariants" +path = "tests/relay_assertion_invariants.rs" + +[[test]] +name = "adversarial_judge" +path = "tests/adversarial_judge.rs" diff --git a/crates/simulation/HOLE_PUNCH_FINDINGS.md b/crates/simulation/HOLE_PUNCH_FINDINGS.md new file mode 100644 index 0000000..dfb8127 --- /dev/null +++ b/crates/simulation/HOLE_PUNCH_FINDINGS.md @@ -0,0 +1,76 @@ +# Hole-punching never succeeded — N3 bundle re-analysis + +Date: 2026-05-22. Bundles: `vastai-N3-1`, `vastai-N3-2`, `vastai-N3-stub` from +`docean:/var/lib/swactor-diag/bundles/`. + +## Finding + +**No node ever held a stable direct path.** Across the 430 s N3-stub run, +every directed edge was `Relay` in every snapshot. The single non-Relay +sighting in all three bundles was stage-0→orchestrator in N3-1 going +`Relay → Mixed → Relay` over a ~24 s window, then collapsing; the reverse +direction never saw Mixed, so it was likely a transient asymmetric reading, +not a real two-way direct path. **Stage↔stage was 100% Relay in every run.** + +## Suspected cause: symmetric (endpoint-dependent) NAT on vast.ai + +Last-snapshot iroh socket counters, N3-stub: + +| node | holepunch_attempts | paths_direct | send_ipv4 | **recv_data_ipv4** | +|--------------|-------------------:|-------------:|----------:|-------------------:| +| orchestrator | 0 | 0 | 199 072 | **0** | +| stage-0 | 3 220 | 0 | 255 282 | **0** | +| stage-1 | 1 072 | 0 | 152 515 | **0** | +| stage-2 | 3 068 | 0 | 241 752 | **0** | + +Cluster-wide: ~7 360 hole-punch attempts, ~850 K direct-path datagrams sent, +**zero received**. `actor_tick_direct_addr_heartbeat = 0` everywhere (the +direct-path keepalive never ticked, because no direct path ever validated). +`portmap.upnp_available = 0`, `pcp_available = 0`, `mapping_failures ≈ +mapping_attempts` on every stage — no NAT control protocol reachable inside +vast.ai containers, so iroh can't request a stable external port. + +Ruled out as causes: + +- **Address discovery worked.** `net_report.reports_full ≥ 1` on every node; + thousands of hole-punch attempts means stages had remote candidates from + the relay's signalling path. +- **UDP isn't blocked.** Every node's `udp_echo` probes to the VPS at + 146.190.110.128:9081 had 100% reply success. Outbound delivers; return on a + pre-opened mapping delivers. +- **Relay path is fine.** All 8 `DialOutcome` events in N3-stub are `Success` + in 1.2–1.9 s. iroh just never upgrades from Relay to Direct. + +The signature — outbound delivers, return-on-existing-mapping delivers, +return-on-newly-punched-port never delivers — is the classic +endpoint-dependent-mapping fingerprint. vast.ai's container egress NAT +appears to pick a different external source port per destination, so the +candidate stage-A learned about stage-B (the port B used reflecting off the +VPS) is not the port B uses sending to A. + +## Implication + +Between vast.ai stages, Direct is structurally unreliable on this provider. +The relay is the path, not a fallback. Provision relay bandwidth/headroom +accordingly and size SWIM timeouts around relay RTT. + +## Data gaps — what would convert inference to proof + +1. **`Tier2Peer.direct_addresses` in snapshots.** Today the peer object + carries only `conn_type` and `relay_urls`. Adding iroh's + `direct_addresses` list (from `Endpoint::remote_info()`) would let us see + which candidates each side learned for each peer, instead of inferring + "they had some" from `holepunch_attempts > 0`. + +2. **UDP-echo source-port reflection.** Have the collector's UDP echo include + the observed `srcAddr:srcPort` in its reply (currently opaque). Probe from + each node to two collector destinations and compare external ports — same + port = endpoint-independent, different = symmetric. One number per node + would answer the NAT-type question definitively rather than by signature. + +3. **Per-edge bandwidth as a packaged metric.** Today reconstructed post-hoc + by summing `MessageSent.size`/`MessageReceived.size` per `(local, peer)`. + Either ship `Tier2Peer { bytes_sent_total, bytes_received_total }` (small + per-peer accumulator on the aggregator; hooks exist in `iroh_driver.rs`), + or document the post-hoc derivation so future analyses don't re-discover + it. diff --git a/crates/simulation/SIM_SPEC.md b/crates/simulation/SIM_SPEC.md index 818cf27..ba51ab6 100644 --- a/crates/simulation/SIM_SPEC.md +++ b/crates/simulation/SIM_SPEC.md @@ -22,25 +22,38 @@ the first such host kind; the second is whatever we need next. ## 0. Motivation The pipeline-parallel-inference example failed eight live N≥3 vast.ai deploys. -The session report (`N3_DEPLOYMENT_REPORT.md`, next to this file) traces the -failure to a SWIM gossip-flap bug ("B1"): in a seven-minute run, the -orchestrator refuted Suspect claims against itself 228 times — roughly once -every 1.8 seconds — and one peer ended the run marked Dead despite probes -succeeding in both directions on both sides of the link. The bug is not -visible in any test we have today. It only appears with three or more peers, -multi-region latency, and enough cumulative gossip state for piggybacked -membership updates to grow into the multi-kilobyte range. +The session report (`N3_DEPLOYMENT_REPORT.md`, next to this file) decomposes +the failure into three independent bugs stacked: -Catching that bug in production costs about two dollars of GPU rental per -attempt, forty-five to ninety minutes of engineer time per iteration, and -produces one non-reproducible bundle of evidence per run. The same source, -run twice, produces different outcomes. +- **Layer A — relay-mediated head-of-line blocking.** iroh 0.96's + `RelayMode::Default` routed gossip through n0's canary relay, which + buffered SWIM traffic for 187 seconds. A shared queue servicing + multiple peers couples otherwise-independent traffic: a 9.8 KB Ack + from one peer delays every probe behind it on the same egress. +- **Layer B — SWIM gossip-flap (B1).** In a seven-minute run, the + orchestrator refuted Suspect claims against itself 228 times — + roughly once every 1.8 seconds — and one peer ended the run marked + Dead despite probes succeeding in both directions on both sides of + the link. The bug only appears with three or more peers, multi- + region latency, and enough cumulative gossip state for piggybacked + membership updates to grow into the multi-kilobyte range. +- **Layer C — internal-cause peer death.** Pipeline stages died + 51–191 seconds into run #2 from worker-process exits, not from + network. The orchestrator recorded "connect timeout" because the + peer was gone. The failure mode is internal: the host emits a + diagnostic record and then halts of its own accord. -The simulator exists to make the iteration loop sub-second and the outcomes -byte-identical for a fixed seed. It is not a complete model of production; -it is the smallest model that lets us tune SWIM without deploying. Future -algorithms layer onto the same engine without changing the SWIM behaviour -this MVP guarantees. +Catching any of these in production costs about two dollars of GPU rental +per attempt, forty-five to ninety minutes of engineer time per iteration, +and produces one non-reproducible bundle of evidence per run. The same +source, run twice, produces different outcomes. + +The simulator exists to make the iteration loop sub-second and the +outcomes byte-identical for a fixed seed. It is not a complete model of +production; it is the smallest model that lets us reproduce A, B, and C +deterministically and tune SWIM (and the relay, and the stage lifecycle) +without deploying. Future algorithms layer onto the same engine without +changing the behaviour this MVP guarantees. --- @@ -48,18 +61,34 @@ this MVP guarantees. The MVP ships when these are simultaneously true. -A property test reproduces the gossip-flap bug deterministically against the -current SWIM source. The same scenario with the same seed produces byte- -identical output across runs and across the architectures we claim to -support. +A property test reproduces the gossip-flap bug (Layer B) deterministically +against the current SWIM source. The same scenario with the same seed +produces byte-identical output across runs and across the architectures we +claim to support. + +A property test reproduces the relay-HOL flap (Layer A) deterministically +against the current SWIM source: a multi-peer scenario routed through one +relay with bounded egress, with cumulative gossip state growing into the +multi-kilobyte range, exhibits probe timeouts that disappear when the +relay's egress capacity is widened or the message-size cap from §10.1 is +enforced. + +A scenario reproduces the run-#2 stage-death failure mode (Layer C): a +stage declared with a scheduled internal exit dies on schedule, emits a +`worker_exited` record with the declared reason, halts, and is observed by +the rest of the cluster through the same diagnostic channel production +uses. The fix workflow does not deploy. A developer writes a property, runs the -sim, sees it fail, edits the SWIM source, re-runs, sees it pass — all -locally, in under a second per iteration. +sim, sees it fail, edits the SWIM source (or the relay policy, or the +stage-supervisor lifecycle), re-runs, sees it pass — all locally, in under +a second per iteration. -The simulator is calibration-grounded against the three N3 bundles we have. -Distributions emitted by the sim, when configured to mirror a given N3 run, -are within declared tolerances of the corresponding live bundle. +The simulator is calibration-grounded against the three N3 bundles we have +(`vastai-N3-1` canary relay + real worker; `vastai-N3-2` own relay + real +worker; `vastai-N3-stub` own relay + stub worker). Distributions emitted +by the sim, when configured to mirror a given N3 run, are within declared +tolerances of the corresponding live bundle. Each known live failure mode has at least one scenario in the library, with a prose comment naming what it reproduces. @@ -125,12 +154,19 @@ given a parsed scenario, run to completion. §4 specifies behaviour. **Network.** A directed-graph link model. Answers send queries deterministically and accepts mutations on a timeline. Holds no schedule of its own; the engine pops events, the network answers questions. §5 -specifies behaviour. +specifies behaviour. The network also owns the **relay vertices** (§5A): a +relay is a first-class vertex of the network graph that is not a host. It +has one ingress queue and one egress queue per outbound link, drains them +at policy-declared capacities, and applies head-of-line ordering. Hosts do +not see whether their traffic was direct or relayed; the relay is opaque +to the host layer. **Host.** An instance of some host kind, one per peer in the scenario. The -host kind for the MVP is the production SWIM state machine wrapped in a -thin adapter. The host trait — what the engine calls and what the host -returns — is §6. +host kinds for the MVP are two: the production SWIM state machine wrapped +in a thin adapter (§6.2) and the **pipeline-stage host kind** (§6A) that +wraps the production stage supervisor lifecycle and emits the diagnostic +records the N3 deployment report's C.1–C.3 findings require. The host +trait — what the engine calls and what the host returns — is §6. **Bundle writer.** The only filesystem-touching component. Receives event and snapshot records from the engine, serializes them to the production @@ -503,13 +539,33 @@ Supported kinds: - `LossBurst { links, prob_ppm, duration_ns }` — override loss probability on named links for a duration. - `RelayBuffer { links, floor_ns, duration_ns }` — impose a minimum - delivery delay on named links for a duration. + delivery delay on named links for a duration. (Legacy per-edge floor; + the relay vertex of §5A models shared-queue HOL more faithfully.) - `PeerKill { peer }` — drop the peer's inbox. In-flight deliveries to the peer are invalidated. The peer's ticks are stopped by the engine. - `PeerResurrect { peer, preserve_state }` — restart the peer. If `preserve_state`, the engine reuses the host instance; otherwise a fresh host of the same kind is instantiated from the scenario's peer declaration. +- `WorkerExit { peer, reason, status_code?, signal? }` — delivers a + `WorkerExit` envelope to the named peer at the mutation's time via the + same recv path that `TimerFired` and `SendFailed` use today. The host + kind decides what to do with it; the stage host (§6A) emits a + `worker_exited` event and returns `Halt`. Targeting a host kind that + does not accept the envelope (e.g. the SWIM kind) aborts the run with a + structured error — silent fallback is the bug class the simulator + exists to prevent. +- `RelayKill { relay }` — the relay stops forwarding. All currently + queued messages are returned in the invalidated-deliveries list. + Subsequent sends through the relay drop with reason `RelayDown`. +- `RelayBoot { relay }` — the relay returns to service. Its queues are + empty; the next forwarded message pays the `cold_start_penalty_ns`. +- `RelayCapacityChange { relay, ingress_capacity_bps?, + egress_capacity_bps_per_link?, queue_depth_bytes? }` — at the named + time, replace any subset of the three relay policy fields (§5A.2). + In-flight messages already past ingress complete at their previously- + computed arrival times; messages that arrive after the mutation use + the new policy. ### 5.6 Determinism within the network @@ -579,6 +635,121 @@ returns. --- +## 5A. The relay vertex + +A relay is a vertex of the network's directed graph that is not a host. +Relay IDs share the ID namespace with host IDs (§5.1 and §8 of this +spec together require uniqueness across both sets). An edge's `from` +and `to` may name either. + +A route between two hosts is either direct — exactly one edge from `A` +to `B` — or relayed — an edge `A→R`, the relay `R`, and an edge +`R→B`. Multi-hop relayed routes (`A→R₁→R₂→B`) are out of scope; the +scenario loader rejects them in §8.2. + +If a scenario declares both a direct `A→B` edge and a relayed `A→R→B` +route, the scenario loader rejects the ambiguity. Each ordered host +pair has at most one route. + +### 5A.2 Relay policy + +Each relay declares the following integer-valued fields: + +- `ingress_capacity_bps` — maximum bytes per second the relay accepts + *across all inbound links combined*. +- `egress_capacity_bps_per_link` — maximum bytes per second the relay + serves *per outbound link*. +- `queue_depth_bytes` — maximum bytes buffered across all egress + queues combined. A message that would push the total beyond this + limit at enqueue time is dropped. +- `queue_discipline` — `Fifo` is the only value the MVP accepts. +- `cold_start_penalty_ns` — extra latency added to the first message + the relay forwards after a `RelayBoot` mutation. + +A relay has no jitter, loss, or cache-state fields of its own; the +edges feeding it carry their own such fields per §5.2. A relay's drop +reasons are exclusively queue-overflow and `RelayDown`; lossy drops +remain a property of edges. + +### 5A.3 The forward algorithm + +When the network receives `send(from, to, byte_len, sent_at_ns)` and +the configured route is relayed through `R`, the composition is: + +1. Resolve the inbound edge `from → R`. Apply §5.4 for the inbound + leg. If that leg drops, the composed send drops with the same + reason; the relay's state is not consulted. +2. At time `arrival_at_R`, attempt to enqueue at the relay. If + `enqueued_bytes + byte_len > queue_depth_bytes`, the composed send + drops with reason `RelayQueueFull` and the relay emits a + `RelayDrop` record (§9.1). +3. Otherwise, compute the ingress serialization end: + `max(arrival_at_R, ingress_queue_tail_ns) + + (byte_len * 1_000_000_000) / ingress_capacity_bps`. +4. Compute the egress serialization start on the outbound link to + `to`: `max(ingress_end, egress_queue_tail_ns[to])`. If the relay + is `Booting`, add `cold_start_penalty_ns` and transition to + `Booted`. Compute `egress_end = egress_start + (byte_len * + 1_000_000_000) / egress_capacity_bps_per_link`. +5. Resolve the outbound edge `R → to` per §5.4 with `sent_at_ns = + egress_end`. Its arrival is the composed send's arrival. +6. The relay emits a `RelayEnqueue` at `arrival_at_R` and a + `RelayDequeue` at `egress_end` (§9.1). + +The composed send returns one `Arrive(at_ns)` or one `Drop(reason)`; +the relay's internal events are surfaced through the side channel +that §3.2 already names for cache state changes. + +A relay's egress and ingress are independent: the ingress can be +serializing a new message while the egress is still draining an old +one. Head-of-line blocking arises only when two messages share an +egress link (or the single ingress). + +### 5A.4 Determinism within the relay + +The relay draws no randomness in the MVP. Iteration over per-egress- +link state inside a relay is by destination host ID in lexicographic +order, per §7.3. + +### 5A.5 Behavioral tests + +The relay's tests assert that it has the properties below; how each +is verified is the test author's call. (Tests live in +`tests/relay_invariants.rs`.) + +- **Composition is transparent to hosts.** A send through a relayed + route returns one `SendOutcome` shaped identically to a direct + send's. The receiving host cannot distinguish a relayed delivery + from a direct one by the message it sees. +- **HOL is observable and bounded.** Two messages sharing an egress + arrive in send order, separated by at least the first message's + egress serialization time. +- **Ingress and egress are independent.** A message destined for peer + X does not delay a message destined for peer Y on the egress side. +- **Queue overflow is exact.** A send that would push + `enqueued_bytes` strictly above `queue_depth_bytes` at its + arrival-at-relay time drops with `RelayQueueFull` and emits a + `RelayDrop` record. A send that exactly fills the queue is + accepted. +- **Cold-start penalty is paid once per boot.** After a `RelayBoot`, + the first forwarded message includes the cold-start penalty; the + second does not. +- **Mutation invalidation is exact.** A `RelayKill` returns exactly + the deliveries in-flight through the relay at the mutation's + virtual time and no others. A `RelayCapacityChange` invalidates no + deliveries. +- **Ambiguous routes are rejected statically.** A scenario declaring + both a direct edge `A→B` and a relayed route `A→R→B`, or two + relayed routes for the same ordered host pair, is rejected by the + loader. +- **No multi-hop in MVP.** A scenario declaring a route that + traverses two relays is rejected by the loader. +- **Determinism.** Same topology, same seed, same query sequence ⇒ + identical composed `SendOutcome` sequence and identical + `RelayEnqueue` / `RelayDequeue` / `RelayDrop` record streams. + +--- + ## 6. Hosting an entity ### 6.1 The host trait @@ -675,6 +846,136 @@ Silent fallback is a test failure. --- +## 6A. The pipeline-stage host kind + +The stage host wraps the production stage supervisor — the worker +lifecycle implemented in `examples/pipeline-parallel-inference` and +running today as `pp-gpu-node`. The wrap is structurally analogous +to §6.2's SWIM host. + +The MVP stage host does not engage in inter-stage application traffic +(activation forwarding, KV-cache updates). The failures Layer C +exhibits are lifecycle failures, not application-protocol failures; +inter-stage traffic is a strict superset and is out of scope for this +extension. + +### 6A.2 Lifecycle + +The stage host moves through four states, each transition emitting +exactly one diagnostic record: + +- `Cold` — initial state on `new_from_config`. No actions until the + first `tick`. +- `Registering` — entered on the first `tick`. The host emits a + `register_name` event (§9.1) with the host's declared name and + address, then transitions to `Running`. +- `Running` — steady state. The host emits no further state- + transition records on its own. +- `Halted` — entered on receipt of a `WorkerExit { reason }` envelope + or after a `PeerKill` mutation. The host emits a `worker_exited` + event (§9.1) with the reason, then returns `Halt`. + +Transitions are linear: `Cold → Registering → Running → Halted`. +There is no resurrection. A `PeerResurrect` mutation against a stage +host produces a fresh instance from the scenario's peer declaration +per §5.5; the resurrected instance starts in `Cold`. + +### 6A.3 Internal-cause exit + +A `WorkerExit { reason }` envelope arrives via the same recv path as +`TimerFired` and `SendFailed`. The stage host's `recv` for that +envelope returns exactly three actions, in order: + +1. `RecordEvent` carrying a `worker_exited` event whose payload + names the reason verbatim. +2. `RecordEvent` carrying a `stage_lifecycle` event from the current + state to `Halted`. +3. `Halt`. + +The order is normative: the event must reach the bundle writer +before the engine acts on the halt. + +A `WorkerExit` mutation is dispatched **synchronously** — the +engine calls the target host's `recv` and processes the returned +actions inside the same `dispatch_mutation` call that records the +mutation, before the main loop pops the next event. This is the +only way to guarantee §6A.6's boundary case: a `WorkerExit` whose +`at_ns` equals the scenario's `duration_ns` must still produce a +`worker_exited` record. Routing the envelope through the queue +(an enqueued `LocalRecv`) would race with the construction-time +`Terminate` at the same virtual time and could silently lose the +event — the exact failure mode the boundary clause forbids. + +The SWIM host kind has no `WorkerExit` semantics. A `WorkerExit` +mutation targeting a SWIM-kind peer aborts the run with a structured +error (`EngineAbort::WorkerExitOnWrongKind`). Silent acceptance is +the failure mode this simulator exists to prevent. + +### 6A.4 Diagnostic surface + +The stage host emits three event kinds in addition to anything the +production stage supervisor already emits: + +- `register_name { name, address, peer_node_id }` — emitted exactly + once per stage instance, at the transition `Registering → Running`. +- `worker_exited { reason, status_code, signal }` — emitted exactly + once per stage instance, immediately before `Halt`. +- `stage_lifecycle { from, to }` — emitted on every state transition + the §6A.2 lifecycle declares. + +Each kind's payload schema is named here for the production schema to +follow. The deployment report's items C.1 and C.3 name +`register_name` and `worker_exited` respectively; this spec fixes +their shapes so the sim and production cannot drift. + +The stage host's `snapshot()` returns a JSON object with the fields: + +- `state` — one of the four §6A.2 lifecycle states. +- `name_registry` — a map of `name → address` for every name this + host has registered. (The deployment report's item C.2 names the + absence of this field as a debugging gap; this spec requires it.) +- `last_exit_reason` — present only when `state == Halted`. + +### 6A.5 Codec + +The stage host kind's codec exists for parity with §3.3. In the MVP +its message-type is empty: the stage host produces no `Send` actions +during its lifecycle. When inter-stage traffic enters scope in a +later revision, the codec gains the production wire format. + +### 6A.6 Behavioral tests + +The stage host's tests assert that it has the properties below. +(Tests live in `tests/stage_host_invariants.rs`.) + +- **Trait conformance.** `kind_tag()` returns `"stage"`, distinct + from every other registered kind's. +- **Lifecycle linearity.** Every stage instance traverses the §6A.2 + states in declared order, never revisits a state, and never skips + one. Each transition emits exactly one `stage_lifecycle` event. +- **`register_name` exactly once.** Across the lifetime of one + instance, exactly one `register_name` event is emitted, at the + `Registering → Running` transition. +- **`worker_exited` exactly once.** Across the lifetime of one + instance, exactly one `worker_exited` event is emitted, + immediately before the `Halt` action that ends the instance. The + event's `reason` field is byte-equal to the mutation's `reason`. +- **Event-before-halt is observable.** A scenario whose + `duration_ns` is the same nanosecond as a `WorkerExit` mutation's + `at_ns` produces a bundle containing the `worker_exited` event. +- **`WorkerExit` against SWIM aborts.** A `WorkerExit` mutation + targeting a SWIM-kind peer aborts the run with a structured error + (`EngineAbort::WorkerExitOnWrongKind`) naming the host's kind and + the mutation's index. +- **Snapshot contains the registry.** A stage host's `snapshot()` + includes a `name_registry` field; every `register_name` event the + host has emitted appears in the map at every subsequent snapshot. +- **Determinism.** Same `HostKindConfig`, same RNG seed, same + envelope sequence ⇒ identical action sequence and identical + emitted event stream. + +--- + ## 7. Determinism This section is normative. A violation is a ship-blocker. @@ -769,15 +1070,32 @@ overrides under `[[links]]` may override any subset. A `[[peers]]` array, each entry: - `id: String`. -- `kind: String` — selects the host kind. +- `kind: String` — selects the host kind. The MVP recognises `"swim"` + and `"stage"`; the relay extension's `stage` kind_config is + `{ name: String, address: String }` (RELAY/STAGE §6A). - `kind_config: { ... }` — host-kind-specific opaque table. -- `initial_state: String` — host-kind-specific. +- `initial_state: String` — host-kind-specific. For `stage`, the only + legal value is `"cold"` (mirrors §6A.2's `Cold` initial state). - `tick_period_ns_override: u64` (optional). +A `[[relays]]` array (relay extension), each entry: + +- `id: String` — unique across the union of peer ids and relay ids. +- `ingress_capacity_bps: u64` — combined ingress bytes/sec. +- `egress_capacity_bps_per_link: u64` — per-outbound-link bytes/sec. +- `queue_depth_bytes: u64` — shared egress buffer bound. +- `cold_start_penalty_ns: u64` (default `0`). + A `[[links]]` array, each entry: - `from: String`. - `to: String`. +- `via: String` (optional) — the ID of a relay through which this + edge is routed. When `via` is set, the edge's `from` and `to` must + both be host IDs; the loader synthesizes the `from → via` and + `via → to` inbound and outbound legs and registers a relayed + `HostRoute` for the host pair. Multiple `via` shorthands through + the same relay are allowed iff their leg policies agree. - Any subset of the §5.2 fields (overrides on top of `[default_link]`). A `[[mutations]]` array, each entry: @@ -812,6 +1130,23 @@ The loader rejects: - A `default_link` field that is non-integer, negative, or in a unit other than the §5.2 names (e.g., `latency_ms` is rejected; only `latency_ns`). +Relay extension rules (added by RELAY/STAGE): + +- Duplicate IDs across the union of `[[peers]]` and `[[relays]]`. +- A `via` reference to a peer ID rather than a relay ID. +- A peer pair declared with both a direct edge and a relayed route + (the loader-level ambiguity check). +- A `[[links]]` entry whose `from` or `to` is a relay and that also + carries a `via` field. +- A multi-hop relayed route (any direct relay→relay edge, which would + make a chained route possible). +- A `worker_exit` mutation whose target peer is not stage-kind. +- A `relay_capacity_change` mutation with no fields set (the mutation + must change at least one of the three policy fields). +- A `stage` peer whose `kind_config.name` is missing or whose + `kind_config.address` is missing or empty (delegated to + `StageHostKindValidator`). + Validation failures produce a structured error with the file path, the offending field, and a one-line explanation. @@ -903,6 +1238,18 @@ Event kinds the MVP emits: - Engine-synthesized cache state changes, dial start and outcome, drop on send, drop on delivery, send-failure errors. - Mutation records. +- Stage host (RELAY/STAGE §6A.4): `register_name`, `worker_exited`, + `stage_lifecycle`. Each carries `kind_tag = "stage"` and the + emitting host's id. +- Relay subsystem (RELAY/STAGE §5A.3): `RelayEnqueue`, `RelayDequeue`, + `RelayDrop`. Each carries `kind_tag = "relay"` and `host_id = null` + (the relay is not a host). + +The stage host's three kinds are introduced *ahead* of production on +the basis of the deployment report's items C.1–C.3. The production +stage supervisor must adopt the same schemas as part of landing this +spec; otherwise the contract test against the production diagnostics +emitter fails and the sim cannot reproduce Layer C without divergence. ### 9.3 Snapshots @@ -992,6 +1339,25 @@ Each assertion kind has a name and a parameter shape. The MVP catalog: kind across the run. - `event_rate { kind, window_ns, max_per_window }` — bounds the rate of an event kind. +- `relay_queue_depth_bounded { relay, max_bytes, window_start_ns?, + window_end_ns? }` (RELAY/STAGE) — across the window (defaults to + the whole run), the named relay's `enqueued_bytes` never exceeds + `max_bytes`. Fails on the first `RelayEnqueue` that pushes the + total above the bound. +- `worker_alive_throughout { peer, window_start_ns, window_end_ns }` + (RELAY/STAGE) — the named peer's lifecycle state remains `Running` + throughout the window. Fails on any `stage_lifecycle` event into + `Halted` whose time falls in the window. The window is inclusive + on both ends: a halt at `window_end_ns` (or at `duration_ns` when + the window spans the whole run) fails the assertion, because the + §6A.3 synchronous dispatch rule guarantees the + `stage_lifecycle → Halted` event appears in the bundle even at + the boundary. +- `name_resolves_within { name, observers, within_ns, from_ns }` + (RELAY/STAGE) — starting at `from_ns`, every observer in + `observers` produces a snapshot whose `name_registry` contains + `name` within `within_ns`. `Inconclusive` if no observer produces + a snapshot in the window. Adding a kind is a deliberate amendment to this section. @@ -1083,7 +1449,25 @@ bundles. The MVP corpus is the three N3 vast.ai bundles described in the deployment report. Each pairs with a scenario in `scenarios/calibration/` -that approximates the conditions under which the bundle was produced. +that approximates the conditions under which the bundle was produced: + +- `vastai-N3-1` ↔ `n3_canary_relay_real_worker.toml`. Canary relay, + real worker. The relay's policy is set to the canary's measured + shape (low egress per link, large queue, multi-hundred-millisecond + cold start). Stage hosts carry the worker-exit timing observed in + the bundle. +- `vastai-N3-2` ↔ `n3_own_relay_real_worker.toml`. Same topology + with the relay's policy widened to the own-relay's measured shape. + Stage hosts carry the same worker-exit timing. +- `vastai-N3-stub` ↔ `n3_own_relay_stub.toml`. Same topology, + widened relay, stage hosts with no `WorkerExit` mutations (full- + run survival). Reproduces the gossip-flap pathology without stage + death. + +Each scenario carries a top-of-file prose comment naming the bundle +it pairs with, the placeholder tolerances that the first calibration +pass will replace with measured numbers, and any base scenario it +extends. ### 11.2 The procedure @@ -1105,6 +1489,20 @@ For each pair, the calibration tool: - Dial-started count. - Per-peer fraction of run time in the Alive state. +Relay extension metrics (RELAY/STAGE §10.2): + +- Relay queue-depth distribution over time (p50, p90, p99 of + `enqueued_bytes`). +- Per-link HOL delay decomposition: each delivery's delay attributed + to base latency, edge-bandwidth serialization, jitter, cold-dial + penalty, relay ingress, relay egress, and active mutations. +- Probe-success-vs-transition ratio: fraction of `Suspect` + transitions in which the observer's and the peer's bidirectional + probes were succeeding at the transition time. +- Worker-exit reason distribution per peer. +- Name-resolution latency per registered name (time from + `register_name` to first observer snapshot containing the name). + Tolerances ship as placeholders informed by intuition; the first calibration pass against the N3 corpus sets the real numbers. @@ -1131,12 +1529,20 @@ simulator does something useful and is testable. | 5 | Assertion evaluator | The gossip-flap reproduction scenario fails on current SWIM, passes after the fix. | | 6 | Bundle writer | A sim bundle renders through the production post-processor. | | 7 | Calibration tool | At least one N3 pair passes calibration. | +| R1 | Relay vertex (§5A) without mutations | A direct send through a relay arrives later than the same send over a direct edge of equal policy by the relay's ingress + egress serialization time. | +| R2 | Relay mutations (§5.5 additions) | A `RelayKill` followed by a `RelayBoot` produces a bundle in which the in-flight messages at kill-time appear in the invalidated-deliveries list and no others. | +| R3 | Stage host kind (§6A) and `WorkerExit` mutation | A scenario with one stage host and one `WorkerExit` mutation produces a bundle containing exactly one `register_name`, one `worker_exited`, and three `stage_lifecycle` records, in §6A.2 order. | +| R4 | Assertion catalog additions (§10.1 R-tail) | The N3-1 calibration scenario fails `relay_queue_depth_bounded` against the canary-relay policy and passes against the own-relay policy. | +| R5 | Scenario loader additions (§8) and bundle additions (§9) | Every shipped calibration scenario parses; bundles render through the production post-processor with the new event kinds passed through unchanged. | +| R6 | Calibration tool extensions (§11) | At least one of the three N3 pairs passes calibration on every declared metric. | -MVP exit is the end of phase 7. Subsequent phases — proptest catalog +MVP exit is the end of phase R6. Subsequent phases — proptest catalog expansion, scenario library growth — are post-MVP. Phases 1, 2, 6 are independently buildable by separate agents from this -spec alone; phases 3 onwards require the prior phase as input. +spec alone; phases 3 onwards require the prior phase as input. The +relay phases R1–R3 are independently buildable from the §5A / §6A +sections alone; R4–R6 require their predecessors. --- @@ -1187,6 +1593,9 @@ must surface in code review. - `examples/pipeline-parallel-inference/N3_DEPLOYMENT_REPORT.md` — source of truth for the live failures the simulator must reproduce. + Items A, B2, C.1, C.2, and C.3 of that report map to §5A, + §11.3 (metrics), §6A.4 + §9.1 (`register_name`), §6A.4 (snapshot + registry), and §6A.4 (`worker_exited`) respectively. - `crates/simulation/NORTH_STAR.md` — the long-term simulator vision. This MVP is a strict subset and does not retract any of its claims. - `crates/simulation/BLOCKED.md` — the staged plan this MVP supersedes @@ -1197,3 +1606,9 @@ must surface in code review. the SWIM host kind wraps. - `crates/distribution/src/diagnostics/` — the schema the bundle must match and the renderer it must render through. +- `crates/distribution/src/bin/swactor-iroh-relay.rs` — the production + relay binary the §5A relay vertex models. The §9.2 parity contract + names this binary's emissions as the production side once it gains + matching schemas. +- `examples/pipeline-parallel-inference/src/bin/pp_gpu_node.rs` — + the production stage supervisor the §6A host kind wraps. diff --git a/crates/simulation/scenarios/calibration/n3_canary_relay_real_worker.toml b/crates/simulation/scenarios/calibration/n3_canary_relay_real_worker.toml new file mode 100644 index 0000000..b625cef --- /dev/null +++ b/crates/simulation/scenarios/calibration/n3_canary_relay_real_worker.toml @@ -0,0 +1,151 @@ +# Calibration scenario paired with bundle `vastai-N3-1`. +# RELAY_SPEC §10.1. +# +# Topology: three SWIM/stage hosts (orchestrator + two pipeline stages) +# routed through one relay. The relay's policy mimics the n0 canary +# cluster as observed in run #1: very low egress per link, a large +# shared queue, and a multi-hundred-millisecond cold-start. Stage hosts +# carry the worker-exit timing observed in the bundle (51s / 75s / 191s +# in run #1's siblings; this scenario uses the most aggressive 51s +# value because it is the assertion that bites first). +# +# Expected verdict on the as-shipped relay policy: `relay_queue_depth_bounded` +# Fails (the canary relay's queue grows past the bound at multi-kilobyte +# gossip volumes — RELAY_SPEC §10.2 "calibration scenario"); the run's +# stage worker_exits surface the C.3 finding deterministically. +# +# All numeric fields below are placeholders. The first calibration pass +# (a follow-up commit per SIM_SPEC §11.3) replaces the placeholder +# tolerances with measured numbers against the actual bundle. Until +# then, these values exist so the scenario parses, runs, and surfaces +# the right shape of failure to a human reader. + +name = "n3_canary_relay_real_worker" +seed = 1 +duration_ns = 425_000_000_000 # 425 s — the live run's duration + +[default_tick] +period_ns = 200_000_000 # 200 ms + +[default_link] +latency_ns = 60_000_000 # 60 ms one-way, multi-region +jitter_stddev_ns = 15_000_000 # 15 ms +loss_prob_ppm = 0 +reorder_prob_ppm = 0 +bandwidth_bps = 25_000_000 # 25 Mb/s +cold_dial_penalty_ns = 200_000_000 +cache_warm_after_ns = 200_000_000 +cache_invalidate_after_idle_ns = 30_000_000_000 + +# Canary relay policy (placeholders pending first calibration pass). +# Low egress per link + a generous queue is what reproduces the +# 187-second buffering pathology observed at t=556598..743981 in +# `vastai-N3-1`. +[[relays]] +id = "canary" +ingress_capacity_bps = 100_000_000 # 100 Mb/s combined ingress +egress_capacity_bps_per_link = 1_000_000 # 1 Mb/s per outbound link — the bottleneck +queue_depth_bytes = 524_288 # 512 KB shared egress buffer +cold_start_penalty_ns = 500_000_000 # 500 ms first-message warmup + +[[peers]] +id = "orchestrator" +kind = "swim" +initial_state = "alive" +kind_config = { probe_interval_ns = 1_000_000_000, suspicion_timeout_ns = 5_000_000_000 } + +[[peers]] +id = "stage_0" +kind = "stage" +initial_state = "cold" +kind_config = { name = "pp-stage-0", address = "10.0.0.10:7700" } + +[[peers]] +id = "stage_1" +kind = "stage" +initial_state = "cold" +kind_config = { name = "pp-stage-1", address = "10.0.0.11:7700" } + +# Every host pair routes through the canary relay. +[[links]] +from = "orchestrator" +to = "stage_0" +via = "canary" +[[links]] +from = "stage_0" +to = "orchestrator" +via = "canary" +[[links]] +from = "orchestrator" +to = "stage_1" +via = "canary" +[[links]] +from = "stage_1" +to = "orchestrator" +via = "canary" +[[links]] +from = "stage_0" +to = "stage_1" +via = "canary" +[[links]] +from = "stage_1" +to = "stage_0" +via = "canary" + +# Stage worker_exits at the live timings (RELAY_SPEC §5.3 / C.3 of the +# deployment report). Used by `worker_alive_throughout` below. +[[mutations]] +at_ns = 51_000_000_000 # 51 s — the earliest of the live worker exits +kind = "worker_exit" +peer = "stage_0" +reason = "tinygrad worker crashed (placeholder)" +status_code = 1 + +[[mutations]] +at_ns = 191_000_000_000 # 191 s — second stage exits later +kind = "worker_exit" +peer = "stage_1" +reason = "tinygrad worker crashed (placeholder)" +status_code = 1 + +# Snapshots scattered so `name_resolves_within` can observe registry +# convergence — or its absence. +[[snapshots]] +at_ns = 1_000_000_000 +[[snapshots]] +at_ns = 10_000_000_000 +[[snapshots]] +at_ns = 50_000_000_000 +[[snapshots]] +at_ns = 200_000_000_000 +[[snapshots]] +at_ns = 400_000_000_000 + +# RELAY_SPEC §7.1 assertions over the calibration run. +# +# `relay_queue_depth_bounded` is the load-bearing assertion: under the +# canary relay's policy, cumulative gossip pushes `enqueued_bytes` +# past the bound, mirroring the 187-second buffering signature. +[[assertions]] +kind = "relay_queue_depth_bounded" +relay = "canary" +max_bytes = 65_536 # 64 KB — placeholder tolerance + +# `worker_alive_throughout` over the first minute fails on stage_0's +# 51s exit, which the deployment report calls out as the canonical +# Layer C symptom. +[[assertions]] +kind = "worker_alive_throughout" +peer = "stage_0" +window_start_ns = 0 +window_end_ns = 60_000_000_000 + +# Name registry convergence: orchestrator should learn pp-stage-0 +# within the resolve deadline. Under the canary buffering this is the +# observable that fails — `pp-entry never resolves` in the live run. +[[assertions]] +kind = "name_resolves_within" +name = "pp-stage-0" +observers = ["orchestrator"] +within_ns = 30_000_000_000 +from_ns = 0 diff --git a/crates/simulation/scenarios/calibration/n3_own_relay_real_worker.toml b/crates/simulation/scenarios/calibration/n3_own_relay_real_worker.toml new file mode 100644 index 0000000..68419e0 --- /dev/null +++ b/crates/simulation/scenarios/calibration/n3_own_relay_real_worker.toml @@ -0,0 +1,143 @@ +# Calibration scenario paired with bundle `vastai-N3-2`. +# RELAY_SPEC §10.1. +# +# Topology: same three SWIM/stage hosts and one relay as +# `n3_canary_relay_real_worker.toml`, but with the relay widened to the +# own-relay's measured shape (high egress per link, modest queue, no +# cold-start penalty). The worker_exit mutations stay — under run #2 +# the relay is no longer the bottleneck but the stages still die at +# 75s / 191s / 51s, proving stage death is a worker bug, not a cluster +# bug (deployment report TL;DR Layer C). +# +# Expected verdict under the widened policy: +# - `relay_queue_depth_bounded` Passes (the own relay does not buffer +# gossip beyond the bound). +# - `worker_alive_throughout` over [0, 60s] still Fails on stage_0's +# 51s exit. Stage death is independent of the relay fix. +# - `name_resolves_within` Passes for pp-stage-0 (the own relay +# forwards SWIM gossip in seconds, not minutes). +# +# Placeholder numerics — first calibration pass replaces them with +# measured values from the bundle in a follow-up commit per +# SIM_SPEC §11.3. + +name = "n3_own_relay_real_worker" +seed = 2 +duration_ns = 412_000_000_000 # 412 s — the live run's duration + +[default_tick] +period_ns = 200_000_000 # 200 ms + +[default_link] +latency_ns = 60_000_000 +jitter_stddev_ns = 15_000_000 +loss_prob_ppm = 0 +reorder_prob_ppm = 0 +bandwidth_bps = 25_000_000 +cold_dial_penalty_ns = 200_000_000 +cache_warm_after_ns = 200_000_000 +cache_invalidate_after_idle_ns = 30_000_000_000 + +# Own-relay policy: wider egress, smaller queue, no cold start. This +# is the only difference from `n3_canary_relay_real_worker.toml`. +[[relays]] +id = "own_relay" +ingress_capacity_bps = 1_000_000_000 # 1 Gb/s combined ingress +egress_capacity_bps_per_link = 100_000_000 # 100 Mb/s per outbound link +queue_depth_bytes = 65_536 # 64 KB +cold_start_penalty_ns = 0 + +[[peers]] +id = "orchestrator" +kind = "swim" +initial_state = "alive" +kind_config = { probe_interval_ns = 1_000_000_000, suspicion_timeout_ns = 5_000_000_000 } + +[[peers]] +id = "stage_0" +kind = "stage" +initial_state = "cold" +kind_config = { name = "pp-stage-0", address = "10.0.0.10:7700" } + +[[peers]] +id = "stage_1" +kind = "stage" +initial_state = "cold" +kind_config = { name = "pp-stage-1", address = "10.0.0.11:7700" } + +[[links]] +from = "orchestrator" +to = "stage_0" +via = "own_relay" +[[links]] +from = "stage_0" +to = "orchestrator" +via = "own_relay" +[[links]] +from = "orchestrator" +to = "stage_1" +via = "own_relay" +[[links]] +from = "stage_1" +to = "orchestrator" +via = "own_relay" +[[links]] +from = "stage_0" +to = "stage_1" +via = "own_relay" +[[links]] +from = "stage_1" +to = "stage_0" +via = "own_relay" + +# Same worker_exits as run #2's live timings: stage_0 at 75s +# (the report quotes 75s for stage-0 in run #2), stage_1 at 191s. +[[mutations]] +at_ns = 75_000_000_000 +kind = "worker_exit" +peer = "stage_0" +reason = "tinygrad worker crashed (placeholder)" +status_code = 1 + +[[mutations]] +at_ns = 191_000_000_000 +kind = "worker_exit" +peer = "stage_1" +reason = "tinygrad worker crashed (placeholder)" +status_code = 1 + +[[snapshots]] +at_ns = 1_000_000_000 +[[snapshots]] +at_ns = 10_000_000_000 +[[snapshots]] +at_ns = 50_000_000_000 +[[snapshots]] +at_ns = 200_000_000_000 +[[snapshots]] +at_ns = 400_000_000_000 + +# `relay_queue_depth_bounded` should Pass now (the calibration pass +# will verify the actual `enqueued_bytes` distribution stays under the +# bound, separating run #2 from run #1). +[[assertions]] +kind = "relay_queue_depth_bounded" +relay = "own_relay" +max_bytes = 65_536 + +# `worker_alive_throughout` still Fails on stage_0's 75s exit — the +# Layer C bug is independent of the relay fix. +[[assertions]] +kind = "worker_alive_throughout" +peer = "stage_0" +window_start_ns = 0 +window_end_ns = 90_000_000_000 + +# Name registry: pp-stage-0 should resolve fast under the own-relay +# policy. +[[assertions]] +kind = "name_resolves_within" +name = "pp-stage-0" +observers = ["orchestrator"] +within_ns = 5_000_000_000 +from_ns = 0 diff --git a/crates/simulation/scenarios/calibration/n3_own_relay_stub.toml b/crates/simulation/scenarios/calibration/n3_own_relay_stub.toml new file mode 100644 index 0000000..964da81 --- /dev/null +++ b/crates/simulation/scenarios/calibration/n3_own_relay_stub.toml @@ -0,0 +1,133 @@ +# Calibration scenario paired with bundle `vastai-N3-stub`. +# RELAY_SPEC §10.1. +# +# Topology: same own-relay shape as `n3_own_relay_real_worker.toml`, +# but with stage hosts that never receive a `worker_exit` mutation — +# the stub worker (`PP_WORKER_STUB=1`) keeps the stage process alive +# for the full run. This isolates Layer B (the SWIM gossip-flap from +# `gossip_flap.toml`) from Layer C (the worker exit). The relay +# bottleneck is gone, the workers stay up, and we observe the +# remaining gossip-flap pathology in isolation. +# +# Expected verdict: +# - `relay_queue_depth_bounded` Passes (no gossip backlog). +# - `worker_alive_throughout` Passes over the full run (stages live). +# - `name_resolves_within` Passes (own-relay forwards in seconds). +# +# The interesting follow-up assertion this scenario unblocks is the +# parent spec's `self_incarnation_bounded` and `no_flap_while_probes_ok` +# — Layer B. Those are already exercised by `gossip_flap.toml`; the +# calibration scenario keeps the topology + relay shape so the §10.2 +# metrics (HOL decomposition, probe_success_vs_transition) compare +# against the bundle on a like-for-like cluster. + +name = "n3_own_relay_stub" +seed = 3 +duration_ns = 420_000_000_000 # 7 min — the live run's duration + +[default_tick] +period_ns = 200_000_000 # 200 ms + +[default_link] +latency_ns = 60_000_000 +jitter_stddev_ns = 15_000_000 +loss_prob_ppm = 0 +reorder_prob_ppm = 0 +bandwidth_bps = 25_000_000 +cold_dial_penalty_ns = 200_000_000 +cache_warm_after_ns = 200_000_000 +cache_invalidate_after_idle_ns = 30_000_000_000 + +[[relays]] +id = "own_relay" +ingress_capacity_bps = 1_000_000_000 +egress_capacity_bps_per_link = 100_000_000 +queue_depth_bytes = 65_536 +cold_start_penalty_ns = 0 + +[[peers]] +id = "orchestrator" +kind = "swim" +initial_state = "alive" +kind_config = { probe_interval_ns = 1_000_000_000, suspicion_timeout_ns = 5_000_000_000 } + +[[peers]] +id = "stage_0" +kind = "stage" +initial_state = "cold" +kind_config = { name = "pp-stage-0", address = "10.0.0.10:7700" } + +[[peers]] +id = "stage_1" +kind = "stage" +initial_state = "cold" +kind_config = { name = "pp-stage-1", address = "10.0.0.11:7700" } + +[[links]] +from = "orchestrator" +to = "stage_0" +via = "own_relay" +[[links]] +from = "stage_0" +to = "orchestrator" +via = "own_relay" +[[links]] +from = "orchestrator" +to = "stage_1" +via = "own_relay" +[[links]] +from = "stage_1" +to = "orchestrator" +via = "own_relay" +[[links]] +from = "stage_0" +to = "stage_1" +via = "own_relay" +[[links]] +from = "stage_1" +to = "stage_0" +via = "own_relay" + +# Crucially: no worker_exit mutations. Stages live the full run. + +[[snapshots]] +at_ns = 1_000_000_000 +[[snapshots]] +at_ns = 30_000_000_000 +[[snapshots]] +at_ns = 120_000_000_000 +[[snapshots]] +at_ns = 300_000_000_000 +[[snapshots]] +at_ns = 419_000_000_000 + +[[assertions]] +kind = "relay_queue_depth_bounded" +relay = "own_relay" +max_bytes = 65_536 + +[[assertions]] +kind = "worker_alive_throughout" +peer = "stage_0" +window_start_ns = 0 +window_end_ns = 420_000_000_000 + +[[assertions]] +kind = "worker_alive_throughout" +peer = "stage_1" +window_start_ns = 0 +window_end_ns = 420_000_000_000 + +[[assertions]] +kind = "name_resolves_within" +name = "pp-stage-0" +observers = ["orchestrator"] +within_ns = 5_000_000_000 +from_ns = 0 + +[[assertions]] +kind = "name_resolves_within" +name = "pp-stage-1" +observers = ["orchestrator"] +within_ns = 5_000_000_000 +from_ns = 0 diff --git a/crates/simulation/scenarios/parity/reference.toml b/crates/simulation/scenarios/parity/reference.toml index 6ce0fff..be7ca4c 100644 --- a/crates/simulation/scenarios/parity/reference.toml +++ b/crates/simulation/scenarios/parity/reference.toml @@ -2,6 +2,10 @@ # The simulator's events.ndjson hash for this scenario is checked in # next to the test (`tests/cross_arch_parity.rs`). A mismatch is either # a deliberate spec amendment with justification, or a determinism bug. +# +# RELAY_SPEC extension: the alpha↔charlie pair is routed through the +# `R` relay so the parity test also gates the relay vertex's +# determinism. Direct edges still cover alpha↔bravo and bravo↔charlie. name = "parity_reference_v1" seed = 0x0dead_beef_cafe_b0 @@ -20,6 +24,13 @@ cold_dial_penalty_ns = 50_000 cache_warm_after_ns = 1_000_000 cache_invalidate_after_idle_ns = 100_000_000_000 +[[relays]] +id = "R" +ingress_capacity_bps = 1_000_000_000 # 1 GB/s combined ingress +egress_capacity_bps_per_link = 100_000_000 # 100 MB/s per outbound link +queue_depth_bytes = 1_000_000 # 1 MB shared queue +cold_start_penalty_ns = 200_000 # 200 µs first-message warmup + [[peers]] id = "alpha" kind = "parity_stub" @@ -38,14 +49,11 @@ kind = "parity_stub" initial_state = "ready" kind_config = { peers = ["alpha", "bravo", "charlie"] } +# Direct edges for alpha↔bravo and bravo↔charlie. [[links]] from = "alpha" to = "bravo" -[[links]] -from = "alpha" -to = "charlie" - [[links]] from = "bravo" to = "alpha" @@ -56,11 +64,19 @@ to = "charlie" [[links]] from = "charlie" -to = "alpha" +to = "bravo" + +# alpha↔charlie is routed through the relay. The loader expands each +# via shorthand into a host→relay leg and a relay→host leg. +[[links]] +from = "alpha" +to = "charlie" +via = "R" [[links]] from = "charlie" -to = "bravo" +to = "alpha" +via = "R" [[snapshots]] at_ns = 5_000_000 diff --git a/crates/simulation/src/bundle.rs b/crates/simulation/src/bundle.rs index d1d713c..7d59b1e 100644 --- a/crates/simulation/src/bundle.rs +++ b/crates/simulation/src/bundle.rs @@ -4,7 +4,7 @@ //! that cross the engine→writer boundary. The on-disk layout of §9 //! lives behind that trait in a later implementation. -use crate::network::{CacheTransition, DropReason}; +use crate::network::{CacheTransition, DropReason, RelayDropReason}; use crate::scenario::Mutation; /// One record the engine produces. The writer is append-only and may @@ -57,6 +57,31 @@ pub enum EventPayload { to: String, warm: bool, }, + /// RELAY_SPEC §9.1. Emitted at the message's arrival at the + /// relay. `kind_tag = "relay"`, `host_id = None`. + RelayEnqueue { + relay: String, + from: String, + to: String, + byte_len: u64, + }, + /// RELAY_SPEC §9.1. Emitted at the message's egress-end time. + /// `kind_tag = "relay"`, `host_id = None`. + RelayDequeue { + relay: String, + from: String, + to: String, + byte_len: u64, + }, + /// RELAY_SPEC §9.1. Emitted on `RelayQueueFull` and on + /// `RelayDown` drops. `kind_tag = "relay"`, `host_id = None`. + RelayDrop { + relay: String, + from: String, + to: String, + byte_len: u64, + reason: RelayDropReason, + }, } /// Why the engine dropped a delivery at arrival time. `Partition` is diff --git a/crates/simulation/src/bundle_file.rs b/crates/simulation/src/bundle_file.rs index e1d45ac..2ce0e34 100644 --- a/crates/simulation/src/bundle_file.rs +++ b/crates/simulation/src/bundle_file.rs @@ -23,7 +23,7 @@ use crate::bundle::{ BundleRecord, BundleWriter, DeliveryDropReason, EventPayload, EventRecord, MutationRecord, SnapshotRecord, }; -use crate::network::{CacheTransition, DropReason}; +use crate::network::{CacheTransition, DropReason, RelayDropReason}; use crate::scenario::{Scenario, to_toml}; /// The version string the manifest records under `simulator_version`. @@ -211,6 +211,28 @@ fn render_event_payload(p: &EventPayload) -> serde_json::Value { "to": to, "warm": warm, }), + EventPayload::RelayEnqueue { relay, from, to, byte_len } => serde_json::json!({ + "kind": "relay_enqueue", + "relay": relay, + "from": from, + "to": to, + "byte_len": byte_len, + }), + EventPayload::RelayDequeue { relay, from, to, byte_len } => serde_json::json!({ + "kind": "relay_dequeue", + "relay": relay, + "from": from, + "to": to, + "byte_len": byte_len, + }), + EventPayload::RelayDrop { relay, from, to, byte_len, reason } => serde_json::json!({ + "kind": "relay_drop", + "relay": relay, + "from": from, + "to": to, + "byte_len": byte_len, + "reason": relay_drop_reason_str(reason), + }), } } @@ -276,6 +298,15 @@ fn drop_reason_str(r: &DropReason) -> &'static str { DropReason::NoRoute => "no_route", DropReason::Partitioned => "partitioned", DropReason::Lossy => "lossy", + DropReason::RelayQueueFull => "relay_queue_full", + DropReason::RelayDown => "relay_down", + } +} + +fn relay_drop_reason_str(r: &RelayDropReason) -> &'static str { + match r { + RelayDropReason::QueueFull => "queue_full", + RelayDropReason::Down => "down", } } diff --git a/crates/simulation/src/engine.rs b/crates/simulation/src/engine.rs index 63bfbca..f491170 100644 --- a/crates/simulation/src/engine.rs +++ b/crates/simulation/src/engine.rs @@ -35,6 +35,17 @@ pub enum EngineAbort { /// A host attempted to send to itself, which the spec leaves /// undefined and we refuse. SelfSend { host: String }, + /// RELAY_SPEC §5.3 — a `WorkerExit` mutation targeted a host of a + /// kind that does not accept the envelope. The MVP only the + /// `stage` host kind accepts `WorkerExit`; anything else aborts + /// rather than silently dropping the envelope (the entire point + /// of the mutation being explicit is that targeting the wrong + /// kind should be loud). + WorkerExitOnWrongKind { + peer: String, + kind: String, + mutation_index: usize, + }, } /// Termination cause; tests inspect this to verify §4.8 / §4.10. @@ -172,8 +183,14 @@ impl Engine { ); e.enqueue(offset, EventKind::Tick { host: peer.id.clone() }); } - for m in &scenario.mutations { - e.enqueue(m.at_ns, EventKind::Mutation { mutation: m.clone() }); + for (idx, m) in scenario.mutations.iter().enumerate() { + e.enqueue( + m.at_ns, + EventKind::Mutation { + mutation: m.clone(), + index: idx, + }, + ); } for s in &scenario.snapshots { e.enqueue(s.at_ns, EventKind::Snapshot); @@ -318,7 +335,11 @@ impl Engine { return TerminationReason::Aborted(abort); } } - EventKind::Mutation { mutation } => self.dispatch_mutation(mutation), + EventKind::Mutation { mutation, index } => { + if let Err(abort) = self.dispatch_mutation(mutation, index) { + return TerminationReason::Aborted(abort); + } + } EventKind::Snapshot => self.dispatch_snapshot(), } // §4.8 early termination — after every dispatch, ask the @@ -451,14 +472,79 @@ impl Engine { self.process_actions(&to, actions, ActionSource::Recv) } - fn dispatch_mutation(&mut self, mutation: Mutation) { + fn dispatch_mutation( + &mut self, + mutation: Mutation, + index: usize, + ) -> Result<(), EngineAbort> { let at = self.now_ns; + // RELAY_SPEC §5.3 / §6.1 — `WorkerExit` is special-cased: + // it is a mutation whose effect is to deliver a `recv` + // envelope to a host, not to mutate the network. Validate + // the target kind here so the abort surfaces before any + // record is written. + if let MutationKind::WorkerExit { + peer, + reason, + status_code, + signal, + } = &mutation.kind + { + let spec_kind = self + .peer_specs + .get(peer) + .map(|s| s.kind.as_str()) + .unwrap_or(""); + if spec_kind != "stage" { + return Err(EngineAbort::WorkerExitOnWrongKind { + peer: peer.clone(), + kind: spec_kind.to_string(), + mutation_index: index, + }); + } + // Record the mutation itself (mirrors the parent's §4.5 + // emission). The WorkerExit envelope is dispatched + // *synchronously* — i.e. the host's `recv` is called + // before this method returns, and the recorded events + // reach the bundle writer before the main loop has a + // chance to pop anything else at the same virtual time. + // + // The synchronous dispatch is normative per §6A.3: + // "the event must reach the bundle writer before the + // engine acts on the halt." With an enqueued LocalRecv, + // a `Terminate` whose sequence number was assigned at + // construction (and so lower than the just-now-enqueued + // LocalRecv) would pop first and the worker_exited + // event would be lost — that is exactly the boundary + // failure §6A.6 names as "Event-before-halt is + // observable." + self.write_record(BundleRecord::Mutation(MutationRecord { + virtual_time_ns: at, + mutation: mutation.clone(), + })); + let envelope = HostMessage::WorkerExit { + reason: reason.clone(), + status_code: *status_code, + signal: *signal, + }; + // Mirror dispatch_local_recv's pre-checks: PeerKill and + // Halt-from-recv both suppress recv. + if self.killed.contains(peer) || self.recv_halted.contains(peer) { + return Ok(()); + } + if let Some(mut host) = self.hosts.remove(peer) { + let actions = host.recv(envelope, at); + self.hosts.insert(peer.clone(), host); + return self.process_actions(peer, actions, ActionSource::Recv); + } + return Ok(()); + } let invalidated = self.network.apply_mutation(&mutation, at); // The drop reason for invalidated deliveries depends on which // mutation invalidated them. Partition → `Partition`, - // PeerKill → `HostKilled`; nothing else invalidates deliveries - // today (LatencySpike / LossBurst / RelayBuffer / Heal / - // PeerResurrect all return an empty list from the network). + // PeerKill → `HostKilled`, RelayKill → `HostKilled` (the + // outbound peer never got the message); nothing else + // invalidates deliveries today. let drop_reason = match &mutation.kind { MutationKind::Partition { .. } => DeliveryDropReason::Partition, MutationKind::PeerKill { .. } => DeliveryDropReason::HostKilled, @@ -544,6 +630,7 @@ impl Engine { })); } self.drain_network_notifications(); + Ok(()) } fn dispatch_snapshot(&mut self) { @@ -697,6 +784,35 @@ impl Engine { kind_tag: "engine".into(), event: EventPayload::DialOutcome { from, to, warm }, }, + NetworkNotification::RelayEnqueue { relay, from, to, byte_len, at_ns } => { + EventRecord { + virtual_time_ns: at_ns, + host_id: None, + kind_tag: "relay".into(), + event: EventPayload::RelayEnqueue { relay, from, to, byte_len }, + } + } + NetworkNotification::RelayDequeue { relay, from, to, byte_len, at_ns } => { + EventRecord { + virtual_time_ns: at_ns, + host_id: None, + kind_tag: "relay".into(), + event: EventPayload::RelayDequeue { relay, from, to, byte_len }, + } + } + NetworkNotification::RelayDrop { + relay, + from, + to, + byte_len, + reason, + at_ns, + } => EventRecord { + virtual_time_ns: at_ns, + host_id: None, + kind_tag: "relay".into(), + event: EventPayload::RelayDrop { relay, from, to, byte_len, reason }, + }, }; self.write_record(BundleRecord::Event(record)); } @@ -767,6 +883,7 @@ enum EventKind { }, Mutation { mutation: Mutation, + index: usize, }, Snapshot, Terminate, diff --git a/crates/simulation/src/evaluator.rs b/crates/simulation/src/evaluator.rs index 3549b9d..806c499 100644 --- a/crates/simulation/src/evaluator.rs +++ b/crates/simulation/src/evaluator.rs @@ -226,6 +226,8 @@ fn payload_to_json(p: &crate::bundle::EventPayload) -> serde_json::Value { DropReason::NoRoute => "no_route", DropReason::Partitioned => "partitioned", DropReason::Lossy => "lossy", + DropReason::RelayQueueFull => "relay_queue_full", + DropReason::RelayDown => "relay_down", }, }), EventPayload::DropOnDelivery { to, reason } => serde_json::json!({ @@ -258,6 +260,31 @@ fn payload_to_json(p: &crate::bundle::EventPayload) -> serde_json::Value { "to": to, "warm": warm, }), + EventPayload::RelayEnqueue { relay, from, to, byte_len } => serde_json::json!({ + "kind": "relay_enqueue", + "relay": relay, + "from": from, + "to": to, + "byte_len": byte_len, + }), + EventPayload::RelayDequeue { relay, from, to, byte_len } => serde_json::json!({ + "kind": "relay_dequeue", + "relay": relay, + "from": from, + "to": to, + "byte_len": byte_len, + }), + EventPayload::RelayDrop { relay, from, to, byte_len, reason } => serde_json::json!({ + "kind": "relay_drop", + "relay": relay, + "from": from, + "to": to, + "byte_len": byte_len, + "reason": match reason { + crate::network::RelayDropReason::QueueFull => "queue_full", + crate::network::RelayDropReason::Down => "down", + }, + }), } } @@ -273,6 +300,10 @@ pub struct SnapshotEntry { pub seq: u32, pub members: BTreeMap, pub self_incarnation: u64, + /// RELAY_SPEC §5.4. Map of `name → address` for every name this + /// host has registered. Empty for hosts that do not register + /// names (e.g. the SWIM host kind). + pub name_registry: BTreeMap, } impl SnapshotEntry { @@ -285,15 +316,29 @@ impl SnapshotEntry { serde_json::from_slice(&rec.snapshot).unwrap_or(serde_json::Value::Null); let members = parse_members(&parsed["members"]); let self_incarnation = parsed["self_incarnation"].as_u64().unwrap_or(0); + let name_registry = parse_name_registry(&parsed["name_registry"]); Self { virtual_time_ns: rec.virtual_time_ns, seq, members, self_incarnation, + name_registry, } } } +fn parse_name_registry(v: &serde_json::Value) -> BTreeMap { + let mut out = BTreeMap::new(); + if let Some(obj) = v.as_object() { + for (name, addr) in obj { + if let Some(s) = addr.as_str() { + out.insert(name.clone(), s.to_string()); + } + } + } + out +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct MemberView { pub state: String, // "Alive" | "Suspect" | "Dead" @@ -366,11 +411,13 @@ fn read_snapshots(root: &Path) -> std::io::Result { let snapshot = &outer["snapshot"]; let members = parse_members(&snapshot["members"]); let self_incarnation = snapshot["self_incarnation"].as_u64().unwrap_or(0); + let name_registry = parse_name_registry(&snapshot["name_registry"]); list.push(SnapshotEntry { virtual_time_ns, seq, members, self_incarnation, + name_registry, }); } list.sort_by_key(|e| e.virtual_time_ns); @@ -470,6 +517,38 @@ fn evaluate_one( "event_rate", eval_event_rate(event_kind, *window_ns, *max_per_window, events), ), + AssertionKind::RelayQueueDepthBounded { + relay, + max_bytes, + window_start_ns, + window_end_ns, + } => ( + "relay_queue_depth_bounded", + eval_relay_queue_depth_bounded( + relay, + *max_bytes, + *window_start_ns, + *window_end_ns, + events, + ), + ), + AssertionKind::WorkerAliveThroughout { + peer, + window_start_ns, + window_end_ns, + } => ( + "worker_alive_throughout", + eval_worker_alive_throughout(peer, *window_start_ns, *window_end_ns, events), + ), + AssertionKind::NameResolvesWithin { + name, + observers, + within_ns, + from_ns, + } => ( + "name_resolves_within", + eval_name_resolves_within(name, observers, *within_ns, *from_ns, snapshots), + ), }; Verdict { name, @@ -877,6 +956,129 @@ fn eval_event_count(event_kind: &str, max: u64, events: &[EventLine]) -> Eval { // ── event_rate ────────────────────────────────────────────────────── +// ── relay_queue_depth_bounded (RELAY_SPEC §7.1, §7.2) ────────────── + +fn eval_relay_queue_depth_bounded( + relay: &str, + max_bytes: u64, + window_start_ns: Option, + window_end_ns: Option, + events: &[EventLine], +) -> Eval { + let start = window_start_ns.unwrap_or(0); + let end = window_end_ns.unwrap_or(u64::MAX); + // Replay relay_enqueue / relay_dequeue events in time order to + // reconstruct `enqueued_bytes`. We sort by virtual_time_ns + + // line_idx so the timeline is stable. + let mut relevant: Vec<&EventLine> = events + .iter() + .filter(|e| { + e.kind_tag == "relay" + && e.event["relay"] == relay + && (e.event["kind"] == "relay_enqueue" || e.event["kind"] == "relay_dequeue") + }) + .collect(); + relevant.sort_by(|a, b| { + a.virtual_time_ns + .cmp(&b.virtual_time_ns) + .then(a.line_idx.cmp(&b.line_idx)) + }); + if relevant.is_empty() { + return inconclusive(); + } + let mut depth: u64 = 0; + for e in &relevant { + let bl = e.event["byte_len"].as_u64().unwrap_or(0); + let in_window = e.virtual_time_ns >= start && e.virtual_time_ns <= end; + match e.event["kind"].as_str() { + Some("relay_enqueue") => { + depth = depth.saturating_add(bl); + if in_window && depth > max_bytes { + return fail(vec![ev_event(e)]); + } + } + Some("relay_dequeue") => { + depth = depth.saturating_sub(bl); + } + _ => {} + } + } + pass() +} + +// ── worker_alive_throughout (RELAY_SPEC §7.1, §7.2) ──────────────── + +fn eval_worker_alive_throughout( + peer: &str, + start: u64, + end: u64, + events: &[EventLine], +) -> Eval { + // Look for stage_lifecycle events into "Halted" for `peer` within + // the window. Any such event ⇒ Fail. Otherwise: Pass if any + // stage_lifecycle for `peer` appears at all (the host registered + // its life), else Inconclusive. + let mut any = false; + let mut evidence = Vec::new(); + for e in events.iter().filter(|e| e.kind_tag == "stage") { + if e.event["kind"] == "stage_lifecycle" && e.host_id.as_deref() == Some(peer) { + any = true; + let to = e.event["to"].as_str().unwrap_or(""); + if to == "Halted" && e.virtual_time_ns >= start && e.virtual_time_ns <= end { + evidence.push(ev_event(e)); + } + } + } + if !any { + return inconclusive(); + } + if evidence.is_empty() { + pass() + } else { + fail(evidence) + } +} + +// ── name_resolves_within (RELAY_SPEC §7.1, §7.2) ──────────────────── + +fn eval_name_resolves_within( + name: &str, + observers: &[String], + within_ns: u64, + from_ns: u64, + snapshots: &SnapshotIndex, +) -> Eval { + let deadline = from_ns.saturating_add(within_ns); + let mut any_observer_snapshot = false; + let mut evidence_fail = Vec::new(); + for obs in observers { + let Some(list) = snapshots.by_host.get(obs) else { + return inconclusive(); + }; + // First snapshot at-or-after from_ns. + let first = list.iter().find(|s| s.virtual_time_ns >= from_ns); + let Some(first) = first else { + return inconclusive(); + }; + any_observer_snapshot = true; + let resolved = list + .iter() + .filter(|s| s.virtual_time_ns >= from_ns && s.virtual_time_ns <= deadline) + .any(|s| s.name_registry.contains_key(name)); + if !resolved { + evidence_fail.push(ev_snapshot(obs, first)); + } + } + if !any_observer_snapshot { + return inconclusive(); + } + if evidence_fail.is_empty() { + pass() + } else { + fail(evidence_fail) + } +} + fn eval_event_rate( event_kind: &str, window_ns: u64, diff --git a/crates/simulation/src/host.rs b/crates/simulation/src/host.rs index bb65f6e..b3e87cf 100644 --- a/crates/simulation/src/host.rs +++ b/crates/simulation/src/host.rs @@ -43,6 +43,15 @@ pub enum HostMessage { /// A `Send` the network refused. The sender — not the destination /// — receives this so the host can react (re-queue, log, etc.). SendFailed { to: HostId, reason: DropReason }, + /// RELAY_SPEC §3.1 / §5.3 — an internal worker-exit signal. The + /// stage host kind consumes this; other kinds refuse it and the + /// engine aborts the run with a structured error if it is + /// delivered to a kind that does not accept it. + WorkerExit { + reason: String, + status_code: Option, + signal: Option, + }, } /// One thing a host's `tick` or `recv` can ask the engine to do. diff --git a/crates/simulation/src/lib.rs b/crates/simulation/src/lib.rs index 00f7741..e9b77a2 100644 --- a/crates/simulation/src/lib.rs +++ b/crates/simulation/src/lib.rs @@ -13,5 +13,6 @@ pub mod parity_host; pub mod property; pub mod rng; pub mod scenario; +pub mod stage_host; pub mod swim_codec; pub mod swim_host; diff --git a/crates/simulation/src/network.rs b/crates/simulation/src/network.rs index a5490c1..513090d 100644 --- a/crates/simulation/src/network.rs +++ b/crates/simulation/src/network.rs @@ -1,9 +1,15 @@ -//! The network (SIM_SPEC §5). +//! The network (SIM_SPEC §5 + RELAY_SPEC §4). //! //! A directed-graph link model with deterministic per-edge state, the //! §5.4 send algorithm, the §5.5 mutation suite, and a §7-conformant //! integer-only computation path (no floats touch any decision). //! +//! RELAY_SPEC §4 adds the relay vertex: a first-class non-host node in +//! the topology with its own ingress and per-egress queues, head-of- +//! line serialization, queue-overflow drops, and cold-start penalty. +//! Relayed routes (RELAY_SPEC §4.1) are composed inside the network's +//! `send`; the engine sees one `SendOutcome` per query regardless. +//! //! The network owns no schedule of its own; the engine pops events and //! queries the network. Each query mutates per-link state but never //! reads from any clock outside the `now_ns` the engine supplies. @@ -11,7 +17,9 @@ use std::collections::{BTreeMap, BTreeSet}; use crate::rng::{SubstreamKey, SubstreamRng, jitter_sample}; -use crate::scenario::{LinkPolicy, LinkRef, Mutation, MutationKind, Scenario}; +use crate::scenario::{ + HostRoute, LinkPolicy, LinkRef, Mutation, MutationKind, Relay, Scenario, +}; // ────────────────────────────────────────────────────────────────────── // Public types @@ -36,12 +44,18 @@ pub enum SendOutcome { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DropReason { - /// The (from, to) pair has no declared edge. + /// The (from, to) pair has no declared edge or route. NoRoute, /// The active partition set cuts this edge. Partitioned, /// The Bernoulli loss draw for the edge fired. Lossy, + /// The relay's queue would overflow if this message were enqueued. + /// RELAY_SPEC §4.4 step 2. + RelayQueueFull, + /// The route is through a relay that has been `RelayKill`-ed. + /// RELAY_SPEC §4.5. + RelayDown, } /// Side-channel notification the engine consumes after each query. @@ -65,6 +79,40 @@ pub enum NetworkNotification { at_ns: u64, warm: bool, }, + /// RELAY_SPEC §4.4 step 7 / §9.1. The message reached the relay + /// and was enqueued (or accounted for in the ingress) at `at_ns`. + RelayEnqueue { + relay: String, + from: String, + to: String, + byte_len: u64, + at_ns: u64, + }, + /// RELAY_SPEC §4.4 step 7 / §9.1. The relay finished serving the + /// message on its outbound egress at `at_ns`. + RelayDequeue { + relay: String, + from: String, + to: String, + byte_len: u64, + at_ns: u64, + }, + /// RELAY_SPEC §4.4 step 2 / §4.5 / §9.1. The relay refused the + /// message at `at_ns`. + RelayDrop { + relay: String, + from: String, + to: String, + byte_len: u64, + reason: RelayDropReason, + at_ns: u64, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RelayDropReason { + QueueFull, + Down, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -105,6 +153,12 @@ pub struct Network { /// owns). The network exposes a list of in-flight deliveries to /// the killed peer for invalidation. killed_peers: BTreeSet, + /// RELAY_SPEC §4 relay vertices, indexed by id. + relays: BTreeMap, + /// RELAY_SPEC §4.1 — per ordered host pair, the resolved route + /// (direct or relayed-through-a-named-relay). Pairs absent from + /// here drop with `NoRoute`. + routes: BTreeMap<(String, String), HostRoute>, seed: u64, next_delivery_id: u64, pending_notifications: Vec, @@ -121,6 +175,45 @@ struct EdgeState { in_flight: Vec, } +#[derive(Debug)] +struct RelayState { + policy: Relay, + /// Per outbound link (keyed by destination node id), the virtual + /// time at which the last scheduled message finishes serialization. + egress_queue_tail_ns: BTreeMap, + /// Across the single shared ingress, the virtual time at which the + /// last scheduled message finishes serialization. + ingress_queue_tail_ns: u64, + /// In-flight messages currently between ingress-enqueue and + /// egress-dequeue. Tracked so `RelayKill` returns the right set. + in_flight: Vec, + boot_state: BootState, + /// Has the relay been killed by a `RelayKill` mutation? After + /// kill, no more messages forward until `RelayBoot`. + killed: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum BootState { + Booted, + /// The relay has just been booted; the next forwarded message + /// pays the cold-start penalty. `since_ns` is informational. + Booting { since_ns: u64 }, +} + +#[derive(Debug, Clone)] +struct RelayInFlight { + delivery_id: DeliveryId, + from_host: String, + to_host: String, + byte_len: u64, + /// Final arrival time at the destination host (post-egress + outbound leg). + arrival_at_ns: u64, + /// Virtual time the message left the relay's egress (used for + /// `enqueued_bytes` bookkeeping in `decrement_after_egress`). + egress_end_ns: u64, +} + #[derive(Debug, Clone, PartialEq, Eq)] enum CacheState { Cold, @@ -132,6 +225,9 @@ enum CacheState { struct InFlight { delivery_id: DeliveryId, scheduled_at_ns: u64, + /// For relayed routes, the relay through which this delivery is + /// being forwarded. `None` for direct edges. + via_relay: Option, } #[derive(Debug, Clone)] @@ -182,6 +278,40 @@ impl Network { }, ); } + let mut relays = BTreeMap::new(); + for r in &scenario.relays { + relays.insert( + r.id.clone(), + RelayState { + policy: r.clone(), + egress_queue_tail_ns: BTreeMap::new(), + ingress_queue_tail_ns: 0, + in_flight: Vec::new(), + boot_state: BootState::Booted, + killed: false, + }, + ); + } + let mut routes = BTreeMap::new(); + for r in &scenario.routes { + routes.insert((r.from().to_string(), r.to().to_string()), r.clone()); + } + // If the scenario has no relays and no `via` shorthand, the + // `routes` field may be empty; in that case build a direct + // route per declared host-to-host edge. This preserves + // backward compatibility with scenarios written before the + // relay extension. + if routes.is_empty() { + for (key, _edge) in edges.iter() { + routes.insert( + key.clone(), + HostRoute::Direct { + from: key.0.clone(), + to: key.1.clone(), + }, + ); + } + } Self { edges, partitioned: BTreeSet::new(), @@ -189,6 +319,8 @@ impl Network { active_loss_burst: Vec::new(), active_relay_buffer: Vec::new(), killed_peers: BTreeSet::new(), + relays, + routes, seed: scenario.seed, next_delivery_id: 0, pending_notifications: Vec::new(), @@ -200,13 +332,47 @@ impl Network { std::mem::take(&mut self.pending_notifications) } - /// Read-only test hook: total number of in-flight deliveries. + /// Read-only test hook: total number of in-flight deliveries + /// across every edge. pub fn in_flight_count(&self) -> usize { self.edges.values().map(|e| e.in_flight.len()).sum() } - /// Per §3.2 / §5.4. + /// Read-only test hook: number of messages currently in the named + /// relay's queues (between ingress enqueue and egress dequeue). + pub fn relay_in_flight_count(&self, relay: &str) -> usize { + self.relays.get(relay).map(|r| r.in_flight.len()).unwrap_or(0) + } + + /// Per §3.2 / §5.4 plus RELAY_SPEC §4.4 composition. pub fn send(&mut self, from: &str, to: &str, byte_len: u64, sent_at_ns: u64) -> SendOutcome { + let route_key = (from.to_string(), to.to_string()); + let Some(route) = self.routes.get(&route_key).cloned() else { + return SendOutcome::Drop { + reason: DropReason::NoRoute, + }; + }; + match route { + HostRoute::Direct { .. } => self.send_direct(from, to, byte_len, sent_at_ns, None), + HostRoute::Relayed { relay, .. } => { + self.send_relayed(from, &relay, to, byte_len, sent_at_ns) + } + } + } + + /// Direct (or single-leg) send along one declared edge. When + /// `relay_context` is `Some`, the in-flight entry is tagged with + /// the originating relay so `RelayKill` can invalidate the right + /// deliveries. The composed `send_relayed` path uses this for the + /// outbound leg. + fn send_direct( + &mut self, + from: &str, + to: &str, + byte_len: u64, + sent_at_ns: u64, + relay_context: Option<&str>, + ) -> SendOutcome { // 1. No declared edge → NoRoute. State unchanged. if !self.edges.contains_key(&(from.to_string(), to.to_string())) { return SendOutcome::Drop { @@ -234,8 +400,7 @@ impl Network { }; } - // Pre-step: idle cooling. If last_send_ns - now > cache_invalidate_after_idle_ns, - // transition to Cold and emit a CacheStateChange. + // Pre-step: idle cooling. self.maybe_idle_cool(&key, sent_at_ns); // 4. serialization_start = max(sent_at, last_arrive_ns). @@ -263,19 +428,16 @@ impl Network { }; let mut arrival = serialization_end.saturating_add(scaled_additive); - // 7. RelayBuffer floor: arrival = max(arrival, sent_at + floor_ns). + // 7. RelayBuffer (legacy per-link floor) — applies to direct + // edges; the new relay vertex has its own delay model. if let Some(floor_ns) = self.effective_relay_floor(&key, sent_at_ns) { arrival = arrival.max(sent_at_ns.saturating_add(floor_ns)); } - // 8. Cold-dial penalty. If the cache is Cold, add the penalty; - // emit DialStart and DialOutcome notifications. + // 8. Cold-dial penalty. let cache_was_cold = matches!(self.edges[&key].cache, CacheState::Cold); if cache_was_cold { arrival = arrival.saturating_add(policy.cold_dial_penalty_ns); - // §5.4 step 8 says "DialOutcome at the arrival time" — - // the only `arrival` in scope at that point is the - // post-penalty value, so capture *after* the add. let dial_outcome_at = arrival; self.pending_notifications.push(NetworkNotification::DialStart { from: from.to_string(), @@ -288,17 +450,11 @@ impl Network { at_ns: dial_outcome_at, warm: true, }); - // §5.4 step 8 — transition Cold → Warming(sent_at_ns). - // No CacheStateChange notification here; per §5.4 step 10 - // the Warmed event fires only when Warming→Warm crosses - // the cache_warm_after_ns threshold. let edge = self.edges.get_mut(&key).unwrap(); edge.cache = CacheState::Warming { since_ns: sent_at_ns }; } - // 9. Reorder draw. If it fires, push arrival past the next - // scheduled delivery on this edge. We pick the latest - // in-flight arrival on the edge plus a small delta. + // 9. Reorder draw. let reorder_draw = { let edge = self.edges.get_mut(&key).unwrap(); edge.rng.next_u32() % 1_000_000 @@ -321,14 +477,6 @@ impl Network { edge.last_send_ns = Some(sent_at_ns); edge.last_arrive_ns = Some(arrival); - // §5.4 step 10 — if Warming and the cumulative warm-after - // threshold has been crossed, transition Warming → Warm and - // emit the single CacheStateChange{Warmed} notification. - // The notification carries the actual transition time - // (`since_ns + cache_warm_after_ns`) rather than the - // observing send's `sent_at_ns`; the bundle reader sees the - // moment the link became warm, not the moment the engine - // happened to detect it. if let CacheState::Warming { since_ns } = edge.cache { if sent_at_ns.saturating_sub(since_ns) >= policy.cache_warm_after_ns { edge.cache = CacheState::Warm; @@ -342,28 +490,223 @@ impl Network { } } - // Record in-flight. let delivery_id = DeliveryId(self.next_delivery_id); self.next_delivery_id += 1; edge.in_flight.push(InFlight { delivery_id, scheduled_at_ns: arrival, + via_relay: relay_context.map(|s| s.to_string()), }); SendOutcome::Arrive { delivery_id, at_ns: arrival } } - /// Inform the network that the engine has delivered (or otherwise - /// removed) a previously-scheduled delivery. The network drops - /// the corresponding in-flight entry; this is how `in_flight` - /// stays accurate for mutation invalidation. - pub fn notify_delivered(&mut self, from: &str, to: &str, delivery_id: DeliveryId) { - if let Some(edge) = self.edges.get_mut(&(from.to_string(), to.to_string())) { + /// RELAY_SPEC §4.4 composition. Inbound leg via direct edge, then + /// relay processing (ingress + egress with optional cold-start), + /// then outbound leg via direct edge. + fn send_relayed( + &mut self, + from: &str, + relay: &str, + to: &str, + byte_len: u64, + sent_at_ns: u64, + ) -> SendOutcome { + // RELAY_SPEC §4.5 — sends through a killed relay drop with + // `RelayDown`. The inbound leg's state is not consulted: the + // relay's deadness is an out-of-band fact about the route. + if self.relays.get(relay).map(|r| r.killed).unwrap_or(false) { + self.pending_notifications + .push(NetworkNotification::RelayDrop { + relay: relay.to_string(), + from: from.to_string(), + to: to.to_string(), + byte_len, + reason: RelayDropReason::Down, + at_ns: sent_at_ns, + }); + return SendOutcome::Drop { + reason: DropReason::RelayDown, + }; + } + + // RELAY_SPEC §4.4 step 1 — inbound leg. Use the *internal* + // send_direct so the inbound edge's state evolves the same way + // a normal direct edge would, but the returned arrival time + // becomes the relay-ingress arrival. + let inbound = self.send_direct(from, relay, byte_len, sent_at_ns, None); + let arrival_at_r = match inbound { + SendOutcome::Arrive { delivery_id, at_ns } => { + // We tracked an in-flight on the inbound edge as a + // bookkeeping artefact; for relayed routes the + // *outbound* leg's in-flight is the canonical one. + // Drop the inbound bookkeeping so PeerKill / Partition + // on the inbound leg do not see a phantom message. + self.discard_inbound_inflight(from, relay, delivery_id); + at_ns + } + SendOutcome::Drop { reason } => return SendOutcome::Drop { reason }, + }; + + // RELAY_SPEC §4.4 step 2 — enqueue at the relay; check + // queue-overflow exact (over-strict: only refuse when adding + // would push beyond the bound). + let Some(relay_state) = self.relays.get_mut(relay) else { + return SendOutcome::Drop { + reason: DropReason::NoRoute, + }; + }; + let cleanup_at = arrival_at_r; + relay_state.cleanup_finished(cleanup_at); + let enqueued_bytes: u64 = relay_state.in_flight.iter().map(|m| m.byte_len).sum(); + if enqueued_bytes.saturating_add(byte_len) > relay_state.policy.queue_depth_bytes { + self.pending_notifications + .push(NetworkNotification::RelayDrop { + relay: relay.to_string(), + from: from.to_string(), + to: to.to_string(), + byte_len, + reason: RelayDropReason::QueueFull, + at_ns: arrival_at_r, + }); + return SendOutcome::Drop { + reason: DropReason::RelayQueueFull, + }; + } + // RELAY_SPEC §4.4 step 3 — ingress serialization. + let ingress_capacity = relay_state.policy.ingress_capacity_bps; + let ingress_serialization = ((byte_len as u128).saturating_mul(1_000_000_000u128) + / (ingress_capacity as u128)) as u64; + let ingress_end = arrival_at_r + .max(relay_state.ingress_queue_tail_ns) + .saturating_add(ingress_serialization); + relay_state.ingress_queue_tail_ns = ingress_end; + + // RELAY_SPEC §4.4 step 4 — egress serialization. Cold start + // penalty fires once per `Booting` state. + let egress_capacity = relay_state.policy.egress_capacity_bps_per_link; + let egress_serialization = ((byte_len as u128).saturating_mul(1_000_000_000u128) + / (egress_capacity as u128)) as u64; + let egress_tail = *relay_state + .egress_queue_tail_ns + .get(to) + .unwrap_or(&0); + let mut egress_start = ingress_end.max(egress_tail); + if let BootState::Booting { .. } = relay_state.boot_state { + egress_start = egress_start.saturating_add(relay_state.policy.cold_start_penalty_ns); + relay_state.boot_state = BootState::Booted; + } + let egress_end = egress_start.saturating_add(egress_serialization); + relay_state + .egress_queue_tail_ns + .insert(to.to_string(), egress_end); + + // RELAY_SPEC §4.4 step 7 — emit enqueue/dequeue notifications. + self.pending_notifications + .push(NetworkNotification::RelayEnqueue { + relay: relay.to_string(), + from: from.to_string(), + to: to.to_string(), + byte_len, + at_ns: arrival_at_r, + }); + self.pending_notifications + .push(NetworkNotification::RelayDequeue { + relay: relay.to_string(), + from: from.to_string(), + to: to.to_string(), + byte_len, + at_ns: egress_end, + }); + + // RELAY_SPEC §4.4 step 5 — outbound leg. The outbound edge's + // own bandwidth model serializes on top of the relay's egress. + let outbound = self.send_direct(relay, to, byte_len, egress_end, Some(relay)); + let final_arrival = match outbound { + SendOutcome::Arrive { at_ns, .. } => at_ns, + SendOutcome::Drop { reason } => { + // The relay scheduling has already happened; we still + // emit the dequeue notification (it represents the + // relay's view) but the composed send drops. + return SendOutcome::Drop { reason }; + } + }; + + // Record on the relay's in-flight so RelayKill can invalidate. + let delivery_id = match self.last_outbound_delivery_id(relay, to) { + Some(d) => d, + None => DeliveryId(self.next_delivery_id.saturating_sub(1)), + }; + let relay_state = self.relays.get_mut(relay).unwrap(); + relay_state.in_flight.push(RelayInFlight { + delivery_id, + from_host: from.to_string(), + to_host: to.to_string(), + byte_len, + arrival_at_ns: final_arrival, + egress_end_ns: egress_end, + }); + + SendOutcome::Arrive { + delivery_id, + at_ns: final_arrival, + } + } + + /// Forget the inbound-leg bookkeeping created by + /// `send_direct(from, relay, …)`. The outbound leg owns the + /// canonical in-flight entry for relayed routes. + fn discard_inbound_inflight(&mut self, from: &str, relay: &str, delivery_id: DeliveryId) { + if let Some(edge) = self + .edges + .get_mut(&(from.to_string(), relay.to_string())) + { edge.in_flight.retain(|f| f.delivery_id != delivery_id); } } - /// Per §3.2 / §5.5. + /// The delivery id the last-issued outbound `send_direct` returned + /// (it pushes onto the outbound edge's `in_flight`; the relay + /// then mirrors that id into its own in-flight). + fn last_outbound_delivery_id(&self, relay: &str, to: &str) -> Option { + self.edges + .get(&(relay.to_string(), to.to_string()))? + .in_flight + .last() + .map(|f| f.delivery_id) + } + + /// Inform the network that the engine has delivered (or otherwise + /// removed) a previously-scheduled delivery. + pub fn notify_delivered(&mut self, from: &str, to: &str, delivery_id: DeliveryId) { + // Direct edge bookkeeping. + if let Some(edge) = self.edges.get_mut(&(from.to_string(), to.to_string())) { + edge.in_flight.retain(|f| f.delivery_id != delivery_id); + } + // For relayed routes, the canonical edge is `relay → to`; the + // engine still calls us with `(from = original sender, to)`. + // Look up the route to find the relay. + let via_relay = self + .routes + .get(&(from.to_string(), to.to_string())) + .and_then(|r| match r { + HostRoute::Relayed { relay, .. } => Some(relay.clone()), + HostRoute::Direct { .. } => None, + }); + if let Some(relay) = via_relay { + if let Some(edge) = self + .edges + .get_mut(&(relay.clone(), to.to_string())) + { + edge.in_flight.retain(|f| f.delivery_id != delivery_id); + } + if let Some(rs) = self.relays.get_mut(&relay) { + rs.in_flight.retain(|m| m.delivery_id != delivery_id); + } + } + } + + /// Per §3.2 / §5.5 + RELAY_SPEC §4.5 / §6.1. pub fn apply_mutation( &mut self, mutation: &Mutation, @@ -439,7 +782,6 @@ impl Network { } MutationKind::PeerKill { peer } => { self.killed_peers.insert(peer.clone()); - // Invalidate every delivery destined to the peer. let mut invalidated = Vec::new(); let edge_keys: Vec<(String, String)> = self .edges @@ -450,8 +792,6 @@ impl Network { for (from, to) in edge_keys { invalidated.extend(self.drain_in_flight_for(&from, &to)); let edge = self.edges.get_mut(&(from.clone(), to.clone())).unwrap(); - // Invalidate cache for that edge per §5.5 (kill - // resets the link). Emit a notification. if !matches!(edge.cache, CacheState::Cold) { edge.cache = CacheState::Cold; self.pending_notifications.push( @@ -470,10 +810,93 @@ impl Network { self.killed_peers.remove(peer); Vec::new() } + MutationKind::WorkerExit { .. } => { + // RELAY_SPEC §6.1 — `WorkerExit` is engine-side; the + // network has nothing to invalidate. The engine + // dispatches it as a recv envelope to the target host. + Vec::new() + } + MutationKind::RelayKill { relay } => self.apply_relay_kill(relay, at_ns), + MutationKind::RelayBoot { relay } => { + self.apply_relay_boot(relay, at_ns); + Vec::new() + } + MutationKind::RelayCapacityChange { + relay, + ingress_capacity_bps, + egress_capacity_bps_per_link, + queue_depth_bytes, + } => { + self.apply_relay_capacity_change( + relay, + *ingress_capacity_bps, + *egress_capacity_bps_per_link, + *queue_depth_bytes, + ); + Vec::new() + } } // `_seed` is captured at construction; we keep it on the type - // so future randomness in mutations (none currently) can derive - // a substream from `("mutation", index)`. + // so future randomness in mutations (none currently do) can + // derive a substream from `("mutation", index)`. + } + + // ── Relay mutation helpers ─────────────────────────────────────── + + fn apply_relay_kill(&mut self, relay: &str, _at_ns: u64) -> Vec { + let Some(rs) = self.relays.get_mut(relay) else { + return Vec::new(); + }; + rs.killed = true; + let drained = std::mem::take(&mut rs.in_flight); + // Each drained entry has a canonical in-flight on the outbound + // edge (relay → to_host). Drop it there too, so the engine + // doesn't think the delivery is still pending. + let invalidated: Vec<_> = drained + .into_iter() + .map(|m| InvalidatedDelivery { + delivery_id: m.delivery_id, + from: m.from_host, + to: m.to_host, + scheduled_at_ns: m.arrival_at_ns, + }) + .collect(); + for inv in &invalidated { + if let Some(edge) = self.edges.get_mut(&(relay.to_string(), inv.to.clone())) { + edge.in_flight.retain(|f| f.delivery_id != inv.delivery_id); + } + } + invalidated + } + + fn apply_relay_boot(&mut self, relay: &str, at_ns: u64) { + if let Some(rs) = self.relays.get_mut(relay) { + rs.killed = false; + rs.in_flight.clear(); + rs.egress_queue_tail_ns.clear(); + rs.ingress_queue_tail_ns = 0; + rs.boot_state = BootState::Booting { since_ns: at_ns }; + } + } + + fn apply_relay_capacity_change( + &mut self, + relay: &str, + ingress: Option, + egress: Option, + depth: Option, + ) { + if let Some(rs) = self.relays.get_mut(relay) { + if let Some(v) = ingress { + rs.policy.ingress_capacity_bps = v; + } + if let Some(v) = egress { + rs.policy.egress_capacity_bps_per_link = v; + } + if let Some(v) = depth { + rs.policy.queue_depth_bytes = v; + } + } } // ── Helpers ────────────────────────────────────────────────────── @@ -547,6 +970,16 @@ impl Network { return Vec::new(); }; let drained = std::mem::take(&mut edge.in_flight); + // For each drained entry, also remove the mirror from the + // relay (if any) so RelayKill semantics stay consistent with + // PeerKill semantics for relayed routes. + for f in &drained { + if let Some(relay) = f.via_relay.clone() { + if let Some(rs) = self.relays.get_mut(&relay) { + rs.in_flight.retain(|m| m.delivery_id != f.delivery_id); + } + } + } drained .into_iter() .map(|f| InvalidatedDelivery { @@ -566,6 +999,15 @@ impl Network { } } +impl RelayState { + /// Drop in-flight entries whose `arrival_at_ns` is at or before + /// `now_ns`. RELAY_SPEC §4.4 step 6 — the relay decrements + /// `enqueued_bytes` lazily once the message clears egress. + fn cleanup_finished(&mut self, now_ns: u64) { + self.in_flight.retain(|m| m.egress_end_ns > now_ns); + } +} + fn sorted_pair(a: &str, b: &str) -> (String, String) { if a <= b { (a.to_string(), b.to_string()) diff --git a/crates/simulation/src/parity_host.rs b/crates/simulation/src/parity_host.rs index daf03b9..45afa4e 100644 --- a/crates/simulation/src/parity_host.rs +++ b/crates/simulation/src/parity_host.rs @@ -147,6 +147,11 @@ impl Host for ParityStubHost { "host": self.id, "to": to, }), + HostMessage::WorkerExit { .. } => { + panic!( + "parity_stub host kind cannot receive WorkerExit; the engine's WorkerExitOnWrongKind gate is broken" + ); + } }; vec![Action::RecordEvent { kind_tag: "parity_stub".into(), diff --git a/crates/simulation/src/property.rs b/crates/simulation/src/property.rs index 9ae2e48..f456f8e 100644 --- a/crates/simulation/src/property.rs +++ b/crates/simulation/src/property.rs @@ -260,10 +260,12 @@ fn generate_scenario( }, default_link: link_policy, peers: peer_records, + relays: Vec::new(), links, mutations: Vec::new(), snapshots: Vec::new(), assertions, + routes: Vec::new(), }; // Round-trip through the loader so we get the same validation // the on-disk path does. diff --git a/crates/simulation/src/scenario.rs b/crates/simulation/src/scenario.rs index 1c5a6f6..b658913 100644 --- a/crates/simulation/src/scenario.rs +++ b/crates/simulation/src/scenario.rs @@ -36,6 +36,8 @@ pub struct Scenario { pub default_tick: DefaultTick, pub default_link: LinkPolicy, pub peers: Vec, + #[serde(default)] + pub relays: Vec, pub links: Vec, #[serde(default)] pub mutations: Vec, @@ -43,6 +45,16 @@ pub struct Scenario { pub snapshots: Vec, #[serde(default)] pub assertions: Vec, + /// Per-ordered-host-pair route resolution. Built by the loader + /// from `links` (and any `via` shorthands the loader expanded into + /// explicit relay legs). For a direct edge, the route names the + /// edge directly; for a relayed route, it names the inbound leg, + /// the relay, and the outbound leg. Networks consume this to + /// avoid re-deriving routes per send. Always skipped on serialize + /// — the loader re-derives it from `links` + `relays`, so + /// emitting it would duplicate the info and risk drift. + #[serde(default, skip_serializing)] + pub routes: Vec, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -82,6 +94,49 @@ pub struct Link { pub policy: LinkPolicy, } +/// A relay vertex (RELAY_SPEC §4). Not a host: it has no host trait, +/// no tick, no recv. It is part of the network topology. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Relay { + pub id: String, + pub ingress_capacity_bps: u64, + pub egress_capacity_bps_per_link: u64, + pub queue_depth_bytes: u64, + #[serde(default)] + pub cold_start_penalty_ns: u64, +} + +/// A resolved route between two hosts. Direct = one edge from host +/// `from` to host `to`. Relayed = inbound edge `from→relay`, then the +/// relay's queues, then outbound edge `relay→to`. Multi-hop relays +/// are rejected by the loader (RELAY_SPEC §4.1). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum HostRoute { + Direct { + from: String, + to: String, + }, + Relayed { + from: String, + relay: String, + to: String, + }, +} + +impl HostRoute { + pub fn from(&self) -> &str { + match self { + HostRoute::Direct { from, .. } | HostRoute::Relayed { from, .. } => from, + } + } + pub fn to(&self) -> &str { + match self { + HostRoute::Direct { to, .. } | HostRoute::Relayed { to, .. } => to, + } + } +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct Mutation { pub at_ns: u64, @@ -120,6 +175,38 @@ pub enum MutationKind { peer: String, preserve_state: bool, }, + /// RELAY_SPEC §6.1. Delivers a `WorkerExit` envelope to the named + /// peer at the mutation's time. Targeting a host kind that does + /// not accept it aborts the run. + WorkerExit { + peer: String, + reason: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + status_code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + signal: Option, + }, + /// RELAY_SPEC §4.5. Relay stops forwarding; queued messages + /// invalidate; later sends drop with `RelayDown`. + RelayKill { + relay: String, + }, + /// RELAY_SPEC §4.5. Relay returns to service in `Booting` state. + /// The next forwarded message pays the cold-start penalty. + RelayBoot { + relay: String, + }, + /// RELAY_SPEC §4.5. Replace any subset of the three policy fields + /// at the mutation's virtual time. + RelayCapacityChange { + relay: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + ingress_capacity_bps: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + egress_capacity_bps_per_link: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + queue_depth_bytes: Option, + }, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -188,6 +275,34 @@ pub enum AssertionKind { window_ns: u64, max_per_window: u64, }, + /// RELAY_SPEC §7.1. The named relay's `enqueued_bytes` never + /// exceeds `max_bytes` across the (defaults: whole-run) window. + RelayQueueDepthBounded { + relay: String, + max_bytes: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + window_start_ns: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + window_end_ns: Option, + }, + /// RELAY_SPEC §7.1. The named peer's lifecycle state remains + /// `Running` throughout the window. Fails on any + /// `stage_lifecycle` event into `Halted` whose time falls in the + /// window. + WorkerAliveThroughout { + peer: String, + window_start_ns: u64, + window_end_ns: u64, + }, + /// RELAY_SPEC §7.1. Starting at `from_ns`, every observer in + /// `observers` produces a snapshot whose `name_registry` + /// contains `name` within `within_ns`. + NameResolvesWithin { + name: String, + observers: Vec, + within_ns: u64, + from_ns: u64, + }, } // ────────────────────────────────────────────────────────────────────── @@ -222,6 +337,7 @@ impl HostKindRegistry { pub fn with_swim() -> Self { let mut reg = Self::default(); reg.register(Box::new(SwimHostKindValidator)); + reg.register(Box::new(StageHostKindValidator)); reg } @@ -274,6 +390,44 @@ impl HostKindValidator for SwimHostKindValidator { } } +/// `stage` host kind validator (RELAY_SPEC §5, §8.1). Requires a +/// `name` (the name the stage registers at `Registering → Running`) +/// and an `address` (the address the stage registers under), both +/// non-empty strings. +pub struct StageHostKindValidator; + +impl HostKindValidator for StageHostKindValidator { + fn kind_tag(&self) -> &'static str { + "stage" + } + fn validate_config(&self, config: &toml::value::Table) -> Result<(), String> { + let name = config + .get("name") + .and_then(|v| v.as_str()) + .ok_or_else(|| "required key missing: name".to_string())?; + if name.is_empty() { + return Err("name must not be empty".to_string()); + } + let address = config + .get("address") + .and_then(|v| v.as_str()) + .ok_or_else(|| "required key missing: address".to_string())?; + if address.is_empty() { + return Err("address must not be empty".to_string()); + } + Ok(()) + } + fn validate_initial_state(&self, state: &str) -> Result<(), String> { + if state == "cold" { + Ok(()) + } else { + Err(format!( + "unknown stage initial_state {state:?}; expected \"cold\"" + )) + } + } +} + fn require_u64(t: &toml::value::Table, key: &str) -> Result { let v = t .get(key) @@ -379,6 +533,8 @@ struct RawScenario { #[serde(default)] peers: Vec, #[serde(default)] + relays: Vec, + #[serde(default)] links: Vec, #[serde(default)] mutations: Vec, @@ -433,6 +589,8 @@ struct RawPeer { struct RawLink { from: String, to: String, + #[serde(default)] + via: Option, #[serde(flatten)] overrides: RawLinkPolicy, } @@ -496,6 +654,7 @@ fn merge(parent: RawScenario, child: RawScenario) -> RawScenario { (Some(p), Some(c)) => Some(merge_policy(p, c)), }, peers: append(parent.peers, child.peers), + relays: append(parent.relays, child.relays), links: append(parent.links, child.links), mutations: append(parent.mutations, child.mutations), snapshots: append(parent.snapshots, child.snapshots), @@ -611,23 +770,74 @@ fn validate( return Err(err(path, "peers", "must declare at least one peer")); } - // Links — endpoints must be declared peers; no duplicate ordered pairs. + // Relays — IDs unique across (peers ∪ relays). RELAY_SPEC §4.1 / + // §8.2. + let mut relay_ids: BTreeSet = BTreeSet::new(); + let mut relays = Vec::with_capacity(raw.relays.len()); + for (i, r) in raw.relays.iter().enumerate() { + let field = |s: &str| format!("relays[{i}].{s}"); + if r.id.is_empty() { + return Err(err(path, field("id"), "must not be empty")); + } + if peer_ids.contains(&r.id) { + return Err(err( + path, + field("id"), + format!( + "relay id {:?} collides with a declared peer id (peer and relay namespaces are shared)", + r.id + ), + )); + } + if !relay_ids.insert(r.id.clone()) { + return Err(err( + path, + field("id"), + format!("duplicate relay id {:?}", r.id), + )); + } + if r.ingress_capacity_bps == 0 { + return Err(err(path, field("ingress_capacity_bps"), "must be > 0")); + } + if r.egress_capacity_bps_per_link == 0 { + return Err(err( + path, + field("egress_capacity_bps_per_link"), + "must be > 0", + )); + } + relays.push(r.clone()); + } + + // Nodes (the union of peers and relays). Edges may touch either. + let node_ids: BTreeSet = peer_ids.union(&relay_ids).cloned().collect(); + + // Links — endpoints must be declared nodes; no duplicate ordered + // pairs. Optional `via` shorthand synthesises two relay legs. + // RELAY_SPEC §4.1 / §8.1 / §8.2. let mut links = Vec::with_capacity(raw.links.len()); let mut seen_edges: BTreeSet<(String, String)> = BTreeSet::new(); + // (from_host, to_host) → relay through which the host pair is routed. + // Only relayed routes from `via` shorthand land here; explicit + // host→relay or relay→host edges go through the standard path. + let mut via_routes: BTreeMap<(String, String), String> = BTreeMap::new(); + // Validate first; deferred resolution into HostRoute happens once + // every link's policy is in hand. + let mut via_links: Vec<(String, String, String, LinkPolicy)> = Vec::new(); for (i, l) in raw.links.iter().enumerate() { let field = |s: &str| format!("links[{i}].{s}"); - if !peer_ids.contains(&l.from) { + if !node_ids.contains(&l.from) { return Err(err( path, field("from"), - format!("references undeclared peer {:?}", l.from), + format!("references undeclared peer or relay {:?}", l.from), )); } - if !peer_ids.contains(&l.to) { + if !node_ids.contains(&l.to) { return Err(err( path, field("to"), - format!("references undeclared peer {:?}", l.to), + format!("references undeclared peer or relay {:?}", l.to), )); } if l.from == l.to { @@ -637,6 +847,50 @@ fn validate( "self-loop links are not permitted", )); } + reject_unknown_fields(path, &format!("links[{i}]"), &l.overrides.extra)?; + let policy = resolve_policy( + path, + &format!("links[{i}]"), + &l.overrides, + Some(default_link), + )?; + if let Some(via) = &l.via { + // `via` shorthand. Both endpoints must be hosts; `via` + // must name a declared relay. The loader synthesizes the + // inbound and outbound legs with the same policy. + if relay_ids.contains(&l.from) || relay_ids.contains(&l.to) { + return Err(err( + path, + field("via"), + format!( + "a link with `via` must have host endpoints; {:?}->{:?} touches a relay", + l.from, l.to + ), + )); + } + if !relay_ids.contains(via) { + return Err(err( + path, + field("via"), + format!("via must reference a declared relay, got {via:?}"), + )); + } + if via_routes + .insert((l.from.clone(), l.to.clone()), via.clone()) + .is_some() + { + return Err(err( + path, + field("via"), + format!( + "duplicate relayed route {:?} -> via {:?} -> {:?}", + l.from, via, l.to + ), + )); + } + via_links.push((l.from.clone(), via.clone(), l.to.clone(), policy)); + continue; + } let key = (l.from.clone(), l.to.clone()); if !seen_edges.insert(key) { return Err(err( @@ -645,24 +899,220 @@ fn validate( format!("duplicate edge {:?} -> {:?}", l.from, l.to), )); } - reject_unknown_fields(path, &format!("links[{i}]"), &l.overrides.extra)?; - let policy = resolve_policy( - path, - &format!("links[{i}]"), - &l.overrides, - Some(default_link), - )?; links.push(Link { from: l.from.clone(), to: l.to.clone(), policy, }); } + // Expand via shorthands. Reject conflicts with explicit edges; + // merge with existing synth edges when policies agree. + // `explicit_edges` is the set declared verbatim in `[[links]]` + // (without `via`); `synth_edges` is what the loader has so far + // synthesized from earlier via shorthands. Overlapping synth + // edges across multiple via routes through the same relay are + // allowed iff their policies match — otherwise we cannot pick a + // single legpolicy for the shared edge. + let explicit_edges = seen_edges.clone(); + let mut synth_edges: BTreeSet<(String, String)> = BTreeSet::new(); + for (from, via, to, policy) in &via_links { + let inbound = (from.clone(), via.clone()); + let outbound = (via.clone(), to.clone()); + if explicit_edges.contains(&inbound) { + return Err(err( + path, + format!("links via {via:?}"), + format!( + "via shorthand conflicts with an explicit edge {from:?} -> {via:?}; declare one or the other" + ), + )); + } + if explicit_edges.contains(&outbound) { + return Err(err( + path, + format!("links via {via:?}"), + format!( + "via shorthand conflicts with an explicit edge {via:?} -> {to:?}; declare one or the other" + ), + )); + } + if synth_edges.contains(&inbound) { + // Already added from another via through this relay; the + // policy must agree. + let existing = links + .iter() + .find(|l| l.from == inbound.0 && l.to == inbound.1) + .expect("synth_edges set ⇒ links entry"); + if existing.policy != *policy { + return Err(err( + path, + format!("links via {via:?}"), + format!( + "multiple via shorthands through {via:?} from {from:?} disagree on the inbound leg policy" + ), + )); + } + } else { + synth_edges.insert(inbound.clone()); + seen_edges.insert(inbound.clone()); + links.push(Link { + from: inbound.0.clone(), + to: inbound.1.clone(), + policy: *policy, + }); + } + if synth_edges.contains(&outbound) { + let existing = links + .iter() + .find(|l| l.from == outbound.0 && l.to == outbound.1) + .expect("synth_edges set ⇒ links entry"); + if existing.policy != *policy { + return Err(err( + path, + format!("links via {via:?}"), + format!( + "multiple via shorthands through {via:?} to {to:?} disagree on the outbound leg policy" + ), + )); + } + } else { + synth_edges.insert(outbound.clone()); + seen_edges.insert(outbound.clone()); + links.push(Link { + from: outbound.0.clone(), + to: outbound.1.clone(), + policy: *policy, + }); + } + } + // Build routes: every ordered host pair with an explicit direct + // edge or a `via` shorthand. Reject ambiguity (both kinds for the + // same pair). Multi-hop relays (which would require an edge from + // one relay to another) are rejected on edge construction below. + let mut routes: BTreeMap<(String, String), HostRoute> = BTreeMap::new(); + for (key, via) in &via_routes { + routes.insert( + key.clone(), + HostRoute::Relayed { + from: key.0.clone(), + relay: via.clone(), + to: key.1.clone(), + }, + ); + } + for link in &links { + if !peer_ids.contains(&link.from) || !peer_ids.contains(&link.to) { + // Edges touching a relay are not by themselves a host + // route — they are legs of a route. Host-pair routes + // come from `via` shorthand or from direct host edges. + continue; + } + let key = (link.from.clone(), link.to.clone()); + if routes.contains_key(&key) { + return Err(err( + path, + "links", + format!( + "ambiguous route for {:?} -> {:?}: declared both directly and through a relay via shorthand", + link.from, link.to + ), + )); + } + routes.insert( + key, + HostRoute::Direct { + from: link.from.clone(), + to: link.to.clone(), + }, + ); + } + // Infer relayed routes from explicit host↔relay edges. For any + // ordered host pair (a, b) without a route yet, look for a relay + // R such that explicit edges a→R and R→b both exist. Exactly one + // such R ⇒ relayed route; more than one ⇒ ambiguous (reject). + // This makes the loader robust against TOML round-trips that + // emit the synth legs explicitly: the loader recovers the same + // routing decision the original via shorthand made. + for a in &peer_ids { + for b in &peer_ids { + if a == b { + continue; + } + let key = (a.clone(), b.clone()); + if routes.contains_key(&key) { + continue; + } + let candidates: Vec<&String> = relay_ids + .iter() + .filter(|r| { + seen_edges.contains(&(a.clone(), (*r).clone())) + && seen_edges.contains(&((*r).clone(), b.clone())) + }) + .collect(); + match candidates.len() { + 0 => {} + 1 => { + routes.insert( + key, + HostRoute::Relayed { + from: a.clone(), + relay: candidates[0].clone(), + to: b.clone(), + }, + ); + } + _ => { + let names: Vec = + candidates.iter().map(|s| format!("{s:?}")).collect(); + return Err(err( + path, + "links", + format!( + "ambiguous route for {:?} -> {:?}: multiple relays ({}) carry both legs", + a, + b, + names.join(", ") + ), + )); + } + } + } + } + // Reject multi-hop relay paths (relay-to-relay edges that would + // allow chaining). MVP rejects any direct relay→relay edge. + for link in &links { + if relay_ids.contains(&link.from) && relay_ids.contains(&link.to) { + return Err(err( + path, + "links", + format!( + "multi-hop relay routes are not supported in the MVP: relay {:?} -> relay {:?}", + link.from, link.to + ), + )); + } + } + let routes_vec: Vec = routes.into_values().collect(); + // Build a quick map peer_id → kind so worker_exit mutations can + // validate that the target is `stage`-kind. RELAY_SPEC §6.2 / + // §8.2. + let peer_kinds: BTreeMap = peers + .iter() + .map(|p| (p.id.clone(), p.kind.clone())) + .collect(); // Mutations: parse from raw table, validate references, validate at_ns. let mut mutations = Vec::with_capacity(raw.mutations.len()); for (i, m) in raw.mutations.iter().enumerate() { - let parsed = parse_mutation(path, i, m, &peer_ids, &seen_edges)?; + let parsed = parse_mutation( + path, + i, + m, + &peer_ids, + &seen_edges, + &relay_ids, + &peer_kinds, + )?; if parsed.at_ns > duration_ns { return Err(err( path, @@ -692,7 +1142,7 @@ fn validate( // Assertions. let mut assertions = Vec::with_capacity(raw.assertions.len()); for (i, a) in raw.assertions.iter().enumerate() { - let parsed = parse_assertion(path, i, a, &peer_ids, duration_ns)?; + let parsed = parse_assertion(path, i, a, &peer_ids, &relay_ids, duration_ns)?; assertions.push(parsed); } @@ -706,10 +1156,12 @@ fn validate( default_tick, default_link, peers, + relays, links, mutations, snapshots, assertions, + routes: routes_vec, }) } @@ -831,6 +1283,8 @@ fn parse_mutation( table: &toml::value::Table, peers: &BTreeSet, edges: &BTreeSet<(String, String)>, + relays: &BTreeSet, + peer_kinds: &BTreeMap, ) -> Result { let field = |s: &str| format!("mutations[{i}].{s}"); let at_ns = require_u64_field(path, &field("at_ns"), table.get("at_ns"))?; @@ -888,6 +1342,112 @@ fn parse_mutation( .and_then(|v| v.as_bool()) .unwrap_or(false), }, + "worker_exit" => { + let peer = peer_field(path, &field("peer"), table.get("peer"), peers)?; + // RELAY_SPEC §8.2 — worker_exit must target a stage peer. + match peer_kinds.get(&peer).map(|s| s.as_str()) { + Some("stage") => {} + Some(other) => { + return Err(err( + path, + field("peer"), + format!( + "worker_exit targets peer {peer:?} of kind {other:?}; only \"stage\" peers accept it" + ), + )); + } + None => { + return Err(err( + path, + field("peer"), + format!("references undeclared peer {peer:?}"), + )); + } + } + let reason = table + .get("reason") + .and_then(|v| v.as_str()) + .ok_or_else(|| err(path, field("reason"), "required string"))? + .to_string(); + let status_code = match table.get("status_code") { + None => None, + Some(v) => Some( + v.as_integer() + .ok_or_else(|| { + err(path, field("status_code"), "must be an integer") + })? + .try_into() + .map_err(|_| { + err(path, field("status_code"), "must fit in i32") + })?, + ), + }; + let signal = match table.get("signal") { + None => None, + Some(v) => Some( + v.as_integer() + .ok_or_else(|| err(path, field("signal"), "must be an integer"))? + .try_into() + .map_err(|_| err(path, field("signal"), "must fit in i32"))?, + ), + }; + MutationKind::WorkerExit { + peer, + reason, + status_code, + signal, + } + } + "relay_kill" => MutationKind::RelayKill { + relay: relay_field(path, &field("relay"), table.get("relay"), relays)?, + }, + "relay_boot" => MutationKind::RelayBoot { + relay: relay_field(path, &field("relay"), table.get("relay"), relays)?, + }, + "relay_capacity_change" => { + let relay = relay_field(path, &field("relay"), table.get("relay"), relays)?; + let opt_u64 = |key: &str| -> Result, LoadError> { + match table.get(key) { + None => Ok(None), + Some(v) => { + let n = v.as_integer().ok_or_else(|| { + err(path, format!("mutations[{i}].{key}"), "must be an integer") + })?; + if n < 0 { + return Err(err( + path, + format!("mutations[{i}].{key}"), + "must be non-negative", + )); + } + if key != "queue_depth_bytes" && n == 0 { + return Err(err( + path, + format!("mutations[{i}].{key}"), + "must be > 0", + )); + } + Ok(Some(n as u64)) + } + } + }; + let ingress = opt_u64("ingress_capacity_bps")?; + let egress = opt_u64("egress_capacity_bps_per_link")?; + let depth = opt_u64("queue_depth_bytes")?; + if ingress.is_none() && egress.is_none() && depth.is_none() { + return Err(err( + path, + field("kind"), + "relay_capacity_change must change at least one of ingress_capacity_bps / egress_capacity_bps_per_link / queue_depth_bytes", + )); + } + MutationKind::RelayCapacityChange { + relay, + ingress_capacity_bps: ingress, + egress_capacity_bps_per_link: egress, + queue_depth_bytes: depth, + } + } other => { return Err(err( path, @@ -983,6 +1543,22 @@ fn peer_field( Ok(s) } +fn relay_field( + path: &Path, + field: &str, + v: Option<&toml::Value>, + relays: &BTreeSet, +) -> Result { + let s = v + .and_then(|x| x.as_str()) + .ok_or_else(|| err(path, field, "required string"))? + .to_string(); + if !relays.contains(&s) { + return Err(err(path, field, format!("references undeclared relay {s:?}"))); + } + Ok(s) +} + fn require_u64_field(path: &Path, field: &str, v: Option<&toml::Value>) -> Result { let n = v .and_then(|x| x.as_integer()) @@ -1029,6 +1605,7 @@ fn parse_assertion( i: usize, table: &toml::value::Table, peers: &BTreeSet, + relays: &BTreeSet, duration_ns: u64, ) -> Result { let field = |s: &str| format!("assertions[{i}].{s}"); @@ -1200,6 +1777,83 @@ fn parse_assertion( table.get("max_per_window"), )?, }, + "relay_queue_depth_bounded" => { + let relay = relay_field(path, &field("relay"), table.get("relay"), relays)?; + let max_bytes = require_u64_field(path, &field("max_bytes"), table.get("max_bytes"))?; + let window_start_ns = match table.get("window_start_ns") { + None => None, + Some(v) => Some(require_u64_field(path, &field("window_start_ns"), Some(v))?), + }; + let window_end_ns = match table.get("window_end_ns") { + None => None, + Some(v) => Some(require_u64_field(path, &field("window_end_ns"), Some(v))?), + }; + if let (Some(s), Some(e)) = (window_start_ns, window_end_ns) { + if s > e { + return Err(err( + path, + field("window_start_ns"), + "window_start_ns must be <= window_end_ns", + )); + } + } + if let Some(e) = window_end_ns { + check_t(e, "window_end_ns")?; + } + AssertionKind::RelayQueueDepthBounded { + relay, + max_bytes, + window_start_ns, + window_end_ns, + } + } + "worker_alive_throughout" => { + let peer = peer_field(path, &field("peer"), table.get("peer"), peers)?; + let window_start_ns = require_u64_field( + path, + &field("window_start_ns"), + table.get("window_start_ns"), + )?; + let window_end_ns = require_u64_field( + path, + &field("window_end_ns"), + table.get("window_end_ns"), + )?; + check_t(window_end_ns, "window_end_ns")?; + if window_start_ns > window_end_ns { + return Err(err( + path, + field("window_start_ns"), + "window_start_ns must be <= window_end_ns", + )); + } + AssertionKind::WorkerAliveThroughout { + peer, + window_start_ns, + window_end_ns, + } + } + "name_resolves_within" => { + let name = table + .get("name") + .and_then(|v| v.as_str()) + .ok_or_else(|| err(path, field("name"), "required string"))? + .to_string(); + if name.is_empty() { + return Err(err(path, field("name"), "must not be empty")); + } + let observers = string_list(path, &field("observers"), table.get("observers"))?; + check_peer_list(&observers, "observers")?; + let within_ns = require_u64_field(path, &field("within_ns"), table.get("within_ns"))?; + let from_ns = require_u64_field(path, &field("from_ns"), table.get("from_ns"))?; + check_t(from_ns.saturating_add(within_ns), "from_ns + within_ns")?; + AssertionKind::NameResolvesWithin { + name, + observers, + within_ns, + from_ns, + } + } other => { return Err(err( path, diff --git a/crates/simulation/src/stage_host.rs b/crates/simulation/src/stage_host.rs new file mode 100644 index 0000000..b399c7b --- /dev/null +++ b/crates/simulation/src/stage_host.rs @@ -0,0 +1,220 @@ +//! Pipeline-stage host kind (RELAY_SPEC §5). +//! +//! Wraps the production stage supervisor lifecycle as a simulator +//! host kind. The MVP host kind models only the lifecycle (Cold → +//! Registering → Running → Halted) and the three diagnostic +//! emissions named in RELAY_SPEC §5.4 (`register_name`, +//! `stage_lifecycle`, `worker_exited`). It does not engage in +//! inter-stage application traffic (§5.1): its codec is intentionally +//! empty for the MVP. +//! +//! The §5.3 internal-cause exit semantics are: a `WorkerExit` +//! envelope arriving via `recv` produces exactly two actions in +//! order — `RecordEvent` carrying a `worker_exited` payload, then +//! `Halt`. The order is normative; reordering would lose the +//! diagnostic on runs that terminate adjacent to the exit time. + +use std::collections::BTreeMap; + +use serde_json::json; + +use crate::host::{ + Action, EventBytes, Host, HostFactory, HostId, HostMessage, KindTag, SnapshotBytes, +}; + +const KIND_TAG: KindTag = "stage"; + +/// RELAY_SPEC §5.2 lifecycle states. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StageState { + Cold, + Registering, + Running, + Halted, +} + +impl StageState { + pub fn as_str(&self) -> &'static str { + match self { + StageState::Cold => "Cold", + StageState::Registering => "Registering", + StageState::Running => "Running", + StageState::Halted => "Halted", + } + } +} + +pub struct StageHost { + id: HostId, + name: String, + address: String, + state: StageState, + name_registry: BTreeMap, + last_exit_reason: Option, +} + +impl StageHost { + pub fn new(id: impl Into, name: impl Into, address: impl Into) -> Self { + Self { + id: id.into(), + name: name.into(), + address: address.into(), + state: StageState::Cold, + name_registry: BTreeMap::new(), + last_exit_reason: None, + } + } + + fn lifecycle_event(&self, from: StageState, to: StageState) -> Action { + Action::RecordEvent { + kind_tag: KIND_TAG.to_string(), + event: encode(&json!({ + "kind": "stage_lifecycle", + "from": from.as_str(), + "to": to.as_str(), + })), + } + } +} + +fn encode(v: &serde_json::Value) -> EventBytes { + serde_json::to_vec(v).expect("stage host event serialises") +} + +impl Host for StageHost { + fn id(&self) -> &str { + &self.id + } + + fn kind_tag(&self) -> KindTag { + KIND_TAG + } + + fn tick(&mut self, _now_ns: u64) -> Vec { + // RELAY_SPEC §5.2 — the first tick drives Cold → Registering + // and immediately Registering → Running. Each transition + // emits exactly one `stage_lifecycle` event; the + // Registering → Running transition additionally emits one + // `register_name`. Subsequent ticks are no-ops. + match self.state { + StageState::Cold => { + let mut actions = Vec::new(); + actions.push(self.lifecycle_event(StageState::Cold, StageState::Registering)); + self.state = StageState::Registering; + // The transition to Running is synchronous in the + // MVP — there is no out-of-band ack to wait for. + let payload = json!({ + "kind": "register_name", + "name": self.name, + "address": self.address, + "peer_node_id": self.id, + }); + actions.push(Action::RecordEvent { + kind_tag: KIND_TAG.to_string(), + event: encode(&payload), + }); + self.name_registry + .insert(self.name.clone(), self.address.clone()); + actions.push(self.lifecycle_event(StageState::Registering, StageState::Running)); + self.state = StageState::Running; + actions + } + _ => Vec::new(), + } + } + + fn recv(&mut self, message: HostMessage, _now_ns: u64) -> Vec { + match message { + HostMessage::WorkerExit { + reason, + status_code, + signal, + } => { + if self.state == StageState::Halted { + return Vec::new(); + } + // RELAY_SPEC §5.3 — the recv for `WorkerExit` returns + // exactly: (1) `RecordEvent` carrying the + // `worker_exited` payload, then (2) `Halt`. The + // lifecycle transition into `Halted` is emitted as + // part of the same envelope so a streaming reader + // sees the canonical Running → Halted edge. + let mut actions = Vec::new(); + let mut payload = serde_json::Map::new(); + payload.insert("kind".to_string(), json!("worker_exited")); + payload.insert("reason".to_string(), json!(reason)); + if let Some(s) = status_code { + payload.insert("status_code".to_string(), json!(s)); + } + if let Some(s) = signal { + payload.insert("signal".to_string(), json!(s)); + } + actions.push(Action::RecordEvent { + kind_tag: KIND_TAG.to_string(), + event: encode(&serde_json::Value::Object(payload)), + }); + actions.push(self.lifecycle_event(self.state, StageState::Halted)); + self.last_exit_reason = Some(reason); + self.state = StageState::Halted; + actions.push(Action::Halt); + actions + } + // App, TimerFired, SendFailed: the MVP stage host kind + // does not engage in inter-stage application traffic + // (§5.1) and does not schedule its own timers. Anything + // arriving here is unsolicited; we drop it without + // emitting actions, but the engine still records the + // envelope's arrival through the bundle writer. + _ => Vec::new(), + } + } + + fn snapshot(&self) -> SnapshotBytes { + // RELAY_SPEC §5.4. Stage snapshot is opaque to the §9 bundle + // schema for SWIM; the evaluator picks `name_registry` and + // optionally `last_exit_reason` from it. + let mut payload = json!({ + "state": self.state.as_str(), + "name_registry": self.name_registry, + "members": {}, + "self_incarnation": 0, + }); + if self.state == StageState::Halted { + if let Some(reason) = &self.last_exit_reason { + payload["last_exit_reason"] = json!(reason); + } + } + serde_json::to_vec(&payload).expect("stage snapshot serialises") + } +} + +// ────────────────────────────────────────────────────────────────────── +// Factory +// ────────────────────────────────────────────────────────────────────── + +pub struct StageHostFactory; + +impl HostFactory for StageHostFactory { + fn kind_tag(&self) -> KindTag { + KIND_TAG + } + fn build( + &self, + host_id: &str, + kind_config: &toml::value::Table, + _peers: &[String], + _tick_period_ns: u64, + ) -> Box { + let name = kind_config + .get("name") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + let address = kind_config + .get("address") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + Box::new(StageHost::new(host_id, name, address)) + } +} diff --git a/crates/simulation/src/swim_host.rs b/crates/simulation/src/swim_host.rs index 642d6a7..a67e993 100644 --- a/crates/simulation/src/swim_host.rs +++ b/crates/simulation/src/swim_host.rs @@ -380,6 +380,19 @@ impl Host for SwimHost { let actions = self.node.report_send_failure(node_id); self.collect_actions(actions) } + // RELAY_SPEC §5.3 — the SWIM host kind has no + // `WorkerExit` semantics. The engine guards this at + // `dispatch_mutation` (aborting before the envelope is + // even routed), so reaching this arm means the engine's + // gate is broken or the host has been reused outside the + // stage-host invariant. Panic loudly rather than silently + // dropping; silent fallback is the bug class the + // simulator exists to prevent. + HostMessage::WorkerExit { .. } => { + panic!( + "SWIM host kind cannot receive WorkerExit; the engine's WorkerExitOnWrongKind gate is broken" + ); + } } } diff --git a/crates/simulation/tests/adversarial_judge.rs b/crates/simulation/tests/adversarial_judge.rs new file mode 100644 index 0000000..e87bfdd --- /dev/null +++ b/crates/simulation/tests/adversarial_judge.rs @@ -0,0 +1,798 @@ +//! Adversarial tests written by the judge. +//! +//! Each test hunts a specific gap the spec/code might have on the +//! purpose the simulator exists to serve. + +use serde_json::Value; +use simulation::bundle::{BundleRecord, EventPayload, SnapshotRecord, VecWriter}; +use simulation::engine::Engine; +use simulation::evaluator::{ + EventLine, Outcome, SnapshotEntry, SnapshotIndex, evaluate, +}; +use simulation::network::{Network, SendOutcome}; +use simulation::scenario::{HostKindRegistry, load_from_str}; +use simulation::stage_host::StageHostFactory; +use std::path::Path; + +fn registry() -> HostKindRegistry { + HostKindRegistry::with_swim() +} + +fn parse(text: &str) -> simulation::scenario::Scenario { + load_from_str(Path::new("(test)"), text, ®istry()) + .expect("scenario must validate") +} + +fn payload_kind(b: &[u8]) -> Option { + let v: Value = serde_json::from_slice(b).ok()?; + v["kind"].as_str().map(|s| s.to_string()) +} + +// ─────────────────────────────────────────────────────────────────── +// Gap 1: BOUNDARY TIMING. +// SIM_SPEC §6A.6 (and the file's own existing test) names: +// "Event-before-halt is observable. A scenario whose `duration_ns` +// is the same nanosecond as a `WorkerExit` mutation's `at_ns` +// produces a bundle containing the `worker_exited` event." +// The existing test (worker_exit_event_is_emitted_before_halt_takes_effect) +// sets worker_exit at half the duration, NOT at the boundary; this +// adversarial test exercises the actual at_ns == duration_ns case. +// ─────────────────────────────────────────────────────────────────── +#[test] +fn worker_exit_at_exact_duration_still_emits_worker_exited_event() { + let text = r#" + name = "boundary" + seed = 1 + duration_ns = 500_000_000 + + [default_tick] + period_ns = 100_000_000 + + [default_link] + latency_ns = 1_000_000 + jitter_stddev_ns = 0 + loss_prob_ppm = 0 + reorder_prob_ppm = 0 + bandwidth_bps = 1_000_000_000 + cold_dial_penalty_ns = 0 + cache_warm_after_ns = 1_000_000_000 + cache_invalidate_after_idle_ns = 10_000_000_000 + + [[peers]] + id = "s" + kind = "stage" + initial_state = "cold" + kind_config = { name = "pp-s", address = "10.0.0.1:7700" } + + [[peers]] + id = "other" + kind = "stage" + initial_state = "cold" + kind_config = { name = "pp-other", address = "10.0.0.2:7700" } + + [[links]] + from = "s" + to = "other" + [[links]] + from = "other" + to = "s" + + # Worker exit at exactly the run's duration. Per §6A.6 the + # event-before-halt rule must still surface the worker_exited + # event in the bundle. + [[mutations]] + at_ns = 500_000_000 + kind = "worker_exit" + peer = "s" + reason = "boundary crash" + "#; + let scen = parse(text); + let writer = VecWriter::default(); + let network = Network::new(&scen); + let mut engine = Engine::new(&scen, network, writer); + engine.register_factory(Box::new(StageHostFactory)); + engine.auto_install_hosts(); + let _ = engine.run(); + let writer = engine.into_writer(); + let worker_exited: Vec<_> = writer + .records + .iter() + .filter_map(|r| match r { + BundleRecord::Event(e) => { + if let EventPayload::Bytes(b) = &e.event { + payload_kind(b) + .filter(|k| k == "worker_exited") + .map(|_| e.host_id.clone()) + } else { + None + } + } + _ => None, + }) + .collect(); + assert_eq!( + worker_exited, + vec![Some("s".to_string())], + "boundary worker_exit must still produce worker_exited; the spec §6A.6 names this exact case" + ); +} + +// ─────────────────────────────────────────────────────────────────── +// Gap 2: SUBSTREAM ISOLATION UNDER STRUCTURAL EDITS. +// SIM_SPEC §7.2 says: "editing one link's policy must not perturb the +// draws on any other link, or every test edit becomes a new random +// universe and bisection is impossible." The companion test in +// network_invariants asserts policy-edit isolation; nothing tests +// that ADDING a new peer/link leaves existing edges' draws unchanged. +// If a sim run's bundle changes when an unrelated peer is added, the +// judge's "structural edit" attack succeeds — the gap §7.2 names is +// real. +// ─────────────────────────────────────────────────────────────────── +fn three_peer_scenario_text(extra_peer: bool) -> String { + let extra_peer_block = if extra_peer { + r#" + [[peers]] + id = "z_unrelated" + kind = "stage" + initial_state = "cold" + kind_config = { name = "pp-z", address = "10.0.0.99:7700" } + "# + } else { + "" + }; + let extra_link_block = if extra_peer { + r#" + [[links]] + from = "z_unrelated" + to = "a" + [[links]] + from = "a" + to = "z_unrelated" + "# + } else { + "" + }; + format!( + r#" + name = "structural_edit" + seed = 42 + duration_ns = 500_000_000 + + [default_tick] + period_ns = 100_000_000 + + [default_link] + latency_ns = 10_000_000 + jitter_stddev_ns = 5_000_000 + loss_prob_ppm = 100_000 + reorder_prob_ppm = 0 + bandwidth_bps = 1_000_000_000 + cold_dial_penalty_ns = 0 + cache_warm_after_ns = 1_000_000_000 + cache_invalidate_after_idle_ns = 10_000_000_000 + + [[peers]] + id = "a" + kind = "stage" + initial_state = "cold" + kind_config = {{ name = "pp-a", address = "10.0.0.1:7700" }} + + [[peers]] + id = "b" + kind = "stage" + initial_state = "cold" + kind_config = {{ name = "pp-b", address = "10.0.0.2:7700" }} + {extra_peer_block} + [[links]] + from = "a" + to = "b" + [[links]] + from = "b" + to = "a" + {extra_link_block} + "# + ) +} + +fn run_collect_relevant_records(text: &str) -> Vec { + let scen = parse(text); + let writer = VecWriter::default(); + let network = Network::new(&scen); + let mut engine = Engine::new(&scen, network, writer); + engine.register_factory(Box::new(StageHostFactory)); + engine.auto_install_hosts(); + let _ = engine.run(); + let writer = engine.into_writer(); + // Project to a host-id-filtered slice that excludes the added + // peer entirely. The §7.2 claim is that the "alphabet a vs b" + // behaviour is unchanged. + let mut out = Vec::new(); + for r in writer.records.iter() { + match r { + BundleRecord::Event(e) => { + let host = e.host_id.as_deref().unwrap_or(""); + if host == "a" || host == "b" || host.is_empty() { + // Bring in the event so we can compare. + let kind = match &e.event { + EventPayload::Bytes(b) => payload_kind(b).unwrap_or_default(), + EventPayload::CacheStateChange { .. } => "cache_state_change".into(), + EventPayload::DialStart { .. } => "dial_start".into(), + EventPayload::DialOutcome { .. } => "dial_outcome".into(), + EventPayload::DropOnSend { from, to, .. } => { + if from == "a" || from == "b" || to == "a" || to == "b" { + "drop_on_send".into() + } else { + continue; + } + } + EventPayload::DropOnDelivery { to, .. } => { + if to == "a" || to == "b" { + "drop_on_delivery".into() + } else { + continue; + } + } + EventPayload::RelayEnqueue { .. } + | EventPayload::RelayDequeue { .. } + | EventPayload::RelayDrop { .. } => continue, + }; + out.push(format!("{}@{}|{}", host, e.virtual_time_ns, kind)); + } + } + _ => {} + } + } + out +} + +#[test] +fn adding_unrelated_peer_does_not_perturb_existing_peers_records() { + let baseline = run_collect_relevant_records(&three_peer_scenario_text(false)); + let with_extra = run_collect_relevant_records(&three_peer_scenario_text(true)); + assert_eq!( + baseline, with_extra, + "Adding an unrelated peer must not perturb the records emitted by `a` and `b` \ + (substream isolation §7.2)." + ); +} + + +// Companion: prove the failure is specifically at the boundary, not +// a bug in the WorkerExit plumbing more broadly. With at_ns one ns +// less than duration_ns, the LocalRecv generated by the mutation +// pops at a strictly earlier instant than Terminate and the event +// is recorded — proving the implementation works correctly off +// the boundary and FAILS exactly on the §6A.6 boundary case. +#[test] +fn worker_exit_one_ns_below_duration_works_correctly() { + let text = r#" + name = "near_boundary" + seed = 1 + duration_ns = 500_000_000 + + [default_tick] + period_ns = 100_000_000 + + [default_link] + latency_ns = 1_000_000 + jitter_stddev_ns = 0 + loss_prob_ppm = 0 + reorder_prob_ppm = 0 + bandwidth_bps = 1_000_000_000 + cold_dial_penalty_ns = 0 + cache_warm_after_ns = 1_000_000_000 + cache_invalidate_after_idle_ns = 10_000_000_000 + + [[peers]] + id = "s" + kind = "stage" + initial_state = "cold" + kind_config = { name = "pp-s", address = "10.0.0.1:7700" } + + [[peers]] + id = "other" + kind = "stage" + initial_state = "cold" + kind_config = { name = "pp-other", address = "10.0.0.2:7700" } + + [[links]] + from = "s" + to = "other" + [[links]] + from = "other" + to = "s" + + [[mutations]] + at_ns = 499_999_999 + kind = "worker_exit" + peer = "s" + reason = "near-boundary crash" + "#; + let scen = parse(text); + let writer = VecWriter::default(); + let network = Network::new(&scen); + let mut engine = Engine::new(&scen, network, writer); + engine.register_factory(Box::new(StageHostFactory)); + engine.auto_install_hosts(); + let _ = engine.run(); + let writer = engine.into_writer(); + let worker_exited: Vec<_> = writer + .records + .iter() + .filter_map(|r| match r { + BundleRecord::Event(e) => { + if let EventPayload::Bytes(b) = &e.event { + payload_kind(b) + .filter(|k| k == "worker_exited") + .map(|_| e.host_id.clone()) + } else { + None + } + } + _ => None, + }) + .collect(); + assert_eq!( + worker_exited, + vec![Some("s".to_string())], + "off-boundary case must emit worker_exited — confirms the boundary failure is the bug" + ); +} + +// ─────────────────────────────────────────────────────────────────── +// Gap 3 (NEW, iter 4): END-TO-END BOUNDARY ENFORCEMENT for the +// `worker_alive_throughout` assertion (SIM_SPEC §10.1). +// +// Spec text (§10.1): "The window is inclusive on both ends: a halt +// at window_end_ns (or at duration_ns when the window spans the +// whole run) fails the assertion, because the §6A.3 synchronous +// dispatch rule guarantees the stage_lifecycle → Halted event +// appears in the bundle even at the boundary." +// +// The previous judge round only verified that the *event* survives +// the boundary. This test ties the two halves together: the +// assertion must surface the boundary halt as `Fail`. If the engine +// drops the lifecycle event (or the evaluator's window comparison +// is exclusive on the upper end), this assertion silently passes +// when a stage actually died at the deadline — exactly the +// invisibility failure the N3 report names. +// ─────────────────────────────────────────────────────────────────── +#[test] +fn worker_alive_throughout_fails_at_exact_duration_boundary() { + let text = r#" + name = "alive_throughout_boundary" + seed = 1 + duration_ns = 500_000_000 + + [default_tick] + period_ns = 100_000_000 + + [default_link] + latency_ns = 1_000_000 + jitter_stddev_ns = 0 + loss_prob_ppm = 0 + reorder_prob_ppm = 0 + bandwidth_bps = 1_000_000_000 + cold_dial_penalty_ns = 0 + cache_warm_after_ns = 1_000_000_000 + cache_invalidate_after_idle_ns = 10_000_000_000 + + [[peers]] + id = "s" + kind = "stage" + initial_state = "cold" + kind_config = { name = "pp-s", address = "10.0.0.1:7700" } + + [[peers]] + id = "other" + kind = "stage" + initial_state = "cold" + kind_config = { name = "pp-other", address = "10.0.0.2:7700" } + + [[links]] + from = "s" + to = "other" + [[links]] + from = "other" + to = "s" + + # Whole-run window plus a halt at the closing instant. The + # spec §10.1 boundary clause says this must Fail. + [[assertions]] + kind = "worker_alive_throughout" + peer = "s" + window_start_ns = 0 + window_end_ns = 500_000_000 + + [[mutations]] + at_ns = 500_000_000 + kind = "worker_exit" + peer = "s" + reason = "boundary crash" + "#; + let scen = parse(text); + let writer = VecWriter::default(); + let network = Network::new(&scen); + let mut engine = Engine::new(&scen, network, writer); + engine.register_factory(Box::new(StageHostFactory)); + engine.auto_install_hosts(); + let _ = engine.run(); + let writer = engine.into_writer(); + + // Reconstruct the evaluator's view from the bundle so the test + // exercises the same path a real run would. + let mut events: Vec = Vec::new(); + let mut snapshots = SnapshotIndex::default(); + let mut line_idx = 0usize; + let mut next_seq: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + for r in writer.records.iter() { + match r { + BundleRecord::Event(e) => { + events.push(EventLine::from_event_record(e, line_idx)); + line_idx += 1; + } + BundleRecord::Mutation(m) => { + events.push(EventLine::from_mutation_record(m, line_idx)); + line_idx += 1; + } + BundleRecord::Snapshot(s) => { + let seq = next_seq.entry(s.host_id.clone()).or_insert(0); + let entry = SnapshotEntry::from_snapshot_record(s, *seq); + *seq += 1; + snapshots + .by_host + .entry(s.host_id.clone()) + .or_default() + .push(entry); + } + } + } + let verdicts = evaluate(&scen, &events, &snapshots); + assert_eq!( + verdicts.len(), + 1, + "exactly one verdict from one assertion" + ); + assert_eq!( + verdicts[0].outcome, + Outcome::Fail, + "spec §10.1 boundary clause: halt at window_end_ns must Fail; \ + got {:?}. If this passes, either the bundle is missing the \ + stage_lifecycle→Halted event at duration_ns (the §6A.3 \ + synchronous-dispatch guarantee failed) or the evaluator's \ + window comparison is exclusive on the upper end (the §10.1 \ + inclusive clause failed). Either way, a stage that died at \ + the resolve deadline is now an invisible failure — the bug \ + class the simulator exists to make visible.", + verdicts[0].outcome, + ); +} + +// ─────────────────────────────────────────────────────────────────── +// Gap 4 (NEW, iter 4): SNAPSHOT AT THE BOUNDARY captures the +// post-halt state. +// +// Spec §4.5 says snapshot "asks every live host for its snapshot()" +// at the scheduled time; §6A.4 says the stage snapshot includes +// `state` and `last_exit_reason` (the latter present only when +// `state == Halted`). Spec §4.1 names mutations as enqueued before +// snapshots — so at the same virtual time a mutation pops first. +// Combined with §6A.3's synchronous WorkerExit dispatch, a snapshot +// at `duration_ns` paired with a `WorkerExit` at `duration_ns` must +// show the stage in `Halted` with `last_exit_reason` populated. +// +// This is the operationally-interesting case: a calibration scenario +// asking "what state did the stage land in at the deadline?" gets a +// truthful answer only if both the synchronous dispatch and the +// snapshot's "live host" inclusion at the boundary work together. +// A killed-but-not-snapshot path would mark the host's terminal +// state as `Running` in the last snapshot — a silent diagnostic +// loss. +// ─────────────────────────────────────────────────────────────────── +#[test] +fn snapshot_at_duration_captures_halted_state_after_worker_exit() { + let text = r#" + name = "snapshot_boundary" + seed = 1 + duration_ns = 500_000_000 + + [default_tick] + period_ns = 100_000_000 + + [default_link] + latency_ns = 1_000_000 + jitter_stddev_ns = 0 + loss_prob_ppm = 0 + reorder_prob_ppm = 0 + bandwidth_bps = 1_000_000_000 + cold_dial_penalty_ns = 0 + cache_warm_after_ns = 1_000_000_000 + cache_invalidate_after_idle_ns = 10_000_000_000 + + [[peers]] + id = "s" + kind = "stage" + initial_state = "cold" + kind_config = { name = "pp-s", address = "10.0.0.1:7700" } + + [[peers]] + id = "other" + kind = "stage" + initial_state = "cold" + kind_config = { name = "pp-other", address = "10.0.0.2:7700" } + + [[links]] + from = "s" + to = "other" + [[links]] + from = "other" + to = "s" + + [[mutations]] + at_ns = 500_000_000 + kind = "worker_exit" + peer = "s" + reason = "deadline crash" + + [[snapshots]] + at_ns = 500_000_000 + "#; + let scen = parse(text); + let writer = VecWriter::default(); + let network = Network::new(&scen); + let mut engine = Engine::new(&scen, network, writer); + engine.register_factory(Box::new(StageHostFactory)); + engine.auto_install_hosts(); + let _ = engine.run(); + let writer = engine.into_writer(); + + // Find the snapshot of `s` at duration_ns and check its `state` + // and `last_exit_reason`. + let snap: Option<&SnapshotRecord> = writer.records.iter().find_map(|r| match r { + BundleRecord::Snapshot(s) + if s.host_id == "s" && s.virtual_time_ns == 500_000_000 => + { + Some(s) + } + _ => None, + }); + let snap = snap.expect( + "spec §4.5: snapshot at duration_ns must fire for every live host; \ + no snapshot for `s` found in the bundle.", + ); + let parsed: Value = serde_json::from_slice(&snap.snapshot).expect("snapshot bytes JSON"); + assert_eq!( + parsed["state"].as_str(), + Some("Halted"), + "spec §6A.4: snapshot must reflect post-WorkerExit state. The mutation pops \ + before the snapshot (lower construction-time seq) and is dispatched \ + synchronously (§6A.3), so by the time the snapshot fires the host is in \ + Halted. Got: {:?}", + parsed["state"], + ); + assert_eq!( + parsed["last_exit_reason"].as_str(), + Some("deadline crash"), + "spec §6A.4: snapshot in Halted state must include last_exit_reason." + ); +} + +// ─────────────────────────────────────────────────────────────────── +// Gap 5 (NEW, iter 4): TWO WORKER EXITS AT THE EXACT BOUNDARY. +// +// The iter-3 fix processes WorkerExit synchronously. With multiple +// WorkerExit mutations at the same virtual time (the boundary), each +// must dispatch in turn and each must record its `worker_exited` +// event before `Terminate` pops. If the synchronous dispatch +// short-circuits on the first mutation (or if a same-time second +// mutation is silently dropped because the engine already considers +// time advanced), Layer C's "multi-stage simultaneous death" gets +// silently truncated to "first stage dies, others vanish." +// ─────────────────────────────────────────────────────────────────── +#[test] +fn two_worker_exits_at_exact_duration_each_emit_worker_exited() { + let text = r#" + name = "two_at_boundary" + seed = 1 + duration_ns = 500_000_000 + + [default_tick] + period_ns = 100_000_000 + + [default_link] + latency_ns = 1_000_000 + jitter_stddev_ns = 0 + loss_prob_ppm = 0 + reorder_prob_ppm = 0 + bandwidth_bps = 1_000_000_000 + cold_dial_penalty_ns = 0 + cache_warm_after_ns = 1_000_000_000 + cache_invalidate_after_idle_ns = 10_000_000_000 + + [[peers]] + id = "s1" + kind = "stage" + initial_state = "cold" + kind_config = { name = "pp-s1", address = "10.0.0.1:7700" } + + [[peers]] + id = "s2" + kind = "stage" + initial_state = "cold" + kind_config = { name = "pp-s2", address = "10.0.0.2:7700" } + + [[links]] + from = "s1" + to = "s2" + [[links]] + from = "s2" + to = "s1" + + [[mutations]] + at_ns = 500_000_000 + kind = "worker_exit" + peer = "s1" + reason = "joint crash 1" + + [[mutations]] + at_ns = 500_000_000 + kind = "worker_exit" + peer = "s2" + reason = "joint crash 2" + "#; + let scen = parse(text); + let writer = VecWriter::default(); + let network = Network::new(&scen); + let mut engine = Engine::new(&scen, network, writer); + engine.register_factory(Box::new(StageHostFactory)); + engine.auto_install_hosts(); + let _ = engine.run(); + let writer = engine.into_writer(); + + let worker_exited_for: std::collections::BTreeSet = writer + .records + .iter() + .filter_map(|r| match r { + BundleRecord::Event(e) => { + if let EventPayload::Bytes(b) = &e.event { + payload_kind(b) + .filter(|k| k == "worker_exited") + .and_then(|_| e.host_id.clone()) + } else { + None + } + } + _ => None, + }) + .collect(); + let expected: std::collections::BTreeSet = + ["s1".to_string(), "s2".to_string()].into_iter().collect(); + assert_eq!( + worker_exited_for, expected, + "spec §6A.6 boundary case generalised to two same-time WorkerExits: both \ + stages must contribute one worker_exited record. If a same-time second \ + mutation is dropped (e.g. because the engine treats `now == duration_ns` \ + as terminal after the first sync dispatch), Layer C's multi-stage \ + deadline-collapse failure mode becomes invisible." + ); +} + +// ─────────────────────────────────────────────────────────────────── +// Gap 6 (NEW, iter 4): RELAY HOL across INDEPENDENT SENDERS to the +// same destination — the exact shape Layer A names. +// +// The N3 report (§A): "A shared queue servicing multiple peers +// couples otherwise-independent traffic: a 9.8 KB Ack from one peer +// delays every probe behind it on the same egress." The existing +// relay HOL test (`head_of_line_is_observable_on_a_shared_egress`) +// only exercises *same source, same destination* — it does not prove +// that the relay's shared egress couples *different* senders. If the +// implementation accidentally partitioned the egress queue per- +// (from, to) pair instead of per-to, the existing test still passes +// while the failure mode the simulator exists to reproduce is +// silently absent. +// +// This test: alpha sends a big message (slow egress to charlie), +// then bravo sends a small message (also to charlie). The small +// message must wait for the big one's egress serialization, because +// they share the egress link relay→charlie. +// ─────────────────────────────────────────────────────────────────── +#[test] +fn relay_egress_hol_couples_independent_senders_on_shared_egress() { + // Counterfactual HOL test: send bravo's small probe alone, then + // re-run the same scenario with alpha's big message preceding + // bravo's. The DIFFERENCE in bravo's arrivals must be at least + // alpha's egress serialization time. This isolates HOL from + // bravo's own egress and from latency. + let text = r#" + name = "egress_hol_two_senders" + seed = 1 + duration_ns = 5_000_000_000 + + [default_tick] + period_ns = 100_000_000 + + [default_link] + # Fast inbound and outbound link bandwidth so the relay's + # egress capacity is the dominant serialization term. + latency_ns = 1_000 + jitter_stddev_ns = 0 + loss_prob_ppm = 0 + reorder_prob_ppm = 0 + bandwidth_bps = 10_000_000_000 + cold_dial_penalty_ns = 0 + cache_warm_after_ns = 1_000_000_000 + cache_invalidate_after_idle_ns = 100_000_000_000 + + [[relays]] + id = "R" + ingress_capacity_bps = 10_000_000_000 + egress_capacity_bps_per_link = 8_000_000 # 8 MB/s per outbound link + queue_depth_bytes = 10_000_000 + cold_start_penalty_ns = 0 + + [[peers]] + id = "alpha" + kind = "stage" + initial_state = "cold" + kind_config = { name = "pp-a", address = "10.0.0.1:7700" } + [[peers]] + id = "bravo" + kind = "stage" + initial_state = "cold" + kind_config = { name = "pp-b", address = "10.0.0.2:7700" } + [[peers]] + id = "charlie" + kind = "stage" + initial_state = "cold" + kind_config = { name = "pp-c", address = "10.0.0.3:7700" } + + [[links]] + from = "alpha" + to = "charlie" + via = "R" + [[links]] + from = "bravo" + to = "charlie" + via = "R" + "#; + let scen = parse(text); + + let big = 100_000u64; // alpha: 100 KB. Egress at 8 MB/s ⇒ 12.5 ms. + let small = 100u64; + + // Run A: bravo alone. + let mut net_alone = Network::new(&scen); + let SendOutcome::Arrive { at_ns: bravo_alone, .. } = + net_alone.send("bravo", "charlie", small, 0) + else { + panic!("bravo→charlie must arrive (alone)"); + }; + + // Run B: alpha first, then bravo on the same Network. + let mut net_coupled = Network::new(&scen); + let SendOutcome::Arrive { .. } = net_coupled.send("alpha", "charlie", big, 0) else { + panic!("alpha→charlie must arrive (coupled)"); + }; + let SendOutcome::Arrive { at_ns: bravo_coupled, .. } = + net_coupled.send("bravo", "charlie", small, 0) + else { + panic!("bravo→charlie must arrive (coupled)"); + }; + + let alpha_egress_serialization_ns = 12_500_000u64; // 100_000 / 8_000_000 * 1e9 + let hol_delay = bravo_coupled - bravo_alone; + assert!( + hol_delay >= alpha_egress_serialization_ns - 100_000, + "spec §5A.3 / Layer A: the shared egress relay→charlie must serialize \ + independent senders. Adding a preceding 100KB alpha→charlie message must \ + delay bravo's small probe by at least alpha's egress serialization \ + (~{alpha_egress_serialization_ns}ns); got hol_delay={hol_delay} \ + (bravo_alone={bravo_alone}, bravo_coupled={bravo_coupled}). \ + If hol_delay is ≪ the floor, the relay's egress queue is \ + (incorrectly) partitioned per source, and the Layer A failure mode \ + (per-peer-independent traffic coupled at a shared egress) cannot be \ + reproduced — defeating the simulator's purpose.", + ); +} diff --git a/crates/simulation/tests/cross_arch_parity.rs b/crates/simulation/tests/cross_arch_parity.rs index e231e11..58fbecd 100644 --- a/crates/simulation/tests/cross_arch_parity.rs +++ b/crates/simulation/tests/cross_arch_parity.rs @@ -33,14 +33,17 @@ use simulation::scenario::{HostKindRegistry, Scenario, load_from_path}; /// SHA-256 of `events.ndjson` for the reference scenario at /// `scenarios/parity/reference.toml`. Checked in; updated only as /// part of a deliberate spec amendment. -// Updated in iteration 10 when DialOutcome's `at_ns` was corrected -// from the pre-penalty arrival to the post-penalty arrival (the -// judge's finding 1 against §5.4 step 8). Bundle events now record -// the message arrival time on the cold-dial path instead of the -// pre-penalty timestamp, so the digest of `events.ndjson` shifts -// for any scenario whose cold-dial penalty is non-zero. +// Updated when the parity reference scenario gained a relayed route +// (RELAY_SPEC §13 — "the cross-architecture parity test extended to +// cover relay-mediated routes"). The alpha↔charlie pair now routes +// through relay `R`, so the bundle stream contains `relay_enqueue` / +// `relay_dequeue` records and the alpha↔charlie arrival times shift +// to reflect the relay's ingress + egress serialization. +// +// Previous value, from iteration 10's DialOutcome correction: +// a76b557d3da7a5d0f393446231669b8f7b1945a251805bed33fa09fb27ab3db0 const EXPECTED_EVENTS_NDJSON_SHA256: &str = - "a76b557d3da7a5d0f393446231669b8f7b1945a251805bed33fa09fb27ab3db0"; + "c7ee2a328c796f482b6c62694fc27936aa58a959b17f59fedd14a3f8c20ad2f2"; fn registry() -> HostKindRegistry { let mut r = HostKindRegistry::with_swim(); diff --git a/crates/simulation/tests/engine_invariants.rs b/crates/simulation/tests/engine_invariants.rs index c8dc47b..8dd9249 100644 --- a/crates/simulation/tests/engine_invariants.rs +++ b/crates/simulation/tests/engine_invariants.rs @@ -96,6 +96,7 @@ enum HostMessageLite { App(Vec), TimerFired { token: u64 }, SendFailed { to: String, reason_tag: &'static str }, + WorkerExit { reason: String }, } struct ScriptedHost { @@ -155,8 +156,13 @@ impl Host for ScriptedHost { DropReason::NoRoute => "no_route", DropReason::Partitioned => "partitioned", DropReason::Lossy => "lossy", + DropReason::RelayQueueFull => "relay_queue_full", + DropReason::RelayDown => "relay_down", }, }, + HostMessage::WorkerExit { reason, .. } => HostMessageLite::WorkerExit { + reason: reason.clone(), + }, }; self.log .lock() diff --git a/crates/simulation/tests/evaluator_invariants.rs b/crates/simulation/tests/evaluator_invariants.rs index 84e23ad..fd70bed 100644 --- a/crates/simulation/tests/evaluator_invariants.rs +++ b/crates/simulation/tests/evaluator_invariants.rs @@ -147,6 +147,7 @@ fn snap(t: u64, seq: u32, members: &[(&str, &str, u64)], self_incarnation: u64) }) .collect(), self_incarnation, + name_registry: std::collections::BTreeMap::new(), } } diff --git a/crates/simulation/tests/relay_assertion_invariants.rs b/crates/simulation/tests/relay_assertion_invariants.rs new file mode 100644 index 0000000..c3d0321 --- /dev/null +++ b/crates/simulation/tests/relay_assertion_invariants.rs @@ -0,0 +1,385 @@ +//! RELAY_SPEC §7.2 behavioural property tests for the three new +//! assertion kinds: `relay_queue_depth_bounded`, +//! `worker_alive_throughout`, and `name_resolves_within`. + +use std::collections::BTreeMap; + +use simulation::evaluator::{EventLine, Outcome, SnapshotEntry, SnapshotIndex, evaluate}; +use simulation::scenario::{ + Assertion, AssertionKind, DefaultTick, HostRoute, Link, LinkPolicy, Peer, Scenario, +}; + +// ────────────────────────────────────────────────────────────────────── +// Fixtures +// ────────────────────────────────────────────────────────────────────── + +fn policy() -> LinkPolicy { + LinkPolicy { + latency_ns: 1, + jitter_stddev_ns: 0, + loss_prob_ppm: 0, + reorder_prob_ppm: 0, + bandwidth_bps: 1_000_000_000, + cold_dial_penalty_ns: 0, + cache_warm_after_ns: 1, + cache_invalidate_after_idle_ns: 1_000_000_000, + } +} + +fn peer(id: &str, kind: &str) -> Peer { + let mut cfg = toml::value::Table::new(); + if kind == "stage" { + cfg.insert("name".into(), toml::Value::String(format!("pp-{id}"))); + cfg.insert( + "address".into(), + toml::Value::String(format!("10.0.0.1:{}", 7000 + id.len())), + ); + } else { + cfg.insert("probe_interval_ns".into(), toml::Value::Integer(1_000)); + cfg.insert("suspicion_timeout_ns".into(), toml::Value::Integer(5_000)); + } + Peer { + id: id.into(), + kind: kind.into(), + kind_config: cfg, + initial_state: if kind == "stage" { "cold" } else { "alive" }.into(), + tick_period_ns_override: None, + } +} + +fn base_scenario(peers: Vec, assertions: Vec) -> Scenario { + let mut links = Vec::new(); + let mut routes = Vec::new(); + for a in &peers { + for b in &peers { + if a.id == b.id { + continue; + } + links.push(Link { + from: a.id.clone(), + to: b.id.clone(), + policy: policy(), + }); + routes.push(HostRoute::Direct { + from: a.id.clone(), + to: b.id.clone(), + }); + } + } + Scenario { + name: "ev".into(), + seed: 1, + duration_ns: 1_000_000_000, + early_terminate_on_all_assertions_resolved: false, + default_tick: DefaultTick { + period_ns: 1_000_000, + }, + default_link: policy(), + peers, + relays: Vec::new(), + links, + mutations: Vec::new(), + snapshots: Vec::new(), + assertions, + routes, + } +} + +fn relay_enqueue(virtual_time_ns: u64, relay: &str, byte_len: u64, line_idx: usize) -> EventLine { + EventLine { + virtual_time_ns, + host_id: None, + kind_tag: "relay".into(), + event: serde_json::json!({ + "kind": "relay_enqueue", + "relay": relay, + "from": "a", + "to": "b", + "byte_len": byte_len, + }), + line_idx, + } +} + +fn relay_dequeue(virtual_time_ns: u64, relay: &str, byte_len: u64, line_idx: usize) -> EventLine { + EventLine { + virtual_time_ns, + host_id: None, + kind_tag: "relay".into(), + event: serde_json::json!({ + "kind": "relay_dequeue", + "relay": relay, + "from": "a", + "to": "b", + "byte_len": byte_len, + }), + line_idx, + } +} + +fn stage_lifecycle(virtual_time_ns: u64, host: &str, to: &str, line_idx: usize) -> EventLine { + EventLine { + virtual_time_ns, + host_id: Some(host.into()), + kind_tag: "stage".into(), + event: serde_json::json!({ + "kind": "stage_lifecycle", + "from": "Running", + "to": to, + }), + line_idx, + } +} + +fn snapshot_with_registry( + virtual_time_ns: u64, + seq: u32, + registry: &[(&str, &str)], +) -> SnapshotEntry { + SnapshotEntry { + virtual_time_ns, + seq, + members: BTreeMap::new(), + self_incarnation: 0, + name_registry: registry + .iter() + .map(|(n, a)| ((*n).into(), (*a).into())) + .collect(), + } +} + +fn idx(snaps: &[(&str, Vec)]) -> SnapshotIndex { + let mut by_host = BTreeMap::new(); + for (h, entries) in snaps { + by_host.insert((*h).to_string(), entries.clone()); + } + SnapshotIndex { by_host } +} + +// ──────────────────────────────────────────────────────────────────── +// RELAY_SPEC §7.2 +// ──────────────────────────────────────────────────────────────────── + +#[test] +fn relay_queue_depth_bounded_passes_when_depth_stays_under_bound() { + // §7.2 "Relay-queue-depth soundness." Pass when the queue never + // exceeds the bound across the window. + let events = vec![ + relay_enqueue(100, "R", 500, 0), + relay_dequeue(200, "R", 500, 1), + relay_enqueue(300, "R", 800, 2), + relay_dequeue(400, "R", 800, 3), + ]; + let scen = base_scenario( + vec![peer("a", "swim"), peer("b", "swim")], + vec![Assertion { + kind: AssertionKind::RelayQueueDepthBounded { + relay: "R".into(), + max_bytes: 1000, + window_start_ns: None, + window_end_ns: None, + }, + }], + ); + let verdicts = evaluate(&scen, &events, &SnapshotIndex::default()); + assert_eq!(verdicts[0].outcome, Outcome::Pass); +} + +#[test] +fn relay_queue_depth_bounded_fails_on_first_enqueue_past_bound() { + // Evidence on Fail references the exact RelayEnqueue that + // crossed the bound. + let events = vec![ + relay_enqueue(100, "R", 600, 0), + relay_enqueue(150, "R", 500, 1), // pushes depth to 1100 > 1000 + relay_dequeue(200, "R", 600, 2), + ]; + let scen = base_scenario( + vec![peer("a", "swim"), peer("b", "swim")], + vec![Assertion { + kind: AssertionKind::RelayQueueDepthBounded { + relay: "R".into(), + max_bytes: 1000, + window_start_ns: None, + window_end_ns: None, + }, + }], + ); + let verdicts = evaluate(&scen, &events, &SnapshotIndex::default()); + assert_eq!(verdicts[0].outcome, Outcome::Fail); + assert_eq!(verdicts[0].evidence.len(), 1); + assert_eq!(verdicts[0].evidence[0].virtual_time_ns, 150); +} + +#[test] +fn relay_queue_depth_bounded_is_inconclusive_with_no_relay_events() { + let scen = base_scenario( + vec![peer("a", "swim"), peer("b", "swim")], + vec![Assertion { + kind: AssertionKind::RelayQueueDepthBounded { + relay: "R".into(), + max_bytes: 100, + window_start_ns: None, + window_end_ns: None, + }, + }], + ); + let verdicts = evaluate(&scen, &[], &SnapshotIndex::default()); + assert_eq!(verdicts[0].outcome, Outcome::Inconclusive); +} + +#[test] +fn relay_queue_depth_bounded_respects_window() { + // An enqueue outside the window should not trigger a fail even + // if it pushes the depth past the bound. + let events = vec![ + relay_enqueue(100, "R", 2000, 0), // outside window — not a fail + relay_dequeue(150, "R", 2000, 1), + relay_enqueue(500, "R", 500, 2), // inside window — within bound + ]; + let scen = base_scenario( + vec![peer("a", "swim"), peer("b", "swim")], + vec![Assertion { + kind: AssertionKind::RelayQueueDepthBounded { + relay: "R".into(), + max_bytes: 1000, + window_start_ns: Some(400), + window_end_ns: Some(1000), + }, + }], + ); + let verdicts = evaluate(&scen, &events, &SnapshotIndex::default()); + assert_eq!(verdicts[0].outcome, Outcome::Pass); +} + +#[test] +fn worker_alive_throughout_passes_on_no_halt_in_window() { + // §7.2 "Worker-alive soundness." Pass when no stage_lifecycle + // event into "Halted" appears in the window. + let events = vec![ + // A non-Halt lifecycle keeps us out of the inconclusive arm. + EventLine { + virtual_time_ns: 100, + host_id: Some("s".into()), + kind_tag: "stage".into(), + event: serde_json::json!({ + "kind": "stage_lifecycle", + "from": "Cold", + "to": "Registering", + }), + line_idx: 0, + }, + ]; + let scen = base_scenario( + vec![peer("s", "stage")], + vec![Assertion { + kind: AssertionKind::WorkerAliveThroughout { + peer: "s".into(), + window_start_ns: 0, + window_end_ns: 1_000, + }, + }], + ); + let verdicts = evaluate(&scen, &events, &SnapshotIndex::default()); + assert_eq!(verdicts[0].outcome, Outcome::Pass); +} + +#[test] +fn worker_alive_throughout_fails_on_halt_in_window() { + let events = vec![ + stage_lifecycle(50, "s", "Registering", 0), + stage_lifecycle(60, "s", "Running", 1), + stage_lifecycle(500, "s", "Halted", 2), + ]; + let scen = base_scenario( + vec![peer("s", "stage")], + vec![Assertion { + kind: AssertionKind::WorkerAliveThroughout { + peer: "s".into(), + window_start_ns: 0, + window_end_ns: 1_000, + }, + }], + ); + let verdicts = evaluate(&scen, &events, &SnapshotIndex::default()); + assert_eq!(verdicts[0].outcome, Outcome::Fail); + assert_eq!(verdicts[0].evidence[0].virtual_time_ns, 500); +} + +#[test] +fn worker_alive_throughout_is_inconclusive_with_no_lifecycle_events() { + let scen = base_scenario( + vec![peer("s", "stage")], + vec![Assertion { + kind: AssertionKind::WorkerAliveThroughout { + peer: "s".into(), + window_start_ns: 0, + window_end_ns: 1_000, + }, + }], + ); + let verdicts = evaluate(&scen, &[], &SnapshotIndex::default()); + assert_eq!(verdicts[0].outcome, Outcome::Inconclusive); +} + +#[test] +fn name_resolves_within_passes_when_every_observer_sees_name_in_window() { + let snapshots = idx(&[ + ("alpha", vec![snapshot_with_registry(100, 0, &[("pp-stage-0", "10.0.0.1:7700")])]), + ("bravo", vec![snapshot_with_registry(150, 0, &[("pp-stage-0", "10.0.0.1:7700")])]), + ]); + let scen = base_scenario( + vec![peer("alpha", "swim"), peer("bravo", "swim")], + vec![Assertion { + kind: AssertionKind::NameResolvesWithin { + name: "pp-stage-0".into(), + observers: vec!["alpha".into(), "bravo".into()], + within_ns: 500, + from_ns: 0, + }, + }], + ); + let verdicts = evaluate(&scen, &[], &snapshots); + assert_eq!(verdicts[0].outcome, Outcome::Pass); +} + +#[test] +fn name_resolves_within_fails_when_an_observer_never_sees_name() { + let snapshots = idx(&[ + ("alpha", vec![snapshot_with_registry(100, 0, &[("pp-stage-0", "addr")])]), + ("bravo", vec![snapshot_with_registry(150, 0, &[("other", "addr")])]), + ]); + let scen = base_scenario( + vec![peer("alpha", "swim"), peer("bravo", "swim")], + vec![Assertion { + kind: AssertionKind::NameResolvesWithin { + name: "pp-stage-0".into(), + observers: vec!["alpha".into(), "bravo".into()], + within_ns: 500, + from_ns: 0, + }, + }], + ); + let verdicts = evaluate(&scen, &[], &snapshots); + assert_eq!(verdicts[0].outcome, Outcome::Fail); +} + +#[test] +fn name_resolves_within_is_inconclusive_when_no_observer_snapshots_in_window() { + let snapshots = idx(&[ + ("alpha", vec![snapshot_with_registry(5, 0, &[("pp-stage-0", "addr")])]), + ]); + let scen = base_scenario( + vec![peer("alpha", "swim")], + vec![Assertion { + kind: AssertionKind::NameResolvesWithin { + name: "pp-stage-0".into(), + observers: vec!["alpha".into()], + within_ns: 5, + from_ns: 100, // no snapshot at or after t=100 + }, + }], + ); + let verdicts = evaluate(&scen, &[], &snapshots); + assert_eq!(verdicts[0].outcome, Outcome::Inconclusive); +} diff --git a/crates/simulation/tests/relay_invariants.rs b/crates/simulation/tests/relay_invariants.rs new file mode 100644 index 0000000..5484205 --- /dev/null +++ b/crates/simulation/tests/relay_invariants.rs @@ -0,0 +1,570 @@ +//! RELAY_SPEC §4.7 / §6.2 behavioural tests for the relay vertex +//! and its mutations. These exercise the network layer directly, +//! mirroring `tests/network_invariants.rs` for the parent §5.7 +//! properties. +//! +//! Each test name maps to a §4.7 / §6.2 property; the assertions +//! avoid mirroring the implementation and focus on what a third +//! party reading the spec would expect to observe. + +use simulation::network::{DropReason, Network, NetworkNotification, RelayDropReason, SendOutcome}; +use simulation::scenario::{HostKindRegistry, load_from_str}; +use std::path::Path; + +const ALL_NOTIFICATIONS: &str = include_str!("../scenarios/parity/reference.toml"); + +fn registry() -> HostKindRegistry { + let mut r = HostKindRegistry::with_swim(); + r.register(Box::new(simulation::parity_host::ParityStubKindValidator)); + r +} + +// Build a minimal scenario from inline TOML so the tests are self- +// contained — they don't reach for shipped scenario files. +fn scenario_from(text: &str) -> simulation::scenario::Scenario { + load_from_str(Path::new("(test)"), text, ®istry()) + .expect("test scenario must validate") +} + +/// Three host topology routed through one relay. Used by most relay +/// tests. +fn one_relay_three_hosts() -> simulation::scenario::Scenario { + scenario_from( + r#" + name = "relay_three_hosts" + seed = 1 + duration_ns = 1_000_000_000 + + [default_tick] + period_ns = 1_000_000 + + [default_link] + latency_ns = 1_000_000 + jitter_stddev_ns = 0 + loss_prob_ppm = 0 + reorder_prob_ppm = 0 + bandwidth_bps = 1_000_000_000 + cold_dial_penalty_ns = 0 + cache_warm_after_ns = 1_000_000_000 + cache_invalidate_after_idle_ns = 10_000_000_000 + + [[relays]] + id = "R" + ingress_capacity_bps = 1_000_000_000 + egress_capacity_bps_per_link = 8_000_000 # 1 MB/s per outbound link + queue_depth_bytes = 1_000_000 + cold_start_penalty_ns = 0 + + [[peers]] + id = "alpha" + kind = "parity_stub" + initial_state = "ready" + kind_config = { peers = ["alpha", "bravo", "charlie"] } + [[peers]] + id = "bravo" + kind = "parity_stub" + initial_state = "ready" + kind_config = { peers = ["alpha", "bravo", "charlie"] } + [[peers]] + id = "charlie" + kind = "parity_stub" + initial_state = "ready" + kind_config = { peers = ["alpha", "bravo", "charlie"] } + + [[links]] + from = "alpha" + to = "bravo" + via = "R" + [[links]] + from = "bravo" + to = "alpha" + via = "R" + [[links]] + from = "alpha" + to = "charlie" + via = "R" + [[links]] + from = "charlie" + to = "alpha" + via = "R" + [[links]] + from = "bravo" + to = "charlie" + via = "R" + [[links]] + from = "charlie" + to = "bravo" + via = "R" + "#, + ) +} + +fn collect_notifications(net: &mut Network) -> Vec { + net.take_pending_notifications() +} + +// ──────────────────────────────────────────────────────────────────── +// RELAY_SPEC §4.7 +// ──────────────────────────────────────────────────────────────────── + +#[test] +fn composition_is_transparent_to_hosts() { + // §4.7 "Composition is transparent to hosts." A `send` over a + // relayed route returns one `SendOutcome` shaped identically to a + // direct send's. The arrival is a single integer; no extra + // structure leaks. + let scen = one_relay_three_hosts(); + let mut net = Network::new(&scen); + let outcome = net.send("alpha", "bravo", 1024, 0); + match outcome { + SendOutcome::Arrive { delivery_id: _, at_ns } => { + assert!(at_ns > 0, "relayed send should produce a positive arrival"); + } + SendOutcome::Drop { reason } => panic!("unexpected drop: {reason:?}"), + } +} + +#[test] +fn relayed_arrival_is_later_than_a_direct_send_of_the_same_size() { + // RELAY_SPEC §11 phase R1: "A direct send through a relay arrives + // later than the same send over a direct edge of equal policy by + // the relay's ingress + egress serialization time." We confirm + // the direction; the exact numeric is the calibration suite's job. + let scen_direct = scenario_from( + r#" + name = "direct" + seed = 1 + duration_ns = 1_000_000_000 + [default_tick] + period_ns = 1_000_000 + [default_link] + latency_ns = 1_000_000 + jitter_stddev_ns = 0 + loss_prob_ppm = 0 + reorder_prob_ppm = 0 + bandwidth_bps = 1_000_000_000 + cold_dial_penalty_ns = 0 + cache_warm_after_ns = 1_000_000_000 + cache_invalidate_after_idle_ns = 10_000_000_000 + [[peers]] + id = "a" + kind = "parity_stub" + initial_state = "ready" + kind_config = { peers = ["a", "b"] } + [[peers]] + id = "b" + kind = "parity_stub" + initial_state = "ready" + kind_config = { peers = ["a", "b"] } + [[links]] + from = "a" + to = "b" + [[links]] + from = "b" + to = "a" + "#, + ); + let scen_relayed = one_relay_three_hosts(); + let mut direct = Network::new(&scen_direct); + let mut relayed = Network::new(&scen_relayed); + let SendOutcome::Arrive { at_ns: direct_arrival, .. } = direct.send("a", "b", 8192, 0) else { + panic!("direct send must arrive"); + }; + let SendOutcome::Arrive { at_ns: relayed_arrival, .. } = + relayed.send("alpha", "bravo", 8192, 0) + else { + panic!("relayed send must arrive"); + }; + assert!( + relayed_arrival > direct_arrival, + "relayed arrival ({relayed_arrival}) should exceed direct arrival ({direct_arrival}) by the relay's serialization time" + ); +} + +#[test] +fn head_of_line_is_observable_on_a_shared_egress() { + // §4.7 "HOL is observable and bounded." Two messages from the + // same sender to the same destination through the relay are + // serialized on the shared egress: the second's arrival is at or + // after the first's arrival plus the first's egress serialization. + let scen = one_relay_three_hosts(); + let mut net = Network::new(&scen); + // 100 KB messages so the egress serialization time dominates. + let SendOutcome::Arrive { at_ns: first, .. } = net.send("alpha", "bravo", 100_000, 0) else { + panic!("first send must arrive"); + }; + let SendOutcome::Arrive { at_ns: second, .. } = net.send("alpha", "bravo", 100_000, 0) else { + panic!("second send must arrive"); + }; + // egress_capacity_bps_per_link in the spec is *bytes* per second + // (SIM_SPEC §5.2 / RELAY_SPEC §4.2). 8_000_000 ⇒ 8 MB/s. 100 KB + // takes 100_000 / 8_000_000 = 12.5 ms = 12_500_000 ns. The gap + // between two shared-egress sends must be ≥ that floor. + let expected_min_gap = 12_500_000u64; + assert!( + second >= first + expected_min_gap, + "second arrival should follow first by ≥ egress serialization (first={first}, second={second}, expected_min_gap={expected_min_gap})" + ); +} + +#[test] +fn ingress_and_egress_are_independent_across_destinations() { + // §4.7 "Ingress and egress are independent." A message destined + // for peer X does not delay a message destined for peer Y on the + // egress side. Two simultaneous sends to *different* + // destinations share only the ingress; the egress queues are + // independent. + let scen = one_relay_three_hosts(); + let mut net = Network::new(&scen); + let SendOutcome::Arrive { at_ns: to_bravo, .. } = net.send("alpha", "bravo", 100_000, 0) else { + panic!("first send must arrive"); + }; + let SendOutcome::Arrive { at_ns: to_charlie, .. } = + net.send("alpha", "charlie", 100_000, 0) + else { + panic!("second send must arrive"); + }; + // Same source so the inbound leg serializes, and the ingress + // serializes, but the egress queues are separate. The second's + // egress should NOT have to wait for the first's egress + // serialization. So the gap should be much smaller than the + // shared-egress case in `head_of_line_is_observable_on_a_shared_egress`. + let shared_egress_min_gap = 100_000_000u64; + let gap = to_charlie - to_bravo; + assert!( + gap < shared_egress_min_gap, + "different-destination gap ({gap}) should be smaller than the shared-egress gap ({shared_egress_min_gap})" + ); +} + +#[test] +fn queue_overflow_drops_with_relay_queue_full() { + // §4.7 "Queue overflow is exact." A send that would push + // `enqueued_bytes` strictly above `queue_depth_bytes` drops with + // `RelayQueueFull` and emits a `RelayDrop` record. + let scen = scenario_from( + r#" + name = "tight_queue" + seed = 1 + duration_ns = 1_000_000_000 + [default_tick] + period_ns = 1_000_000 + [default_link] + latency_ns = 1_000_000 + jitter_stddev_ns = 0 + loss_prob_ppm = 0 + reorder_prob_ppm = 0 + bandwidth_bps = 1_000_000_000 + cold_dial_penalty_ns = 0 + cache_warm_after_ns = 1_000_000_000 + cache_invalidate_after_idle_ns = 10_000_000_000 + [[relays]] + id = "R" + ingress_capacity_bps = 1_000_000_000 + egress_capacity_bps_per_link = 8_000 # 1 KB/s; egress can't drain + queue_depth_bytes = 1024 # 1 KB total queue + cold_start_penalty_ns = 0 + [[peers]] + id = "a" + kind = "parity_stub" + initial_state = "ready" + kind_config = { peers = ["a", "b"] } + [[peers]] + id = "b" + kind = "parity_stub" + initial_state = "ready" + kind_config = { peers = ["a", "b"] } + [[links]] + from = "a" + to = "b" + via = "R" + [[links]] + from = "b" + to = "a" + via = "R" + "#, + ); + let mut net = Network::new(&scen); + // Three 512 B messages back to back: first fits, second fills the + // queue (1024 total), third overflows. + let SendOutcome::Arrive { .. } = net.send("a", "b", 512, 0) else { + panic!("first 512B should fit"); + }; + let SendOutcome::Arrive { .. } = net.send("a", "b", 512, 0) else { + panic!("second 512B should fit (queue = 1024 total)"); + }; + let outcome = net.send("a", "b", 1, 0); + assert!( + matches!(outcome, SendOutcome::Drop { reason: DropReason::RelayQueueFull }), + "third send must overflow with RelayQueueFull, got {outcome:?}" + ); + let notes = collect_notifications(&mut net); + let drops: Vec<_> = notes + .iter() + .filter_map(|n| { + if let NetworkNotification::RelayDrop { + relay, + reason: RelayDropReason::QueueFull, + .. + } = n + { + Some(relay.clone()) + } else { + None + } + }) + .collect(); + assert_eq!( + drops, + vec!["R".to_string()], + "exactly one RelayDrop{{QueueFull}} notification expected" + ); +} + +#[test] +fn cold_start_penalty_is_paid_exactly_once_per_boot() { + // §4.7 "Cold-start penalty is paid once per boot." After a + // RelayBoot, the first forwarded message includes the penalty; + // the second does not. + let scen = scenario_from( + r#" + name = "boot_penalty" + seed = 1 + duration_ns = 1_000_000_000 + [default_tick] + period_ns = 1_000_000 + [default_link] + latency_ns = 1_000_000 + jitter_stddev_ns = 0 + loss_prob_ppm = 0 + reorder_prob_ppm = 0 + bandwidth_bps = 1_000_000_000 + cold_dial_penalty_ns = 0 + cache_warm_after_ns = 1_000_000_000 + cache_invalidate_after_idle_ns = 10_000_000_000 + [[relays]] + id = "R" + ingress_capacity_bps = 1_000_000_000 + egress_capacity_bps_per_link = 1_000_000_000 + queue_depth_bytes = 100_000 + cold_start_penalty_ns = 50_000_000 # 50ms + [[peers]] + id = "a" + kind = "parity_stub" + initial_state = "ready" + kind_config = { peers = ["a", "b"] } + [[peers]] + id = "b" + kind = "parity_stub" + initial_state = "ready" + kind_config = { peers = ["a", "b"] } + [[links]] + from = "a" + to = "b" + via = "R" + [[links]] + from = "b" + to = "a" + via = "R" + [[mutations]] + at_ns = 100_000_000 + kind = "relay_kill" + relay = "R" + [[mutations]] + at_ns = 200_000_000 + kind = "relay_boot" + relay = "R" + "#, + ); + use simulation::scenario::{Mutation, MutationKind}; + let mut net = Network::new(&scen); + // Boot the relay at t=200ms (simulating engine dispatch). + net.apply_mutation( + &Mutation { + at_ns: 200_000_000, + kind: MutationKind::RelayBoot { + relay: "R".into(), + }, + }, + 200_000_000, + ); + let SendOutcome::Arrive { at_ns: first, .. } = net.send("a", "b", 1024, 300_000_000) else { + panic!("first post-boot send must arrive"); + }; + let SendOutcome::Arrive { at_ns: second, .. } = net.send("a", "b", 1024, 400_000_000) else { + panic!("second post-boot send must arrive"); + }; + // The first should include the 50ms penalty; the second should + // not, so first - sent ≥ 50ms and second - sent < 50ms. + let first_relative = first - 300_000_000; + let second_relative = second - 400_000_000; + assert!( + first_relative >= 50_000_000, + "first post-boot send should include the 50ms cold-start penalty (relative={first_relative}ns)" + ); + assert!( + second_relative < 50_000_000, + "second post-boot send should NOT include the 50ms cold-start penalty (relative={second_relative}ns)" + ); +} + +#[test] +fn relay_kill_drops_subsequent_sends_with_relay_down() { + // §4.7 "Mutation invalidation is exact." After a RelayKill, sends + // through the relay drop with RelayDown. + let scen = scenario_from( + r#" + name = "kill_then_send" + seed = 1 + duration_ns = 1_000_000_000 + [default_tick] + period_ns = 1_000_000 + [default_link] + latency_ns = 1_000_000 + jitter_stddev_ns = 0 + loss_prob_ppm = 0 + reorder_prob_ppm = 0 + bandwidth_bps = 1_000_000_000 + cold_dial_penalty_ns = 0 + cache_warm_after_ns = 1_000_000_000 + cache_invalidate_after_idle_ns = 10_000_000_000 + [[relays]] + id = "R" + ingress_capacity_bps = 1_000_000_000 + egress_capacity_bps_per_link = 1_000_000_000 + queue_depth_bytes = 100_000 + cold_start_penalty_ns = 0 + [[peers]] + id = "a" + kind = "parity_stub" + initial_state = "ready" + kind_config = { peers = ["a", "b"] } + [[peers]] + id = "b" + kind = "parity_stub" + initial_state = "ready" + kind_config = { peers = ["a", "b"] } + [[links]] + from = "a" + to = "b" + via = "R" + [[links]] + from = "b" + to = "a" + via = "R" + "#, + ); + use simulation::scenario::{Mutation, MutationKind}; + let mut net = Network::new(&scen); + net.apply_mutation( + &Mutation { + at_ns: 50_000_000, + kind: MutationKind::RelayKill { + relay: "R".into(), + }, + }, + 50_000_000, + ); + let outcome = net.send("a", "b", 100, 60_000_000); + assert!( + matches!(outcome, SendOutcome::Drop { reason: DropReason::RelayDown }), + "post-kill send must drop with RelayDown, got {outcome:?}" + ); +} + +#[test] +fn relay_capacity_change_affects_only_future_sends() { + // §4.7 "Mutation invalidation is exact." A RelayCapacityChange + // invalidates no deliveries; it affects only sends that begin + // after its time. We assert that pre-change in-flight count is + // unchanged across the mutation. + let scen = one_relay_three_hosts(); + use simulation::scenario::{Mutation, MutationKind}; + let mut net = Network::new(&scen); + // Queue up a delivery, then apply the policy change. + let _ = net.send("alpha", "bravo", 1024, 0); + let before = net.relay_in_flight_count("R"); + let invalidated = net.apply_mutation( + &Mutation { + at_ns: 100_000, + kind: MutationKind::RelayCapacityChange { + relay: "R".into(), + ingress_capacity_bps: Some(1), + egress_capacity_bps_per_link: Some(1), + queue_depth_bytes: Some(1), + }, + }, + 100_000, + ); + let after = net.relay_in_flight_count("R"); + assert!(invalidated.is_empty(), "RelayCapacityChange invalidates no deliveries"); + assert_eq!( + before, after, + "in-flight relay state must not change across RelayCapacityChange" + ); +} + +#[test] +fn relay_kill_invalidates_in_flight_through_that_relay() { + // §4.7 "Mutation invalidation is exact." A RelayKill returns + // exactly the deliveries in-flight through the relay at the + // mutation's virtual time and no others. + let scen = one_relay_three_hosts(); + use simulation::scenario::{Mutation, MutationKind}; + let mut net = Network::new(&scen); + let SendOutcome::Arrive { delivery_id, .. } = net.send("alpha", "bravo", 1024, 0) else { + panic!("send must arrive"); + }; + let invalidated = net.apply_mutation( + &Mutation { + at_ns: 1, + kind: MutationKind::RelayKill { + relay: "R".into(), + }, + }, + 1, + ); + assert_eq!(invalidated.len(), 1, "exactly one in-flight delivery"); + assert_eq!(invalidated[0].delivery_id, delivery_id); + // Relay's in-flight bookkeeping is drained. + assert_eq!(net.relay_in_flight_count("R"), 0); +} + +#[test] +fn determinism_relayed_send_sequences_are_identical_across_runs() { + // §4.7 "Determinism." Same topology, same seed, same query + // sequence ⇒ identical composed SendOutcome sequence. + let scen = one_relay_three_hosts(); + let run = |s: &simulation::scenario::Scenario| -> Vec { + let mut net = Network::new(s); + let mut arrivals = Vec::new(); + for (i, (from, to)) in [ + ("alpha", "bravo"), + ("bravo", "charlie"), + ("charlie", "alpha"), + ("alpha", "charlie"), + ] + .iter() + .enumerate() + { + let outcome = net.send(from, to, 4096, (i as u64) * 1_000); + if let SendOutcome::Arrive { at_ns, .. } = outcome { + arrivals.push(at_ns); + } else { + panic!("unexpected drop in determinism test"); + } + } + arrivals + }; + let a = run(&scen); + let b = run(&scen); + assert_eq!(a, b, "two runs of the same scenario must produce identical arrivals"); +} + +// Smoke check: this constant is referenced to suppress the +// otherwise-unused `include_str!` import. It's a sanity guard that +// the parity scenario lives where we expect. +#[test] +fn parity_scenario_text_includes_default_link_header() { + assert!(ALL_NOTIFICATIONS.contains("default_link")); +} diff --git a/crates/simulation/tests/relay_scenario_validation.rs b/crates/simulation/tests/relay_scenario_validation.rs new file mode 100644 index 0000000..7d527d3 --- /dev/null +++ b/crates/simulation/tests/relay_scenario_validation.rs @@ -0,0 +1,290 @@ +//! RELAY_SPEC §8.3 scenario-loader behavioural tests. +//! +//! Each test pairs with a rule §8.2 names; the body asserts that the +//! rule fires (the malformed scenario rejects) and that a sibling +//! scenario obeying the rule loads cleanly. + +use simulation::scenario::{HostKindRegistry, load_from_str}; +use std::path::Path; + +fn registry() -> HostKindRegistry { + HostKindRegistry::with_swim() +} + +fn try_load(text: &str) -> Result { + load_from_str(Path::new("(test)"), text, ®istry()).map_err(|e| e.to_string()) +} + +fn must_load(text: &str) -> simulation::scenario::Scenario { + try_load(text).expect("scenario must validate") +} + +fn must_reject(text: &str, rule_substring: &str) -> String { + let err = try_load(text).expect_err("scenario must reject"); + assert!( + err.contains(rule_substring), + "expected error to mention {rule_substring:?}, got: {err}" + ); + err +} + +const HEADER: &str = r#" +name = "test" +seed = 1 +duration_ns = 1_000_000_000 +[default_tick] +period_ns = 1_000_000 +[default_link] +latency_ns = 1_000_000 +jitter_stddev_ns = 0 +loss_prob_ppm = 0 +reorder_prob_ppm = 0 +bandwidth_bps = 1_000_000_000 +cold_dial_penalty_ns = 0 +cache_warm_after_ns = 1_000_000_000 +cache_invalidate_after_idle_ns = 10_000_000_000 +"#; + +const SWIM_PEERS_A_B: &str = r#" +[[peers]] +id = "a" +kind = "swim" +initial_state = "alive" +kind_config = { probe_interval_ns = 1_000_000, suspicion_timeout_ns = 5_000_000 } +[[peers]] +id = "b" +kind = "swim" +initial_state = "alive" +kind_config = { probe_interval_ns = 1_000_000, suspicion_timeout_ns = 5_000_000 } +"#; + +// ──────────────────────────────────────────────────────────────────── +// RELAY_SPEC §8.3 — Validation is complete for relay rules +// ──────────────────────────────────────────────────────────────────── + +#[test] +fn rejects_duplicate_id_across_peers_and_relays() { + let text = format!( + "{HEADER}{SWIM_PEERS_A_B} + [[relays]] + id = \"a\" + ingress_capacity_bps = 1_000_000 + egress_capacity_bps_per_link = 1_000_000 + queue_depth_bytes = 1_000 + " + ); + must_reject(&text, "collides with a declared peer id"); +} + +#[test] +fn rejects_via_pointing_at_a_peer_instead_of_a_relay() { + // §8.2 — a via reference to a non-relay id is rejected. + let text = format!( + "{HEADER}{SWIM_PEERS_A_B} + [[peers]] + id = \"c\" + kind = \"swim\" + initial_state = \"alive\" + kind_config = {{ probe_interval_ns = 1_000_000, suspicion_timeout_ns = 5_000_000 }} + [[links]] + from = \"a\" + to = \"c\" + via = \"b\" + " + ); + must_reject(&text, "via must reference a declared relay"); +} + +#[test] +fn rejects_ambiguous_route_direct_and_via() { + // §4.1 — a host pair declared with both a direct edge and a + // relayed route via shorthand is rejected. + let text = format!( + "{HEADER}{SWIM_PEERS_A_B} + [[relays]] + id = \"R\" + ingress_capacity_bps = 1_000_000 + egress_capacity_bps_per_link = 1_000_000 + queue_depth_bytes = 1_000 + [[links]] + from = \"a\" + to = \"b\" + [[links]] + from = \"a\" + to = \"b\" + via = \"R\" + " + ); + must_reject(&text, "ambiguous route"); +} + +#[test] +fn rejects_link_with_relay_endpoint_and_via_field() { + // §8.2 — a link with `via` must have host endpoints. + let text = format!( + "{HEADER}{SWIM_PEERS_A_B} + [[relays]] + id = \"R\" + ingress_capacity_bps = 1_000_000 + egress_capacity_bps_per_link = 1_000_000 + queue_depth_bytes = 1_000 + [[links]] + from = \"a\" + to = \"R\" + via = \"R\" + " + ); + must_reject(&text, "host endpoints"); +} + +#[test] +fn rejects_multi_hop_relay_to_relay_edge() { + // §8.2 — multi-hop relayed routes (an edge between two relays) + // are not supported in the MVP. + let text = format!( + "{HEADER}{SWIM_PEERS_A_B} + [[relays]] + id = \"R1\" + ingress_capacity_bps = 1_000_000 + egress_capacity_bps_per_link = 1_000_000 + queue_depth_bytes = 1_000 + [[relays]] + id = \"R2\" + ingress_capacity_bps = 1_000_000 + egress_capacity_bps_per_link = 1_000_000 + queue_depth_bytes = 1_000 + [[links]] + from = \"R1\" + to = \"R2\" + " + ); + must_reject(&text, "multi-hop"); +} + +#[test] +fn rejects_worker_exit_targeting_non_stage_peer() { + // §8.2 — a worker_exit mutation whose target peer is not + // stage-kind is rejected at load time. + let text = format!( + "{HEADER}{SWIM_PEERS_A_B} + [[mutations]] + at_ns = 100 + kind = \"worker_exit\" + peer = \"a\" + reason = \"x\" + " + ); + must_reject(&text, "only \"stage\" peers accept it"); +} + +#[test] +fn relay_capacity_change_with_no_fields_is_rejected() { + let text = format!( + "{HEADER}{SWIM_PEERS_A_B} + [[relays]] + id = \"R\" + ingress_capacity_bps = 1_000_000 + egress_capacity_bps_per_link = 1_000_000 + queue_depth_bytes = 1_000 + [[mutations]] + at_ns = 100 + kind = \"relay_capacity_change\" + relay = \"R\" + " + ); + must_reject(&text, "must change at least one"); +} + +#[test] +fn accepts_valid_relay_scenario_with_via_shorthand() { + let text = format!( + "{HEADER}{SWIM_PEERS_A_B} + [[relays]] + id = \"R\" + ingress_capacity_bps = 1_000_000 + egress_capacity_bps_per_link = 1_000_000 + queue_depth_bytes = 1_000 + [[links]] + from = \"a\" + to = \"b\" + via = \"R\" + [[links]] + from = \"b\" + to = \"a\" + via = \"R\" + " + ); + let scen = must_load(&text); + assert_eq!(scen.relays.len(), 1); + // Loader expanded shorthand into 4 explicit legs. + assert_eq!(scen.links.len(), 4); + // Two relayed routes (a↔b through R). + assert_eq!(scen.routes.len(), 2); +} + +#[test] +fn accepts_stage_peer_with_well_formed_kind_config() { + let text = format!( + "{HEADER} + [[peers]] + id = \"s\" + kind = \"stage\" + initial_state = \"cold\" + kind_config = {{ name = \"pp-stage\", address = \"10.0.0.1:7700\" }} + " + ); + let scen = must_load(&text); + assert_eq!(scen.peers.len(), 1); + assert_eq!(scen.peers[0].kind, "stage"); +} + +#[test] +fn stage_kind_config_missing_name_is_rejected_via_kind_validator() { + // §8.3 "Stage kind-config is delegated." The kind validator, + // not the loader's generic rule, surfaces the missing-name + // message. + let text = format!( + "{HEADER} + [[peers]] + id = \"s\" + kind = \"stage\" + initial_state = \"cold\" + kind_config = {{ address = \"10.0.0.1:7700\" }} + " + ); + must_reject(&text, "required key missing: name"); +} + +#[test] +fn merge_preserves_relays_from_base() { + // §8.3 "Merge preserves relays." We construct base + child via + // file paths so the loader's resolve_extends path runs. + use std::fs; + use tempfile::tempdir; + let dir = tempdir().unwrap(); + let base_path = dir.path().join("base.toml"); + let child_path = dir.path().join("child.toml"); + let base = format!( + "{HEADER}{SWIM_PEERS_A_B} + [[relays]] + id = \"R\" + ingress_capacity_bps = 1_000_000 + egress_capacity_bps_per_link = 1_000_000 + queue_depth_bytes = 1_000 + [[links]] + from = \"a\" + to = \"b\" + via = \"R\" + [[links]] + from = \"b\" + to = \"a\" + via = \"R\" + " + ); + fs::write(&base_path, &base).unwrap(); + let child = "[base]\nextends = \"base.toml\"\n"; + fs::write(&child_path, child).unwrap(); + let scen = simulation::scenario::load_from_path(&child_path, ®istry()) + .expect("child must validate via base"); + assert_eq!(scen.relays.len(), 1, "child inherits the base's relay"); +} diff --git a/crates/simulation/tests/stage_host_invariants.rs b/crates/simulation/tests/stage_host_invariants.rs new file mode 100644 index 0000000..bad0032 --- /dev/null +++ b/crates/simulation/tests/stage_host_invariants.rs @@ -0,0 +1,405 @@ +//! RELAY_SPEC §5.6 behavioural tests for the stage host kind. +//! +//! The stage host is exercised both as a unit (direct calls to +//! `tick` / `recv` / `snapshot`) and as a participant in an engine +//! run (`WorkerExit` mutation routing, abort-on-wrong-kind). + +use serde_json::Value; +use simulation::bundle::{BundleRecord, EventPayload, VecWriter}; +use simulation::engine::{Engine, EngineAbort, TerminationReason}; +use simulation::host::{Action, Host, HostMessage}; +use simulation::network::Network; +use simulation::scenario::{HostKindRegistry, Mutation, MutationKind, load_from_str}; +use simulation::stage_host::{StageHost, StageHostFactory, StageState}; +use std::path::Path; + +fn registry() -> HostKindRegistry { + HostKindRegistry::with_swim() +} + +fn parse(text: &str) -> simulation::scenario::Scenario { + load_from_str(Path::new("(test)"), text, ®istry()) + .expect("scenario must validate") +} + +fn payload_kind(b: &[u8]) -> String { + serde_json::from_slice::(b).unwrap()["kind"] + .as_str() + .unwrap() + .to_string() +} + +// ──────────────────────────────────────────────────────────────────── +// RELAY_SPEC §5.6 +// ──────────────────────────────────────────────────────────────────── + +#[test] +fn trait_conformance_kind_tag_is_stage() { + let host = StageHost::new("h1", "name", "addr"); + assert_eq!(host.kind_tag(), "stage"); + assert_eq!(host.id(), "h1"); +} + +#[test] +fn first_tick_emits_lifecycle_and_register_name_in_order() { + // §5.2 — Cold → Registering, register_name, Registering → Running. + // The three RecordEvent actions appear in declared order; the + // host emits no `Send` or `Halt` on this tick. + let mut host = StageHost::new("stage_0", "pp-stage-0", "10.0.0.10:7700"); + let actions = host.tick(1_000_000); + assert_eq!(actions.len(), 3); + let mut kinds = Vec::new(); + for action in &actions { + match action { + Action::RecordEvent { kind_tag, event } => { + assert_eq!(kind_tag, "stage"); + kinds.push(payload_kind(event)); + } + other => panic!("first-tick action must be RecordEvent, got {other:?}"), + } + } + assert_eq!( + kinds, + vec![ + "stage_lifecycle".to_string(), + "register_name".to_string(), + "stage_lifecycle".to_string(), + ] + ); +} + +#[test] +fn subsequent_ticks_are_noops_once_running() { + // §5.6 "Lifecycle linearity." After the first tick takes us to + // Running, subsequent ticks emit no actions until WorkerExit + // arrives. + let mut host = StageHost::new("s", "n", "a"); + let _ = host.tick(0); + for t in 1..10 { + assert!(host.tick(t).is_empty()); + } +} + +#[test] +fn worker_exit_emits_worker_exited_then_lifecycle_then_halt() { + // §5.3 — recv(WorkerExit) returns exactly RecordEvent + // (worker_exited), RecordEvent (stage_lifecycle → Halted), Halt. + // Order is normative. + let mut host = StageHost::new("s", "n", "a"); + let _ = host.tick(0); + let actions = host.recv( + HostMessage::WorkerExit { + reason: "crashed".into(), + status_code: Some(2), + signal: None, + }, + 100_000, + ); + assert_eq!(actions.len(), 3); + match &actions[0] { + Action::RecordEvent { kind_tag, event } => { + assert_eq!(kind_tag, "stage"); + assert_eq!(payload_kind(event), "worker_exited"); + let parsed: Value = serde_json::from_slice(event).unwrap(); + assert_eq!(parsed["reason"], "crashed"); + assert_eq!(parsed["status_code"], 2); + } + other => panic!("action[0] must be RecordEvent(worker_exited), got {other:?}"), + } + match &actions[1] { + Action::RecordEvent { kind_tag, event } => { + assert_eq!(kind_tag, "stage"); + assert_eq!(payload_kind(event), "stage_lifecycle"); + let parsed: Value = serde_json::from_slice(event).unwrap(); + assert_eq!(parsed["to"], "Halted"); + } + other => panic!("action[1] must be RecordEvent(stage_lifecycle), got {other:?}"), + } + assert!(matches!(actions[2], Action::Halt)); +} + +#[test] +fn snapshot_includes_name_registry_after_registration() { + // §5.6 "Snapshot contains the registry." Once the stage has + // registered, every subsequent snapshot includes the name. + let mut host = StageHost::new("s", "my-name", "10.0.0.1:7700"); + let _ = host.tick(0); + let snap = host.snapshot(); + let parsed: Value = serde_json::from_slice(&snap).unwrap(); + assert_eq!(parsed["name_registry"]["my-name"], "10.0.0.1:7700"); + assert_eq!(parsed["state"], "Running"); +} + +#[test] +fn snapshot_includes_last_exit_reason_after_halt() { + let mut host = StageHost::new("s", "n", "a"); + let _ = host.tick(0); + let _ = host.recv( + HostMessage::WorkerExit { + reason: "stopped".into(), + status_code: None, + signal: None, + }, + 100_000, + ); + let snap = host.snapshot(); + let parsed: Value = serde_json::from_slice(&snap).unwrap(); + assert_eq!(parsed["state"], "Halted"); + assert_eq!(parsed["last_exit_reason"], "stopped"); +} + +#[test] +fn determinism_same_inputs_produce_same_action_sequence() { + let mk = || StageHost::new("h", "n", "a"); + let go = |mut h: StageHost| -> Vec { + let mut out = Vec::new(); + for action in h.tick(0) { + if let Action::RecordEvent { event, .. } = action { + out.push(payload_kind(&event)); + } + } + for action in h.recv( + HostMessage::WorkerExit { + reason: "x".into(), + status_code: None, + signal: None, + }, + 10, + ) { + match action { + Action::RecordEvent { event, .. } => out.push(payload_kind(&event)), + Action::Halt => out.push("Halt".into()), + _ => {} + } + } + out + }; + assert_eq!(go(mk()), go(mk())); +} + +// ──────────────────────────────────────────────────────────────────── +// Engine integration: WorkerExit mutation routing +// ──────────────────────────────────────────────────────────────────── + +fn one_stage_scenario(worker_exit_at_ns: u64) -> simulation::scenario::Scenario { + parse(&format!( + r#" + name = "one_stage" + seed = 1 + duration_ns = 1_000_000_000 + + [default_tick] + period_ns = 100_000_000 + + [default_link] + latency_ns = 1_000_000 + jitter_stddev_ns = 0 + loss_prob_ppm = 0 + reorder_prob_ppm = 0 + bandwidth_bps = 1_000_000_000 + cold_dial_penalty_ns = 0 + cache_warm_after_ns = 1_000_000_000 + cache_invalidate_after_idle_ns = 10_000_000_000 + + [[peers]] + id = "s" + kind = "stage" + initial_state = "cold" + kind_config = {{ name = "pp-s", address = "10.0.0.1:7700" }} + + [[peers]] + id = "other" + kind = "stage" + initial_state = "cold" + kind_config = {{ name = "pp-other", address = "10.0.0.2:7700" }} + + [[links]] + from = "s" + to = "other" + [[links]] + from = "other" + to = "s" + + [[mutations]] + at_ns = {worker_exit_at_ns} + kind = "worker_exit" + peer = "s" + reason = "internal crash" + "# + )) +} + +#[test] +fn worker_exit_event_is_emitted_before_halt_takes_effect() { + // §5.6 "Event-before-halt is observable." A scenario whose + // duration matches the worker_exit time still contains the + // `worker_exited` event in its bundle. + let scen = one_stage_scenario(500_000_000); + let writer = VecWriter::default(); + let network = Network::new(&scen); + let mut engine = Engine::new(&scen, network, writer); + engine.register_factory(Box::new(StageHostFactory)); + engine.auto_install_hosts(); + let _ = engine.run(); + let writer = engine.into_writer(); + let worker_exited = writer + .records + .iter() + .filter_map(|r| match r { + BundleRecord::Event(e) => { + if let EventPayload::Bytes(b) = &e.event { + let v: Value = serde_json::from_slice(b).ok()?; + if v["kind"] == "worker_exited" { + return Some(e.host_id.clone()); + } + } + None + } + _ => None, + }) + .collect::>(); + assert_eq!( + worker_exited, + vec![Some("s".to_string())], + "exactly one worker_exited event from the targeted stage" + ); +} + +#[test] +fn worker_exit_against_swim_aborts_run() { + // §5.6 "WorkerExit against SWIM aborts." A WorkerExit mutation + // targeting a SWIM-kind peer aborts the run with a structured + // error naming the kind and the mutation index. + // + // We construct the scenario in code (bypassing the loader's + // own §8.2 guard, which already rejects this at load time) so + // the engine's runtime gate is what the test actually exercises. + use simulation::scenario::{ + DefaultTick, HostRoute, Link, LinkPolicy, Peer, Scenario, + }; + let policy = LinkPolicy { + latency_ns: 1_000_000, + jitter_stddev_ns: 0, + loss_prob_ppm: 0, + reorder_prob_ppm: 0, + bandwidth_bps: 1_000_000_000, + cold_dial_penalty_ns: 0, + cache_warm_after_ns: 1_000_000_000, + cache_invalidate_after_idle_ns: 10_000_000_000, + }; + let mut config = toml::value::Table::new(); + config.insert("probe_interval_ns".into(), toml::Value::Integer(1_000_000)); + config.insert( + "suspicion_timeout_ns".into(), + toml::Value::Integer(5_000_000), + ); + let scen = Scenario { + name: "swim_worker_exit".into(), + seed: 1, + duration_ns: 1_000_000_000, + early_terminate_on_all_assertions_resolved: false, + default_tick: DefaultTick { + period_ns: 100_000_000, + }, + default_link: policy, + peers: vec![Peer { + id: "x".into(), + kind: "swim".into(), + kind_config: config, + initial_state: "alive".into(), + tick_period_ns_override: None, + }], + relays: Vec::new(), + links: vec![Link { + from: "x".into(), + to: "x".into(), + policy, + }], + mutations: vec![Mutation { + at_ns: 10_000_000, + kind: MutationKind::WorkerExit { + peer: "x".into(), + reason: "should abort".into(), + status_code: None, + signal: None, + }, + }], + snapshots: vec![], + assertions: vec![], + routes: vec![HostRoute::Direct { + from: "x".into(), + to: "x".into(), + }], + }; + let writer = VecWriter::default(); + let network = Network::new(&scen); + let mut engine = Engine::new(&scen, network, writer); + // Don't install any SWIM host — engine still has the peer spec + // by kind. The abort fires at mutation dispatch, before any + // host call. + let result = engine.run(); + assert!( + matches!( + result, + TerminationReason::Aborted(EngineAbort::WorkerExitOnWrongKind { + ref kind, + mutation_index: 0, + .. + }) if kind == "swim" + ), + "expected WorkerExitOnWrongKind abort, got {result:?}" + ); +} + +#[test] +fn lifecycle_stages_appear_in_order_in_bundle() { + // §5.6 "Lifecycle linearity." End-to-end through the engine, the + // stage host emits Cold→Registering, Registering→Running, and + // eventually Running→Halted lifecycle events in that order. + let scen = one_stage_scenario(500_000_000); + let writer = VecWriter::default(); + let network = Network::new(&scen); + let mut engine = Engine::new(&scen, network, writer); + engine.register_factory(Box::new(StageHostFactory)); + engine.auto_install_hosts(); + let _ = engine.run(); + let writer = engine.into_writer(); + let lifecycle: Vec<(String, String)> = writer + .records + .iter() + .filter_map(|r| match r { + BundleRecord::Event(e) if e.host_id.as_deref() == Some("s") => { + if let EventPayload::Bytes(b) = &e.event { + let v: Value = serde_json::from_slice(b).ok()?; + if v["kind"] == "stage_lifecycle" { + return Some(( + v["from"].as_str().unwrap_or("").to_string(), + v["to"].as_str().unwrap_or("").to_string(), + )); + } + } + None + } + _ => None, + }) + .collect(); + assert_eq!( + lifecycle, + vec![ + ("Cold".to_string(), "Registering".to_string()), + ("Registering".to_string(), "Running".to_string()), + ("Running".to_string(), "Halted".to_string()), + ] + ); +} + +#[test] +fn cold_state_string_is_well_known() { + // Sanity: the lifecycle state strings are stable contract. + // RELAY_SPEC §5.2 names them; tests outside this file + // (assertion evaluators, calibration scenarios) match on them. + assert_eq!(StageState::Cold.as_str(), "Cold"); + assert_eq!(StageState::Registering.as_str(), "Registering"); + assert_eq!(StageState::Running.as_str(), "Running"); + assert_eq!(StageState::Halted.as_str(), "Halted"); +}