diff --git a/.gitignore b/.gitignore index caee45d..8328a5b 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,9 @@ doc/ # Temporary files, for example, from tests. tmp/ +# TLC's default state directory when the formal model is run by hand. +/states/ + # If the VM crashes, it generates a dump, let's ignore it too. erl_crash.dump diff --git a/CHANGELOG.md b/CHANGELOG.md index e01b07f..4d6e920 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,17 +1,66 @@ ## Unreleased +- Add layered anti-entropy qualification: three-node StreamData lifecycle + models, seeded adversarial transport histories, TLA+ models for convergence, + chunk assembly, and permanent peer eviction, plus a Docker-backed Jepsen + oracle across distribution, a test-only sideband TCP lane, and + lossy/reordering transports. + `mix test` is the every-PR ExUnit/property/checker gate and `mix test.soak` + runs the six-profile nightly/release campaign. +- **Breaking**: move the replica transport API from + `Group.Replica.Transport.*` to `Group.Transport.*`; the default adapter is + now `Group.Transport.DistErl`. The boundary also names logical direction + rather than implementation mechanics: adapters implement `outgoing/5`, + sideband adapters use `Group.Transport.Outbox.push/5`, and receiving adapters + call `Group.Transport.incoming/4` or `incoming_batch/4`. No compatibility + aliases are provided. +- Rename the internal replica wire helper from `Group.Replica.Protocol` to + `Group.Replica.WireProtocol` to avoid overloading Elixir protocol terminology. + The standalone TCP adapter is retained only as hidden test infrastructure; + Group ships the transport contract, dist-Erlang adapter, and outbox helper. +- **Breaking**: replica protocol v3 streams exact snapshots as provisional, + transport-neutral byte-targeted chunks (`1 MiB` by default) followed by an + independently retryable terminal manifest. The sender scans once and retains + only its current chunk; a concurrent mutation suppresses commit. Receivers + stage in reusable shard-owned private ETS and advance the cursor only after + one exact, authority-fenced assembly is complete. Chunk/commit loss, + duplication, reordering, conflicting retransmission, supersession, expiry, + and shard crashes remain repairable by anti-entropy. Sideband transports can + use per-shard local outboxes for bounded batching without adding a hop to the + default dist-Erlang adapter. Late-starting replica lanes now rebuild their + view from shared exact authority when startup fanout races registration. +- Replace replica state sends/snapshots with per-origin, generation- and + cluster-epoch-fenced streams: sequenced deltas repair gaps from a bounded + oplog and fall back to exact origin snapshots after pruning. Replica data now + uses a pluggable nonblocking transport (dist Erlang by default via + `send_nosuspend`), while dist Erlang remains the control plane. Nonblocking + control heartbeats lease peer state, requesting a fresh authoritative hello + on generation or epoch-revision changes, so a stopped Group on a connected + VM cannot leave permanent registry or membership rows. Reconnects also sweep + superseded per-shard receive cursors and reconstruct epochless PG rows, so + reordered cluster controls cannot strand live rows from an older epoch. Full + epoch authority is installed once by shard 0; matching data shards exchange + constant-size lane hellos and retain shard-to-shard transport ordering. + Authority capture is serialized with epoch activation, and exact versus + incrementally observed revisions are tracked separately so a concurrent + partial snapshot cannot be mistaken for complete authority. +- Registry authority is retained per origin separately from the visible + winner. Conflict callbacks select the winner; Group now records and + propagates an authoritative loser delete, and each owner node terminates only + its own losing process. This also applies to custom conflict callbacks. +- Add `Group.monitor_generation/1` so long-lived registration owners can + terminate and re-register when the local membership ETS generation is lost. - **Breaking**: `Group.disconnect/3` now discards the complete local view of each departed cluster — remote entries included, and monitors receive `:unregistered`/`:left` events for them — instead of removing only locally owned rows. Reconnecting resyncs through the normal snapshot exchange. `connect`/`disconnect` also raise `ArgumentError` for non-binary cluster names instead of silently tolerating them. -- The built-in registry conflict resolver now consistently includes the winner's metadata in - the losing process's `{:group_registry_conflict, key, winner_meta}` exit reason. Custom - `resolve_registry_conflict` callbacks remain responsible for any process exits they require. +- The registry conflict resolver now consistently includes the winner's metadata in + the losing process's `{:group_registry_conflict, key, winner_meta}` exit reason. - **Breaking**: `Group.dispatch/4` remote sends and process-DOWN replication are now - non-suspending and never auto-connect — on a busy or disconnected distribution link the - message is dropped, the link is force-disconnected, and bounded reconnect retries begin (the - same policy replication lanes have used since 0.1.8). Previously dispatch could block the - caller and initiate new connections. + non-suspending and never auto-connect. Busy dispatch drops still force a disconnect and + bounded reconnect retry; replica messages are dropped and repaired by anti-entropy without + disturbing the dist-Erlang control connection. Previously dispatch could block the caller + and initiate new connections. - Configured function-form `extract_meta` callbacks are now applied on reads and lifecycle events (previously they were silently ignored and full metadata was exposed), and invalid `:extract_meta` values raise `ArgumentError` at startup. diff --git a/CLAUDE.md b/CLAUDE.md index 29fb3e3..90041ce 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,208 +1,318 @@ -# Group — CLAUDE.md +# Group — Maintainer Architecture Guide -## What is Group +## What Group Is -Distributed process registry + process groups + lifecycle monitoring + isolated subclusters. +Group is an eventually consistent distributed process registry, process-group +service, lifecycle monitor, and named-subcluster layer. Erlang distribution is +the membership and authority control plane. Replica state moves over a +configurable, nonblocking data transport and converges through sequenced +anti-entropy streams. ## Project Structure ``` lib/ - group.ex — Public API: register, join, members, monitor, dispatch, connect/disconnect - group/event.ex — %Group.Event{} struct - group/supervisor.ex — Top-level supervisor (rest_for_one: Data → PeerReconnect → Replica.Supervisor → Registry → ClusterLease) - group/cluster_lease.ex — Local named-cluster TTL sweeper - group/peer_reconnect.ex — Bounded reconnect loop after busy distribution links - group/replica/ - data.ex — GenServer that owns ETS tables and serializes shared membership mutations. - supervisor.ex — one_for_one supervisor for Replica shards - group/replica.ex — Sharded GenServer: replication, peer discovery, conflict resolution, monitoring - group/application.ex — Empty app supervisor (Group instances are started by consumers) + group.ex — public API and configuration docs + group/supervisor.ex — rest_for_one instance supervisor + group/cluster_lease.ex — local named-cluster TTL policy + group/peer_reconnect.ex — bounded retry for busy dispatch links + group/replica.ex — sharded writes, control, AE, projection + group/replica/data.ex — ETS owner, journal, authority, indexes + group/replica/wire_protocol.ex — wire version, stream identity, mutations + group/replica/snapshot.ex — byte-targeted snapshot chunks/staging + group/transport.ex — replica transport contract + group/transport/dist_erl.ex — default dist-Erlang adapter + group/transport/outbox.ex — optional lossy sideband outboxes test/ - test_helper.exs — Ensures EPMD/distribution are running; disables prevent_overlapping_partitions - group_test.exs — Local tests (async: true) - distributed_test.exs — Multi-node tests using OTP :peer - support/ - test_cluster.ex — Peer node helpers (start_peers, spawn_register, spawn_join, etc.) - test_conflict_resolver.ex — Custom resolver for tests -priv/bench/ — Benchmarks (run_local.sh, run_distributed.sh) + support/test_tcp_transport.ex — test-only independent-socket adapter + replica_model_property_test.exs — shrinkable real-node lifecycle model + replica_adversarial_test.exs — seeded three-node transport chaos + replica_snapshot_* — chunk/assembly failure coverage + formal/ — TLC protocol, assembly, eviction models + jepsen/ — independent-VM lifecycle oracle/campaign +priv/bench/ — local and distributed benchmark project ``` -## Running Tests +## Required Gates ```bash -mix test # all tests -mix test test/group_test.exs # local only -mix test test/distributed_test.exs # distributed only -``` - -Tests require `elixirc_paths(:test)` includes `test/support/`. Distributed tests use OTP `:peer` module for real Erlang nodes. - -## Running Benchmarks +mix test # every-PR ExUnit/property/chaos/pure-checker gate +mix test.soak # nightly mutation/live-checker + six-profile Jepsen campaign -```bash -cd priv/bench && ./run_local.sh -cd priv/bench && ./run_distributed.sh +# Focused development +mix test test/group_test.exs +mix test test/distributed_test.exs +mix test test/replica_model_property_test.exs ``` -## Architecture +`mix test` accepts normal Mix test paths/options and then runs the pure Jepsen +checker qualification. `mix test.soak` runs that gate first, kills every +defined protocol mutant, proves the live checker rejects injected corruption, +then runs twenty five-minute histories for distribution/TCP/chaos × +mixed/permanent scenarios. Chaos/mixed must exercise a multi-record delta +repair; the remaining profiles retain the one-record stress configuration. +See `test/README.md`, `test/formal/README.md`, and `test/jepsen/README.md`. -### Supervision Tree +## Supervision and Ownership ``` Group.Supervisor (rest_for_one) -├── Group.Replica.Data — owns all ETS tables; serializes membership mutations -├── Group.PeerReconnect — bounded reconnects after busy-link disconnects -├── Group.Replica.Supervisor — one_for_one, N shard GenServers -│ ├── Replica shard 0 -│ ├── Replica shard 1 +├── optional sideband transport manager + per-shard outboxes +├── Group.Replica.Data +├── Group.PeerReconnect +├── Group.Replica.Supervisor +│ ├── Group.Replica shard 0 +│ ├── Group.Replica shard 1 │ └── ... -├── Registry (Elixir) — :duplicate, for monitor subscriptions -└── Group.ClusterLease — local named-cluster TTL sweeper +├── Registry — local monitor subscriptions +└── Group.ClusterLease — local named-cluster TTL sweeper ``` -`rest_for_one` means: if Data dies, every later child restarts. If PeerReconnect dies, the Replica supervisor and monitor Registry restart. If Replica.Supervisor dies, Registry and ClusterLease restart. Replica shards rebuild local process monitors from surviving ETS after restart. - -### Sharding - -`phash2({cluster, key}, num_shards)` routes to shard. Default 8 shards. Must match across all nodes (validated on peer_connect). Including cluster in hash avoids false contention between default and named cluster operations. - -### ETS Tables (per shard × 4 + 3 shared) - -| Table | Type | Key | Tuple | -|-------|------|-----|-------| -| `reg_by_key` | `:set` | `{cluster, key}` | `{{cluster, key}, pid, meta, time, node}` | -| `reg_by_pid` | `:ordered_set` | `{pid, cluster, key}` | `{{pid, cluster, key}, meta, time, node}` | -| `pg_by_key` | `:ordered_set` | `{cluster, key, pid}` | `{{cluster, key, pid}, meta, time, node}` | -| `pg_by_pid` | `:ordered_set` | `{pid, cluster, key}` | `{{pid, cluster, key}, meta, time, node}` | -| `cluster_nodes` | `:bag` | cluster | `{cluster, node}` | -| `node_clusters` | `:bag` | node | `{node, cluster}` | -| `cluster_leases` | `:set` | cluster | `{cluster, ttl_ms, expires_at}` | - -**Why ordered_set for by_pid tables**: Contiguous range scans for `entries_by_pid` and `delete_all_for_pid` (process death cleanup). Also enables efficient existence checks with `select(..., 1)`. - -**Why ordered_set for pg_by_key**: `pg_members/4` scans `{cluster, key, *}` as a contiguous range — O(members in group), not O(table). - -**Why set for reg_by_key**: Only needs direct lookup/delete by `{cluster, key}` — O(1). - -All tables: `:public`, `read_concurrency: true`, `decentralized_counters: true`. Per-key registry/PG writes serialize through the owning Replica shard; shared cluster-membership dual-index mutations serialize through `Group.Replica.Data`. No table enables `write_concurrency`. - -### Reads vs Writes - -- **Reads** (`lookup`, `members`, `local_registry_count`) go directly to ETS — no GenServer involved -- **Writes** (`register`, `join`, `leave`, `unregister`) go through the shard's - local request lane (`send` + monitor + tagged reply), not `GenServer.call` -- **Replication** arrives as `handle_info` messages on shard GenServers - -### Config - -Stored in `persistent_term` keyed by `{Group, name}`. Includes shard/buffering/reconnect settings plus `log`, and optionally `extract_meta` and `resolve_registry_conflict`. - -## Key Protocols - -### Peer Discovery (per-shard, on nodeup/init) - -1. Each shard sends `{:peer_connect, pid, shard, num_shards, clusters}` to its counterpart -2. Receiver adds sender to nil cluster ETS, computes shared clusters, sends `{:peer_connect_ack, ...}` -3. Both sides send `{:cluster_state, cluster, reg_data, pg_data}` for each shared cluster -4. `merge_remote_cluster_data` applies data: new entries insert, conflicts go through `resolve_conflict` - -### Replication (steady state) +`Group.Replica.Data` owns every ETS table. A shard restart therefore preserves +the tables, repairs interrupted index/journal work, replays appended-but-not- +applied local mutations, and rebuilds monitors for locally owned processes. +Restart repair streams each primary materialized table once while rebuilding +its existing reverse index; it does not allocate a whole-table list or retain +additional per-origin indexes. +The optional transport precedes Data so losing its manager restarts the whole +instance and cannot leave stale transport sessions attached to retained state. + +Reads use public ETS directly. Writes route through +`:erlang.phash2({cluster, key}, num_shards)` to a local shard request lane. +Shard counts must match across peers. + +## ETS State + +Per shard: + +| Table | Key | Role | +|---|---|---| +| `reg_by_key` | `{cluster, key}` | visible registry winner | +| `reg_by_pid` | `{pid, cluster, key}` | visible reverse index | +| `reg_claim_by_key` | `{cluster, key, origin, generation, epoch}` | authoritative claims | +| `reg_claim_by_pid` | `{pid, cluster, key, origin, generation, epoch}` | claim reverse index | +| `pg_by_key` | `{cluster, key, pid}` | visible PG membership | +| `pg_by_pid` | `{pid, cluster, key}` | PG reverse index | +| `replica_stream_meta` | stream id | head, retained floor, journal position | +| `replica_oplog` | `{stream, sequence}` | retained mutation record | +| `replica_oplog_order` | append id | shard-wide pruning order | +| `replica_cursor` | stream id | highest contiguous sequence applied | + +Shared tables hold cluster/node indexes, local TTL leases, local and remote +cluster epochs, closed-cluster barriers, origin generations, exact/observed +authority revisions, installed lane views, and journal metadata. + +Per-shard tables omit `write_concurrency` because one shard serializes their +writes. The shared replication metadata table uses +`write_concurrency: :auto`: every shard atomically updates only its own +`{:append_counter, shard}` object. Cross-shard arrival order has no semantic +meaning; an append id exists only to bound that shard's oplog across streams. + +## Authority and Stream Identity + +Every mutation belongs to: -After discovery, writes replicate in two stages: -- nil cluster: uses `state.remote_shards` map -- Named clusters: uses `cluster_nodes` ETS table -- Sender batches: `replicate_registry_batch`, `replicate_pg_batch` -- Receiver buffers registry and PG lanes separately, bulk-applies ETS writes, - then takes a bounded fairness turn for local work -- Remote shard sends use `send_nosuspend(..., [:noconnect])`; a `false` result - force-disconnects that node and enters bounded reconnect retries for that - peer only - -### Conflict Resolution - -`resolve_conflict/5` and the batched equivalent handle all registry key conflicts (live replicated registry ops and partition-heal `merge_remote_cluster_data`). - -- Default resolver: most recent timestamp wins; pid ordering tiebreaker on equal timestamps -- **Tiebreaker MUST be deterministic across all nodes**: `time2 > time1 or (time2 == time1 and pid2 > pid1)`. Using `>=` causes mutual kill. -- Custom: `resolve_registry_conflict: {mod, func, extra_args}` option; the callback owns any process exits -- The default resolver kills the loser with `{:group_registry_conflict, key, winner_meta}` -- Runs synchronously inside shard GenServer — must return quickly - -### Named Cluster Connect/Disconnect - -- `connect/2`: adds to ETS, picks random shard S, S notifies remote S, remote acks with bundled data + fans out to siblings -- `disconnect/2`: removes local membership, calls ALL local shards to purge the complete replicated cluster view, and has shard 0 broadcast to remotes -- `connect(..., ttl: ms)`: still checks `cluster_nodes` first, so an already-connected - cluster stays an ETS-fast noop and does not refresh the TTL -- TTL rows are local policy only; they do not change `cluster_nodes` / - `node_clusters` semantics -- On TTL expiry, `Group.ClusterLease` disconnects only if the local node has no - cluster-scoped monitors, no local registrations, and no local PG memberships - in that cluster. Otherwise it extends the lease by one TTL interval. - -### Nodedown / Process Death - -- `nodedown`: every shard calls `purge_cluster_node` (unconditional — not gated on shard 0) then `purge_node` -- Process DOWN: `delete_all_for_pids` + one non-suspending `replicate_process_down_batch` per target peer -- Remote shard DOWN (monitored pid): treated like nodedown for that node - -### Monitor Events - -Events delivered as `{:group, [%Group.Event{}, ...], %{name: name}}`. Batched per handler turn: -- Single ops: one event per message -- Bulk ops (nodedown, process DOWN, cluster_state merge): all events in one message - -Patterns: `:all`, `{:exact, key}`, `{:prefix, "prefix/"}` - -### Prefix Queries - -`Group.members(name, "prefix/")` scans ALL shards (can't hash prefix to one shard). Uses ETS range guards: -```elixir -{:andalso, {:>=, :"$1", prefix}, {:<, :"$1", next_binary_prefix(prefix)}} ``` -where `next_binary_prefix` increments the last byte of the prefix string. - -Keys ending with `"/"` are rejected by `validate_key!/1` in register/unregister/join/leave — trailing slash is reserved for prefix queries. - -### Dispatch - -`dispatch/4` sends to all members (registry + PG). Groups remote PG members by node and sends one non-suspending `{:group_dispatch, pids, message}` per remote node (O(nodes), not O(members)). The target shard is chosen by `phash2(self(), num_shards)` for per-sender ordering. - -`dispatch_local/4` skips cross-node messaging. - -## ETS Match Spec Patterns - -- Use `{:==, :"$N", value}` for runtime variables in guards (e.g., filtering by node) -- **NOT** `{:const, value}` — it's invalid in ETS match specs -- Literal Elixir variables interpolate directly into match pattern tuple positions as exact-match filters -- For result bodies, runtime values can't be embedded directly — use `Enum.map` post-select - -## Distributed Test Patterns +{group, origin_node, origin_generation, shard, cluster, cluster_epoch} +``` -- `Group.TestCluster.start_peers(N)` starts N real Erlang nodes via OTP `:peer` -- All helpers (`spawn_register`, `spawn_join`, `spawn_monitor_forwarder`) use `:erpc.call` with compiled modules from `test/support/` -- `Node.spawn` with anonymous functions won't work — remote node needs the defining module's beam file -- `assert_eventually/2` polls with retries for async replication -- `flush_shards/2` sends a mailbox barrier through each shard so buffered sender - and receiver replication work is flushed too -- `assert_ets_consistent/1` verifies dual-index tables match -- Partition tests use 3 nodes (isolate 1 from other 2). 2-node partitions are unreliable because the test node bridges them. -- `Supervisor.start_link` links to caller — in RPC context, must `Process.unlink(pid)` or supervisor dies on RPC return -- `test_helper.exs`: starts EPMD when needed, calls `Node.start/2`, sets the cookie, and disables `prevent_overlapping_partitions` +The origin appends a strictly increasing sequence before applying the +materialized change. Generation fences a restarted Group instance. A +named-cluster epoch fences close/reopen. The nil cluster uses the origin +generation as its epoch. + +Shard 0 installs one exact node-wide authority snapshot: + +- origin generation; +- complete active cluster→epoch set; +- exact authority revision; and +- opaque transport descriptor. + +The exact authority and its shared-cluster forward/reverse indexes are one +serialized Data mutation. A concurrent local connect is ordered wholly before +or after that mutation, so exact authority cannot exist without the matching +replica route indefinitely. +Local activation projects its epoch, self route, and all already-exact remote +routes at that same boundary. Local deactivation removes admission and queues +old-epoch cleanup to every shard before replying; callers and timed-out aliases +are not lifecycle coordinators. + +Other shards exchange constant-size lane hellos. Shared authority is not enough +to accept data: each shard records an installed lane view only after it has +purged streams outside that authority. Exact and incrementally observed +revisions are distinct from the complete applied revision. A persisted +`{generation, revision}` hint atomically fences every lane when any heartbeat +observes newer authority. Incremental controls compare-and-install their +expected revision against the applied/observed/hinted state in one Data turn, +so a raced heartbeat cannot publish partial authority. Hints refine only a +peer with prior exact authority; after retirement, delayed heartbeats or lane +hellos cannot recreate authority, a route, or a lease. Only the exact +dist-Erlang hello reintroduces the peer. If a lane hello precedes exact +authority during discovery, authority fanout immediately re-probes that lane; +the rejected hello is never retained as a route. + +`peer_connect` and `peer_connect_ack` are discovery hints, not authority. +Unsequenced legacy state messages are ignored; all replica recovery uses +heads, contiguous deltas, or exact snapshots. + +## Anti-Entropy + +Replica data messages are: + +- `heads`: stream, retained floor, and head; +- `delta_batch`: one or more contiguous stream runs; +- `need`: the receiver's next missing sequence; and +- `snapshot_chunk`: one byte-targeted part of an exact origin slice. + +A receiver advances its cursor only through a contiguous prefix. Duplicates are +idempotent; gaps request the missing suffix. Periodic heads recover a dropped +tail even if no later write occurs. If the requested sequence is below the +bounded oplog floor, the origin sends an exact snapshot containing only its own +claims and memberships for that shard/cluster stream. Absence is deletion. + +The oplog is bounded per shard and never waits for acknowledgements. There are +no leaders, quorums, replicated per-entry tombstones, known-member lists, or +retention barriers. A slow peer cannot pin memory. + +Snapshots are transport-neutral and byte-targeted (1 MiB by default). A single +oversized row is one chunk. Multi-chunk receivers stage rows in shard-owned +private ETS and preserve the old visible slice/cursor until the complete, +authority-valid manifest is present. Loss, duplication, reordering, +supersession, stale authority, expiry, and shard crash must leave no partial +visible state. Staging expires after one peer-lease interval without progress. +At most one sender worker per shard captures rows off the control process and +sends only if the stream identity and fully-applied head remain unchanged after +the scan; otherwise anti-entropy retries. Sender capture and receiver staging +retain completed byte-bounded chunks in unnamed private ETS tables owned by +their worker/shard. Only one chunk is assembled or copied on a process heap at +a time. These tables are ephemeral: explicit completion/retry cleanup deletes +them, and owner death deletes them automatically. Monitor events produced by a +large exact install are staged in bounded private-ETS batches until the cursor +commits. + +## Nonblocking Transport + +All cross-node Group control sends use +`:erlang.send_nosuspend(..., [:noconnect])`. The default distribution replica +adapter sends directly the same way and adds no local hop. `:busy` and +`:disconnected` mean “drop this message”; periodic anti-entropy repairs it. + +A sideband adapter may use one local `Group.Transport.Outbox` per shard. +Outboxes batch by peer, bound admitted messages, impose deadlines, and run +bounded socket work outside Group shards. Queue overflow, expiry, or socket +backpressure drops the batch. +The test suite's hidden TCP adapter exercises a genuinely independent socket +lane; it is validation infrastructure, not a supported production transport. + +Transport ordering is not required for correctness. Per-shard ordered delivery +is a fast path; stream sequences reject duplicate/out-of-order data, and +generation/epoch/lane fences handle control/data reordering. A transport must +reassemble any transport segmentation before `incoming_batch/4`. + +## Registry Projection and Process Ownership + +Registry claims remain authoritative per origin even when hidden by another +origin's visible winner. The configured callback deterministically ranks each +claim independently; Group selects the maximum `{rank, pid}` so the result is +associative and commutative. Group owns lifecycle effects: each losing origin +appends its own authoritative unregister and exits only its local process with +`{:group_registry_conflict, key, winner_meta}`. + +A node never monitors or exits another node's member processes. Local shards +monitor locally owned registration/PG pids. Local `DOWN` appends ordered +unregister/leave mutations before deletion and replication. Remote owner death +arrives through those records; `nodedown` or lease expiry is the fallback for +an origin that cannot emit them. + +## Peer and Cluster Lifecycle + +- Dist-Erlang `nodedown` immediately purges that node's visible rows, claims, + cursors, authority, cluster routing, and sideband session on every shard. +- Constant-size control heartbeats cover the case where the Erlang node remains + connected but its Group instance disappears. Peer-lease expiry performs the + same complete purge. +- After a receiver shard restart, retained authority/view metadata reconstructs + leases for origins whose ETS rows survived, so a permanently disappeared + Group is still evicted without scanning registry or PG data. +- Each lane deletes its own persisted view only after purging its rows and + cursors; shard 0 never erases a sibling's restart breadcrumb. If shard 0 + restarts with `observed_revision != exact_revision`, it also reconstructs the + pending exact-hello repair obligation. Persisted hints reconstruct the same + bounded lease if a lane crashes after fencing authority but before updating + its in-memory deadline. +- Discovery probes continue after expiry. A returning current/new generation + is fenced, installs authority per lane, and reconstructs through deltas or an + exact snapshot. Delayed cleanup rechecks shared authority before deleting + routes, and a lane hello is not reported `peer_up` until authority admits it. +- Incremental named-cluster open/close controls are generation fenced and + batched, but shard 0 is their only node-wide authority writer; controls + arriving on other lanes are forwarded locally. Revisions must be contiguous. + A gap fences every lane and requests an exact hello instead of applying a + partial authority set. +- Local cluster close uses a temporary all-shard completion barrier. The last + shard removes routing/epoch rows; restart repair completes abandoned closes, + and reconnect waits so an old close cannot erase new writes. + +TTL leases are local policy only. On expiry, Group disconnects a named cluster +only when no local registrations, PG memberships, or cluster monitors remain. + +## Batching and Fairness + +Registry and PG mutations share one outbound sender buffer so local mailbox +order is retained. It flushes on size, age, idle timer, and before control or +routing barriers. Receivers apply contiguous runs in bulk and emit lifecycle +events in operation batches. After replicated work, each shard takes a bounded +FIFO local-request turn to prevent replica pressure from starving callers. + +## Distributed Test Rules + +- Use three peers for partition/recovery tests. Two nodes cannot cover an + independent survivor while an origin and receiver disagree. +- Remote helpers must be compiled under `test/support/`; call them with MFA + through `:erpc`. +- Unlink supervisors started inside RPC helpers. +- `flush_shards/2` is only a mailbox/barrier aid; convergence assertions must + still wait for AE and inspect exact public/internal state. +- Always assert dead owners are absent, retained owners are alive, claims and + projections agree, cursors are contiguous, no partial snapshot remains, and + retired origins have no rows. ## Critical Invariants -1. **purge_cluster_node is unconditional** — every shard calls it on nodedown/DOWN, not just shard 0. Late peer_connect on non-zero shard can re-add dead node after shard 0 cleaned it. -2. **Dispatch :unregistered for evicted local pid** in "remote wins" branch of resolve_conflict — monitors need to see the eviction. -3. **merge_remote_cluster_data uses Enum.reduce** (not `for`) to thread `{state, events}` through, since resolve_conflict modifies `state.monitors`. -4. **Additive merge only** — cluster_state merge inserts but never deletes. Local named-cluster disconnect therefore purges the complete local cluster view, and replicated batches are membership-gated at apply time. -5. **PG tables have no overwrite conflicts** — `pg_by_key` key includes pid: `{cluster, key, pid}`. -6. **Named-cluster data is membership-gated** — both `cluster_state` and buffered replicated operations reject data for clusters the local node has left. +1. A cursor never advances across a gap or before a full exact snapshot commits. +2. Exact snapshots replace one origin slice; they are never additive merges. +3. Authority requires generation, complete applied epoch revision, a matching + persisted observation hint, and installed lane readiness. Observed + heartbeats alone are not authority; the last exact snapshot revision remains + separate while a contiguous incremental update is applied. Exact + authority and its shared-cluster routing projection are installed atomically. + Local activation is projected atomically; deactivation cleanup is durable + before its initiating caller can disappear. +4. A stale generation, epoch, lane, shard, transitive pid, or stream whose + origin differs from the transport-reported source is rejected before + applying replica data. Group trusts the adapter's source identity; + authenticating a sideband peer belongs to that transport. +5. Registry claims are retained per origin until that origin deletes them or is + retired; the visible winner is reconstructible from remaining claims. A + remote winner is serially revalidated against shared and lane authority + immediately before any local losing owner is retired. Keys reconciled while + a retained claim is fenced by an authority gap are reprojected when that + lane installs the exact view; a current cursor must not strand an older + visible winner. +6. Only an owner node monitors, retires, or exits its member processes. +7. Oplog pruning is local and bounded; lagging peers use exact snapshot repair. +8. `nodedown` and peer-lease expiry purge every public and internal reference + to the retired origin. A lane retains its persisted restart breadcrumb until + its own rows/cursors are gone. Remote shard death cannot leave state + permanently; lease expiry or fenced rediscovery completes cleanup/recovery. +9. Snapshot staging is private, all-or-nothing, authority fenced, and expiring. +10. Local append/journal repair makes an appended mutation either replayable or + durably applied after a shard crash. +11. Cross-node control and replica calls never block a Group shard. +12. Cluster close completion survives caller timeout and shard restart. ## Logging -- `log:` option: `:info` (default), `false` (routine logs disabled), `:verbose` (all shards) -- `log/2` (normal), `log_verbose/2` (verbose only), `log_once/2` (shard 0 only) use `Logger.info`; `:verbose` is Group's own flag, not a Logger level -- Registry conflicts are unconditional `Logger.error` events; busy distribution links are unconditional `Logger.warning` events -- Runtime change: `Group.log_level(name, level)` +`log: :info | :verbose | false` controls routine Group logs and can be changed +with `Group.log_level/2`. Registry conflicts remain errors and busy dispatch +links remain warnings regardless of the routine level. diff --git a/README.md b/README.md index c3d8fce..1d08668 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ lifecycle monitoring, and isolated subclusters for Elixir. No external dependenc where only connected nodes participate. - **Sharded writes** — writes fan out across N GenServer shards to reduce contention. Reads go directly to ETS. +- **Nonblocking anti-entropy** — replica sends never wait on a remote socket; + sequenced deltas, bounded oplogs, and exact snapshots repair dropped work. ## Installation @@ -212,14 +214,19 @@ delivers an event with `previous_meta` set to the old value. All operations are **eventually consistent**: - Writes (`register`, `join`, etc.) return immediately after updating local ETS. -- Changes replicate to other nodes asynchronously over Erlang distribution. +- Changes replicate asynchronously over a configurable, nonblocking replica + transport. Erlang distribution remains the membership/control plane. - During network partitions, nodes may have divergent views. -- When partitions heal, state is re-synced via `cluster_state` messages. +- When connectivity returns, per-origin stream heads repair missing sequence + ranges from a bounded oplog; a lag beyond the retained prefix falls back to + an exact snapshot of that origin's shard/cluster slice. +- A dist-Erlang `nodedown` removes that node's view immediately. If the Erlang + node remains connected but its Group instance stops responding, a bounded + control-plane lease removes the same state and later discovery can rebuild it. - Registry conflicts (same key registered on two nodes during a partition) can be resolved with a configurable `resolve_registry_conflict` callback. The - built-in resolver kills the losing process with - `{:group_registry_conflict, key, winner_meta}`; custom resolvers control any - process exits themselves. + callback selects a winner; each origin retires and terminates only its own + losing process with `{:group_registry_conflict, key, winner_meta}`. ## Configuration @@ -238,7 +245,12 @@ All operations are **eventually consistent**: replicated_sender_flush_interval: 5, busy_dist_retry_attempts: 300, busy_dist_retry_interval: 1_000, - replicated_pg_receiver_local_request_quota: 8 + replicated_pg_receiver_local_request_quota: 8, + replica_transport: Group.Transport.DistErl, + replicated_oplog_max_entries: 65_536, + replicated_snapshot_chunk_target_bytes: 1_048_576, + replicated_anti_entropy_interval: 1_000, + replicated_peer_lease_timeout: 15_000 } ``` @@ -255,11 +267,13 @@ All operations are **eventually consistent**: `Logger.error` events and busy distribution links remain `Logger.warning` events. The level can be changed at runtime with `Group.log_level/2`. - **`resolve_registry_conflict`** — `{module, function, extra_args}` callback - invoked as `apply(mod, fun, [name, key, {pid1, meta1, time1}, {pid2, meta2, time2} | extra_args])`. - Called when partition healing or concurrent registration finds the same key - registered on two nodes. Must return the winning pid and is responsible for - any process exits it requires. Runs synchronously inside the shard GenServer — - must return quickly and never block. + invoked once per competing claim as + `apply(mod, fun, [name, key, {pid, meta, time} | extra_args])`. It must return + a deterministic Erlang term used as the claim's rank. Group chooses the + maximum `{rank, pid}`, making the winner independent of delivery order and + grouping. Group records an authoritative delete and terminates a losing owner + only on that owner's local node. The callback runs synchronously inside the + shard GenServer, so it must return quickly and never block. - **`extract_meta`** — `{module, function, args}` or `fun(meta)` applied to metadata on reads and lifecycle events. Useful for stripping internal fields. - **`replicated_pg_receiver_buffer_size`** — max buffered replicated PG @@ -275,18 +289,44 @@ All operations are **eventually consistent**: - **`replicated_sender_flush_interval`** — max outbound buffer age in milliseconds. Defaults to 5. - **`busy_dist_retry_attempts`** — reconnect attempts after a non-suspending - remote send reports a busy link. Defaults to 300. -- **`busy_dist_retry_interval`** — milliseconds between busy-link reconnect - attempts. Defaults to 1,000. -- **`replicated_pg_receiver_local_request_quota`** — local PG requests drained - after each replicated receiver turn. Defaults to 8. + remote dispatch reports a busy dist link. Defaults to 300. Replica transport + messages are simply dropped and repaired instead of forcing a disconnect. +- **`busy_dist_retry_interval`** — milliseconds between dispatch busy-link + reconnect attempts. Defaults to 1,000. +- **`replicated_pg_receiver_local_request_quota`** — legacy-named quota for + queued local shard requests drained per fairness turn while replica data or + cluster controls are busy. Defaults to 8. +- **`replica_transport`** — a module implementing + `Group.Transport`, or `{module, opts}`. The default + `Group.Transport.DistErl` adapter uses `:erlang.send_nosuspend/3`; adapters + must return promptly with `:ok`, `:busy`, or `:disconnected`. Dropped and busy + messages are repaired by anti-entropy. Sideband implementations can use + `Group.Transport.Outbox` to move bounded batching and socket work outside the + Group shard. +- **`replicated_oplog_max_entries`** — maximum retained replica records per + shard across all local streams. Defaults to 65,536. Pruning never waits for + peer acknowledgements; a peer behind the retained floor receives an exact + snapshot. +- **`replicated_snapshot_chunk_target_bytes`** — target maximum encoded size + of each exact-snapshot message. Defaults to 1 MiB and applies above every + transport, including dist Erlang. A single row larger than the target is + sent alone. Receivers stage provisional chunks in shard-owned private ETS and + replace visible state only after the complete slice and its terminal manifest + are present. +- **`replicated_anti_entropy_interval`** — interval in milliseconds for stream + head advertisements and nonblocking control heartbeats. Defaults to 1,000. +- **`replicated_peer_lease_timeout`** — time without a dist-Erlang control + heartbeat before state owned by that Group peer is purged. Defaults to 15,000 + and must exceed the anti-entropy interval. Probes continue after expiry so a + Group restart on a still-connected VM recovers automatically. ## Architecture ``` Group.Supervisor (:"my_app_group_sup") -├── Group.Replica.Data — owns ETS tables and serializes membership writes -├── Group.PeerReconnect — bounded recovery after busy distribution links +├── optional transport child — sideband manager and per-shard outboxes +├── Group.Replica.Data — owns ETS, journal, generations, and epochs +├── Group.PeerReconnect — bounded recovery after busy remote dispatch ├── Group.Replica.Supervisor — supervises N shard GenServers │ ├── Group.Replica (shard 0) │ ├── Group.Replica (shard 1) @@ -310,24 +350,44 @@ contention for unrelated keys. ### ETS Tables -Each shard owns 4 ETS tables: +Each shard has materialized read indexes plus authority/recovery indexes: | Table | Type | Key | Purpose | |---|---|---|---| | `reg_by_key` | `:set` | `{cluster, key}` | Registry lookup — O(1) | | `reg_by_pid` | `:ordered_set` | `{pid, cluster, key}` | Reverse index for death cleanup | +| `reg_claim_by_key` | `:ordered_set` | `{cluster, key, origin, generation, epoch}` | One authoritative registry claim per origin | +| `reg_claim_by_pid` | `:ordered_set` | `{pid, cluster, key, origin, generation, epoch}` | Reverse claim index for owner death and repair | | `pg_by_key` | `:ordered_set` | `{cluster, key, pid}` | Group membership lookup | | `pg_by_pid` | `:ordered_set` | `{pid, cluster, key}` | Reverse index for death cleanup | +| `replica_stream_meta` | `:set` | `stream_id` | Local stream head, retained floor, and applied journal position | +| `replica_oplog` | `:ordered_set` | `{stream_id, sequence}` | Bounded sequenced mutation records | +| `replica_oplog_order` | `:ordered_set` | `append_id` | Shard-wide pruning order across streams | +| `replica_cursor` | `:set` | `stream_id` | Highest contiguous remote sequence applied | -Plus 3 shared tables: +Registry claim tables retain one authoritative claim per origin independently +of the visible winner. Stream metadata, oplog, append-order, and receive-cursor +tables support crash replay and gap repair. Keeping claims separate from the +single visible `reg_by_key` projection prevents a losing-but-still-live remote +claim from being forgotten before its owner emits an authoritative delete. + +The node also has shared control/authority tables: - `cluster_nodes` (`:bag`, cluster→nodes) - `node_clusters` (`:bag`, node→clusters) - `cluster_leases` (`:set`, cluster→`{ttl_ms, expires_at}`) for local `connect(..., ttl: ms)` policy - -`cluster_nodes` / `node_clusters` remain the authoritative cluster-membership -tables. `cluster_leases` is only local lease metadata used by the sweeper. +- `replication_meta` (`:set`) for the local generation, authority revisions, + per-lane installed views, journal metadata, and one atomic append counter per + shard +- `local_cluster_epochs` and `closed_local_cluster_epochs` (`:set`) for active + and closing local named-cluster lifetimes +- `remote_cluster_epochs` (`:set`) for exact generation-fenced remote authority + +`cluster_nodes` / `node_clusters` are the routing projection read by APIs and +replication fanout. Generation-fenced local/remote epoch tables are the +authority used to install that projection. `cluster_leases` is only local +policy metadata used by the sweeper. `Group.Replica.Data` owns all tables and is supervised with `rest_for_one` so tables survive shard crashes. @@ -340,28 +400,168 @@ nodes. This handshake: 1. Validates that shard counts match (raises on mismatch). 2. Exchanges cluster membership lists. -3. Each shard sends its locally owned registry and group slice in a - `cluster_state` message for each shared cluster. - -This is how a new node catches up to the existing cluster state. +3. Shard 0 exchanges protocol version, origin generation, and one complete + active named-cluster epoch snapshot per node. Matching data shards exchange + only constant-size lane/transport descriptors tied to that authority + revision. + +Constant-size heartbeats renew the peer lease. If an origin generation or +cluster-epoch revision changes, the receiver requests a fresh authoritative +hello; if heartbeats stop, lease expiry purges that origin's complete local +view and discovery probes allow it to rejoin later. + +Incremental cluster open/close controls are generation fenced, receiver +batched, and installed by shard 0 into one node-wide authority table. The +highest observed revision keeps heartbeats constant-size during a burst; after +the burst becomes quiet, one authoritative hello closes any gaps left by +dropped or reordered controls. Per-shard view rows record only constant-size +lane readiness; they do not copy the epoch map. The highest observed +incremental revision, complete applied revision, and last exact revision are +tracked separately. Data installs a contiguous incremental batch only if its +expected generation/revision still matches the applied authority, observation, +and persisted hint in the same serialized callback; a raced heartbeat rejects +the whole batch. A persisted `{generation, revision}` hint fences every lane +when any heartbeat observes newer authority. It can refine only an already +known peer: after complete retirement, delayed heartbeats and lane hellos cannot +recreate authority, a transport route, or a lease. Only an exact dist-Erlang +hello reintroduces the peer. Discovery hints never mutate membership on their +own. +The exact authority and its shared-cluster forward/reverse index rows are +replaced in one serialized Data operation. This closes the race where a local +cluster connect and a remote exact install could each miss the other's state +and permanently omit a valid replica route. Local activation likewise installs +its epoch, self route, and already-exact remote routes in one Data turn. Local +deactivation removes admission and queues idempotent old-epoch cleanup on every +shard before returning; API timeout or caller death cannot strand rows, routes, +or a close barrier. Authority installation then fans a local fence to every +lane, which sweeps only that lane's retained receive +streams. Shared authority may become visible before that fanout reaches a lane, +but the lane's constant-size view is not marked installed until its purge +finishes; data validation requires that marker. A heartbeat or lane hello can +confirm an installed view but cannot promote a pending one. Because PG rows +intentionally do not carry protocol epochs, a superseded origin/cluster slice +is cleared and its current cursor reset so the next head reconstructs it from +retained deltas or an exact snapshot. + +If a receiver shard restarts while Data retains remote rows, it reconstructs +lease candidates from constant-size authority/view metadata rather than +scanning registry or PG entries. A live Group refreshes through discovery; a +Group that never returns is purged after the normal bounded lease timeout. +Each lane removes that persisted view only after purging its own rows and +cursors, so shard 0 cannot erase a suspended sibling's restart breadcrumb. +Shard 0 also reconstructs any pending applied/observed-versus-exact authority +repair. Persisted hints seed the same bounded retirement lease after a lane +restart, and final route cleanup rechecks that no newer exact authority or hint +was installed while an older cleanup caller was delayed. +Registry conflicts reconciled during that temporary authority gap retain a +bounded set of affected keys in the receiving lane. Installing the exact view +reprojects those keys before normal processing resumes, so retained current +claims cannot be hidden forever behind a stale visible winner and authority +repair never requires scanning every claim in the shard. + +Replica state itself does not travel on the control plane. Once the hello is +fenced, stream-head exchange on the replica transport catches the peer up. ### Replication -After the initial sync, steady-state changes propagate through separate sender -and receiver batching lanes: +Every local mutation is first appended to a stream identified by +`{group, origin_node, origin_generation, shard, cluster, cluster_epoch}` and a +strictly increasing sequence number. It is then applied to the materialized +ETS view and batched into one delta message per target. Process-death registry +and PG removals can share one record and retain their one-event-batch behavior. + +Receivers advance a cursor only across a contiguous sequence prefix. A gap +requests the missing suffix. Repeated head advertisements recover a dropped +tail even when no later write occurs. If the requested sequence is older than +the bounded oplog floor, the origin sends an exact snapshot of only its own +registry claims and PG memberships; absence from that snapshot is a delete. + +There are no leaders, quorum acknowledgements, per-entry replicated tombstones, +or known-membership retention barriers. Oplog memory is bounded locally and +independently of slow peers. Deletes are normal ordered records while retained, +and exact snapshots close gaps after pruning. Exact snapshots are split into +transport-neutral byte-bounded provisional chunks followed by a small terminal +manifest. Loss, duplication, reordering, or receipt of every chunk without that +commit leaves the old visible slice and cursor untouched. Incomplete staging +expires after a peer-lease interval without progress and is destroyed +automatically with its owning shard. Rejected chunks or manifests, nodedown, +generation replacement, and retired epochs destroy matching staging +immediately. Named-cluster close uses only a temporary +local shard-completion barrier; the final shard removes it and all routing rows, +including after a caller timeout or shard restart. Reconnect waits for that +barrier so a prior close cannot erase newly accepted writes. + +The sender flush timer is mainly a fallback for idle periods. The unified +outbound buffer also flushes immediately when it hits the configured size, +when a new enqueue finds the buffer already past its flush interval, and before +control or routing work such as cluster connect/disconnect or peer-protocol +handling. + +Transport ordering is not required for correctness: each shard serializes +writes, each stream numbers them, and receivers reject gaps and duplicates. +Per-shard ordered delivery is still a useful fast path. Cross-stream order is +not a correctness dependency; cluster epochs reject data racing a disconnect +or reconnect, and generation fencing rejects data from a restarted origin. +An alternative sideband adapter passes incoming messages to +`Group.Transport.incoming/4` locally. Configure a custom adapter while +authority and membership remain on dist Erlang: -- local writes enqueue outbound registry or PG replication in shard-local sender - buffers -- sender flushes group those ops by target node and send one - `replicate_registry_batch` or `replicate_pg_batch` message per remote node -- remote shards buffer those replicated registry / PG ops receiver-side, apply - them in FIFO order with bulk ETS operations, then take a bounded fairness turn - before yielding back to the mailbox +```elixir +replica_transport: + {MyApp.GroupTransport, + [outbox_batch_size: 64, outbox_batch_bytes: 1_048_576, + outbox_flush_interval: 1, outbox_deadline: 100, + outbox_max_messages: 1_024]} +``` -The sender flush timer is mainly a fallback for idle periods. Outbound buffers -also flush immediately when they hit the configured size, when a new enqueue -finds the buffer already past its flush interval, and before control or -routing work such as cluster connect/disconnect or peer-protocol handling. +The default `Group.Transport.DistErl` adapter sends directly to the remote shard and +does not pay for a local outbox. Sideband adapters can delegate `outgoing/5` to +`Group.Transport.Outbox.push/5` and supervise one outbox per shard +with `Group.Transport.Outbox.child_spec/1`. An outbox groups messages by +target and invokes the adapter's `send_batch/4` callback. Calls that expire or +return `:busy`/`:disconnected` are dropped without a local retry; the next +anti-entropy exchange repairs them. + +Exact-snapshot scans run in at most one off-shard worker per shard. The worker +validates the stream identity and fully-applied head both before and after one +pass over the registry and PG tables. It streams each completed chunk +immediately and retains only the current byte-targeted chunk on its heap. A +concurrent write suppresses the terminal manifest, so every provisional chunk +remains invisible and anti-entropy retries the newer head. Backpressure resumes +from the first unsent chunk; if only the manifest was backpressured, the sender +retains that small tuple and retries it without rescanning. + +The receiver necessarily retains one complete candidate in private ETS before +beginning exact replacement: absence from the committed candidate is a delete. +It also stores minimal row-presence markers to reject mixed/duplicate +assemblies and bounded monitor-event batches until the cursor becomes visible. +Transfer tables are cleared and pooled by the shard after completion or +rejection instead of being created for every retry; shard death destroys active +and pooled tables automatically. Sender memory is therefore O(chunk size), +while receiver staging is O(the exact origin slice), with neither becoming a +new authority source. + +The optional `peer_up/5` and `peer_down/4` callbacks report one shard lane at a +time. A sideband adapter that shares a single node connection must retain it +while any reported lane remains live and release it after the last lane goes +down. `incoming/4` and `incoming_batch/4` return `:disconnected` and drop when +their local shard is restarting; an ingress reader must treat that as an +expected lossy delivery outcome. Group invokes `peer_up/5` and records an +outbound lane only after exact/current authority admits it; a delayed lane hello +after peer retirement remains a side-effect-free request for exact authority. +When a hello legitimately outruns first-time exact authority, authority fanout +immediately re-probes that shard without retaining a speculative route or +waiting for the periodic anti-entropy interval. + +A message-oriented backend fits this callback shape by obtaining a connection +once from `init_outbox/3`, then sending each `send_batch/4` result to a +registered incoming name on the target node. Queue pressure maps to `:busy` and +a missing session maps to `:disconnected`. The adapter passes its peer node as +`source_node`; Group verifies that stream origins and member pids match it. +Exact snapshots are already bounded by Group. A transport with a smaller +maximum frame may additionally segment an encoded batch, but it must completely +reassemble that batch before calling +`Group.Transport.incoming_batch/4`. ### Named Cluster TTL Leases @@ -380,25 +580,34 @@ no longer care about a cluster. ### Process Death Cleanup -Shards monitor all registered/joined processes. On `DOWN`, the shard: +Shards monitor only locally owned registered/joined processes. A node never +monitors or exits another node's member processes. On a local owner `DOWN`, the +shard: 1. Removes entries from both the primary and reverse-index ETS tables. -2. Groups removed entries by peer and sends one non-suspending process-down batch per peer. +2. Appends authoritative unregister/leave mutations before deleting the rows, + then sends one non-suspending sequenced delta batch per peer. 3. Fires `:unregistered` / `:left` events to local monitors. -### Node Disconnect +### Peer Removal and Recovery On `nodedown`, each shard purges all entries owned by the disconnected node -from its ETS tables and fires events for each removed entry. +from its ETS tables, claims, cursors, and authority indexes and fires events for +each removed entry. If dist Erlang stays connected but a Group instance or its +control lane disappears, heartbeat lease expiry performs the same complete +purge. Discovery probes continue after expiry; a returning instance announces +a new or current generation and anti-entropy reconstructs its live state. ## Testing ```bash mix test +mix test.soak # nightly mutation/live-checker and six-profile Jepsen qualification ``` -See [`test/README.md`](test/README.md) for details on the distributed test -infrastructure. +See [`test/README.md`](test/README.md) for the every-PR gate, shrinkable +StreamData lifecycle-model tests, bounded TLA+ models, and the nightly +three-node Jepsen transport/lifecycle campaign. ## Benchmarks diff --git a/lib/group.ex b/lib/group.ex index 27ce1eb..97b5cda 100644 --- a/lib/group.ex +++ b/lib/group.ex @@ -10,15 +10,18 @@ defmodule Group do ## Consistency Model - All operations are **eventually consistent**. The built-in replication layer uses - Erlang distribution to propagate state across nodes, which means: + All operations are **eventually consistent**. Erlang distribution remains + the membership/control plane; replica state uses a configurable nonblocking + transport with sequenced anti-entropy streams. This means: - Writes (register, join, etc.) return immediately after local update - - Other nodes receive updates asynchronously via Erlang distribution + - Other nodes receive updates asynchronously over the replica transport - During network partitions, nodes may have divergent views - - When partitions heal, conflicts are resolved. The built-in resolver kills - the losing process with `{:group_registry_conflict, key, winner_meta}`; - custom resolvers control any process exits themselves + - When connectivity heals, stream gaps are repaired from a bounded oplog or + an exact per-origin snapshot; dropped replica sends are therefore safe + - Registry conflicts resolve deterministically. Each losing origin records an + authoritative delete and terminates only its own local process with + `{:group_registry_conflict, key, winner_meta}` ## Clusters @@ -169,6 +172,15 @@ defmodule Group do - **Memberships** are stored in replicated, sharded ETS indexes and are automatically cleaned up when member processes die. + + - **Process ownership is local**: a shard monitors and exits only processes + owned by its own node. Remote lifecycle changes arrive as sequenced replica + records or are removed by `nodedown`/peer-lease expiry. + + - **Replica recovery is bounded and leaderless**: per-origin sequence gaps use + retained deltas and then exact origin-slice snapshots after oplog pruning. + Persisted generation/revision hints fence every lane across authority races; + only an exact dist-Erlang hello can reintroduce a fully retired peer. """ alias Group.Replica @@ -193,12 +205,14 @@ defmodule Group do replication messages. `false` disables routine info/verbose logs; registry conflicts and busy distribution links still emit error/warning logs. - `:resolve_registry_conflict` — `{module, function, extra_args}` callback invoked when - two nodes hold the same registry key (partition heal or concurrent registration). - Called as `apply(module, function, [name, key, {pid1, meta1, time1}, {pid2, meta2, time2} | extra_args])`. - Must return the winner pid and is responsible for any process exits it requires. When - no callback is configured, the built-in resolver kills the loser with - `{:group_registry_conflict, key, winner_meta}`. **Important:** This callback runs - synchronously inside the shard GenServer — it must + multiple origins hold the same registry key after partition healing or concurrent + registration. Called once per claim as + `apply(module, function, [name, key, {pid, meta, time} | extra_args])` and must + return a deterministic Erlang term used as that claim's rank. Group chooses the + maximum `{rank, pid}` so every node obtains the same winner regardless of claim + arrival or grouping. It records an authoritative delete and terminates a losing + process only on its owner node. + **Important:** This callback runs synchronously inside the shard GenServer — it must return quickly and never block. Any information needed for the decision should be carried in the registration metadata, not fetched at resolution time. - `:extract_meta` — `{module, function, args}` or a one-argument function to @@ -218,13 +232,32 @@ defmodule Group do buffer replicated outbound ops before flushing during idle periods. Sender buffers also flush on size, overdue enqueue, and control/routing barriers (default: `5`) - - `:busy_dist_retry_attempts` — max reconnect attempts after a shard hits - `send_nosuspend == false` to a remote node and forces a disconnect + - `:busy_dist_retry_attempts` — max reconnect attempts after a remote + `Group.dispatch/4` send reports a busy dist link and forces a disconnect (default: `300`) - - `:busy_dist_retry_interval` — interval in milliseconds between reconnect - attempts after a busy-dist disconnect (default: `1_000`) - - `:replicated_pg_receiver_local_request_quota` — max queued local PG shard requests - drained after each replicated PG flush before yielding (default: `8`) + - `:busy_dist_retry_interval` — interval in milliseconds between dispatch + busy-link reconnect attempts (default: `1_000`) + - `:replicated_pg_receiver_local_request_quota` — legacy-named quota for queued + local shard requests drained in each fairness turn, including while replica + data or cluster controls are busy (default: `8`) + - `:replica_transport` — replica data transport module or `{module, opts}` tuple. + Defaults to `Group.Transport.DistErl`. The transport must be + nonblocking and may return `:busy`; anti-entropy repairs dropped messages. + Sideband transports can use `Group.Transport.Outbox` for lossy, + batched, per-shard isolation without adding a hop to the default transport. + - `:replicated_oplog_max_entries` — maximum retained replica records per shard + before old prefixes are pruned and lagging peers require a snapshot + (default: `65_536`) + - `:replicated_snapshot_chunk_target_bytes` — target maximum encoded size of + each transport-neutral exact-snapshot chunk (default: `1_048_576`). A + single registry or membership row larger than the target remains one chunk; + a separate small terminal manifest commits the complete candidate. + - `:replicated_anti_entropy_interval` — milliseconds between repeated stream + head advertisements (default: `1_000`) + - `:replicated_peer_lease_timeout` — milliseconds without a dist-Erlang + replica heartbeat before remote Group state is purged (default: `15_000`). + Must be greater than `:replicated_anti_entropy_interval`. Discovery probes + an expired peer so a restarted Group can recover automatically. """ def child_spec(opts) do name = Keyword.fetch!(opts, :name) @@ -233,6 +266,26 @@ defmodule Group do def start_link(opts), do: Group.Supervisor.start_link(opts) + @doc """ + Monitors the process that owns the local Group membership generation. + + All local registry and process-group entries are stored in ETS tables owned + by this process. If it exits, those entries no longer exist even when their + owner processes remain alive. Long-lived owners can use this monitor to + terminate and re-register against the next Group generation. + + A generation that exits between lookup and monitor creation still produces + the normal immediate `:DOWN` message for the returned monitor reference. + + Returns `{:ok, pid, monitor_ref}` or `{:error, :not_running}`. + """ + def monitor_generation(name) when is_atom(name) do + case GenServer.whereis(Data.data_name(name)) do + pid when is_pid(pid) -> {:ok, pid, Process.monitor(pid)} + nil -> {:error, :not_running} + end + end + # =========================================================================== # Cluster Management (Node <-> Cluster) # =========================================================================== @@ -419,10 +472,14 @@ defmodule Group do when is_atom(name) and is_binary(key) and is_map(meta) and is_list(opts) do validate_key!(key) cluster = Keyword.get(opts, :cluster) - validate_cluster_connected!(name, cluster) + epoch = validate_cluster_connected!(name, cluster) shard = Replica.shard_for(name, cluster, key) - Replica.local_request(shard, {:register, cluster, key, self(), meta}, call_timeout(opts)) + Replica.local_request( + shard, + {:register, cluster, epoch, key, self(), meta}, + call_timeout(opts) + ) end @doc """ @@ -453,10 +510,10 @@ defmodule Group do when is_atom(name) and is_binary(key) and is_list(opts) do validate_key!(key) cluster = Keyword.get(opts, :cluster) - validate_cluster_connected!(name, cluster) + epoch = validate_cluster_connected!(name, cluster) shard = Replica.shard_for(name, cluster, key) - Replica.local_request(shard, {:unregister, cluster, key}, call_timeout(opts)) + Replica.local_request(shard, {:unregister, cluster, epoch, key}, call_timeout(opts)) end @doc """ @@ -610,10 +667,14 @@ defmodule Group do is_list(opts) do validate_key!(group) cluster = Keyword.get(opts, :cluster) - validate_cluster_connected!(name, cluster) + epoch = validate_cluster_connected!(name, cluster) shard = Replica.shard_for(name, cluster, group) - Replica.local_request(shard, {:join, cluster, group, self(), meta}, call_timeout(opts)) + Replica.local_request( + shard, + {:join, cluster, epoch, group, self(), meta}, + call_timeout(opts) + ) end @doc """ @@ -641,10 +702,10 @@ defmodule Group do when is_atom(name) and is_binary(group) and is_list(opts) do validate_key!(group) cluster = Keyword.get(opts, :cluster) - validate_cluster_connected!(name, cluster) + epoch = validate_cluster_connected!(name, cluster) shard = Replica.shard_for(name, cluster, group) - Replica.local_request(shard, {:leave, cluster, group, self()}, call_timeout(opts)) + Replica.local_request(shard, {:leave, cluster, epoch, group, self()}, call_timeout(opts)) end # =========================================================================== @@ -1074,13 +1135,14 @@ defmodule Group do @doc false def connect_clusters(name, clusters, timeout) when is_atom(name) and is_list(clusters) and is_integer(timeout) do - Data.add_cluster_node(name, clusters, node()) + timeout = Data.await_closed_local_clusters(name, clusters, timeout) + epochs = Data.activate_local_clusters_durable(name, clusters) notify_shard = :rand.uniform(get_config(name).num_shards) - 1 Replica.local_request( Replica.shard_name(name, notify_shard), - {:cluster_connect, clusters}, + {:cluster_connect, clusters, epochs}, timeout ) end @@ -1088,17 +1150,9 @@ defmodule Group do @doc false def disconnect_clusters(name, clusters, timeout) when is_atom(name) and is_list(clusters) and is_integer(timeout) do - Data.remove_cluster_node(name, clusters, node()) - - num_shards = get_config(name).num_shards - - shard_names = for i <- 0..(num_shards - 1), do: Replica.shard_name(name, i) - - Replica.local_request_all( - shard_names, - {:cluster_disconnect, clusters}, - timeout - ) + _epochs = Data.deactivate_local_clusters_durable(name, clusters) + _remaining_timeout = Data.await_closed_local_clusters(name, clusters, timeout) + :ok end # =========================================================================== @@ -1170,12 +1224,16 @@ defmodule Group do |> List.flatten() end - defp validate_cluster_connected!(_name, nil), do: :ok + defp validate_cluster_connected!(name, nil), do: Data.generation(name) defp validate_cluster_connected!(name, cluster) do - unless node() in Data.cluster_nodes(name, cluster) do - raise ArgumentError, - "not connected to cluster #{inspect(cluster)}. Call Group.connect(#{inspect(name)}, #{inspect(cluster)}) first" + case Data.local_cluster_epoch(name, cluster) do + nil -> + raise ArgumentError, + "not connected to cluster #{inspect(cluster)}. Call Group.connect(#{inspect(name)}, #{inspect(cluster)}) first" + + epoch -> + epoch end end diff --git a/lib/group/replica.ex b/lib/group/replica.ex index 933c795..5ce1de7 100644 --- a/lib/group/replica.ex +++ b/lib/group/replica.ex @@ -6,428 +6,207 @@ defmodule Group.Replica do @process_down_batch_size 32 @replicated_pg_receiver_flush_timer :flush_replicated_pg_receiver_buffer @replicated_registry_receiver_flush_timer :flush_replicated_registry_receiver_buffer - @replicated_pg_broadcast_flush_timer :flush_replicated_pg_broadcast_buffer - @replicated_registry_broadcast_flush_timer :flush_replicated_registry_broadcast_buffer + @replica_broadcast_flush_timer :flush_replica_broadcast_buffer + @anti_entropy_timer :group_replica_anti_entropy @local_request_tag :group_local_request @local_reply_tag :group_local_reply + @protocol_version Group.Replica.WireProtocol.version() + @priority_control_quota 64 + @incoming_batch_quota 64 + @snapshot_event_batch_size 512 _archdoc = ~S""" - Sharded GenServer: peer discovery, replication, monitoring, conflict resolution. - - One per shard. Registered as :"#{name}_replica_#{shard_index}". - - ## Message Protocol - - | Message | Direction | Purpose | - |------------------------------------------------------------|----------------|----------------------------------| - | `{:peer_connect, pid, shard, num_shards, clusters}` | A→B (per-shard)| Establish peer relationship | - | `{:peer_connect_ack, pid, shard, num_shards, clusters}` | B→A (per-shard)| Acknowledge peer | - | `{:cluster_state, cluster, reg_data, pg_data}` | both | Per-cluster data snapshot | - | `{:replicate_registry_batch, ops}` | broadcast | Propagate batched registry ops | - | `{:replicate_pg_batch, ops}` | broadcast | Propagate batched PG ops | - | `{:cluster_connect, clusters, pid}` | S→remote S | Node joining named clusters | - | `{:cluster_connect_ack, clusters, pid, cluster_data}` | S→remote S | Ack + bundled shard data | - | `{:cluster_disconnect, clusters, pid}` | shard 0→remote | Node leaving named clusters | - | `{:send_cluster_data, clusters, target_node}` | local fan-out | Notify siblings: send shard data | - | `{:group_dispatch, pids, message}` | caller→remote | Per-node fan-out for dispatch | - - ## Protocol Flows - - ### 1. Peer Discovery (nodeup or init) - - Triggered by `nodeup` or `init`. Each shard independently discovers its - counterpart on the remote node. Both sides exchange cluster lists, then send - per-cluster `cluster_state` snapshots for shared clusters (always includes - nil). Merge applies data; conflicts with local entries go through - `resolve_conflict`. - - Node A shard i Node B shard i - ──────────── ──────────── - │ │ - │ {:peer_connect, pid, i, N, clusters} │ - │──────────────────────────────────────>│ - │ │── add A to nil cluster ETS - │ │── compute shared clusters - │ │── add A to shared named clusters - │ │── monitor A's shard pid - │ │ - │ {:peer_connect_ack, pid, i, N, clusters} - │<──────────────────────────────────────│ - │── add B to nil cluster ETS │ - │── compute shared clusters │ {:cluster_state, C, reg, pg} - │── add B to shared named clusters │──────────────────────────────>│ - │── monitor B's shard pid │ (one per shared cluster) │ - │ │ │ - │ {:cluster_state, C, reg, pg} │ │ - │──────────────────────────────>│ │ │ - │ (one per shared cluster) │ │ │ - │ │ │ │ - ▼ ▼ ▼ ▼ - merge_remote_cluster_data merge_remote_cluster_data - ├─ no conflict: insert ├─ no conflict: insert - ├─ local vs remote: resolve_conflict (default kills loser; re-broadcast winner) - └─ both remote: timestamp wins └─ both remote: timestamp wins - - ### 2. Steady-State Replication - - After peer discovery, local writes enqueue outbound registry or PG replication - in separate sender buffers. Flushes group those ops by target node and send - one `{:replicate_registry_batch, ops}` or `{:replicate_pg_batch, ops}` per - remote node. The nil cluster uses `remote_shards` (per-shard map); named - clusters use `cluster_nodes` ETS. Reads (`lookup`, `members`) go directly to - ETS — no GenServer involved. - - Node A shard i Node B shard i - ──────────── ──────────── - │ │ - Group.register(name, key, meta) │ - │── ETS insert (by_key + by_pid) │ - │── monitor pid │ - │── enqueue sender-side registry op │ - │ │ - │ {:replicate_registry_batch, [ ... ]} - │──────────────────────────────────────>│ - │ │── enqueue receiver-side - │ │ registry ops - │ │── flush in FIFO order - │ │ ├─ nil: insert - │ │ ├─ same pid: update - │ │ ├─ local conflict: - │ │ │ resolve_conflict() - │ │ └─ both remote: - │ │ timestamp wins - │ │ - Group.join(name, group, meta) │ - │── ETS insert (by_key + by_pid) │ - │── enqueue sender-side PG op │ - │ │ - │ {:replicate_pg_batch, [ ... ]} - │──────────────────────────────────────>│ - │ │── enqueue receiver-side PG - │ │ ops - │ │── flush in FIFO order - │ │ (no overwrite conflict - │ │ for PG) - │ │ - Group.dispatch(name, group, msg) │ - │── send directly to local pids │ - │── group remote pids by node │ - │── hash self() to pick shard j │ - │ │ - │ {:group_dispatch, pids, msg} Node B shard j - │────────────────────────────────────────>│ - │ │── send msg to each local pid - │ │ - - Dispatch groups remote PG members by node and sends one - `:group_dispatch` message per remote node, reducing cross-node - messages from O(members) to O(nodes). The target shard is chosen - by hashing the caller's pid (`phash2(self(), num_shards)`), so - back-to-back dispatches from the same caller always route through - the same shard, preserving per-sender message ordering. - - ### 3. Named Cluster Connect (random shard S + fan-out) - - `Group.connect/2` adds local node to ETS, picks random shard S, and sends - one GenServer.call. Shard S notifies remote shard S, which acks with bundled - data and fans out to siblings. Randomizing S load-balances across shards - when many concurrent connects happen. - - Node A Node B - ────── ────── - Group.connect(name, "game") - │── ETS: add self to "game" - │── pick random shard S - │ - Shard S Shard S - ─────── ─────── - │ │ - │ {:cluster_connect, ["game"], pid} │ - │──────────────────────────────────────>│ - │ │── ETS: add A to "game" - │ │── bundle shard S data - │ │ - │ {:cluster_connect_ack, ["game"], pid, [{cluster, reg, pg}]} - │<──────────────────────────────────────│ - │ │ - │ Shard S sends to siblings: - │ {:send_cluster_data, ["game"], A} - │ │ - │ Shards 0..N (except S): - │ │── {:cluster_state, "game", reg, pg} - │ │──────────────────────────────>│ - │ │ (to matching A shard) │ - │ │ │ - │── merge bundled ack data │ - │── ETS: add B to "game" │ - │── send shard S cluster_state ────────>│ - │── fan out to siblings: │ - │ {:send_cluster_data, ["game"], B} │ - │ │ - Shards 0..N (except S): │ - │── {:cluster_state, "game", reg, pg} │ - │──────────────────────────────────────>│ - │ (to matching B shard) │ - - ### 4. Named Cluster Disconnect (all shards local + shard 0 broadcast) - - `Group.disconnect/2` removes local node from ETS, then calls ALL local shards - to purge their entries. Only shard 0 broadcasts to remote shard 0, which fans - out to siblings for per-shard purge. - - Node A Node B - ────── ────── - Group.disconnect(name, "game") - │── ETS: remove self from "game" - │ - Shards 0..N (all called): - │── purge own entries for "game"+A - │── dispatch :unregistered/:left events - │ - Shard 0 only: - │ {:cluster_disconnect, ["game"], pid} - │──────────────────────────────────────>│ Shard 0 - │ │── ETS: remove A from "game" - │ │── fan out to siblings: - │ │ {:cluster_disconnect, ["game"], pid} - │ │ - │ Shards 0..N: - │ │── purge entries for "game"+A - │ │── dispatch events - - ### 5. Partition Heal (peer discovery re-runs + conflict resolution) - - When a partition heals, `nodeup` triggers peer discovery on both sides. - Both exchange `cluster_state` snapshots. Registry key conflicts where the - existing entry is local go through `resolve_conflict` — the same path used - for live contention. The default resolver kills the loser process. - - The tiebreaker must be deterministic regardless of which node is resolving. - The default uses timestamp comparison, with pid ordering as a tiebreaker - when timestamps are equal (`pid2 > pid1`). Erlang pids have a total order - (by node name then id), so this produces the same winner on all nodes. - Using a perspective-dependent tiebreaker (e.g. "remote wins on ties") would - cause mutual kill — both nodes pick the other's pid, both processes die. - - Node A Node B - ────── ────── - (partition: A and B both register key K) - A has: {K, pid_a, time_a, local} B has: {K, pid_b, time_b, local} - │ │ - ─────── partition heals (nodeup) ──────────── - │ │ - │ peer_connect / peer_connect_ack │ - │<─────────────────────────────────────>│ - │ │ - │ {:cluster_state, nil, [{K, pid_b, ...}], []} - │<──────────────────────────────────────│ - │ │ - │ {:cluster_state, nil, [{K, pid_a, ...}], []} - │──────────────────────────────────────>│ - │ │ - merge: K exists locally merge: K exists locally - resolve_conflict( resolve_conflict( - local={pid_a, time_a}, local={pid_b, time_b}, - remote={pid_b, time_b}) remote={pid_a, time_a}) - │ │ - (assuming time_b > time_a): (assuming time_b > time_a): - pid_b wins (remote) pid_b wins (local) - ├─ kill pid_a ├─ kill pid_a (cross-node, idempotent) - ├─ delete pid_a entry ├─ re-insert pid_b with new timestamp - ├─ insert pid_b └─ re-broadcast pid_b - ├─ demonitor pid_a in a registry batch - ├─ dispatch :unregistered(pid_a) │──────────────────────────>│ - │ │ (arrives as same-pid │ - │ │ update — harmless) │ - - ### 6. Nodedown / Process Death Cleanup - - `nodedown` purges all remote node data. Local process `DOWN` purges the pid's - entries and broadcasts unregister/leave to cluster members. - - Node A Node B dies - ────── ────────── - │ X - {:nodedown, B} │ - │ │ - All shards (each independently): │ - │── purge_cluster_node(B) │ - │ (remove B from all cluster_nodes │ - │ and node_clusters — idempotent, │ - │ guards against late peer_connect) │ - │ │ - │── purge_node(shard, B) │ - │ (scan by_key for node==B, │ - │ delete from both by_key + by_pid) │ - │── dispatch :unregistered/:left events │ - │── remove B from remote_shards │ - - ────────────────────────────────────────────── - - Local process dies Node B - ────────────────── ────── - {:DOWN, mref, :process, pid, reason} │ - │ │ - Owning shard: │ - │── delete_all_for_pid(shard, pid) │ - │ (scan by_pid, delete from by_key, │ - │ match_delete from by_pid) │ - │── enqueue unregister/leave ops │ - │ into replicated sender batches ──────>│── delete if pid matches - │── demonitor pid │ - │── dispatch events │ - - ## Cluster Membership Tracking - - The nil cluster is tracked in ETS (cluster_nodes table), maintained by the - peer_connect protocol. Nodes are added on peer discovery and removed on - nodedown/shard death. This allows Group.nodes/1 to return actual Group peers - rather than all Erlang nodes. - - ## Sharding - - Each key is routed to a shard via `:erlang.phash2({cluster, key}, num_shards)`. - Including `cluster` in the hash input means the same key string in different - clusters may land on different shards — this is intentional so named-cluster - operations don't create false contention with nil-cluster operations. - - `phash2` produces near-uniform distribution across shards for diverse keyspaces. - With 10K distinct keys across 2–8 shards, observed deviation from perfect - uniformity is <2%. In practice, real workloads with varied key prefixes will - see balanced shard load. - - **Hot keys:** A single extremely popular key (e.g. a chat room - with thousands of joins/leaves) always hashes to one shard, so all *writes* - for that key serialize through that shard's GenServer. However, *reads* — - `Group.lookup/3` and `Group.members/3` — go directly to ETS and bypass the - GenServer entirely. Since reads typically dominate, a hot key's impact on - overall throughput is limited to write-heavy scenarios. Adding more shards - does not help a single hot key (it still lands on one shard), but it does - reduce contention between unrelated keys. - - Shard counts must match across all nodes in a cluster. The peer_connect - handshake validates `num_shards` and raises on mismatch, since a disagreement - would route the same key to different shards on different nodes, breaking - replication consistency. - - ## Conflict Resolution is Synchronous - - The `:resolve_registry_conflict` callback runs synchronously inside the shard - GenServer's `handle_info` (during replicated registry apply or - `merge_remote_cluster_data`). - This is intentional: the resolver's return value determines ETS mutations (delete - loser entry, insert winner, demonitor evicted local pid, re-broadcast winner) that - must happen atomically within a single `handle_info` turn. Making the resolver async - would open a window where another replicated registry update, `DOWN`, or `cluster_state` - for the same key could race with the pending resolution, corrupting the dual-index - ETS tables. - - Consequence: a blocking resolver stalls the **entire shard** — no registrations, - joins, replication, or cleanup can proceed on that shard until the callback returns. - Callers must ensure their resolver returns quickly. Any information needed for the - decision (e.g. priority, version, creation time) should be carried in the - registration metadata, not fetched at resolution time. - - ## Monitor Event Delivery - - Lifecycle events (`:registered`, `:unregistered`, `:joined`, `:left`) are delivered - to `Group.monitor/3` subscribers in a batched diff of `{:group, events, info}` tuples. - Each GenServer handler invocation is a natural batch boundary: - - - **Single local operations** (register, join, leave, unregister): build one - event, deliver one tuple with one event per matching subscriber. - - **Buffered replicated operations** (batched replicated registry and PG - ops): receiver shards may accumulate several ops before flushing, then - deliver one tuple per subscriber containing the ordered events from that - flush. - - **Bulk operations** (nodedown, process DOWN, cluster_disconnect, cluster_state - merge): accumulate events into a local variable, then deliver one tuple per - subscriber containing all matching events from that handler turn. - - Events are built by `build_event/6`, accumulated in reverse via prepend, and - flushed by `notify_monitors/2` which reverses once, resolves only the monitor - keys that can match each event (`:all`, `{:exact, key}`, and the key's - slash-terminated prefixes), caches those lookups per batch, and sends one - `{:group, events, %{name: name}}` per subscriber. Both functions are private to - this module. - - `resolve_conflict/5` returns `{state, event_or_nil}` so callers can accumulate - the event. `merge_remote_cluster_data/5` threads `{state, events}` through its - reduce, generating `:registered`/`:joined` events for new entries and conflict - events for existing ones. This means `cluster_state` merges (peer discovery, - partition heal, `Group.connect`) produce batched diffs with all new entries. - `build_purged_events/5` takes an events accumulator and prepends purged-entry - events to it. - - ## Replicated Sender Buffering - - Local writes stage outbound replicated registry and PG operations in separate - sender buffers. On flush, those ops are grouped by target node so one shard - send can carry many logical replication updates. - - Sender-side buffering is not timer-only. A sender buffer flushes when: - - - the lane reaches `replicated_sender_buffer_size` - - a new enqueue notices the lane is already older than - `replicated_sender_flush_interval` - - a control or topology path crosses the sender barrier (`cluster_connect`, - `cluster_disconnect`, peer protocol, `cluster_state`, `DOWN`, `nodedown`, - process-down cleanup, explicit mailbox barriers) - - terminate runs - - The timer is therefore a fallback for idle periods, not the only flush - trigger. A very long single GenServer callback can still delay all of these - flush paths until that callback returns; batching and fairness only apply - between mailbox turns. - - Remote shard sends use `send_nosuspend(..., [:noconnect])`. If a send returns - `false`, Group treats that as a degraded link: it drops the unsent message, - force-disconnects the Erlang node, and enters a bounded reconnect loop for - that node. Recovery only starts from this explicit busy-send path; ordinary - `nodedown` events do not start reconnect retries on their own. - - ## Replicated Receiver Fairness - - Receiver-side batching solves the apply-cost problem for both replicated PG - and replicated registry traffic, but a hot stream in either lane can still - monopolize the shard if every completed replication turn is immediately - followed by another one. - - To keep local latency-sensitive writes from sitting behind an unbounded remote - backlog, the shard gives a bounded local request turn after each completed - replicated PG or replicated registry apply turn: - - - one bounded replicated lane turn (PG or registry) - - then drain any already-waiting cluster/protocol messages - - then drain up to `replicated_pg_receiver_local_request_quota` local PG - `join` / `leave` requests, or one local non-PG request - - then yield back to the GenServer loop - - Contiguous local PG `join` / `leave` requests from that bounded local turn are - staged against an in-memory view and applied with bulk ETS operations, while - replicated registry flushes are staged against an in-memory view per - `{cluster, key}` and applied with bulk ETS operations. Other local request - types still execute sequentially in FIFO order. - - Local callers use an explicit request/reply lane (`send` + monitor + tagged - reply) rather than `GenServer.call/3`, so the replica can selectively receive - one local request turn without reaching into `'$gen_call'` internals. - - The fairness model ensures ordering is preserved where correctness matters: - - - all public local shard calls get protection from replicated PG and registry - backlog, but earlier cluster/protocol messages still run first, avoiding - stale ordering around disconnect, peer discovery, and cluster sync - - FIFO is preserved within the local lane because the selective receive matches - a single broad `@local_request_tag` shape and therefore takes the oldest - queued local request in the mailbox - - local PG batching does not reorder within that local lane; it only batches - contiguous `join` / `leave` messages already collected in FIFO order + Sharded control process for local writes, replica transport, anti-entropy, + process monitoring, and registry conflict projection. + + There is one process per shard, registered as + :"#{name}_replica_#{shard_index}". Reads bypass it and use the materialized + ETS indexes owned by Group.Replica.Data. + + ## Authority and identity + + Each locally owned mutation belongs to one stream: + + {group, origin_node, origin_generation, shard, cluster, cluster_epoch} + + The shard assigns a strictly increasing sequence number and appends the record + to its write-ahead oplog before changing materialized ETS. Data owns the + journal, so a shard crash replays any appended-but-unapplied record before + rebuilding local process monitors. + + A registry's authoritative claims are stored per origin separately from its + single visible winner. Conflict selection computes one deterministic rank per + claim and chooses the maximum `{rank, pid}`, so selection is associative, + commutative, and independent of delivery order. + When a local claim loses, only its owner node appends the authoritative + unregister and terminates the local process. Immediately before that + irreversible step, Data serially revalidates the remote winner against the + node-wide authority and this lane's installed view; an authority change + restarts selection without killing either owner. Retaining hidden remote + claims until their origin deletes them prevents a later winner change from + orphaning or permanently forgetting a live claim. Local unregister and + process-DOWN paths always re-project every affected key after deleting their + claims. On a shard restart, the complete claim/key union is projected again, + closing the crash window between journal application and the materialized + winner write. + + PG memberships need no winner projection: the origin stream owns exactly the + rows whose member processes live on that origin node. + + ## Wire protocol + + Dist Erlang remains the control plane: + + - peer_connect / peer_connect_ack discover matching shards and clusters. + - shard 0 exchanges replica_hello authority containing the origin generation + and complete active named-cluster epoch set exactly once per node. + - matching nonzero shards exchange constant-size replica_lane_hello messages + containing their transport descriptor and the authority revision they use. + - replica_cluster_open / replica_cluster_close fence named-cluster lifetimes. + - constant-size periodic heartbeats provide a bounded peer lease without + creating remote process monitors. A generation/epoch-revision mismatch + requests a fresh authoritative hello. + + Data installs an exact remote authority and its shared-cluster dual-index + projection in one serialized operation. A concurrent local cluster connect + therefore observes either the old authority and is covered by the install, + or the new authority and projects the peer itself; neither ordering can leave + exact authority permanently disconnected from replica routing. + Local activation uses the same Data serialization point to install its epoch, + self route, and every already-exact remote route. Local deactivation removes + admission and enqueues idempotent cleanup to every shard before Data replies. + The public API still waits for the shard barrier, but caller survival is not a + correctness dependency; shard restart repair consumes the same close marker. + + One persisted `{generation, revision}` authority hint is the cross-lane + fence. A heartbeat or lane hello may advance it only for a peer that already + has exact authority; Data invalidates every installed lane view in the same + turn. A delayed hint after complete retirement is discovery only: it cannot + recreate authority, a transport route, or an unbounded lease. Only the + dist-Erlang exact hello reintroduces that peer. Contiguous incremental cluster + controls compare their expected generation/revision against the applied + authority, observed revision, and hint inside one Data callback. A raced + heartbeat therefore rejects the complete incremental write instead of + installing a partial epoch set. + + Replica state uses the configured Group.Transport: + + - heads advertises {stream, retained_floor, head}. + - delta_batch carries one or more contiguous stream runs. + - need requests the receiver's next missing sequence. + - snapshot_chunk carries a byte-bounded part of one exact origin slice when + the requested prefix has already been pruned. snapshot_commit carries the + independently retryable terminal row/chunk counts. Receivers expose nothing + until one valid manifest and every provisional chunk are present. + + Every stream field is validated against the source node and + current generation/epoch. An old generation, a closed epoch, a wrong shard, + or a transitive claim for another node's pid is rejected. Control/data + reordering is safe: early messages are ignored and repeated heads repair them; + late messages fail their generation or epoch fence. Snapshot chunks may be + lost, duplicated, reordered, or mixed across retransmissions at the same + stream head; exact row counts, set insertion, and conflicting-retransmission + checks prevent partial or mixed commits. Rejected chunks/manifests clear their + staging immediately. Node loss, + generation replacement, and retired cluster streams discard matching partial + assemblies immediately rather than waiting for their inactivity deadline. + Unsequenced legacy state messages and malformed replica frames are rejected + before they can mutate a cursor or materialized row. + + ## Bounded recovery + + The oplog is bounded per shard, not by peer acknowledgements. A dropped tail + is found by periodic heads. A gap inside the retained range is repaired with + bounded delta batches. A gap below the retained floor receives the existing + full-sync primitive, narrowed to an exact origin/shard/cluster snapshot. + Absence from that snapshot is deletion, so no tombstones are required. + + There is no leader, quorum, retention ACK, or requirement to know all members. + A slow or disconnected peer cannot pin memory. When it returns it repairs from + deltas when possible and a snapshot otherwise. + + ## Nonblocking transport and ordering + + All cross-node control messages use :erlang.send_nosuspend/3 with :noconnect. + The default replica adapter does the same. Transport callbacks return :ok, + :busy, or :disconnected; failure drops the message and anti-entropy repairs it. + Replica shards never remotely monitor or exit member processes. + + Optional peer lifecycle callbacks are per shard lane. A sideband adapter that + shares one node connection across shards keeps that route until its final + live lane goes down; one flapping shard cannot disconnect healthy lanes. A + lane hello calls `peer_up` and enters `remote_shards` only after exact/current + authority admits that lane, so delayed post-retirement hellos cannot strand + an unleased transport route. If a lane hello arrives before exact authority, + authority fanout immediately repeats shard-local discovery instead of waiting + for the periodic anti-entropy probe. + + The transport need not order messages for correctness. The local shard + serializes writes, sequence numbers establish per-stream order, and receivers + reject duplicates and gaps. TCP shard-to-shard ordering remains the efficient + fast path. No semantic operation spans clusters, so cross-stream ordering is + unnecessary; generation and cluster-epoch fences cover lifecycle races. + + ## Batching and fairness + + Local writes share one outbound sender buffer so registry and PG mutations + retain mailbox order. Flushes group records by target and stream. Size, age, + control/routing barriers, and idle timers bound the delay. + + Incremental cluster controls are generation fenced, receiver batched, and + installed only by shard 0 into the node-wide authority table; controls that + arrive on another lane are forwarded locally to that single owner. Revisions + must be contiguous. A gap records the highest observation, fences every lane, + and requests an exact hello instead of applying a partial authority set. The + last exact revision, complete applied revision, and highest observed revision + are distinct. The Data owner compare-and-installs an incremental batch only + when its expected revision still matches the applied revision and persisted + hint. Shard-local lane readiness is separate from shared authority, so no + epoch map is copied per shard. On shard restart, constant-size retained + authority/view metadata seeds leases for origins whose rows survived in Data; + a disappeared Group is still retired after the normal bounded timeout. Each + lane deletes its own view only after purging its rows/cursors, so shard 0 + cannot erase a sibling's restart breadcrumb. Shard 0 also reconstructs an + exact-hello obligation whenever the persisted observed authority revision is + newer than the last exact revision. If a registry conflict is reconciled + while one retained claim is temporarily fenced by such an authority gap, the + lane remembers only that key and reprojects it when the exact view is + installed. This avoids a shard-wide claim scan while ensuring a current + cursor can never strand an old visible winner. + + Exact-snapshot scans run in at most one off-shard worker per shard. The worker + validates the local stream identity and fully-applied head before and after a + single pass, sends completed provisional chunks immediately, and retains only + one byte-targeted chunk. Overlapping writes suppress the terminal commit and + periodic anti-entropy retries the newer head. A busy chunk resumes at its + index; a busy commit retains only its small manifest. Receivers retain the + complete candidate and exact-install events in private ETS because exact + absence-as-delete replacement must start from a complete set. Cleared tables are + pooled for reuse; shard death deletes active and pooled tables. This keeps + sender memory bounded at million-to-tens-of-millions scale without putting + whole-snapshot heaps on the control process. + + Incoming PG mutations retain the bulk receiver lane. Contiguous registry + records in one stream run are projected together and emit one monitor event + batch. A mixed process-down record applies maximal same-domain segments in + wire order and emits one combined batch. + + After replicated work, the shard takes a bounded local-request turn before + yielding. Priority-control recursion is also quota-bounded. FIFO is preserved + within the local request lane, while protocol and cluster barriers flush + earlier buffered state first. + + Snapshot staging is owned by the receiving shard, expires after a peer lease + without progress, and disappears automatically if the shard crashes. Commit + first replaces the numeric receive cursor with a durable + `{:snapshot_installing, sequence}` marker. Startup repair sees that marker, + removes any partially replaced registry/PG slice, and clears the cursor so + anti-entropy requests the exact state again. """ require Logger - alias Group.Replica.Data + alias Group.Replica.{Data, Snapshot, WireProtocol} defstruct [ :name, @@ -440,24 +219,35 @@ defmodule Group.Replica do :replicated_sender_buffer_size, :replicated_sender_flush_interval, :replicated_pg_receiver_local_request_quota, + :replicated_oplog_max_entries, + :replicated_snapshot_chunk_target_bytes, + :replicated_anti_entropy_interval, + :replicated_peer_lease_timeout, + :replica_transport, + :replica_transport_opts, + :anti_entropy_ref, :pending_replicated_pg_started_at, :pending_replicated_pg_flush_ref, :pending_replicated_registry_started_at, :pending_replicated_registry_flush_ref, - :pending_replicated_pg_broadcast_started_at, - :pending_replicated_pg_broadcast_flush_ref, - :pending_replicated_registry_broadcast_started_at, - :pending_replicated_registry_broadcast_flush_ref, + :pending_replica_broadcast_started_at, + :pending_replica_broadcast_flush_ref, pending_replicated_pg_len: 0, pending_replicated_pg_ops: [], pending_replicated_registry_len: 0, pending_replicated_registry_ops: [], - pending_replicated_pg_broadcast_len: 0, - pending_replicated_pg_broadcast_ops: [], - pending_replicated_registry_broadcast_len: 0, - pending_replicated_registry_broadcast_ops: [], + pending_replica_broadcast_len: 0, + pending_replica_broadcast_ops: [], remote_shards: %{}, - monitors: %{} + peer_last_seen: %{}, + cluster_control_dirty: %{}, + authority_dirty_notified: MapSet.new(), + pending_registry_reprojections: %{}, + monitors: %{}, + snapshot_transfers: %{}, + snapshot_staging_pool: [], + snapshot_send: nil, + snapshot_send_offsets: %{} ] def start_link(opts) do @@ -507,6 +297,16 @@ defmodule Group.Replica do :ok end + @doc false + def local_cast(shard_name, request) when is_atom(shard_name) or is_pid(shard_name) do + case GenServer.whereis(shard_name) do + nil -> :ok + pid -> send(pid, {@local_request_tag, :noreply, request}) + end + + :ok + end + # ===================================================================== # GenServer callbacks # ===================================================================== @@ -536,12 +336,68 @@ defmodule Group.Replica do replicated_sender_buffer_size: config.replicated_sender_buffer_size, replicated_sender_flush_interval: config.replicated_sender_flush_interval, replicated_pg_receiver_local_request_quota: - config.replicated_pg_receiver_local_request_quota + config.replicated_pg_receiver_local_request_quota, + replicated_oplog_max_entries: config.replicated_oplog_max_entries, + replicated_snapshot_chunk_target_bytes: config.replicated_snapshot_chunk_target_bytes, + replicated_anti_entropy_interval: config.replicated_anti_entropy_interval, + replicated_peer_lease_timeout: config.replicated_peer_lease_timeout, + replica_transport: elem(config.replica_transport, 0), + replica_transport_opts: elem(config.replica_transport, 1) } + state = schedule_anti_entropy(state) + + # Repair any interrupted multi-table journal/index mutation, complete + # write-ahead records left unapplied by a shard crash, then rebuild local + # process monitors from the surviving materialized tables. + :ok = Data.repair_local_replica_journal(name, shard_index) + state = replay_local_journal(state) + :ok = Data.repair_shard_indexes(name, shard_index) + {state, _events} = rebuild_registry_projections(state) + + _completed_clusters = + Data.mark_closed_cluster_shard( + name, + Data.closed_local_cluster_epochs(name), + shard_index + ) + # Rebuild monitors from any surviving ETS data (after shard crash/restart) state = rebuild_monitors(state) + # A shard can die after replica rows are materialized but before it handles + # the peer's retirement. The ETS owner survives a shard restart, while the + # in-memory lease map does not. Reconstruct every retained origin as a lease + # candidate: live Groups answer the normal discovery probe below and refresh + # the lease; permanently disappeared Groups are purged when it expires. + restarted_at = monotonic_millis() + retained_origins = Data.retained_replica_origins(name, shard_index) + + cluster_control_dirty = + if shard_index == 0 do + Enum.reduce(retained_origins, %{}, fn origin, dirty -> + exact = Data.remote_cluster_epoch_exact_revision(name, origin) + observed = Data.remote_cluster_epoch_observed_revision(name, origin) + known_generation = Data.remote_generation(name, origin) + hint = Data.remote_replica_authority_hint(name, origin) + + if (not is_nil(observed) and observed != exact) or + (not is_nil(hint) and elem(hint, 0) != known_generation) do + Map.put(dirty, origin, restarted_at) + else + dirty + end + end) + else + %{} + end + + state = %{ + state + | peer_last_seen: Map.new(retained_origins, &{&1, restarted_at}), + cluster_control_dirty: cluster_control_dirty + } + log_once(state, fn -> "#{log_prefix(state)} started (shards=#{num_shards})" end) # Discover peers on all known nodes @@ -567,12 +423,12 @@ defmodule Group.Replica do # ===================================================================== @impl true - def handle_call({:register, _, _, _, _} = request, _from, state) do + def handle_call({:register, _, _, _, _, _} = request, _from, state) do {reply, state} = process_local_request(state, request) {:reply, reply, state} end - def handle_call({:unregister, _, _} = request, _from, state) do + def handle_call({:unregister, _, _, _} = request, _from, state) do {reply, state} = process_local_request(state, request) {:reply, reply, state} end @@ -581,12 +437,12 @@ defmodule Group.Replica do # Process group calls # ===================================================================== - def handle_call({:join, _, _, _, _} = request, _from, state) do + def handle_call({:join, _, _, _, _, _} = request, _from, state) do {reply, state} = process_local_request(state, request) {:reply, reply, state} end - def handle_call({:leave, _, _, _} = request, _from, state) do + def handle_call({:leave, _, _, _, _} = request, _from, state) do {reply, state} = process_local_request(state, request) {:reply, reply, state} end @@ -595,11 +451,21 @@ defmodule Group.Replica do # Cluster connect/disconnect (broadcast to all shards, rare operation) # ===================================================================== + def handle_call({:cluster_connect, _, _} = request, _from, state) do + {reply, state} = process_local_request(state, request) + {:reply, reply, state} + end + def handle_call({:cluster_connect, _} = request, _from, state) do {reply, state} = process_local_request(state, request) {:reply, reply, state} end + def handle_call({:cluster_disconnect, _, _} = request, _from, state) do + {reply, state} = process_local_request(state, request) + {:reply, reply, state} + end + def handle_call({:cluster_disconnect, _} = request, _from, state) do {reply, state} = process_local_request(state, request) {:reply, reply, state} @@ -610,21 +476,571 @@ defmodule Group.Replica do # ===================================================================== @impl true - def handle_info({:replicate_registry_batch, ops}, state) do - state = flush_pending_replicated_sender_barrier(state) + def handle_info( + {:replica_hello, remote_pid, version, generation, epoch_revision, cluster_epochs, + transport_id, transport_descriptor}, + %{shard_index: 0} = state + ) do + state = flush_pending_replicated_message_barrier(state) + remote_node = node(remote_pid) + + known_generation = Data.remote_generation(state.name, remote_node) + + hinted_generation = + case Data.remote_replica_authority_hint(state.name, remote_node) do + {hinted_generation, _revision} -> hinted_generation + nil -> known_generation + end + + observed_revision = Data.remote_cluster_epoch_observed_revision(state.name, remote_node) + authoritative_revision = Data.remote_cluster_epoch_revision(state.name, remote_node) + + exact_revision = Data.remote_cluster_epoch_exact_revision(state.name, remote_node) + + stale_generation? = + not is_nil(hinted_generation) and hinted_generation != generation and + not WireProtocol.generation_newer?(generation, hinted_generation) + + stale_revision? = + known_generation == generation and + Enum.any?([observed_revision, authoritative_revision], fn + revision when is_integer(revision) -> epoch_revision < revision + _ -> false + end) + + cond do + version != WireProtocol.version() or transport_id != state.replica_transport.id() or + not WireProtocol.valid_generation?(generation) -> + Logger.error( + "#{log_prefix_shard(state)} incompatible replica protocol/transport from #{inspect(remote_node)}" + ) + + {:noreply, state} + + stale_generation? or stale_revision? -> + {:noreply, state} + + known_generation == generation and exact_revision == epoch_revision -> + # Requests and heartbeats may race while one large authority snapshot + # is being installed. Once this exact revision is present, another + # identical hello is only a lease/descriptor refresh; reinstalling its + # full epoch set would serialize every shard behind redundant ETS work. + state = + if replica_view_current?(state, remote_node) do + state + else + install_current_replica_lane(state, remote_node, generation) + end + + if replica_authority_current?(state, remote_node, generation, epoch_revision) and + replica_view_current?(state, remote_node) do + state = notify_replica_transport_peer_up(state, remote_node, transport_descriptor) + + {:noreply, + %{ + state + | remote_shards: Map.put(state.remote_shards, remote_node, remote_pid), + peer_last_seen: Map.put(state.peer_last_seen, remote_node, monotonic_millis()), + cluster_control_dirty: Map.delete(state.cluster_control_dirty, remote_node) + }} + else + {:noreply, + state + |> mark_cluster_control_dirty(remote_node) + |> request_replica_authority(remote_node)} + end + + true -> + {:noreply, + install_replica_authority( + state, + remote_pid, + generation, + epoch_revision, + cluster_epochs, + transport_id, + transport_descriptor + )} + end + end + + def handle_info( + {:replica_hello, remote_pid, _version, _generation, _epoch_revision, _cluster_epochs, + _transport_id, _transport_descriptor}, + state + ) do + # Full authority is installed only by shard 0. A full hello delivered to a + # data lane cannot be used as its lane identity because remote_pid belongs + # to the remote control shard, not this matching shard. + {:noreply, request_replica_authority(state, node(remote_pid))} + end + + def handle_info( + {:replica_lane_hello, remote_pid, version, generation, epoch_revision, transport_id, + transport_descriptor}, + state + ) do + state = flush_pending_replicated_message_barrier(state) + remote_node = node(remote_pid) + + if version == WireProtocol.version() and transport_id == state.replica_transport.id() do + state = observe_replica_authority_hint(state, remote_node, generation, epoch_revision) + + cond do + replica_authority_current?(state, remote_node, generation, epoch_revision) and + replica_view_current?(state, remote_node) -> + state = + state + |> notify_replica_transport_peer_up(remote_node, transport_descriptor) + |> put_remote_shard(remote_node, remote_pid) + |> purge_remote_streams_outside_authority(remote_node) + |> touch_replica_peer(remote_node) + |> Map.update!(:cluster_control_dirty, &Map.delete(&1, remote_node)) + |> send_replica_heads(remote_node) + + {:noreply, state} + + replica_exact_authority_current?(state, remote_node, generation, epoch_revision) -> + # The shared authority can arrive before this sibling is registered, + # so shard-zero fanout is intentionally lossy at startup. Rebuild + # this lane directly from the exact shared authority. + {:noreply, + state + |> notify_replica_transport_peer_up(remote_node, transport_descriptor) + |> put_remote_shard(remote_node, remote_pid) + |> install_current_replica_lane(remote_node, generation)} + + true -> + # A lane hello is only a hint until node-wide exact authority exists. + # In particular, a delayed hello after retirement must not recreate + # an unleased route that can live forever and suppress rediscovery. + {:noreply, request_replica_authority(state, remote_node)} + end + else + Logger.error( + "#{log_prefix_shard(state)} incompatible replica protocol/transport from #{inspect(remote_node)}" + ) + + {:noreply, state} + end + end + + def handle_info( + {:replica_authority_installed_local, remote_node, generation, epoch_revision, + old_generation, stale_epochs}, + state + ) do + state = flush_pending_replicated_message_barrier(state) + + if replica_authority_current?(state, remote_node, generation, epoch_revision) do + state = maybe_purge_remote_generation(state, remote_node, old_generation, generation) + + state = + if old_generation == generation do + state + |> purge_closed_remote_epochs(remote_node, stale_epochs) + |> purge_remote_streams_outside_authority(remote_node) + else + state + end + + state = install_replica_view(state, remote_node, generation) + + state = %{ + state + | cluster_control_dirty: Map.delete(state.cluster_control_dirty, remote_node), + authority_dirty_notified: MapSet.delete(state.authority_dirty_notified, remote_node) + } + + state = + if Map.has_key?(state.remote_shards, remote_node) do + state + |> touch_replica_peer(remote_node) + |> send_replica_heads(remote_node) + else + # A lane hello can legitimately outrun shard zero's exact authority. + # The hello is not retained as a route, because a delayed hello after + # retirement must not recreate an unleased peer. Once exact authority + # reaches this lane, repeat shard-local discovery immediately instead + # of waiting for the next anti-entropy probe. + send_remote_shard_message( + state, + remote_node, + {:peer_connect, self(), state.shard_index, state.num_shards, + Data.my_clusters(state.name)} + ) + + state + end + + {:noreply, state} + else + {:noreply, state} + end + end + + def handle_info( + {:replica_cluster_open, remote_pid, generation, revision, epochs} = control, + state + ) do + state = flush_pending_replicated_message_barrier(state) + + if state.shard_index == 0 do + remote_node = node(remote_pid) + + controls = + collect_replica_cluster_controls( + :replica_cluster_open, + remote_pid, + generation, + [{revision, epochs}], + state.replicated_sender_buffer_size - 1 + ) + + case accepted_replica_cluster_epochs(state, remote_node, generation, controls) do + {:accept, expected_revision, observed_revision, epochs} -> + case Data.put_remote_cluster_epochs( + state.name, + state.shard_index, + remote_node, + generation, + expected_revision, + observed_revision, + epochs + ) do + {:ok, stale} -> + shared = + Enum.filter(epochs, fn {cluster, _epoch} -> + node() in Data.cluster_nodes(state.name, cluster) + end) + + Data.add_cluster_node(state.name, Enum.map(shared, &elem(&1, 0)), remote_node) + + fan_out_to_siblings( + state, + {:replica_cluster_open_control_local, remote_node, generation, observed_revision, + epochs, stale, Enum.map(shared, &elem(&1, 0))} + ) + + state = + state + |> mark_authority_dirty(remote_node) + |> purge_closed_remote_epochs(remote_node, stale) + |> purge_superseded_remote_streams(remote_node, epochs) + |> purge_remote_streams_outside_authority(remote_node) + + state = install_replica_view(state, remote_node, generation) + state = send_replica_heads(state, remote_node, Enum.map(shared, &elem(&1, 0))) + {:noreply, take_one_local_request_turn(state)} + + :stale -> + {:noreply, + state + |> mark_authority_dirty(remote_node) + |> request_replica_authority(remote_node) + |> take_one_local_request_turn()} + end + + :stale -> + {:noreply, take_one_local_request_turn(state)} + + :refresh -> + {:noreply, + state + |> request_replica_authority(remote_node) + |> take_one_local_request_turn()} + + {:gap, observed_revision} -> + :ok = + Data.observe_remote_cluster_epoch_revision( + state.name, + remote_node, + observed_revision + ) - {state, flushed?} = enqueue_replicated_registry_ops(state, ops) + {:noreply, + state + |> mark_authority_dirty(remote_node) + |> request_replica_authority(remote_node) + |> take_one_local_request_turn()} + end + else + # Incremental authority is node-wide and therefore has one local owner. + # Forward a control that arrived on another lane instead of racing its + # check/update against shard 0. + _ = send_local_control_message(state, control) + {:noreply, take_one_local_request_turn(state)} + end + end + + def handle_info({:replica_authority_dirty_local, remote_node}, %{shard_index: 0} = state) do + {:noreply, mark_cluster_control_dirty(state, remote_node)} + end + + def handle_info({:replica_authority_dirty_local, remote_node}, state) do + _ = + send_local_control_message(state, {:replica_authority_dirty_local, remote_node}) - state = if flushed?, do: take_priority_turn(state), else: state {:noreply, state} end - def handle_info({:replicate_pg_batch, ops}, state) do - state = flush_pending_replicated_sender_barrier(state) + def handle_info( + {:replica_cluster_open_control_local, remote_node, generation, revision, epochs, stale, + shared}, + state + ) do + state = flush_pending_replicated_message_barrier(state) + + state = + if replica_authority_current?(state, remote_node, generation, revision) do + state = + state + |> purge_closed_remote_epochs(remote_node, stale) + |> purge_superseded_remote_streams(remote_node, epochs) + |> purge_remote_streams_outside_authority(remote_node) + + state + |> install_replica_view(remote_node, generation) + |> send_replica_heads(remote_node, shared) + else + state + end + + {:noreply, take_one_local_request_turn(state)} + end + + def handle_info( + {:replica_cluster_close, remote_pid, generation, revision, epochs}, + %{shard_index: 0} = state + ) do + state = flush_pending_replicated_message_barrier(state) + remote_node = node(remote_pid) + + controls = + collect_replica_cluster_controls( + :replica_cluster_close, + remote_pid, + generation, + [{revision, epochs}], + state.replicated_sender_buffer_size - 1 + ) + + case accepted_replica_cluster_epochs(state, remote_node, generation, controls) do + {:accept, expected_revision, observed_revision, epochs} -> + case Data.close_remote_cluster_epochs( + state.name, + 0, + remote_node, + generation, + expected_revision, + observed_revision, + epochs + ) do + {:ok, closed} -> + if state.shard_index == 0 do + Data.remove_cluster_node(state.name, Enum.map(closed, &elem(&1, 0)), remote_node) + end + + fan_out_to_siblings( + state, + {:replica_cluster_close_control_local, remote_node, generation, observed_revision, + closed} + ) + + state = + state + |> mark_cluster_control_dirty(remote_node) + |> purge_closed_remote_epochs(remote_node, closed) + |> purge_remote_streams_outside_authority(remote_node) + + state = install_replica_view(state, remote_node, generation) + {:noreply, take_one_local_request_turn(state)} + + :stale -> + {:noreply, + state + |> mark_authority_dirty(remote_node) + |> request_replica_authority(remote_node) + |> take_one_local_request_turn()} + end + + :stale -> + {:noreply, take_one_local_request_turn(state)} + + :refresh -> + {:noreply, + state + |> request_replica_authority(remote_node) + |> take_one_local_request_turn()} + + {:gap, observed_revision} -> + :ok = + Data.observe_remote_cluster_epoch_revision( + state.name, + remote_node, + observed_revision + ) + + {:noreply, + state + |> mark_cluster_control_dirty(remote_node) + |> request_replica_authority(remote_node) + |> take_one_local_request_turn()} + end + end + + def handle_info({:replica_cluster_close, remote_pid, _generation, _revision, _epochs}, state) do + {:noreply, request_replica_authority(state, node(remote_pid))} + end + + def handle_info( + {:replica_cluster_close_control_local, remote_node, generation, revision, closed}, + state + ) do + state = flush_pending_replicated_message_barrier(state) + + state = + if replica_authority_current?(state, remote_node, generation, revision) do + state = + state + |> purge_closed_remote_epochs(remote_node, closed) + |> purge_remote_streams_outside_authority(remote_node) + + install_replica_view(state, remote_node, generation) + else + state + end + + {:noreply, take_one_local_request_turn(state)} + end + + def handle_info( + {:replica_heartbeat, remote_pid, version, generation, epoch_revision, transport_id, + transport_descriptor}, + state + ) do + remote_node = node(remote_pid) + + compatible? = + version == WireProtocol.version() and transport_id == state.replica_transport.id() + + state = + if compatible? do + observe_replica_authority_hint(state, remote_node, generation, epoch_revision) + else + state + end + + state = + cond do + compatible? and + replica_authority_current?(state, remote_node, generation, epoch_revision) and + replica_view_current?(state, remote_node) -> + state + |> notify_replica_transport_peer_up(remote_node, transport_descriptor) + |> put_remote_shard(remote_node, remote_pid) + |> touch_replica_peer(remote_node) + + compatible? and + replica_authority_current?(state, remote_node, generation, epoch_revision) -> + state + + true -> + request_replica_authority(state, remote_node) + end + + {:noreply, state} + end + + def handle_info({:replica_hello_request, remote_pid}, state) do + if state.shard_index == 0 do + {:noreply, send_replica_hello(state, node(remote_pid))} + else + _ = send_local_control_message(state, {:replica_hello_request, remote_pid}) + {:noreply, state} + end + end + + def handle_info({:group_replica_frame, remote_pid, message}, state) when is_pid(remote_pid) do + remote_node = node(remote_pid) + state = handle_replica_message(state, remote_node, message) + {:noreply, take_priority_turn(state)} + end + + def handle_info({:group_replica_frame, remote_node, message}, state) + when is_atom(remote_node) do + state = handle_replica_message(state, remote_node, message) + {:noreply, take_priority_turn(state)} + end + + def handle_info({:group_replica_batch, remote_node, messages}, state) + when is_atom(remote_node) and is_list(messages) do + {turn, remaining} = Enum.split(messages, @incoming_batch_quota) + state = Enum.reduce(turn, state, &handle_replica_message(&2, remote_node, &1)) + + if remaining != [] do + send(self(), {:group_replica_batch, remote_node, remaining}) + end + + {:noreply, take_priority_turn(state)} + end + + def handle_info( + {:replica_snapshot_send_complete, token, worker, snapshot_key, result}, + %{snapshot_send: {worker, token, snapshot_key}} = state + ) do + offsets = + case result do + :complete -> + Map.delete(state.snapshot_send_offsets, snapshot_key) + + {:resume, chunk_index} -> + if current_snapshot_send?(state, snapshot_key) do + Map.put(state.snapshot_send_offsets, snapshot_key, {:chunk, chunk_index}) + else + Map.delete(state.snapshot_send_offsets, snapshot_key) + end + + {:resume_commit, manifest} -> + if current_snapshot_send?(state, snapshot_key) do + Map.put(state.snapshot_send_offsets, snapshot_key, {:commit, manifest}) + else + Map.delete(state.snapshot_send_offsets, snapshot_key) + end + + :retry -> + if current_snapshot_send?(state, snapshot_key) do + state.snapshot_send_offsets + else + Map.delete(state.snapshot_send_offsets, snapshot_key) + end + end + + {:noreply, %{state | snapshot_send: nil, snapshot_send_offsets: offsets}} + end - {state, flushed?} = enqueue_replicated_pg_ops(state, ops) + def handle_info( + {:replica_snapshot_send_complete, _token, _worker, _snapshot_key, _result}, + state + ) do + {:noreply, state} + end + + def handle_info({@anti_entropy_timer, ref}, state) do + state = + if state.anti_entropy_ref == ref do + state + |> expire_stale_snapshot_transfers() + |> expire_stale_replica_peers() + |> probe_replica_peers() + |> request_quiet_cluster_hellos() + |> broadcast_replica_heartbeats() + |> broadcast_replica_heads() + |> schedule_anti_entropy() + else + state + end - state = if flushed?, do: take_priority_turn(state), else: state {:noreply, state} end @@ -637,6 +1053,10 @@ defmodule Group.Replica do {:noreply, process_local_request_turn(state, [{{:alias, alias_ref}, request}])} end + def handle_info({@local_request_tag, :noreply, request}, state) do + {:noreply, process_local_request_turn(state, [{:noreply, request}])} + end + # ===================================================================== # Peer discovery protocol # ===================================================================== @@ -655,21 +1075,17 @@ defmodule Group.Replica do %{name: name, shard_index: shard} = state remote_node = node(remote_pid) - # Compute shared clusters and add the remote node to nil plus every shared - # named cluster in one serialized membership mutation. + # Compute shared clusters for diagnostics only. The generation-fenced hello + # is the sole authority that mutates peer and cluster membership. Keeping + # discovery hints side-effect free prevents a delayed pre-restart + # peer_connect from permanently re-adding stale cluster rows. my_clusters = Data.my_clusters(name) shared = compute_shared_clusters(my_clusters, remote_clusters) - Data.add_cluster_node(name, [nil | Enum.reject(shared, &is_nil/1)], remote_node) - already_known = Map.has_key?(state.remote_shards, remote_node) - - state = - if already_known do - state - else - Process.monitor(remote_pid) - %{state | remote_shards: Map.put(state.remote_shards, remote_node, remote_pid)} - end + # Replica peers are addressed by registered `{name, node}` and established + # only by replica_hello. Do not remotely monitor the shard PID: creating a + # remote monitor itself emits a distribution signal and may suspend on a + # busy dist connection. # Send ack with our cluster list send_to_peer( @@ -678,14 +1094,12 @@ defmodule Group.Replica do {:peer_connect_ack, self(), shard, state.num_shards, my_clusters} ) + send_replica_hello(state, remote_node) + log_once(state, fn -> "#{log_prefix(state)} peer_connect from #{remote_node} (#{length(shared)} shared clusters)" end) - # Send cluster_state for all shared clusters in one pass (single table scan - # instead of one scan per cluster — O(N) vs O(C×N)) - send_cluster_states(state, shared, remote_node) - {:noreply, state} end @@ -709,28 +1123,16 @@ defmodule Group.Replica do %{name: name} = state remote_node = node(remote_pid) - # Compute shared clusters and add the remote node to nil plus every shared - # named cluster in one serialized membership mutation. + # Discovery acknowledgements are hints only; replica_hello is the sole + # generation-fenced authority for peer and cluster membership. my_clusters = Data.my_clusters(name) shared = compute_shared_clusters(my_clusters, remote_clusters) - Data.add_cluster_node(name, [nil | Enum.reject(shared, &is_nil/1)], remote_node) - - already_known = Map.has_key?(state.remote_shards, remote_node) - - state = - if already_known do - state - else - Process.monitor(remote_pid) - %{state | remote_shards: Map.put(state.remote_shards, remote_node, remote_pid)} - end log_once(state, fn -> "#{log_prefix(state)} peer_connect_ack from #{remote_node} (#{length(shared)} shared clusters)" end) - # Send cluster_state for all shared clusters in one pass - send_cluster_states(state, shared, remote_node) + send_replica_hello(state, remote_node) {:noreply, state} end @@ -741,155 +1143,33 @@ defmodule Group.Replica do end # ===================================================================== - # Cluster state (unified handler for peer discovery + cluster join) + # Node up/down # ===================================================================== - def handle_info({:cluster_state, cluster, reg_data, pg_data}, state) do + def handle_info({:nodeup, remote_node}, state) do state = flush_pending_replicated_message_barrier(state) - %{name: name} = state - - # Guard: skip merge for named clusters we're not a member of - if cluster_member?(name, cluster) do - log_once(state, fn -> - "#{log_prefix(state)} cluster_state cluster=#{inspect(cluster)} (#{length(reg_data)} reg, #{length(pg_data)} pg entries)" - end) + %{shard_index: shard, name: name} = state - log_verbose(state, fn -> - "#{log_prefix_shard(state)} merging cluster=#{inspect(cluster)} (#{length(reg_data)} reg, #{length(pg_data)} pg entries)" - end) + send_remote_shard_message( + state, + remote_node, + {:peer_connect, self(), shard, state.num_shards, Data.my_clusters(name)} + ) - {state, events} = merge_remote_cluster_data(state, cluster, reg_data, pg_data) - notify_monitors(name, events) - {:noreply, state} - else - {:noreply, state} - end + {:noreply, state} end - # ===================================================================== - # Cluster connect/disconnect from remote - # ===================================================================== - - def handle_info({:cluster_connect, clusters, remote_pid}, state) do - state = flush_pending_replicated_message_barrier(state) - %{name: name} = state - remote_node = node(remote_pid) - - shared = - Enum.filter(clusters, fn c -> - node() in Data.cluster_nodes(name, c) - end) - - log_once(state, fn -> - "#{log_prefix(state)} #{remote_node} cluster_connect #{inspect(shared)} (#{length(shared)}/#{length(clusters)} shared)" - end) - - if shared != [] do - Data.add_cluster_node(name, shared, remote_node) - - # Bundle this shard's cluster data directly into the ack (one cross-node - # message instead of ack + N separate cluster_state messages) - {reg_by_cluster, pg_by_cluster} = - Data.local_data_by_cluster(name, state.shard_index, shared) - - cluster_data = - for cluster <- shared do - reg_data = Map.get(reg_by_cluster, cluster, []) - pg_data = Map.get(pg_by_cluster, cluster, []) - {cluster, reg_data, pg_data} - end - - send_to_peer(state, remote_node, {:cluster_connect_ack, shared, self(), cluster_data}) - fan_out_to_siblings(state, {:send_cluster_data, shared, remote_node}) - end - - {:noreply, state} - end - - def handle_info({:cluster_connect_ack, clusters, remote_pid, cluster_data}, state) do - state = flush_pending_replicated_message_barrier(state) - %{name: name} = state - remote_node = node(remote_pid) - - # Guard: skip if remote node went down (nodedown race) or if we left the - # cluster (connect+disconnect race). Without these, a delayed ack would - # re-add a dead/irrelevant node to cluster_nodes permanently. - {state, events} = - if Map.has_key?(state.remote_shards, remote_node) do - active = Enum.filter(clusters, fn c -> node() in Data.cluster_nodes(name, c) end) - - if active != [] do - Data.add_cluster_node(name, active, remote_node) - - # Merge the data bundled in the ack - {new_state, events} = - Enum.reduce(cluster_data, {state, []}, fn {cluster, reg_data, pg_data}, - {acc_state, acc_events} -> - if cluster in active and (reg_data != [] or pg_data != []) do - merge_remote_cluster_data(acc_state, cluster, reg_data, pg_data, acc_events) - else - {acc_state, acc_events} - end - end) - - send_cluster_states(new_state, active, remote_node) - fan_out_to_siblings(new_state, {:send_cluster_data, active, remote_node}) - {new_state, events} - else - {state, []} - end - else - {state, []} - end - - notify_monitors(name, events) - {:noreply, state} - end - - def handle_info({:cluster_disconnect, clusters, remote_pid}, state) do + def handle_info({:nodedown, dead_node}, state) do state = flush_pending_replicated_message_barrier(state) + state = discard_snapshot_transfers_for_source(state, dead_node) + state = discard_snapshot_send_offsets_for_target(state, dead_node) + state = discard_pending_registry_reprojections(state, dead_node) %{name: name, shard_index: shard} = state - remote_node = node(remote_pid) - - log_once(state, fn -> - "#{log_prefix(state)} #{remote_node} cluster_disconnect #{inspect(clusters)}" - end) - - if shard == 0 do - Data.remove_cluster_node(name, clusters, remote_node) - fan_out_to_siblings(state, {:cluster_disconnect, clusters, remote_pid}) - end - - events = - Enum.reduce(clusters, [], fn cluster, acc -> - {purged_reg, purged_pg} = purge_cluster_entries(name, shard, cluster, remote_node) - build_purged_events(name, purged_reg, purged_pg, :cluster_disconnect, acc) - end) - - notify_monitors(name, events) - {:noreply, state} - end - - # ===================================================================== - # Node up/down - # ===================================================================== - - def handle_info({:nodeup, remote_node}, state) do - state = flush_pending_replicated_message_barrier(state) - %{shard_index: shard, name: name} = state - - send_remote_shard_message( - state, - remote_node, - {:peer_connect, self(), shard, state.num_shards, Data.my_clusters(name)} - ) - {:noreply, state} - end - - def handle_info({:nodedown, dead_node}, state) do - state = flush_pending_replicated_message_barrier(state) - %{name: name, shard_index: shard} = state + # Cursor absence is the durable restart marker for an incomplete or + # retiring remote stream. Clear it before touching materialized rows so a + # shard crash at any later purge step finishes that retirement on restart. + Data.delete_replica_cursors_for_origin(name, shard, dead_node) # Remove cluster memberships from shared tables. Every shard calls this # unconditionally (not just shard 0) to handle the race where a non-zero @@ -900,14 +1180,41 @@ defmodule Group.Replica do # Purge all data from the dead node {purged_reg, purged_pg} = Data.purge_node(name, shard, dead_node) + affected_claims = Data.purge_registry_claims_for_origin(name, shard, dead_node) log_once(state, fn -> "#{log_prefix(state)} nodedown #{dead_node} (purged #{length(purged_reg)} reg, #{length(purged_pg)} pg entries)" end) events = build_purged_events(name, purged_reg, purged_pg, :nodedown) + + {state, events} = + Enum.reduce(affected_claims, {state, events}, fn {cluster, key}, {acc, inner_events} -> + reconcile_registry_projection(acc, cluster, key, :nodedown, inner_events) + end) + notify_monitors(name, events) - state = %{state | remote_shards: Map.delete(state.remote_shards, dead_node)} + + state = %{ + state + | remote_shards: Map.delete(state.remote_shards, dead_node), + peer_last_seen: Map.delete(state.peer_last_seen, dead_node), + cluster_control_dirty: Map.delete(state.cluster_control_dirty, dead_node), + authority_dirty_notified: MapSet.delete(state.authority_dirty_notified, dead_node) + } + + Data.delete_remote_replica_info(name, shard, dead_node) + + if function_exported?(state.replica_transport, :peer_down, 4) do + :ok = + state.replica_transport.peer_down( + name, + dead_node, + shard, + state.replica_transport_opts + ) + end + {:noreply, state} end @@ -919,77 +1226,59 @@ defmodule Group.Replica do state = flush_pending_replicated_message_barrier(state) %{name: name, shard_index: shard} = state - remote_node = node(pid) - - if remote_node != node() and Map.get(state.remote_shards, remote_node) == pid do - # Remote shard process died — purge its cluster memberships and node data. - # Unconditional (not gated on shard 0) — same reasoning as nodedown handler. - Data.purge_cluster_node(name, remote_node) - {purged_reg, purged_pg} = Data.purge_node(name, shard, remote_node) + # Replica shards intentionally never create remote process monitors: their + # liveness is generation/lease fenced and remote monitoring can itself + # suspend on a busy distribution connection. Consequently every genuine + # DOWN handled here belongs to a locally owned registry/PG process. + if Map.has_key?(state.monitors, pid) do + {downs, monitors} = + collect_local_process_downs( + [{pid, reason}], + state.monitors, + @process_down_batch_size - 1 + ) - log_verbose(state, fn -> - "#{log_prefix_shard(state)} remote_shard_down #{remote_node} (purged #{length(purged_reg)} reg, #{length(purged_pg)} pg)" - end) + pids = Enum.map(downs, &elem(&1, 0)) + reason_by_pid = Map.new(downs) + {visible_reg, pending_pg} = Data.entries_for_pids(name, shard, pids) - events = build_purged_events(name, purged_reg, purged_pg, {:nodedown, remote_node}) - notify_monitors(name, events) - state = %{state | remote_shards: Map.delete(state.remote_shards, remote_node)} - state = %{state | monitors: Map.delete(state.monitors, pid)} - {:noreply, state} - else - if Map.has_key?(state.monitors, pid) do - {downs, monitors} = - collect_local_process_downs( - [{pid, reason}], - state.monitors, - @process_down_batch_size - 1 - ) + claimed_reg = + Data.local_registry_claims_by_pids(name, shard, pids) + |> Enum.map(fn {pid, cluster, key, meta, _generation, _epoch} -> + {pid, cluster, key, meta} + end) - pids = Enum.map(downs, &elem(&1, 0)) - reason_by_pid = Map.new(downs) - {purged_reg, purged_pg} = Data.delete_all_for_pids(name, shard, pids) + pending_reg = Enum.uniq(visible_reg ++ claimed_reg) - log_verbose(state, fn -> - "#{log_prefix_shard(state)} process_down_batch pids=#{length(downs)} (#{length(purged_reg) + length(purged_pg)} entries cleaned)" - end) + sequenced_downs = + append_process_down_records(state, reason_by_pid, pending_reg, pending_pg) - broadcast_process_down_batch(state, reason_by_pid, purged_reg, purged_pg) - events = build_process_down_events(name, purged_reg, purged_pg, reason_by_pid) - notify_monitors(name, events) - state = %{state | monitors: Map.drop(monitors, pids)} - {:noreply, state} - else - {:noreply, state} - end - end - end + {purged_reg, purged_pg} = Data.delete_all_for_pids(name, shard, pids) - def handle_info({:replicate_process_down_batch, reg_entries, pg_entries}, state) do - state = flush_pending_replicated_message_barrier(state) - %{name: name, shard_index: shard} = state + log_verbose(state, fn -> + "#{log_prefix_shard(state)} process_down_batch pids=#{length(downs)} (#{length(purged_reg) + length(purged_pg)} entries cleaned)" + end) - log_verbose(state, fn -> - "#{log_prefix_shard(state)} replicate_process_down_batch (#{length(reg_entries)} reg, #{length(pg_entries)} pg)" - end) + state = finish_process_down_records(state, sequenced_downs) - deleted_reg = Data.registry_delete_matching_many(name, shard, reg_entries) - deleted_pg = Data.pg_delete_matching_many(name, shard, pg_entries) - events = build_process_down_batch_events(name, deleted_reg, deleted_pg) - notify_monitors(name, events) - {:noreply, state} - end + affected_registry_keys = + pending_reg + |> Enum.map(fn {_pid, cluster, key, _meta} -> {cluster, key} end) + |> Enum.uniq() - def handle_info({:send_cluster_data, clusters, target_node}, state) do - state = flush_pending_replicated_message_barrier(state) - %{name: name} = state + {state, projection_events} = + reconcile_registry_keys(state, affected_registry_keys, reason, []) - active = Enum.filter(clusters, fn c -> node() in Data.cluster_nodes(name, c) end) + events = + projection_events ++ + build_process_down_events(name, purged_reg, purged_pg, reason_by_pid) - if active != [] do - send_cluster_states(state, active, target_node) + notify_monitors(name, events) + state = %{state | monitors: Map.drop(monitors, pids)} + {:noreply, state} + else + {:noreply, state} end - - {:noreply, state} end def handle_info({:group_dispatch, pids, message}, state) do @@ -1024,21 +1313,10 @@ defmodule Group.Replica do {:noreply, state} end - def handle_info({@replicated_pg_broadcast_flush_timer, flush_ref}, state) do - state = - if state.pending_replicated_pg_broadcast_flush_ref == flush_ref do - flush_pending_replicated_pg_broadcast(state) - else - state - end - - {:noreply, state} - end - - def handle_info({@replicated_registry_broadcast_flush_timer, flush_ref}, state) do + def handle_info({@replica_broadcast_flush_timer, flush_ref}, state) do state = - if state.pending_replicated_registry_broadcast_flush_ref == flush_ref do - flush_pending_replicated_registry_broadcast(state) + if state.pending_replica_broadcast_flush_ref == flush_ref do + flush_pending_replica_broadcast(state) else state end @@ -1178,18 +1456,14 @@ defmodule Group.Replica do :ok end + defp reply_local_request(:noreply, _reply), do: :ok + defp process_local_request_turn( state, - [{_reply_to, request} | _] = initial_messages + initial_messages ) do remaining = - case local_request_domain(request) do - :pg -> - max(state.replicated_pg_receiver_local_request_quota - length(initial_messages), 0) - - :other -> - 0 - end + max(state.replicated_pg_receiver_local_request_quota - length(initial_messages), 0) messages = collect_local_request_messages(initial_messages, remaining) process_local_request_messages(state, messages) @@ -1205,6 +1479,9 @@ defmodule Group.Replica do {@local_request_tag, alias_ref, request} when is_reference(alias_ref) -> collect_local_request_messages([{{:alias, alias_ref}, request} | acc], remaining - 1) + + {@local_request_tag, :noreply, request} -> + collect_local_request_messages([{:noreply, request} | acc], remaining - 1) after 0 -> Enum.reverse(acc) @@ -1269,24 +1546,70 @@ defmodule Group.Replica do end defp process_local_request_without_barrier(state, request) do + with :ok <- validate_local_mutation_epoch(state, request) do + case request do + {:register, cluster, _epoch, key, pid, meta} -> + do_register(state, cluster, key, pid, meta) + + {:unregister, cluster, _epoch, key} -> + do_unregister(state, cluster, key) + + {:join, cluster, _epoch, key, pid, meta} -> + do_join(state, cluster, key, pid, meta) + + {:leave, cluster, _epoch, key, pid} -> + do_leave(state, cluster, key, pid) + + {:cluster_connect, clusters} -> + do_cluster_connect(state, clusters) + + {:cluster_connect, clusters, epochs} -> + do_cluster_connect(state, clusters, epochs) + + {:cluster_disconnect, clusters} -> + do_cluster_disconnect(state, clusters) + + {:cluster_disconnect, clusters, epochs} -> + do_cluster_disconnect(state, clusters, epochs) + + _ -> + {{:error, :invalid_local_request}, state} + end + else + {:error, reason} -> {{:error, reason}, state} + end + end + + defp validate_local_mutation_epoch(state, request) do case request do - {:register, cluster, key, pid, meta} -> - do_register(state, cluster, key, pid, meta) + {op, cluster, epoch, _key, _pid, _meta} when op in [:register, :join] -> + validate_local_mutation_epoch(state, cluster, epoch) + + {op, cluster, epoch, _key} when op == :unregister -> + validate_local_mutation_epoch(state, cluster, epoch) - {:unregister, cluster, key} -> - do_unregister(state, cluster, key) + {op, cluster, epoch, _key, _pid} when op == :leave -> + validate_local_mutation_epoch(state, cluster, epoch) - {:join, cluster, key, pid, meta} -> - do_join(state, cluster, key, pid, meta) + {op, _cluster, _key, _pid, _meta} when op in [:register, :join] -> + {:error, :stale_cluster_epoch} - {:leave, cluster, key, pid} -> - do_leave(state, cluster, key, pid) + {op, _cluster, _key} when op == :unregister -> + {:error, :stale_cluster_epoch} - {:cluster_connect, clusters} -> - do_cluster_connect(state, clusters) + {op, _cluster, _key, _pid} when op == :leave -> + {:error, :stale_cluster_epoch} - {:cluster_disconnect, clusters} -> - do_cluster_disconnect(state, clusters) + _ -> + :ok + end + end + + defp validate_local_mutation_epoch(state, cluster, epoch) do + if Data.local_cluster_epoch(state.name, cluster) == epoch do + :ok + else + {:error, :stale_cluster_epoch} end end @@ -1334,56 +1657,10 @@ defmodule Group.Replica do |> flush_pending_replicated_barrier() end - defp flush_pending_replicated_sender_barrier( - %{ - pending_replicated_pg_broadcast_len: 0, - pending_replicated_registry_broadcast_len: 0 - } = state - ), - do: state - - defp flush_pending_replicated_sender_barrier( - %{pending_replicated_pg_broadcast_len: 0, pending_replicated_registry_broadcast_len: len} = - state - ) - when len > 0, - do: flush_pending_replicated_registry_broadcast(state) - - defp flush_pending_replicated_sender_barrier( - %{pending_replicated_pg_broadcast_len: len, pending_replicated_registry_broadcast_len: 0} = - state - ) - when len > 0, - do: flush_pending_replicated_pg_broadcast(state) - - defp flush_pending_replicated_sender_barrier(state) do - if state.pending_replicated_pg_broadcast_started_at <= - state.pending_replicated_registry_broadcast_started_at do - state - |> flush_pending_replicated_pg_broadcast() - |> flush_pending_replicated_registry_broadcast() - else - state - |> flush_pending_replicated_registry_broadcast() - |> flush_pending_replicated_pg_broadcast() - end - end - - defp flush_pending_replicated_pg_broadcast_barrier( - %{pending_replicated_pg_broadcast_len: 0} = state - ), - do: state - - defp flush_pending_replicated_pg_broadcast_barrier(state), - do: flush_pending_replicated_pg_broadcast(state) - - defp flush_pending_replicated_registry_broadcast_barrier( - %{pending_replicated_registry_broadcast_len: 0} = state - ), - do: state + defp flush_pending_replicated_sender_barrier(%{pending_replica_broadcast_len: 0} = state), + do: state - defp flush_pending_replicated_registry_broadcast_barrier(state), - do: flush_pending_replicated_registry_broadcast(state) + defp flush_pending_replicated_sender_barrier(state), do: flush_pending_replica_broadcast(state) defp process_pg_local_request_batch(state, messages) do %{name: name, shard_index: shard} = state @@ -1395,91 +1672,109 @@ defmodule Group.Replica do {%{}, [], [], [], %{}, MapSet.new()}, fn {reply_to, request}, {entries, replies, events, broadcasts, new_monitors, maybe_demonitor_pids} -> - case request do - {:join, cluster, key, pid, meta} -> - member = {cluster, key, pid} - {initial, current} = local_pg_batch_entry(entries, name, shard, member) - - case current do - nil -> - time = System.system_time() - new_monitors = ensure_local_batch_monitor(state, new_monitors, pid) - - { - Map.put(entries, member, {initial, {meta, time, local_node}}), - [{reply_to, :ok} | replies], - [ - build_event(name, :joined, key, pid, meta, %{ - previous_meta: nil, - cluster: cluster - }) - | events - ], - [ - {:join, cluster, key, pid, meta, time, :join, node(pid)} | broadcasts - ], - new_monitors, - maybe_demonitor_pids - } - - {old_meta, _time, _node} when old_meta == meta -> - {entries, [{reply_to, :ok} | replies], events, broadcasts, new_monitors, - maybe_demonitor_pids} - - {old_meta, _time, _node} -> - time = System.system_time() - - { - Map.put(entries, member, {initial, {meta, time, local_node}}), - [{reply_to, :ok} | replies], - [ - build_event(name, :joined, key, pid, meta, %{ - previous_meta: old_meta, - cluster: cluster - }) - | events - ], - [ - {:join, cluster, key, pid, meta, time, :update, node(pid)} | broadcasts - ], - new_monitors, - maybe_demonitor_pids - } - end - - {:leave, cluster, key, pid} -> - member = {cluster, key, pid} - {initial, current} = local_pg_batch_entry(entries, name, shard, member) - - case current do - nil -> - {entries, [{reply_to, {:error, :not_in_group}} | replies], events, broadcasts, - new_monitors, maybe_demonitor_pids} - - {meta, _time, _node} -> - { - Map.put(entries, member, {initial, nil}), - [{reply_to, :ok} | replies], - [ - build_event(name, :left, key, pid, meta, %{reason: :leave, cluster: cluster}) - | events - ], - [ - {:leave, cluster, key, pid, meta, :leave} | broadcasts - ], - new_monitors, - MapSet.put(maybe_demonitor_pids, pid) - } - end + if validate_local_mutation_epoch(state, request) != :ok do + {entries, [{reply_to, {:error, :stale_cluster_epoch}} | replies], events, broadcasts, + new_monitors, maybe_demonitor_pids} + else + case request do + {:join, cluster, _epoch, key, pid, meta} -> + member = {cluster, key, pid} + {initial, current} = local_pg_batch_entry(entries, name, shard, member) + + case current do + nil -> + time = System.system_time() + new_monitors = ensure_local_batch_monitor(state, new_monitors, pid) + + { + Map.put(entries, member, {initial, {meta, time, local_node}}), + [{reply_to, :ok} | replies], + [ + build_event(name, :joined, key, pid, meta, %{ + previous_meta: nil, + cluster: cluster + }) + | events + ], + [ + {:join, cluster, key, pid, meta, time, :join, node(pid)} | broadcasts + ], + new_monitors, + maybe_demonitor_pids + } + + {old_meta, _time, _node} when old_meta == meta -> + {entries, [{reply_to, :ok} | replies], events, broadcasts, new_monitors, + maybe_demonitor_pids} + + {old_meta, _time, _node} -> + time = System.system_time() + + { + Map.put(entries, member, {initial, {meta, time, local_node}}), + [{reply_to, :ok} | replies], + [ + build_event(name, :joined, key, pid, meta, %{ + previous_meta: old_meta, + cluster: cluster + }) + | events + ], + [ + {:join, cluster, key, pid, meta, time, :update, node(pid)} | broadcasts + ], + new_monitors, + maybe_demonitor_pids + } + end + + {:leave, cluster, _epoch, key, pid} -> + member = {cluster, key, pid} + {initial, current} = local_pg_batch_entry(entries, name, shard, member) + + case current do + nil -> + {entries, [{reply_to, {:error, :not_in_group}} | replies], events, broadcasts, + new_monitors, maybe_demonitor_pids} + + {meta, _time, _node} -> + { + Map.put(entries, member, {initial, nil}), + [{reply_to, :ok} | replies], + [ + build_event(name, :left, key, pid, meta, %{ + reason: :leave, + cluster: cluster + }) + | events + ], + [ + {:leave, cluster, key, pid, meta, :leave} | broadcasts + ], + new_monitors, + MapSet.put(maybe_demonitor_pids, pid) + } + end + end end end ) + sequenced_broadcasts = + broadcasts + |> Enum.reverse() + |> Enum.map(&append_local_replica_record(state, &1)) + {insert_entries, delete_entries} = pg_batch_diff(entries) Data.pg_delete_many(name, shard, delete_entries) Data.pg_insert_many(name, shard, insert_entries) state = finalize_local_batch_monitors(state, new_monitors, maybe_demonitor_pids) - state = send_local_batch_broadcasts(state, broadcasts) + + state = + Enum.reduce(sequenced_broadcasts, state, fn record, acc -> + finish_local_replica_record(acc, record, :pg) + end) + notify_monitors(name, events) reply_local_requests(replies) state @@ -1604,19 +1899,63 @@ defmodule Group.Replica do end defp enqueue_broadcast_op(state, {:register, _cluster, _key, _pid, _meta, _time, _node} = op), - do: enqueue_replicated_registry_broadcast(state, op) + do: sequence_and_enqueue_broadcast(state, op, :registry) defp enqueue_broadcast_op(state, {:unregister, _cluster, _key, _pid, _meta, _reason} = op), - do: enqueue_replicated_registry_broadcast(state, op) + do: sequence_and_enqueue_broadcast(state, op, :registry) defp enqueue_broadcast_op( state, {:join, _cluster, _key, _pid, _meta, _time, _reason, _node} = op ), - do: enqueue_replicated_pg_broadcast(state, op) + do: sequence_and_enqueue_broadcast(state, op, :pg) defp enqueue_broadcast_op(state, {:leave, _cluster, _key, _pid, _meta, _reason} = op), - do: enqueue_replicated_pg_broadcast(state, op) + do: sequence_and_enqueue_broadcast(state, op, :pg) + + defp sequence_and_enqueue_broadcast(state, op, domain) do + record = append_local_replica_record(state, op) + finish_local_replica_record(state, record, domain) + end + + defp append_local_replica_record(state, op) do + cluster = WireProtocol.op_cluster(op) + + case Data.local_stream_id(state.name, state.shard_index, cluster) do + nil -> + nil + + stream_id -> + {seq, mutations} = + Data.append_replica_record(state.name, state.shard_index, stream_id, [op]) + + {:sequenced, stream_id, seq, mutations} + end + end + + defp finish_local_replica_record(state, nil, _domain), do: state + + defp finish_local_replica_record( + state, + {:sequenced, stream_id, seq, mutations} = sequenced, + domain + ) do + apply_registry_claim_mutations(state, stream_id, seq, mutations) + + :ok = Data.mark_local_replica_applied(state.name, state.shard_index, stream_id, seq) + + :ok = + Data.prune_replica_oplog( + state.name, + state.shard_index, + state.replicated_oplog_max_entries + ) + + case domain do + :registry -> enqueue_replicated_registry_broadcast(state, sequenced) + :pg -> enqueue_replicated_pg_broadcast(state, sequenced) + end + end defp reply_local_requests(replies) do Enum.each(Enum.reverse(replies), fn {reply_to, reply} -> @@ -1624,8 +1963,8 @@ defmodule Group.Replica do end) end - defp local_request_domain({:join, _cluster, _key, _pid, _meta}), do: :pg - defp local_request_domain({:leave, _cluster, _key, _pid}), do: :pg + defp local_request_domain({:join, _cluster, _epoch, _key, _pid, _meta}), do: :pg + defp local_request_domain({:leave, _cluster, _epoch, _key, _pid}), do: :pg defp local_request_domain(_request), do: :other defp do_register(state, cluster, key, pid, meta) do @@ -1634,6 +1973,8 @@ defmodule Group.Replica do case Data.registry_lookup(name, shard, cluster, key) do nil -> time = System.system_time() + op = {:register, cluster, key, pid, meta, time, node(pid)} + record = append_local_replica_record(state, op) mref = monitor_pid(state, pid) Data.registry_insert(name, shard, cluster, key, pid, meta, time, node(pid)) @@ -1641,11 +1982,7 @@ defmodule Group.Replica do "#{log_prefix_shard(state)} register key=#{inspect(key)} pid=#{inspect(pid)} cluster=#{inspect(cluster)}" end) - state = - enqueue_replicated_registry_broadcast( - state, - {:register, cluster, key, pid, meta, time, node(pid)} - ) + state = finish_local_replica_record(state, record, :registry) state = put_monitor(state, pid, mref) @@ -1660,17 +1997,15 @@ defmodule Group.Replica do {^pid, old_meta, _time, _node} -> time = System.system_time() + op = {:register, cluster, key, pid, meta, time, node(pid)} + record = append_local_replica_record(state, op) Data.registry_insert(name, shard, cluster, key, pid, meta, time, node(pid)) log_verbose(state, fn -> "#{log_prefix_shard(state)} re-register key=#{inspect(key)} pid=#{inspect(pid)} cluster=#{inspect(cluster)}" end) - state = - enqueue_replicated_registry_broadcast( - state, - {:register, cluster, key, pid, meta, time, node(pid)} - ) + state = finish_local_replica_record(state, record, :registry) event = build_event(name, :registered, key, pid, meta, %{ @@ -1691,6 +2026,8 @@ defmodule Group.Replica do case Data.registry_lookup(name, shard, cluster, key) do {pid, meta, _time, entry_node} when entry_node == node() -> + op = {:unregister, cluster, key, pid, meta, :unregister} + record = append_local_replica_record(state, op) Data.registry_delete(name, shard, cluster, key, pid) state = maybe_demonitor_pid(state, name, shard, pid) @@ -1698,11 +2035,7 @@ defmodule Group.Replica do "#{log_prefix_shard(state)} unregister key=#{inspect(key)} pid=#{inspect(pid)} cluster=#{inspect(cluster)}" end) - state = - enqueue_replicated_registry_broadcast( - state, - {:unregister, cluster, key, pid, meta, :unregister} - ) + state = finish_local_replica_record(state, record, :registry) event = build_event(name, :unregistered, key, pid, meta, %{ @@ -1710,7 +2043,10 @@ defmodule Group.Replica do cluster: cluster }) - notify_monitors(name, [event]) + {state, projection_events} = + reconcile_registry_keys(state, [{cluster, key}], :unregister, []) + + notify_monitors(name, projection_events ++ [event]) {:ok, state} nil -> @@ -1727,6 +2063,8 @@ defmodule Group.Replica do case Data.pg_lookup(name, shard, cluster, key, pid) do nil -> time = System.system_time() + op = {:join, cluster, key, pid, meta, time, :join, node(pid)} + record = append_local_replica_record(state, op) mref = monitor_pid(state, pid) Data.pg_insert(name, shard, cluster, key, pid, meta, time, node(pid)) @@ -1734,11 +2072,7 @@ defmodule Group.Replica do "#{log_prefix_shard(state)} join key=#{inspect(key)} pid=#{inspect(pid)} cluster=#{inspect(cluster)}" end) - state = - enqueue_replicated_pg_broadcast( - state, - {:join, cluster, key, pid, meta, time, :join, node(pid)} - ) + state = finish_local_replica_record(state, record, :pg) state = put_monitor(state, pid, mref) @@ -1753,17 +2087,15 @@ defmodule Group.Replica do {old_meta, _time, _node} -> time = System.system_time() + op = {:join, cluster, key, pid, meta, time, :update, node(pid)} + record = append_local_replica_record(state, op) Data.pg_insert(name, shard, cluster, key, pid, meta, time, node(pid)) log_verbose(state, fn -> "#{log_prefix_shard(state)} re-join key=#{inspect(key)} pid=#{inspect(pid)} cluster=#{inspect(cluster)}" end) - state = - enqueue_replicated_pg_broadcast( - state, - {:join, cluster, key, pid, meta, time, :update, node(pid)} - ) + state = finish_local_replica_record(state, record, :pg) event = build_event(name, :joined, key, pid, meta, %{previous_meta: old_meta, cluster: cluster}) @@ -1781,6 +2113,8 @@ defmodule Group.Replica do {{:error, :not_in_group}, state} {meta, _time, _node} -> + op = {:leave, cluster, key, pid, meta, :leave} + record = append_local_replica_record(state, op) Data.pg_delete(name, shard, cluster, key, pid) state = maybe_demonitor_pid(state, name, shard, pid) @@ -1788,11 +2122,7 @@ defmodule Group.Replica do "#{log_prefix_shard(state)} leave key=#{inspect(key)} pid=#{inspect(pid)} cluster=#{inspect(cluster)}" end) - state = - enqueue_replicated_pg_broadcast( - state, - {:leave, cluster, key, pid, meta, :leave} - ) + state = finish_local_replica_record(state, record, :pg) event = build_event(name, :left, key, pid, meta, %{reason: :leave, cluster: cluster}) notify_monitors(name, [event]) @@ -1800,7 +2130,15 @@ defmodule Group.Replica do end end - defp do_cluster_connect(state, clusters) do + defp do_cluster_connect(state, clusters), + do: + do_cluster_connect( + state, + clusters, + Enum.map(clusters, &{&1, Data.local_cluster_epoch(state.name, &1)}) + ) + + defp do_cluster_connect(state, clusters, epochs) do state = flush_pending_replicated_sender_barrier(state) %{name: name} = state @@ -1811,15 +2149,74 @@ defmodule Group.Replica do peers = Data.cluster_nodes(name, nil) -- [node()] for target_node <- peers do - send_remote_shard_message(state, target_node, {:cluster_connect, clusters, self()}) + send_remote_shard_message( + state, + target_node, + {:replica_cluster_open, self(), Data.generation(name), + Data.local_cluster_epoch_revision(name), epochs} + ) end {:ok, state} end - defp do_cluster_disconnect(state, clusters) do + defp do_cluster_disconnect(state, clusters), + do: + do_cluster_disconnect( + state, + clusters, + Enum.map(clusters, &{&1, Data.closed_local_cluster_epoch(state.name, &1)}) + ) + + defp do_cluster_disconnect(state, _clusters, epochs) do + epochs = + Enum.filter(epochs, fn + {cluster, epoch} when not is_nil(epoch) -> + Data.closed_local_cluster_pending?( + state.name, + cluster, + epoch, + state.shard_index + ) + + _ -> + false + end) + + case epochs do + [] -> {:ok, state} + epochs -> do_cluster_disconnect_epochs(state, epochs) + end + end + + defp do_cluster_disconnect_epochs(state, epochs) do state = flush_pending_replicated_sender_barrier(state) %{name: name, shard_index: shard} = state + clusters = Enum.map(epochs, &elem(&1, 0)) + + closed_streams = + Enum.flat_map(epochs, fn + {cluster, epoch} when not is_nil(epoch) -> + [ + WireProtocol.stream_id( + name, + node(), + Data.generation(name), + shard, + cluster, + epoch + ) + ] + + _ -> + [] + end) + + state = discard_snapshot_send_offsets_for_streams(state, closed_streams) + + # See the nodedown ordering note: restart repair rejects remote rows whose + # receive cursor was cleared before this cluster-wide purge. + :ok = Data.delete_replica_cursors_for_clusters(name, shard, clusters) log_once(state, fn -> "#{log_prefix(state)} cluster_disconnect #{inspect(clusters)}" @@ -1827,6 +2224,7 @@ defmodule Group.Replica do {events, local_pids} = Enum.reduce(clusters, {[], MapSet.new()}, fn cluster, {events, local_pids} -> + _affected_keys = Data.purge_registry_claims_for_cluster(name, shard, cluster) {purged_reg, purged_pg} = purge_cluster_entries(name, shard, cluster, :all) local_pids = @@ -1847,10 +2245,27 @@ defmodule Group.Replica do maybe_demonitor_pid(acc, name, shard, pid) end) + # Disconnect purges every origin's materialized rows for these clusters. + # Forget their receive cursors as well: if this node later reconnects while + # a remote origin kept the same epoch, its advertised head must rebuild the + # rows instead of being mistaken for data we still retain. if shard == 0 do - broadcast_to_peers(state, {:cluster_disconnect, clusters, self()}) + broadcast_to_peers( + state, + {:replica_cluster_close, self(), Data.generation(name), + Data.local_cluster_epoch_revision(name), epochs} + ) end + Enum.each(epochs, fn + {cluster, epoch} when not is_nil(epoch) -> + Data.drop_local_stream(name, shard, cluster, epoch) + + _ -> + :ok + end) + + _completed_clusters = Data.mark_closed_cluster_shard(name, epochs, shard) notify_monitors(name, events) {:ok, state} end @@ -1919,66 +2334,35 @@ defmodule Group.Replica do end defp enqueue_replicated_pg_broadcast(state, op), - do: enqueue_replicated_pg_broadcasts(state, [op]) + do: enqueue_replica_broadcasts(state, [op]) + + defp enqueue_replicated_registry_broadcast(state, op), + do: enqueue_replica_broadcasts(state, [op]) - defp enqueue_replicated_pg_broadcasts(state, ops) do - state = flush_pending_replicated_registry_broadcast_barrier(state) + defp enqueue_replica_broadcasts(state, ops) do now = System.monotonic_time(:millisecond) ops_len = length(ops) state = - case state.pending_replicated_pg_broadcast_len do + case state.pending_replica_broadcast_len do 0 -> - %{state | pending_replicated_pg_broadcast_started_at: now} - |> schedule_replicated_pg_broadcast_flush() - |> Map.put(:pending_replicated_pg_broadcast_ops, Enum.reverse(ops)) - |> Map.put(:pending_replicated_pg_broadcast_len, ops_len) + %{state | pending_replica_broadcast_started_at: now} + |> schedule_replica_broadcast_flush() + |> Map.put(:pending_replica_broadcast_ops, Enum.reverse(ops)) + |> Map.put(:pending_replica_broadcast_len, ops_len) len -> %{ state - | pending_replicated_pg_broadcast_ops: - Enum.reverse(ops, state.pending_replicated_pg_broadcast_ops), - pending_replicated_pg_broadcast_len: len + ops_len + | pending_replica_broadcast_ops: + Enum.reverse(ops, state.pending_replica_broadcast_ops), + pending_replica_broadcast_len: len + ops_len } end - if state.pending_replicated_pg_broadcast_len >= state.replicated_sender_buffer_size or - pending_replicated_pg_broadcast_due?(state, now) do - flush_pending_replicated_pg_broadcast(state) - else - state - end - end - - defp enqueue_replicated_registry_broadcast(state, op), - do: enqueue_replicated_registry_broadcasts(state, [op]) - - defp enqueue_replicated_registry_broadcasts(state, ops) do - state = flush_pending_replicated_pg_broadcast_barrier(state) - now = System.monotonic_time(:millisecond) - ops_len = length(ops) - - state = - case state.pending_replicated_registry_broadcast_len do - 0 -> - %{state | pending_replicated_registry_broadcast_started_at: now} - |> schedule_replicated_registry_broadcast_flush() - |> Map.put(:pending_replicated_registry_broadcast_ops, Enum.reverse(ops)) - |> Map.put(:pending_replicated_registry_broadcast_len, ops_len) - - len -> - %{ - state - | pending_replicated_registry_broadcast_ops: - Enum.reverse(ops, state.pending_replicated_registry_broadcast_ops), - pending_replicated_registry_broadcast_len: len + ops_len - } - end - - if state.pending_replicated_registry_broadcast_len >= state.replicated_sender_buffer_size or - pending_replicated_registry_broadcast_due?(state, now) do - flush_pending_replicated_registry_broadcast(state) + if state.pending_replica_broadcast_len >= state.replicated_sender_buffer_size or + pending_replica_broadcast_due?(state, now) do + flush_pending_replica_broadcast(state) else state end @@ -1997,53 +2381,80 @@ defmodule Group.Replica do |> take_one_local_request_turn() end - defp take_priority_control_turn(state) do + defp take_priority_control_turn(state), + do: take_priority_control_turn(state, @priority_control_quota) + + defp take_priority_control_turn(state, 0), do: state + + defp take_priority_control_turn(state, remaining) do receive do {:peer_connect, _remote_pid, _remote_shard_index, _remote_num_shards, _remote_clusters} = msg -> state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) + take_priority_control_turn(state, remaining - 1) {:peer_connect_ack, _remote_pid, _remote_shard_index, _remote_num_shards, _remote_clusters} = msg -> state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) + take_priority_control_turn(state, remaining - 1) + + {:replica_hello, _remote_pid, _version, _generation, _epoch_revision, _cluster_epochs, + _transport_id, _descriptor} = msg -> + state = process_inline_priority_message(state, msg) + take_priority_control_turn(state, remaining - 1) + + {:replica_lane_hello, _remote_pid, _version, _generation, _epoch_revision, _transport_id, + _descriptor} = msg -> + state = process_inline_priority_message(state, msg) + take_priority_control_turn(state, remaining - 1) + + {:replica_authority_installed_local, _remote_node, _generation, _epoch_revision, + _old_generation, _stale_epochs} = msg -> + state = process_inline_priority_message(state, msg) + take_priority_control_turn(state, remaining - 1) + + {:replica_authority_dirty_local, _remote_node} = msg -> + state = process_inline_priority_message(state, msg) + take_priority_control_turn(state, remaining - 1) - {:cluster_connect, _clusters, _remote_pid} = msg -> + {:replica_cluster_open_control_local, _remote_node, _generation, _revision, _epochs, _stale, + _shared} = msg -> state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) + take_priority_control_turn(state, remaining - 1) - {:cluster_connect_ack, _clusters, _remote_pid, _cluster_data} = msg -> + {:replica_cluster_close_control_local, _remote_node, _generation, _revision, _closed} = + msg -> state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) + take_priority_control_turn(state, remaining - 1) - {:cluster_disconnect, _clusters, _remote_pid} = msg -> + {:replica_heartbeat, _remote_pid, _version, _generation, _epoch_revision, _transport_id, + _transport_descriptor} = msg -> state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) + take_priority_control_turn(state, remaining - 1) - {:cluster_state, _cluster, _reg_data, _pg_data} = msg -> + {:replica_hello_request, _remote_pid} = msg -> state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) + take_priority_control_turn(state, remaining - 1) - {:send_cluster_data, _clusters, _target_node} = msg -> + {:replica_cluster_open, _remote_pid, _generation, _revision, _epochs} = msg -> state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) + take_priority_control_turn(state, remaining - 1) - {:replicate_process_down_batch, _reg_entries, _pg_entries} = msg -> + {:replica_cluster_close, _remote_pid, _generation, _revision, _epochs} = msg -> state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) + take_priority_control_turn(state, remaining - 1) {:DOWN, _mref, :process, _pid, _reason} = msg -> state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) + take_priority_control_turn(state, remaining - 1) {:nodeup, _remote_node} = msg -> state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) + take_priority_control_turn(state, remaining - 1) {:nodedown, _remote_node} = msg -> state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) + take_priority_control_turn(state, remaining - 1) after 0 -> state @@ -2058,6 +2469,9 @@ defmodule Group.Replica do {@local_request_tag, alias_ref, request} when is_reference(alias_ref) -> process_local_request_turn(state, [{{:alias, alias_ref}, request}]) + + {@local_request_tag, :noreply, request} -> + process_local_request_turn(state, [{:noreply, request}]) after 0 -> state @@ -2098,38 +2512,20 @@ defmodule Group.Replica do %{state | pending_replicated_registry_flush_ref: flush_ref} end - defp schedule_replicated_pg_broadcast_flush(%{replicated_sender_flush_interval: 0} = state) do - %{state | pending_replicated_pg_broadcast_flush_ref: nil} - end - - defp schedule_replicated_pg_broadcast_flush(state) do - flush_ref = make_ref() - - Process.send_after( - self(), - {@replicated_pg_broadcast_flush_timer, flush_ref}, - state.replicated_sender_flush_interval - ) - - %{state | pending_replicated_pg_broadcast_flush_ref: flush_ref} - end - - defp schedule_replicated_registry_broadcast_flush( - %{replicated_sender_flush_interval: 0} = state - ) do - %{state | pending_replicated_registry_broadcast_flush_ref: nil} + defp schedule_replica_broadcast_flush(%{replicated_sender_flush_interval: 0} = state) do + %{state | pending_replica_broadcast_flush_ref: nil} end - defp schedule_replicated_registry_broadcast_flush(state) do + defp schedule_replica_broadcast_flush(state) do flush_ref = make_ref() Process.send_after( self(), - {@replicated_registry_broadcast_flush_timer, flush_ref}, + {@replica_broadcast_flush_timer, flush_ref}, state.replicated_sender_flush_interval ) - %{state | pending_replicated_registry_broadcast_flush_ref: flush_ref} + %{state | pending_replica_broadcast_flush_ref: flush_ref} end defp pending_replicated_pg_due?(%{pending_replicated_pg_len: 0}, _now), do: false @@ -2147,24 +2543,12 @@ defmodule Group.Replica do state.replicated_registry_receiver_flush_interval end - defp pending_replicated_pg_broadcast_due?(%{pending_replicated_pg_broadcast_len: 0}, _now), + defp pending_replica_broadcast_due?(%{pending_replica_broadcast_len: 0}, _now), do: false - defp pending_replicated_pg_broadcast_due?(state, now) do - state.replicated_sender_flush_interval == 0 or - now - state.pending_replicated_pg_broadcast_started_at >= - state.replicated_sender_flush_interval - end - - defp pending_replicated_registry_broadcast_due?( - %{pending_replicated_registry_broadcast_len: 0}, - _now - ), - do: false - - defp pending_replicated_registry_broadcast_due?(state, now) do + defp pending_replica_broadcast_due?(state, now) do state.replicated_sender_flush_interval == 0 or - now - state.pending_replicated_registry_broadcast_started_at >= + now - state.pending_replica_broadcast_started_at >= state.replicated_sender_flush_interval end @@ -2222,47 +2606,24 @@ defmodule Group.Replica do } end - defp flush_pending_replicated_pg_broadcast(%{pending_replicated_pg_broadcast_len: 0} = state), + defp flush_pending_replica_broadcast(%{pending_replica_broadcast_len: 0} = state), do: state - defp flush_pending_replicated_pg_broadcast(state) do - ops = Enum.reverse(state.pending_replicated_pg_broadcast_ops) - - log_verbose(state, fn -> - "#{log_prefix_shard(state)} flush_replicated_pg_broadcast_buffer ops=#{length(ops)}" - end) - - send_replicated_pg_batches(state, ops) - - %{ - state - | pending_replicated_pg_broadcast_ops: [], - pending_replicated_pg_broadcast_len: 0, - pending_replicated_pg_broadcast_started_at: nil, - pending_replicated_pg_broadcast_flush_ref: nil - } - end - - defp flush_pending_replicated_registry_broadcast( - %{pending_replicated_registry_broadcast_len: 0} = state - ), - do: state - - defp flush_pending_replicated_registry_broadcast(state) do - ops = Enum.reverse(state.pending_replicated_registry_broadcast_ops) + defp flush_pending_replica_broadcast(state) do + ops = Enum.reverse(state.pending_replica_broadcast_ops) log_verbose(state, fn -> - "#{log_prefix_shard(state)} flush_replicated_registry_broadcast_buffer ops=#{length(ops)}" + "#{log_prefix_shard(state)} flush_replica_broadcast_buffer ops=#{length(ops)}" end) - send_replicated_registry_batches(state, ops) + send_replicated_batches(state, ops) %{ state - | pending_replicated_registry_broadcast_ops: [], - pending_replicated_registry_broadcast_len: 0, - pending_replicated_registry_broadcast_started_at: nil, - pending_replicated_registry_broadcast_flush_ref: nil + | pending_replica_broadcast_ops: [], + pending_replica_broadcast_len: 0, + pending_replica_broadcast_started_at: nil, + pending_replica_broadcast_flush_ref: nil } end @@ -2381,22 +2742,28 @@ defmodule Group.Replica do member = {cluster, key, pid} {initial, current} = replicated_pg_entry(entries, name, shard, member) - previous_meta = - case {current, reason} do - {{old_meta, _old_time, _old_node}, :update} -> old_meta - _ -> nil - end + case current do + {^meta, ^time, ^entry_node} -> + {entries, events} + + _ -> + previous_meta = + case {current, reason} do + {{old_meta, _old_time, _old_node}, :update} -> old_meta + _ -> nil + end - event = - build_event(name, :joined, key, pid, meta, %{ - previous_meta: previous_meta, - cluster: cluster - }) + event = + build_event(name, :joined, key, pid, meta, %{ + previous_meta: previous_meta, + cluster: cluster + }) - updated_entries = - Map.put(entries, member, {initial, {meta, time, entry_node}}) + updated_entries = + Map.put(entries, member, {initial, {meta, time, entry_node}}) - {updated_entries, [event | events]} + {updated_entries, [event | events]} + end {:leave, cluster, key, pid, meta, reason}, {entries, events} -> member = {cluster, key, pid} @@ -2477,20 +2844,33 @@ defmodule Group.Replica do end) end - defp send_replicated_pg_batches(state, ops) do + defp send_replicated_batches(state, ops) do ops - |> group_broadcast_ops_by_target(state, &pg_op_cluster/1) + |> group_broadcast_ops_by_target(state, &sequenced_op_cluster/1) |> Enum.each(fn {target_node, target_ops} -> - send_remote_shard_message(state, target_node, {:replicate_pg_batch, target_ops}) + send_replica_delta_batch(state, target_node, target_ops) end) end - defp send_replicated_registry_batches(state, ops) do - ops - |> group_broadcast_ops_by_target(state, ®istry_op_cluster/1) - |> Enum.each(fn {target_node, target_ops} -> - send_remote_shard_message(state, target_node, {:replicate_registry_batch, target_ops}) - end) + defp send_replica_delta_batch(state, target_node, sequenced_ops) do + runs = + sequenced_ops + |> Enum.group_by(fn {:sequenced, stream_id, _seq, _mutations} -> stream_id end) + |> Enum.map(fn {stream_id, records} -> + records = + records + |> Enum.map(fn {:sequenced, ^stream_id, seq, mutations} -> {seq, mutations} end) + |> Enum.sort_by(&elem(&1, 0)) + + {first_seq, _mutations} = hd(records) + + {_floor, head, _applied} = + Data.replica_stream_head(state.name, state.shard_index, stream_id) + + {stream_id, first_seq, records, head} + end) + + outgoing_replica_message(state, target_node, {:delta_batch, WireProtocol.version(), runs}) end defp group_broadcast_ops_by_target(ops, state, cluster_fun) do @@ -2521,7 +2901,7 @@ defmodule Group.Replica do target_nodes = case cluster do nil -> - for {target_node, _pid} <- state.remote_shards, do: target_node + for {target_node, _last_seen} <- state.peer_last_seen, do: target_node _cluster -> for target_node <- Data.cluster_nodes(state.name, cluster), @@ -2538,11 +2918,19 @@ defmodule Group.Replica do defp pg_op_cluster({:leave, cluster, _key, _pid, _meta, _reason}), do: cluster + defp pg_op_cluster({:sequenced, _stream_id, _seq, [op | _]}), do: pg_op_cluster(op) + defp registry_op_cluster({:register, cluster, _key, _pid, _meta, _time, _entry_node}), do: cluster defp registry_op_cluster({:unregister, cluster, _key, _pid, _meta, _reason}), do: cluster + defp registry_op_cluster({:sequenced, _stream_id, _seq, [op | _]}), + do: registry_op_cluster(op) + + defp sequenced_op_cluster({:sequenced, stream_id, _seq, _mutations}), + do: WireProtocol.stream_cluster(stream_id) + defp replicated_op_for_active_cluster?(name, op, cluster_fun) when is_function(cluster_fun, 1) do case cluster_fun.(op) do @@ -2557,7 +2945,7 @@ defmodule Group.Replica do defp broadcast_to_peers(state, message) do for {target_node, _pid} <- state.remote_shards do - send_remote_shard_message(state, target_node, message) + send_remote_control_message(state, target_node, message) end end @@ -2569,177 +2957,2120 @@ defmodule Group.Replica do :ok false -> - Group.PeerReconnect.busy_link(state.name, target_node) - :ok + :busy end end - defp broadcast_process_down_batch(state, reason_by_pid, reg_entries, pg_entries) do - messages = - Enum.reduce(reg_entries, %{}, fn {pid, cluster, key, meta}, acc -> - accumulate_process_down_entry( - acc, - process_down_targets(state, cluster), - {:reg, pid, cluster, key, meta, Map.fetch!(reason_by_pid, pid)} + defp send_remote_control_message(state, target_node, message) do + control_name = shard_name(state.name, 0) + + case :erlang.send_nosuspend({control_name, target_node}, message, [:noconnect]) do + true -> :ok + false -> :busy + end + end + + defp outgoing_replica_message(state, target_node, message) do + case state.replica_transport.outgoing( + state.name, + target_node, + state.shard_index, + message, + state.replica_transport_opts + ) do + :ok -> :ok + :busy -> :ok + :disconnected -> :ok + end + + state + end + + defp install_replica_authority( + state, + remote_pid, + generation, + epoch_revision, + cluster_epochs, + _transport_id, + transport_descriptor + ) do + remote_node = node(remote_pid) + + case Data.put_remote_replica_info( + state.name, + 0, + remote_node, + generation, + epoch_revision, + cluster_epochs + ) do + :stale -> + state + |> mark_cluster_control_dirty(remote_node) + |> request_replica_authority(remote_node) + + {old_generation, stale_epochs} -> + finish_replica_authority_install( + state, + remote_pid, + remote_node, + generation, + epoch_revision, + transport_descriptor, + old_generation, + stale_epochs ) - end) - |> then(fn acc -> - Enum.reduce(pg_entries, acc, fn {pid, cluster, key, meta}, inner -> - accumulate_process_down_entry( - inner, - process_down_targets(state, cluster), - {:pg, pid, cluster, key, meta, Map.fetch!(reason_by_pid, pid)} - ) - end) - end) + end + end - Enum.each(messages, fn {target_node, {reg_entries, pg_entries}} -> - send_remote_shard_message( + defp finish_replica_authority_install( + state, + remote_pid, + remote_node, + generation, + epoch_revision, + transport_descriptor, + old_generation, + stale_epochs + ) do + state = maybe_purge_remote_generation(state, remote_node, old_generation, generation) + state = notify_replica_transport_peer_up(state, remote_node, transport_descriptor) + + state = + if old_generation == generation do + state + |> purge_closed_remote_epochs(remote_node, stale_epochs) + |> purge_remote_streams_outside_authority(remote_node) + else + state + end + + state = install_replica_view(state, remote_node, generation) + + if replica_authority_current?(state, remote_node, generation, epoch_revision) and + replica_view_current?(state, remote_node) do + state = %{ + state + | remote_shards: Map.put(state.remote_shards, remote_node, remote_pid), + peer_last_seen: Map.put(state.peer_last_seen, remote_node, monotonic_millis()), + cluster_control_dirty: Map.delete(state.cluster_control_dirty, remote_node) + } + + fan_out_to_siblings( state, - target_node, - {:replicate_process_down_batch, Enum.reverse(reg_entries), Enum.reverse(pg_entries)} + {:replica_authority_installed_local, remote_node, generation, epoch_revision, + old_generation, stale_epochs} ) - end) - end - defp process_down_targets(state, nil) do - for {target_node, _pid} <- state.remote_shards, do: target_node + send_replica_heads(state, remote_node) + else + state + |> mark_cluster_control_dirty(remote_node) + |> request_replica_authority(remote_node) + end end - defp process_down_targets(%{name: name}, cluster) do - for target_node <- Data.cluster_nodes(name, cluster), target_node != node(), do: target_node + defp notify_replica_transport_peer_up(state, remote_node, transport_descriptor) do + if function_exported?(state.replica_transport, :peer_up, 5) do + :ok = + state.replica_transport.peer_up( + state.name, + remote_node, + state.shard_index, + transport_descriptor, + state.replica_transport_opts + ) + end + + state end - defp accumulate_process_down_entry(acc, target_nodes, {:reg, pid, cluster, key, meta, reason}) do - Enum.reduce(target_nodes, acc, fn target_node, inner -> - Map.update( - inner, + defp send_replica_hello(state, target_node) do + descriptor = state.replica_transport.descriptor(state.name, state.replica_transport_opts) + + if state.shard_index == 0 do + {generation, epoch_revision, cluster_epochs} = + Data.local_replica_authority(state.name) + + send_remote_control_message( + state, target_node, - {[{pid, cluster, key, meta, reason}], []}, - fn {reg_entries, pg_entries} -> - {[{pid, cluster, key, meta, reason} | reg_entries], pg_entries} - end + {:replica_hello, self(), WireProtocol.version(), generation, epoch_revision, + cluster_epochs, state.replica_transport.id(), descriptor} ) - end) - end - - defp accumulate_process_down_entry(acc, target_nodes, {:pg, pid, cluster, key, meta, reason}) do - Enum.reduce(target_nodes, acc, fn target_node, inner -> - Map.update( - inner, + else + send_remote_shard_message( + state, target_node, - {[], [{pid, cluster, key, meta, reason}]}, - fn {reg_entries, pg_entries} -> - {reg_entries, [{pid, cluster, key, meta, reason} | pg_entries]} - end + {:replica_lane_hello, self(), WireProtocol.version(), Data.generation(state.name), + Data.local_cluster_epoch_revision(state.name), state.replica_transport.id(), descriptor} ) - end) + end + + state end - defp collect_local_process_downs(acc, monitors, 0), do: {Enum.reverse(acc), monitors} + defp request_replica_authority(state, remote_node) do + send_remote_control_message(state, remote_node, {:replica_hello_request, self()}) + state + end - defp collect_local_process_downs(acc, monitors, remaining) do - receive do - {:DOWN, _mref, :process, pid, reason} when is_map_key(monitors, pid) -> - collect_local_process_downs([{pid, reason} | acc], monitors, remaining - 1) - after - 0 -> - {Enum.reverse(acc), monitors} + defp replica_authority_current?(state, remote_node, generation, epoch_revision) do + Data.remote_generation(state.name, remote_node) == generation and + Data.remote_cluster_epoch_observed_revision(state.name, remote_node) == epoch_revision and + Data.remote_replica_authority_hint(state.name, remote_node) == + {generation, epoch_revision} + end + + defp replica_exact_authority_current?(state, remote_node, generation, epoch_revision) do + replica_authority_current?(state, remote_node, generation, epoch_revision) and + Data.remote_cluster_epoch_exact_revision(state.name, remote_node) == epoch_revision + end + + defp replica_view_current?(state, remote_node) do + known_generation = Data.remote_generation(state.name, remote_node) + observed_revision = Data.remote_cluster_epoch_observed_revision(state.name, remote_node) + + Data.remote_replica_authority_hint(state.name, remote_node) == + {known_generation, observed_revision} and + Data.remote_cluster_epoch_revision(state.name, remote_node) == observed_revision and + Data.remote_view_generation(state.name, state.shard_index, remote_node) == + known_generation and + Data.remote_view_cluster_epoch_revision(state.name, state.shard_index, remote_node) == + Data.remote_cluster_epoch_exact_revision(state.name, remote_node) and + Data.remote_view_observed_revision(state.name, state.shard_index, remote_node) == + observed_revision + end + + defp observe_replica_authority_hint(state, remote_node, generation, epoch_revision) do + if WireProtocol.valid_generation?(generation) and is_integer(epoch_revision) and + epoch_revision >= 0 and + Data.observe_remote_replica_hint( + state.name, + remote_node, + generation, + epoch_revision + ) do + # A heartbeat or lane hello can outrun a dropped authority control. Data + # has already fenced every affected lane; retain an exact-hello retry + # obligation independently of this one best-effort request. + state + |> ensure_replica_peer_retirement_deadline(remote_node) + |> mark_authority_dirty(remote_node) + |> request_replica_authority(remote_node) + else + state end end - defp build_process_down_events(name, reg_entries, pg_entries, reason_by_pid) do - events = - Enum.reduce(reg_entries, [], fn {pid, cluster, key, meta}, acc -> - [ - build_event(name, :unregistered, key, pid, meta, %{ - reason: Map.fetch!(reason_by_pid, pid), - cluster: cluster - }) - | acc - ] - end) + defp install_replica_view(state, remote_node, generation) do + case Data.put_remote_view_info( + state.name, + state.shard_index, + remote_node, + generation, + Data.remote_cluster_epoch_exact_revision(state.name, remote_node), + Data.remote_cluster_epoch_observed_revision(state.name, remote_node) + ) do + :ok -> + reproject_pending_registry_keys(state, remote_node) - Enum.reduce(pg_entries, events, fn {pid, cluster, key, meta}, acc -> - [ - build_event(name, :left, key, pid, meta, %{ - reason: Map.fetch!(reason_by_pid, pid), - cluster: cluster - }) - | acc - ] - end) + :stale -> + state + |> mark_authority_dirty(remote_node) + |> request_replica_authority(remote_node) + end end - defp build_process_down_batch_events(name, reg_entries, pg_entries) do - events = - Enum.reduce(reg_entries, [], fn {pid, cluster, key, meta, reason}, acc -> - [ - build_event(name, :unregistered, key, pid, meta, %{reason: reason, cluster: cluster}) - | acc - ] - end) + defp install_current_replica_lane(state, remote_node, generation) do + old_generation = + Data.remote_view_generation(state.name, state.shard_index, remote_node) - Enum.reduce(pg_entries, events, fn {pid, cluster, key, meta, reason}, acc -> - [build_event(name, :left, key, pid, meta, %{reason: reason, cluster: cluster}) | acc] - end) + state = maybe_purge_remote_generation(state, remote_node, old_generation, generation) + state = purge_remote_streams_outside_authority(state, remote_node) + state = install_replica_view(state, remote_node, generation) + + if replica_view_current?(state, remote_node) do + state + |> touch_replica_peer(remote_node) + |> Map.update!(:cluster_control_dirty, &Map.delete(&1, remote_node)) + |> send_replica_heads(remote_node) + else + state + end end - defp fan_out_to_siblings(state, message) do - %{name: name, shard_index: shard_index, num_shards: num_shards} = state + defp schedule_anti_entropy(state) do + ref = make_ref() - for i <- 0..(num_shards - 1), i != shard_index do - send(shard_name(name, i), message) - end + Process.send_after( + self(), + {@anti_entropy_timer, ref}, + state.replicated_anti_entropy_interval + ) + + %{state | anti_entropy_ref: ref} end - defp monitor_pid(state, pid) do - Map.get(state.monitors, pid) || Process.monitor(pid) + defp monotonic_millis, do: System.monotonic_time(:millisecond) + + defp put_remote_shard(state, remote_node, remote_pid) do + %{state | remote_shards: Map.put(state.remote_shards, remote_node, remote_pid)} end - defp put_monitor(state, pid, mref) do - %{state | monitors: Map.put_new(state.monitors, pid, mref)} + defp touch_replica_peer(state, remote_node) do + %{state | peer_last_seen: Map.put(state.peer_last_seen, remote_node, monotonic_millis())} end - defp maybe_demonitor_pid(state, name, shard, pid) do - case Data.maybe_demonitor(name, shard, pid) do - :ok -> - case Map.pop(state.monitors, pid) do - {nil, monitors} -> - %{state | monitors: monitors} + defp ensure_replica_peer_retirement_deadline(state, remote_node) do + %{ + state + | peer_last_seen: Map.put_new(state.peer_last_seen, remote_node, monotonic_millis()) + } + end - {mref, monitors} -> - Process.demonitor(mref, [:flush]) - %{state | monitors: monitors} - end + defp collect_replica_cluster_controls(_tag, _remote_pid, _generation, acc, 0), + do: Enum.reverse(acc) - :still_monitored -> - state + defp collect_replica_cluster_controls(tag, remote_pid, generation, acc, remaining) do + receive do + {^tag, ^remote_pid, ^generation, revision, epochs} -> + collect_replica_cluster_controls( + tag, + remote_pid, + generation, + [{revision, epochs} | acc], + remaining - 1 + ) + after + 0 -> Enum.reverse(acc) end end - defp rebuild_monitors(state) do - %{name: name, shard_index: shard} = state - local = node() + defp accepted_replica_cluster_epochs(state, remote_node, generation, controls) do + case { + Data.remote_replica_authority_hint(state.name, remote_node), + Data.remote_view_generation(state.name, state.shard_index, remote_node) + } do + {{^generation, _hinted_revision}, ^generation} -> + expected_revision = + Data.remote_cluster_epoch_observed_revision(state.name, remote_node) - # Scan all entries in this shard's tables and re-establish monitors - # Only monitor local pids — remote pids are cleaned up by their owning - # node's DOWN handler (broadcast) or by nodedown. - reg_table = Data.reg_by_pid_table(name, shard) + accepted = + Enum.filter(controls, fn {revision, _epochs} -> + is_nil(expected_revision) or revision > expected_revision + end) + |> Enum.sort_by(&elem(&1, 0)) + |> Enum.uniq_by(&elem(&1, 0)) - monitors = - try do - :ets.foldl( - fn {{pid, _cluster, _key}, _meta, _time, entry_node}, acc -> - if entry_node == local and not Map.has_key?(acc, pid) do - mref = Process.monitor(pid) - Map.put(acc, pid, mref) + case accepted do + [] -> + :stale + + accepted -> + next_revision = (expected_revision || -1) + 1 + + if contiguous_cluster_controls?(accepted, next_revision) do + observed_revision = accepted |> List.last() |> elem(0) + + epochs = + accepted + |> Enum.flat_map(&elem(&1, 1)) + |> Map.new() + |> Map.to_list() + + {:accept, expected_revision, observed_revision, epochs} else - acc + {:gap, accepted |> List.last() |> elem(0)} end - end, - %{}, - reg_table + end + + _other_authority -> + :refresh + end + end + + defp contiguous_cluster_controls?(controls, expected_revision) do + Enum.reduce_while(controls, expected_revision, fn + {^expected_revision, _epochs}, ^expected_revision -> {:cont, expected_revision + 1} + _control, _expected -> {:halt, false} + end) != false + end + + defp mark_cluster_control_dirty(state, remote_node) do + %{ + state + | cluster_control_dirty: + Map.put(state.cluster_control_dirty, remote_node, monotonic_millis()) + } + end + + defp mark_authority_dirty(%{shard_index: 0} = state, remote_node) do + mark_cluster_control_dirty(state, remote_node) + end + + defp mark_authority_dirty(state, remote_node) do + if MapSet.member?(state.authority_dirty_notified, remote_node) do + state + else + case send_local_control_message(state, {:replica_authority_dirty_local, remote_node}) do + :ok -> + %{ + state + | authority_dirty_notified: MapSet.put(state.authority_dirty_notified, remote_node) + } + + :disconnected -> + state + end + end + end + + defp request_quiet_cluster_hellos(state) do + now = monotonic_millis() + + dirty = + Enum.reduce(state.cluster_control_dirty, %{}, fn {remote_node, last_activity}, acc -> + cond do + is_nil(Data.remote_generation(state.name, remote_node)) and + is_nil(Data.remote_replica_authority_hint(state.name, remote_node)) -> + acc + + now - last_activity >= state.replicated_anti_entropy_interval -> + request_replica_authority(state, remote_node) + Map.put(acc, remote_node, now) + + true -> + Map.put(acc, remote_node, last_activity) + end + end) + + %{state | cluster_control_dirty: dirty} + end + + defp broadcast_replica_heartbeats(state) do + transport_id = state.replica_transport.id() + descriptor = state.replica_transport.descriptor(state.name, state.replica_transport_opts) + + Enum.reduce(state.remote_shards, state, fn {target_node, _pid}, acc -> + send_remote_shard_message( + acc, + target_node, + {:replica_heartbeat, self(), WireProtocol.version(), Data.generation(acc.name), + Data.local_cluster_epoch_revision(acc.name), transport_id, descriptor} + ) + + acc + end) + end + + defp probe_replica_peers(state) do + Enum.each(Node.list(), fn remote_node -> + unless Map.has_key?(state.remote_shards, remote_node) do + send_remote_shard_message( + state, + remote_node, + {:peer_connect, self(), state.shard_index, state.num_shards, + Data.my_clusters(state.name)} + ) + end + end) + + state + end + + defp broadcast_replica_heads(state) do + peers = Map.keys(state.peer_last_seen) + + heads_by_target = + state.name + |> Data.replica_stream_heads(state.shard_index) + |> Enum.reduce(%{}, fn {stream_id, _floor, _head} = head, acc -> + if current_local_replica_stream?(state, stream_id) do + targets = + case WireProtocol.stream_cluster(stream_id) do + nil -> + peers + + cluster -> + state.name + |> Data.cluster_nodes(cluster) + |> Enum.filter(&Map.has_key?(state.peer_last_seen, &1)) + end + + Enum.reduce(targets, acc, fn target_node, inner -> + Map.update(inner, target_node, [head], &[head | &1]) + end) + else + acc + end + end) + + Enum.reduce(heads_by_target, state, fn {target_node, heads}, acc -> + outgoing_replica_message( + acc, + target_node, + {:heads, WireProtocol.version(), Enum.reverse(heads)} + ) + end) + end + + defp expire_stale_replica_peers(state) do + now = monotonic_millis() + + Enum.reduce(state.peer_last_seen, state, fn {remote_node, last_seen}, acc -> + if now - last_seen > acc.replicated_peer_lease_timeout do + expire_replica_peer(acc, remote_node) + else + acc + end + end) + end + + defp expire_stale_snapshot_transfers(%{snapshot_transfers: transfers} = state) + when map_size(transfers) == 0, + do: state + + defp expire_stale_snapshot_transfers(state) do + now = monotonic_millis() + + Enum.reduce(state.snapshot_transfers, state, fn {key, transfer}, acc -> + if now - transfer.last_progress > acc.replicated_peer_lease_timeout do + discard_snapshot_transfer(acc, key) + else + acc + end + end) + end + + defp discard_snapshot_transfer(state, key) do + case Map.pop(state.snapshot_transfers, key) do + {nil, transfers} -> + %{state | snapshot_transfers: transfers} + + {transfer, transfers} -> + :ok = Snapshot.clear_staging_table(transfer.table) + :ok = Snapshot.clear_staging_table(transfer.events) + + %{ + state + | snapshot_transfers: transfers, + snapshot_staging_pool: [ + {transfer.table, transfer.events} | state.snapshot_staging_pool + ] + } + end + end + + defp discard_snapshot_transfers_for_source(state, source_node) do + Enum.reduce(state.snapshot_transfers, state, fn + {{^source_node, _stream_id} = key, _transfer}, acc -> + discard_snapshot_transfer(acc, key) + + {_key, _transfer}, acc -> + acc + end) + end + + defp discard_snapshot_transfers_for_streams(state, stream_ids) do + stream_ids = MapSet.new(stream_ids) + + Enum.reduce(state.snapshot_transfers, state, fn + {{_source_node, stream_id} = key, _transfer}, acc -> + if MapSet.member?(stream_ids, stream_id) do + discard_snapshot_transfer(acc, key) + else + acc + end + end) + end + + defp discard_snapshot_send_offsets_for_target(state, target_node) do + offsets = + Map.reject(state.snapshot_send_offsets, fn + {{^target_node, _stream_id, _head}, _chunk_index} -> true + {_key, _chunk_index} -> false + end) + + %{state | snapshot_send_offsets: offsets} + end + + defp discard_snapshot_send_offsets_for_streams(state, stream_ids) do + stream_ids = MapSet.new(stream_ids) + + offsets = + Map.reject(state.snapshot_send_offsets, fn + {{_target_node, stream_id, _head}, _chunk_index} -> + MapSet.member?(stream_ids, stream_id) + end) + + %{state | snapshot_send_offsets: offsets} + end + + defp expire_replica_peer(state, remote_node) do + state = discard_snapshot_transfers_for_source(state, remote_node) + state = discard_snapshot_send_offsets_for_target(state, remote_node) + state = discard_pending_registry_reprojections(state, remote_node) + %{name: name, shard_index: shard} = state + + Data.delete_replica_cursors_for_origin(name, shard, remote_node) + + {purged_reg, purged_pg} = Data.purge_node(name, shard, remote_node) + affected_claims = Data.purge_registry_claims_for_origin(name, shard, remote_node) + events = build_purged_events(name, purged_reg, purged_pg, :peer_lease_expired) + + {state, events} = + Enum.reduce(affected_claims, {state, events}, fn {cluster, key}, {acc, inner_events} -> + reconcile_registry_projection(acc, cluster, key, :peer_lease_expired, inner_events) + end) + + notify_monitors(name, events) + + retirement = Data.expire_remote_replica_lane(name, shard, remote_node) + if retirement == :node_retired, do: Data.purge_cluster_node(name, remote_node) + + if function_exported?(state.replica_transport, :peer_down, 4) do + :ok = + state.replica_transport.peer_down( + name, + remote_node, + shard, + state.replica_transport_opts + ) + end + + %{ + state + | remote_shards: Map.delete(state.remote_shards, remote_node), + peer_last_seen: Map.delete(state.peer_last_seen, remote_node), + cluster_control_dirty: Map.delete(state.cluster_control_dirty, remote_node), + authority_dirty_notified: MapSet.delete(state.authority_dirty_notified, remote_node) + } + end + + defp send_replica_heads(state, target_node) do + send_replica_heads(state, target_node, :all) + end + + defp send_replica_heads(state, target_node, clusters) do + heads = replica_heads_for_clusters(state, target_node, clusters) + + if heads == [] do + state + else + outgoing_replica_message(state, target_node, {:heads, WireProtocol.version(), heads}) + end + end + + defp replica_heads_for_clusters(state, target_node, :all) do + state.name + |> Data.replica_stream_heads(state.shard_index) + |> Enum.filter(fn {stream_id, _floor, _head} -> + replica_stream_target?(state, stream_id, target_node) + end) + end + + defp replica_heads_for_clusters(state, target_node, clusters) when is_list(clusters) do + Enum.flat_map(clusters, fn cluster -> + case Data.local_stream_id(state.name, state.shard_index, cluster) do + nil -> + [] + + stream_id -> + {floor, head, _applied} = + Data.replica_stream_head(state.name, state.shard_index, stream_id) + + if head > 0 and replica_stream_target?(state, stream_id, target_node) do + [{stream_id, floor, head}] + else + [] + end + end + end) + end + + defp replica_stream_target?(state, stream_id, target_node) do + current_local_replica_stream?(state, stream_id) and + case WireProtocol.stream_cluster(stream_id) do + nil -> Map.has_key?(state.peer_last_seen, target_node) + cluster -> target_node in Data.cluster_nodes(state.name, cluster) + end + end + + defp current_local_replica_stream?(state, stream_id) do + WireProtocol.valid_stream_id?(stream_id) and + WireProtocol.stream_name(stream_id) == state.name and + WireProtocol.stream_origin(stream_id) == node() and + WireProtocol.stream_shard(stream_id) == state.shard_index and + WireProtocol.stream_generation(stream_id) == Data.generation(state.name) and + WireProtocol.stream_epoch(stream_id) == + Data.local_cluster_epoch(state.name, WireProtocol.stream_cluster(stream_id)) + end + + defp valid_remote_stream?(state, source_node, stream_id) do + if WireProtocol.valid_stream_id?(stream_id) do + cluster = WireProtocol.stream_cluster(stream_id) + + WireProtocol.stream_name(stream_id) == state.name and + WireProtocol.stream_origin(stream_id) == source_node and + WireProtocol.stream_shard(stream_id) == state.shard_index and + replica_view_current?(state, source_node) and + WireProtocol.stream_generation(stream_id) == + Data.remote_generation(state.name, source_node) and + WireProtocol.stream_epoch(stream_id) == + Data.remote_cluster_epoch(state.name, source_node, cluster) and + (is_nil(cluster) or cluster_member?(state.name, cluster)) + else + false + end + end + + defp handle_replica_message(state, source_node, {:heads, version, heads}) + when version == @protocol_version and is_list(heads) do + if Enum.all?(heads, &valid_replica_head?/1) do + handle_replica_heads(state, source_node, heads) + else + state + end + end + + defp handle_replica_message(state, source_node, {:delta_batch, version, runs}) + when version == @protocol_version and is_list(runs) do + if Enum.all?(runs, &valid_replica_delta_run?/1) do + state = flush_pending_replicated_sender_barrier(state) + + Enum.reduce(runs, state, fn {stream_id, _first_seq, records, advertised_head}, acc -> + apply_replica_delta_run(acc, source_node, stream_id, records, advertised_head) + end) + else + state + end + end + + defp handle_replica_message(state, source_node, {:need, version, stream_id, next_seq}) + when version == @protocol_version and is_integer(next_seq) and next_seq > 0 do + if WireProtocol.valid_stream_id?(stream_id) and + WireProtocol.stream_origin(stream_id) == node() and + WireProtocol.stream_shard(stream_id) == state.shard_index and + replica_stream_target?(state, stream_id, source_node) do + send_replica_repair(state, source_node, stream_id, next_seq) + else + state + end + end + + defp handle_replica_message(state, source_node, {:needs, version, needs}) + when version == @protocol_version and is_list(needs) do + if Enum.all?(needs, &valid_replica_need?/1) do + send_replica_repairs(state, source_node, needs) + else + state + end + end + + defp handle_replica_message( + state, + source_node, + {:snapshot_chunk, version, stream_id, snapshot_seq, chunk_index, reg_data, pg_data} + ) + when version == @protocol_version and is_integer(snapshot_seq) and snapshot_seq >= 0 and + is_integer(chunk_index) and is_list(reg_data) and is_list(pg_data) do + if valid_snapshot_stream?(state, source_node, stream_id, snapshot_seq) and + valid_snapshot_chunk?(chunk_index, reg_data, pg_data) and + valid_snapshot_rows?(state, source_node, stream_id, reg_data, pg_data) do + stage_replica_snapshot_chunk( + state, + source_node, + stream_id, + snapshot_seq, + chunk_index, + reg_data, + pg_data + ) + else + state + end + end + + defp handle_replica_message( + state, + source_node, + {:snapshot_commit, version, stream_id, snapshot_seq, chunk_count, registry_count, + pg_count} + ) + when version == @protocol_version and is_integer(snapshot_seq) and snapshot_seq >= 0 and + is_integer(chunk_count) and is_integer(registry_count) and is_integer(pg_count) do + if valid_snapshot_stream?(state, source_node, stream_id, snapshot_seq) and + valid_snapshot_commit_manifest?(chunk_count, registry_count, pg_count) do + stage_replica_snapshot_commit( + state, + source_node, + stream_id, + snapshot_seq, + chunk_count, + registry_count, + pg_count + ) + else + state + end + end + + defp handle_replica_message(state, _source_node, _message), do: state + + defp handle_replica_heads(state, source_node, heads) do + needs = + Enum.flat_map(heads, fn {stream_id, _floor, head} -> + if valid_remote_stream?(state, source_node, stream_id) do + cursor = Data.replica_cursor(state.name, state.shard_index, stream_id) + if head > cursor, do: [{stream_id, cursor + 1}], else: [] + else + [] + end + end) + + needs + |> Enum.chunk_every(state.replicated_sender_buffer_size) + |> Enum.reduce(state, fn chunk, acc -> + outgoing_replica_message(acc, source_node, {:needs, WireProtocol.version(), chunk}) + end) + end + + defp valid_snapshot_stream?(state, source_node, stream_id, snapshot_seq) do + valid_remote_stream?(state, source_node, stream_id) and + snapshot_seq > Data.replica_cursor(state.name, state.shard_index, stream_id) + end + + defp valid_snapshot_chunk?(chunk_index, reg_data, pg_data) do + chunk_row_count = length(reg_data) + length(pg_data) + chunk_index > 0 and (chunk_row_count > 0 or chunk_index == 1) + end + + defp valid_snapshot_commit_manifest?(chunk_count, registry_count, pg_count) do + total_count = registry_count + pg_count + + chunk_count > 0 and registry_count >= 0 and pg_count >= 0 and + chunk_count <= max(total_count, 1) and + (total_count > 0 or chunk_count == 1) + end + + defp valid_snapshot_rows?(state, source_node, stream_id, reg_data, pg_data) do + cluster = WireProtocol.stream_cluster(stream_id) + + Enum.all?(reg_data, fn + {key, pid, meta, time} + when is_binary(key) and is_pid(pid) and is_map(meta) and is_integer(time) -> + node(pid) == source_node and + shard_index_for(cluster, key, state.num_shards) == state.shard_index + + _other -> + false + end) and + Enum.all?(pg_data, fn + {key, pid, meta, time} + when is_binary(key) and is_pid(pid) and is_map(meta) and is_integer(time) -> + node(pid) == source_node and + shard_index_for(cluster, key, state.num_shards) == state.shard_index + + _other -> + false + end) and unique_snapshot_rows?(reg_data, pg_data) + end + + defp unique_snapshot_rows?(reg_data, pg_data) do + reg_data + |> MapSet.new(fn {key, _pid, _meta, _time} -> key end) + |> MapSet.size() == length(reg_data) and + pg_data + |> MapSet.new(fn {key, pid, _meta, _time} -> {key, pid} end) + |> MapSet.size() == length(pg_data) + end + + defp valid_replica_head?({stream_id, floor, head}) do + WireProtocol.valid_stream_id?(stream_id) and is_integer(floor) and floor >= 1 and + is_integer(head) and head >= 0 and floor <= head + 1 + end + + defp valid_replica_head?(_head), do: false + + defp valid_replica_need?({stream_id, next_seq}) do + WireProtocol.valid_stream_id?(stream_id) and is_integer(next_seq) and next_seq > 0 + end + + defp valid_replica_need?(_need), do: false + + defp valid_replica_delta_run?({stream_id, first_seq, records, advertised_head}) + when is_integer(first_seq) and first_seq > 0 and is_list(records) and records != [] and + is_integer(advertised_head) and advertised_head >= first_seq do + WireProtocol.valid_stream_id?(stream_id) and + valid_replica_record_sequence?(records, first_seq, advertised_head) + end + + defp valid_replica_delta_run?(_run), do: false + + defp valid_replica_record_sequence?(records, first_seq, advertised_head) do + case Enum.reduce_while(records, first_seq, fn + {seq, mutations}, expected when seq == expected and is_list(mutations) -> + {:cont, expected + 1} + + _record, _expected -> + {:halt, :invalid} + end) do + next_seq when is_integer(next_seq) -> next_seq - 1 <= advertised_head + :invalid -> false + end + end + + defp stage_replica_snapshot_chunk( + state, + source_node, + stream_id, + snapshot_seq, + chunk_index, + reg_data, + pg_data + ) do + key = {source_node, stream_id} + + case snapshot_transfer(state, key, snapshot_seq) do + {:ignore, state} -> + state + + {:ok, state, transfer} -> + cond do + MapSet.member?(transfer.received, chunk_index) and + Snapshot.chunk_matches?(transfer.table, chunk_index, reg_data, pg_data) -> + state + + MapSet.member?(transfer.received, chunk_index) -> + discard_snapshot_transfer(state, key) + + not snapshot_chunk_within_manifest?(transfer, chunk_index, reg_data, pg_data) -> + discard_snapshot_transfer(state, key) + + true -> + case Snapshot.stage_rows(transfer.table, chunk_index, reg_data, pg_data) do + :ok -> + transfer = %{ + transfer + | received: MapSet.put(transfer.received, chunk_index), + registry_seen: transfer.registry_seen + length(reg_data), + pg_seen: transfer.pg_seen + length(pg_data), + last_progress: monotonic_millis() + } + + state = put_snapshot_transfer(state, key, transfer) + maybe_commit_snapshot_transfer(state, key, source_node, stream_id) + + {:error, :duplicate_row} -> + discard_snapshot_transfer(state, key) + end + end + end + end + + defp stage_replica_snapshot_commit( + state, + source_node, + stream_id, + snapshot_seq, + chunk_count, + registry_count, + pg_count + ) do + key = {source_node, stream_id} + manifest = {chunk_count, registry_count, pg_count} + + case snapshot_transfer(state, key, snapshot_seq) do + {:ignore, state} -> + state + + {:ok, state, %{manifest: nil} = transfer} -> + if snapshot_transfer_within_manifest?(transfer, manifest) do + transfer = %{transfer | manifest: manifest, last_progress: monotonic_millis()} + state = put_snapshot_transfer(state, key, transfer) + maybe_commit_snapshot_transfer(state, key, source_node, stream_id) + else + discard_snapshot_transfer(state, key) + end + + {:ok, state, %{manifest: ^manifest}} -> + maybe_commit_snapshot_transfer(state, key, source_node, stream_id) + + {:ok, state, _conflicting_transfer} -> + discard_snapshot_transfer(state, key) + end + end + + defp snapshot_transfer(state, key, snapshot_seq) do + case Map.get(state.snapshot_transfers, key) do + nil -> + {state, transfer} = new_snapshot_transfer(state, snapshot_seq) + {:ok, put_snapshot_transfer(state, key, transfer), transfer} + + %{snapshot_seq: existing_seq} when existing_seq > snapshot_seq -> + {:ignore, state} + + %{snapshot_seq: existing_seq} when existing_seq < snapshot_seq -> + state = discard_snapshot_transfer(state, key) + {state, transfer} = new_snapshot_transfer(state, snapshot_seq) + {:ok, put_snapshot_transfer(state, key, transfer), transfer} + + %{snapshot_seq: ^snapshot_seq} = transfer -> + {:ok, state, transfer} + end + end + + defp new_snapshot_transfer(state, snapshot_seq) do + {table, events, pool} = + case state.snapshot_staging_pool do + [{table, events} | pool] -> {table, events, pool} + [] -> {Snapshot.new_staging_table(), Snapshot.new_event_table(), []} + end + + transfer = %{ + snapshot_seq: snapshot_seq, + manifest: nil, + registry_seen: 0, + pg_seen: 0, + received: MapSet.new(), + last_progress: monotonic_millis(), + table: table, + events: events + } + + {%{state | snapshot_staging_pool: pool}, transfer} + end + + defp put_snapshot_transfer(state, key, transfer) do + %{state | snapshot_transfers: Map.put(state.snapshot_transfers, key, transfer)} + end + + defp maybe_commit_snapshot_transfer(state, key, source_node, stream_id) do + transfer = Map.fetch!(state.snapshot_transfers, key) + + case transfer.manifest do + {chunk_count, registry_count, pg_count} -> + if MapSet.size(transfer.received) == chunk_count and + transfer.registry_seen == registry_count and transfer.pg_seen == pg_count do + commit_snapshot_transfer(state, key, source_node, stream_id, transfer) + else + state + end + + nil -> + state + end + end + + defp snapshot_chunk_within_manifest?(%{manifest: nil}, _chunk_index, _reg_data, _pg_data), + do: true + + defp snapshot_chunk_within_manifest?( + %{manifest: {chunk_count, registry_count, pg_count}} = transfer, + chunk_index, + reg_data, + pg_data + ) do + chunk_index <= chunk_count and + transfer.registry_seen + length(reg_data) <= registry_count and + transfer.pg_seen + length(pg_data) <= pg_count + end + + defp snapshot_transfer_within_manifest?( + transfer, + {chunk_count, registry_count, pg_count} + ) do + Enum.all?(transfer.received, &(&1 <= chunk_count)) and + transfer.registry_seen <= registry_count and transfer.pg_seen <= pg_count + end + + defp commit_snapshot_transfer(state, key, source_node, stream_id, transfer) do + {chunk_count, _registry_count, _pg_count} = transfer.manifest + + state = + if valid_snapshot_stream?(state, source_node, stream_id, transfer.snapshot_seq) do + state = flush_pending_replicated_barrier(state) + cluster = WireProtocol.stream_cluster(stream_id) + + :ok = + Data.begin_replica_snapshot_install( + state.name, + state.shard_index, + stream_id, + transfer.snapshot_seq + ) + + event_buffer = Snapshot.new_event_buffer(transfer.events) + + {state, event_buffer} = + Data.replace_registry_claims_for_stream_from_staging( + state.name, + state.shard_index, + stream_id, + transfer.snapshot_seq, + transfer.table, + chunk_count, + {state, event_buffer}, + fn key, {acc, buffer} -> + {acc, events} = reconcile_registry_projection(acc, cluster, key, :reconcile, []) + {acc, Snapshot.buffer_events(Enum.reverse(events), buffer)} + end, + fn claims, {acc, buffer} -> + reconcile_registry_snapshot_batch(acc, cluster, stream_id, claims, buffer) + end + ) + + event_buffer = + replace_remote_pg_snapshot_from_staging( + state, + source_node, + cluster, + transfer.table, + chunk_count, + event_buffer + ) + + _event_buffer = Snapshot.finish_event_buffer(event_buffer) + + :ok = + Data.put_replica_cursor( + state.name, + state.shard_index, + stream_id, + transfer.snapshot_seq + ) + + notify_snapshot_events(state.name, transfer.events) + state + else + state + end + + discard_snapshot_transfer(state, key) + end + + defp apply_replica_delta_run(state, source_node, stream_id, records, advertised_head) do + if valid_remote_stream?(state, source_node, stream_id) do + cursor = Data.replica_cursor(state.name, state.shard_index, stream_id) + records = Enum.drop_while(records, fn {seq, _mutations} -> seq <= cursor end) + + case records do + [] -> + state + + [{first_seq, _mutations} | _] when first_seq > cursor + 1 -> + request_replica_need(state, source_node, stream_id, cursor + 1) + + _ -> + {contiguous, _next_seq} = take_contiguous_replica_records(records, cursor + 1, []) + + {accepted, rejected} = + Enum.split_while(contiguous, fn {_seq, mutations} -> + valid_replica_mutations?(state, stream_id, mutations) + end) + + if rejected != [] do + Logger.error( + "#{log_prefix_shard(state)} rejected replica record with invalid origin/cluster authority from #{inspect(source_node)}" + ) + end + + state = + if accepted == [] do + state + else + :ok = Data.ensure_replica_cursor(state.name, state.shard_index, stream_id) + apply_received_replica_records(state, stream_id, accepted) + end + |> flush_pending_replicated_barrier() + + case List.last(accepted) do + nil -> + if rejected == [] do + state + else + request_replica_need(state, source_node, stream_id, cursor + 1) + end + + {last_seq, _mutations} -> + :ok = Data.put_replica_cursor(state.name, state.shard_index, stream_id, last_seq) + + if last_seq < advertised_head or length(accepted) < length(records) do + request_replica_need(state, source_node, stream_id, last_seq + 1) + else + state + end + end + end + else + state + end + end + + defp take_contiguous_replica_records([], next_seq, acc), do: {Enum.reverse(acc), next_seq} + + defp take_contiguous_replica_records([{seq, mutations} | rest], seq, acc) do + take_contiguous_replica_records(rest, seq + 1, [{seq, mutations} | acc]) + end + + defp take_contiguous_replica_records(_records, next_seq, acc), + do: {Enum.reverse(acc), next_seq} + + defp valid_replica_mutations?(state, stream_id, mutations) do + origin = WireProtocol.stream_origin(stream_id) + cluster = WireProtocol.stream_cluster(stream_id) + + mutations != [] and + Enum.all?( + mutations, + &valid_replica_mutation?( + &1, + cluster, + origin, + state.shard_index, + state.num_shards + ) + ) + end + + defp valid_replica_mutation?( + {:register, cluster, key, pid, meta, time, entry_node}, + cluster, + origin, + shard, + num_shards + ) + when is_binary(key) and is_pid(pid) and is_map(meta) and is_integer(time), + do: + node(pid) == origin and entry_node == origin and + shard_index_for(cluster, key, num_shards) == shard + + defp valid_replica_mutation?( + {:unregister, cluster, key, pid, meta, _reason}, + cluster, + origin, + shard, + num_shards + ) + when is_binary(key) and is_pid(pid) and is_map(meta), + do: node(pid) == origin and shard_index_for(cluster, key, num_shards) == shard + + defp valid_replica_mutation?( + {:join, cluster, key, pid, meta, time, _reason, entry_node}, + cluster, + origin, + shard, + num_shards + ) + when is_binary(key) and is_pid(pid) and is_map(meta) and is_integer(time), + do: + node(pid) == origin and entry_node == origin and + shard_index_for(cluster, key, num_shards) == shard + + defp valid_replica_mutation?( + {:leave, cluster, key, pid, meta, _reason}, + cluster, + origin, + shard, + num_shards + ) + when is_binary(key) and is_pid(pid) and is_map(meta), + do: node(pid) == origin and shard_index_for(cluster, key, num_shards) == shard + + defp valid_replica_mutation?(_mutation, _cluster, _origin, _shard, _num_shards), do: false + + defp apply_received_replica_records(state, stream_id, records) do + records + |> Enum.chunk_by(fn {_seq, mutations} -> replica_record_domain(mutations) end) + |> Enum.reduce(state, fn records, acc -> + case records |> hd() |> elem(1) |> replica_record_domain() do + :registry -> + acc = flush_pending_replicated_pg_barrier(acc) + + {acc, events} = + Enum.reduce(records, {acc, []}, fn {seq, mutations}, {inner, events} -> + {inner, record_events} = + apply_received_registry_claims(inner, stream_id, seq, mutations) + + {inner, record_events ++ events} + end) + + notify_monitors(acc.name, events) + acc + + _pg_or_mixed -> + Enum.reduce(records, acc, fn {seq, mutations}, inner -> + apply_received_replica_record(inner, stream_id, seq, mutations) + end) + end + end) + end + + defp replica_record_domain(mutations) do + case mutations |> Enum.map(&replica_mutation_domain/1) |> Enum.uniq() do + [domain] -> domain + [] -> :empty + _ -> :mixed + end + end + + defp enqueue_received_replica_mutation(state, {:register, _, _, _, _, _, _} = op) do + {state, _flushed?} = enqueue_replicated_registry_ops(state, [op]) + state + end + + defp enqueue_received_replica_mutation(state, {:unregister, _, _, _, _, _} = op) do + {state, _flushed?} = enqueue_replicated_registry_ops(state, [op]) + state + end + + defp enqueue_received_replica_mutation(state, {:join, _, _, _, _, _, _, _} = op) do + {state, _flushed?} = enqueue_replicated_pg_ops(state, [op]) + state + end + + defp enqueue_received_replica_mutation(state, {:leave, _, _, _, _, _} = op) do + {state, _flushed?} = enqueue_replicated_pg_ops(state, [op]) + state + end + + defp apply_received_replica_record(state, stream_id, seq, mutations) do + domains = mutations |> Enum.map(&replica_mutation_domain/1) |> Enum.uniq() + + if length(domains) > 1 do + apply_received_mixed_replica_record(state, stream_id, seq, mutations) + else + apply_received_homogeneous_replica_record(state, stream_id, seq, mutations, domains) + end + end + + defp apply_received_homogeneous_replica_record( + state, + stream_id, + seq, + mutations, + [:registry] + ) do + state = flush_pending_replicated_pg_barrier(state) + {state, _events} = apply_received_registry_claims(state, stream_id, seq, mutations) + state + end + + defp apply_received_homogeneous_replica_record(state, _stream_id, _seq, mutations, [:pg]) do + Enum.reduce(mutations, state, fn op, acc -> + enqueue_received_replica_mutation(acc, op) + end) + end + + defp apply_received_homogeneous_replica_record(state, _stream_id, _seq, [], []), do: state + + # Process death may remove registry and PG rows in one authoritative record. + # Apply its maximal same-domain segments in wire order and emit one monitor + # batch, preserving the existing process-down batching contract. + defp apply_received_mixed_replica_record(state, stream_id, seq, mutations) do + state = flush_pending_replicated_barrier(state) + + {state, events} = + mutations + |> Enum.chunk_by(&replica_mutation_domain/1) + |> Enum.reduce({state, []}, fn segment, {acc, events} -> + case replica_mutation_domain(hd(segment)) do + :registry -> + {acc, segment_events} = apply_received_registry_claims(acc, stream_id, seq, segment) + {acc, segment_events ++ events} + + :pg -> + {insert_entries, delete_entries, segment_events} = + apply_replicated_pg_ops(acc.name, acc.shard_index, segment) + + Data.pg_delete_many(acc.name, acc.shard_index, delete_entries) + Data.pg_insert_many(acc.name, acc.shard_index, insert_entries) + {acc, segment_events ++ events} + end + end) + + notify_monitors(state.name, events) + state + end + + defp replica_mutation_domain({:register, _, _, _, _, _, _}), do: :registry + defp replica_mutation_domain({:unregister, _, _, _, _, _}), do: :registry + defp replica_mutation_domain({:join, _, _, _, _, _, _, _}), do: :pg + defp replica_mutation_domain({:leave, _, _, _, _, _}), do: :pg + + defp apply_received_registry_claims(state, stream_id, seq, ops) do + keys = + Enum.map(ops, fn + {:register, _cluster, key, pid, meta, time, _entry_node} -> + Data.put_registry_claim( + state.name, + state.shard_index, + stream_id, + seq, + key, + pid, + meta, + time + ) + + key + + {:unregister, _cluster, key, pid, _meta, _reason} -> + Data.delete_registry_claim(state.name, state.shard_index, stream_id, seq, key, pid) + key + end) + |> Enum.uniq() + + cluster = WireProtocol.stream_cluster(stream_id) + + Enum.reduce(keys, {state, []}, fn key, {acc, events} -> + reconcile_registry_projection(acc, cluster, key, :reconcile, events) + end) + end + + defp send_replica_repair(state, target_node, stream_id, next_seq) do + send_replica_repairs(state, target_node, [{stream_id, next_seq}]) + end + + defp request_replica_need(state, target_node, stream_id, next_seq) do + outgoing_replica_message( + state, + target_node, + {:needs, WireProtocol.version(), [{stream_id, next_seq}]} + ) + end + + defp send_replica_repairs(state, target_node, needs) do + {state, runs} = + Enum.reduce(needs, {state, []}, fn {stream_id, next_seq}, {acc, runs} -> + if WireProtocol.stream_origin(stream_id) == node() and + WireProtocol.stream_shard(stream_id) == acc.shard_index and + replica_stream_target?(acc, stream_id, target_node) do + case replica_repair(acc, target_node, stream_id, next_seq) do + {:run, run} -> {acc, [run | runs]} + {:state, acc} -> {acc, runs} + end + else + {acc, runs} + end + end) + + case runs do + [] -> + state + + runs -> + outgoing_replica_message( + state, + target_node, + {:delta_batch, WireProtocol.version(), Enum.reverse(runs)} + ) + end + end + + defp replica_repair(state, target_node, stream_id, next_seq) do + {floor, head, _applied} = + Data.replica_stream_head(state.name, state.shard_index, stream_id) + + cond do + next_seq > head -> + {:state, state} + + next_seq >= floor -> + records = + Data.replica_records( + state.name, + state.shard_index, + stream_id, + next_seq, + state.replicated_sender_buffer_size + ) + + case records do + [] -> + {:state, send_replica_snapshot(state, target_node, stream_id, head)} + + [{first_seq, _} | _] -> + {:run, {stream_id, first_seq, records, head}} + end + + true -> + {:state, send_replica_snapshot(state, target_node, stream_id, head)} + end + end + + defp send_replica_snapshot(state, target_node, stream_id, head) do + case state.snapshot_send do + {worker, _token, _snapshot_key} when is_pid(worker) -> + if Process.alive?(worker), + do: state, + else: start_replica_snapshot_send(state, target_node, stream_id, head) + + nil -> + start_replica_snapshot_send(state, target_node, stream_id, head) + end + end + + defp start_replica_snapshot_send(state, target_node, stream_id, head) do + owner = self() + token = make_ref() + snapshot_key = {target_node, stream_id, head} + + offsets = + state.snapshot_send_offsets + |> Enum.reject(fn + {{^target_node, ^stream_id, other_head}, _chunk_index} -> other_head != head + {_other_key, _chunk_index} -> false + end) + |> Map.new() + + resume = Map.get(offsets, snapshot_key, {:chunk, 1}) + + snapshot_context = %{ + name: state.name, + shard_index: state.shard_index, + replicated_snapshot_chunk_target_bytes: state.replicated_snapshot_chunk_target_bytes, + replica_transport: state.replica_transport, + replica_transport_opts: state.replica_transport_opts + } + + worker = + spawn(fn -> + result = + try do + stream_replica_snapshot( + snapshot_context, + target_node, + stream_id, + head, + resume + ) + catch + kind, reason -> + Logger.error( + "#{log_prefix_shard(snapshot_context)} snapshot capture failed: " <> + Exception.format_banner(kind, reason) + ) + + :retry + end + + send( + owner, + {:replica_snapshot_send_complete, token, self(), snapshot_key, result} + ) + end) + + %{state | snapshot_send: {worker, token, snapshot_key}, snapshot_send_offsets: offsets} + end + + defp stream_replica_snapshot( + state, + target_node, + stream_id, + head, + {:commit, manifest} + ) do + if current_snapshot_send?(state, target_node, stream_id, head) do + send_replica_snapshot_commit(state, target_node, stream_id, head, manifest) + else + :complete + end + end + + defp stream_replica_snapshot(state, target_node, stream_id, head, {:chunk, start_index}) do + if current_snapshot_send?(state, target_node, stream_id, head) do + do_stream_replica_snapshot(state, target_node, stream_id, head, start_index) + else + :complete + end + end + + defp do_stream_replica_snapshot(state, target_node, stream_id, head, start_index) do + cluster = WireProtocol.stream_cluster(stream_id) + envelope_bytes = Snapshot.stream_envelope_bytes(stream_id, head) + + emit = fn registry, pg, chunk_index -> + message = + {:snapshot_chunk, WireProtocol.version(), stream_id, head, chunk_index, registry, pg} + + case state.replica_transport.outgoing( + state.name, + target_node, + state.shard_index, + message, + state.replica_transport_opts + ) do + :ok -> :ok + result when result in [:busy, :disconnected] -> throw({:snapshot_resume, chunk_index}) + end + end + + try do + stream = + Snapshot.new_stream( + state.replicated_snapshot_chunk_target_bytes, + envelope_bytes, + start_index, + emit + ) + + stream = + Data.reduce_registry_claim_batches_for_stream( + state.name, + state.shard_index, + stream_id, + stream, + &Snapshot.stream_registry_many/2 + ) + + stream = + Data.reduce_pg_entry_batches_for_origin( + state.name, + state.shard_index, + cluster, + node(), + stream, + &Snapshot.stream_pg_many/2 + ) + + stream = Snapshot.finish_stream(stream) + manifest = {stream.chunk_count, stream.registry_count, stream.pg_count} + + # Chunks are provisional. Appending advances the head before changing + # materialized rows and advances `applied` only afterward, so an + # unchanged fully-applied head proves the completed scan is one exact + # state at `head`. Only the terminal commit makes those chunks visible. + if current_snapshot_send?(state, target_node, stream_id, head) do + send_replica_snapshot_commit(state, target_node, stream_id, head, manifest) + else + :complete + end + catch + {:snapshot_resume, chunk_index} -> {:resume, chunk_index} + end + end + + defp send_replica_snapshot_commit( + state, + target_node, + stream_id, + head, + {chunk_count, registry_count, pg_count} = manifest + ) do + message = + {:snapshot_commit, WireProtocol.version(), stream_id, head, chunk_count, registry_count, + pg_count} + + case state.replica_transport.outgoing( + state.name, + target_node, + state.shard_index, + message, + state.replica_transport_opts + ) do + :ok -> :complete + result when result in [:busy, :disconnected] -> {:resume_commit, manifest} + end + end + + defp current_snapshot_send?(state, {target_node, stream_id, head}), + do: current_snapshot_send?(state, target_node, stream_id, head) + + defp current_snapshot_send?(state, target_node, stream_id, head) do + cluster = WireProtocol.stream_cluster(stream_id) + + {_floor, current_head, applied} = + Data.replica_stream_head(state.name, state.shard_index, stream_id) + + Data.local_stream_id(state.name, state.shard_index, cluster) == stream_id and + current_head == head and applied == head and + target_node in Data.cluster_nodes(state.name, cluster) + end + + defp replace_remote_pg_snapshot_from_staging( + state, + source_node, + cluster, + staging_table, + _chunk_count, + event_buffer + ) do + event_buffer = + Snapshot.reduce_pg_batches(staging_table, event_buffer, fn rows, buffer -> + {inserts, buffer} = + Enum.reduce(rows, {[], buffer}, fn {key, pid, meta, time}, {inserts, inner} -> + case Data.pg_lookup(state.name, state.shard_index, cluster, key, pid) do + nil -> + event = build_event(state.name, :joined, key, pid, meta, %{cluster: cluster}) + + { + [{cluster, key, pid, meta, time, source_node} | inserts], + Snapshot.buffer_event(event, inner) + } + + {^meta, ^time, ^source_node} -> + {inserts, inner} + + {old_meta, _old_time, old_source} when old_source != source_node -> + Logger.error( + "#{log_prefix_shard(state)} repaired PG row with impossible stored origin " <> + "#{inspect(old_source)} while installing exact snapshot from " <> + inspect(source_node) + ) + + event = + build_event(state.name, :joined, key, pid, meta, %{ + previous_meta: old_meta, + cluster: cluster + }) + + { + [{cluster, key, pid, meta, time, source_node} | inserts], + Snapshot.buffer_event(event, inner) + } + + {old_meta, _old_time, ^source_node} -> + inner = + if old_meta != meta do + Snapshot.buffer_event( + build_event(state.name, :joined, key, pid, meta, %{ + previous_meta: old_meta, + cluster: cluster + }), + inner + ) + else + inner + end + + {[{cluster, key, pid, meta, time, source_node} | inserts], inner} + end + end) + + :ok = Data.pg_insert_many(state.name, state.shard_index, Enum.reverse(inserts)) + buffer + end) + + event_buffer = + Data.fold_pg_entries_for_origin( + state.name, + state.shard_index, + cluster, + source_node, + event_buffer, + fn {key, pid, old_meta, _old_time}, buffer -> + if Snapshot.member_pg?(staging_table, key, pid) do + buffer + else + :ok = Data.pg_delete(state.name, state.shard_index, cluster, key, pid) + + event = + build_event(state.name, :left, key, pid, old_meta, %{ + reason: :reconcile, + cluster: cluster + }) + + Snapshot.buffer_event(event, buffer) + end + end + ) + + event_buffer + end + + defp maybe_purge_remote_generation(state, _remote_node, nil, _generation), do: state + + defp maybe_purge_remote_generation(state, _remote_node, generation, generation), do: state + + defp maybe_purge_remote_generation(state, remote_node, _old_generation, _generation) do + state = discard_snapshot_transfers_for_source(state, remote_node) + state = discard_pending_registry_reprojections(state, remote_node) + Data.delete_replica_cursors_for_origin(state.name, state.shard_index, remote_node) + {_reg, _pg} = Data.purge_node(state.name, state.shard_index, remote_node) + + affected = + Data.purge_registry_claims_for_origin( + state.name, + state.shard_index, + remote_node + ) + + {state, events} = + Enum.reduce(affected, {state, []}, fn {cluster, key}, {acc, inner_events} -> + reconcile_registry_projection(acc, cluster, key, :nodedown, inner_events) + end) + + notify_monitors(state.name, events) + state + end + + defp purge_closed_remote_epochs(state, _remote_node, []), do: state + + defp purge_closed_remote_epochs(state, remote_node, cluster_epochs) do + generation = Data.remote_generation(state.name, remote_node) + + stream_ids = + Enum.map(cluster_epochs, fn {cluster, epoch} -> + WireProtocol.stream_id( + state.name, + remote_node, + generation, + state.shard_index, + cluster, + epoch + ) + end) + + state = discard_snapshot_transfers_for_streams(state, stream_ids) + + Enum.each(stream_ids, fn stream_id -> + :ok = Data.delete_replica_cursor(state.name, state.shard_index, stream_id) + end) + + affected_keys = + Data.purge_registry_claims_for_streams( + state.name, + state.shard_index, + stream_ids + ) + + clusters = cluster_epochs |> Enum.map(&elem(&1, 0)) |> Enum.uniq() + + purged_pg = + Data.delete_pg_for_origin_clusters( + state.name, + state.shard_index, + clusters, + remote_node + ) + + events = build_purged_events(state.name, [], purged_pg, :cluster_disconnect, []) + + {state, events} = + Enum.reduce(affected_keys, {state, events}, fn {cluster, key}, + {inner_state, inner_events} -> + reconcile_registry_projection( + inner_state, + cluster, + key, + :cluster_disconnect, + inner_events + ) + end) + + notify_monitors(state.name, events) + + state + end + + defp purge_superseded_remote_streams(state, remote_node, current_epochs) do + generation = Data.remote_generation(state.name, remote_node) + + current_epochs = Map.new(current_epochs) + + superseded = + Enum.flat_map(current_epochs, fn {cluster, current_epoch} -> + current_stream = + WireProtocol.stream_id( + state.name, + remote_node, + generation, + state.shard_index, + cluster, + current_epoch + ) + + state.name + |> Data.replica_cursor_streams_for_origin_cluster( + state.shard_index, + remote_node, + cluster + ) + |> Enum.reject(&(&1 == current_stream)) + end) + + purge_superseded_remote_streams(state, remote_node, current_epochs, superseded) + end + + defp purge_remote_streams_outside_authority(state, remote_node) do + generation = Data.remote_generation(state.name, remote_node) + + streams = + Data.replica_cursor_streams_for_origin( + state.name, + state.shard_index, + remote_node + ) + + # A shard only needs authority for clusters for which it has retained + # receive state. This keeps local fanout proportional to actual shard data, + # rather than rebuilding the node-wide epoch map in every lane. + current_epochs = + streams + |> Enum.map(&WireProtocol.stream_cluster/1) + |> Enum.uniq() + |> Map.new(fn cluster -> + {cluster, Data.remote_cluster_epoch(state.name, remote_node, cluster)} + end) + + superseded = + Enum.reject(streams, fn stream_id -> + WireProtocol.stream_generation(stream_id) == generation and + Map.get(current_epochs, WireProtocol.stream_cluster(stream_id)) == + WireProtocol.stream_epoch(stream_id) + end) + + purge_superseded_remote_streams(state, remote_node, current_epochs, superseded) + end + + defp purge_superseded_remote_streams(state, _remote_node, _current_epochs, []), do: state + + defp purge_superseded_remote_streams( + state, + remote_node, + current_epochs, + superseded + ) do + state = discard_snapshot_transfers_for_streams(state, superseded) + generation = Data.remote_generation(state.name, remote_node) + + superseded + |> Enum.group_by(&WireProtocol.stream_cluster/1) + |> Enum.reduce(state, fn {cluster, cluster_streams}, acc -> + Enum.each(cluster_streams, fn stream_id -> + :ok = Data.delete_replica_cursor(state.name, state.shard_index, stream_id) + end) + + case Map.get(current_epochs, cluster) do + nil -> + :ok + + current_epoch -> + current_stream = + WireProtocol.stream_id( + state.name, + remote_node, + generation, + state.shard_index, + cluster, + current_epoch + ) + + :ok = Data.delete_replica_cursor(state.name, state.shard_index, current_stream) + end + + affected_keys = + Data.purge_registry_claims_for_streams( + state.name, + state.shard_index, + cluster_streams + ) + + # PG rows do not carry their stream epoch. Remove the origin/cluster + # slice and reset the current cursor so its exact state is rebuilt by + # the next head advertisement (delta when retained, snapshot after + # pruning). Registry claims do carry epochs and are removed narrowly. + purged_pg = + Data.delete_pg_for_origin_clusters( + state.name, + state.shard_index, + [cluster], + remote_node + ) + + events = build_purged_events(state.name, [], purged_pg, :cluster_disconnect, []) + + {acc, events} = + Enum.reduce(affected_keys, {acc, events}, fn {affected_cluster, key}, + {inner, inner_events} -> + reconcile_registry_projection( + inner, + affected_cluster, + key, + :cluster_disconnect, + inner_events + ) + end) + + notify_monitors(state.name, events) + + acc + end) + end + + defp append_process_down_records(state, reason_by_pid, reg_entries, pg_entries) do + mutations_by_cluster = + Enum.reduce(reg_entries, %{}, fn {pid, cluster, key, meta}, acc -> + op = {:unregister, cluster, key, pid, meta, Map.fetch!(reason_by_pid, pid)} + Map.update(acc, cluster, [op], &[op | &1]) + end) + |> then(fn acc -> + Enum.reduce(pg_entries, acc, fn {pid, cluster, key, meta}, inner -> + op = {:leave, cluster, key, pid, meta, Map.fetch!(reason_by_pid, pid)} + Map.update(inner, cluster, [op], &[op | &1]) + end) + end) + + Enum.flat_map(mutations_by_cluster, fn {cluster, mutations} -> + case Data.local_stream_id(state.name, state.shard_index, cluster) do + nil -> + [] + + stream_id -> + {seq, mutations} = + Data.append_replica_record( + state.name, + state.shard_index, + stream_id, + Enum.reverse(mutations) + ) + + [{:sequenced, stream_id, seq, mutations}] + end + end) + end + + defp finish_process_down_records(state, records) do + Enum.each(records, fn {:sequenced, stream_id, seq, mutations} -> + apply_registry_claim_mutations(state, stream_id, seq, mutations) + :ok = Data.mark_local_replica_applied(state.name, state.shard_index, stream_id, seq) + end) + + :ok = + Data.prune_replica_oplog( + state.name, + state.shard_index, + state.replicated_oplog_max_entries + ) + + records + |> Enum.reduce(%{}, fn {:sequenced, _stream_id, _seq, [op | _]} = record, acc -> + cluster = WireProtocol.op_cluster(op) + + Enum.reduce(process_down_targets(state, cluster), acc, fn target_node, inner -> + Map.update(inner, target_node, [record], &[record | &1]) + end) + end) + |> Enum.reduce(state, fn {target_node, target_records}, acc -> + send_replica_delta_batch(acc, target_node, Enum.reverse(target_records)) + end) + end + + defp process_down_targets(state, nil) do + for {target_node, _last_seen} <- state.peer_last_seen, do: target_node + end + + defp process_down_targets(%{name: name}, cluster) do + for target_node <- Data.cluster_nodes(name, cluster), target_node != node(), do: target_node + end + + defp collect_local_process_downs(acc, monitors, 0), do: {Enum.reverse(acc), monitors} + + defp collect_local_process_downs(acc, monitors, remaining) do + receive do + {:DOWN, _mref, :process, pid, reason} when is_map_key(monitors, pid) -> + collect_local_process_downs([{pid, reason} | acc], monitors, remaining - 1) + after + 0 -> + {Enum.reverse(acc), monitors} + end + end + + defp build_process_down_events(name, reg_entries, pg_entries, reason_by_pid) do + events = + Enum.reduce(reg_entries, [], fn {pid, cluster, key, meta}, acc -> + [ + build_event(name, :unregistered, key, pid, meta, %{ + reason: Map.fetch!(reason_by_pid, pid), + cluster: cluster + }) + | acc + ] + end) + + Enum.reduce(pg_entries, events, fn {pid, cluster, key, meta}, acc -> + [ + build_event(name, :left, key, pid, meta, %{ + reason: Map.fetch!(reason_by_pid, pid), + cluster: cluster + }) + | acc + ] + end) + end + + defp fan_out_to_siblings(state, message) do + %{name: name, shard_index: shard_index, num_shards: num_shards} = state + + for i <- 0..(num_shards - 1), i != shard_index do + case Process.whereis(shard_name(name, i)) do + pid when is_pid(pid) -> send(pid, message) + nil -> :ok + end + end + end + + defp send_local_control_message(state, message) do + case Process.whereis(shard_name(state.name, 0)) do + pid when is_pid(pid) -> + send(pid, message) + :ok + + nil -> + :disconnected + end + end + + defp monitor_pid(state, pid) do + Map.get(state.monitors, pid) || Process.monitor(pid) + end + + defp put_monitor(state, pid, mref) do + %{state | monitors: Map.put_new(state.monitors, pid, mref)} + end + + defp maybe_demonitor_pid(state, name, shard, pid) do + case Data.maybe_demonitor(name, shard, pid) do + :ok -> + case Map.pop(state.monitors, pid) do + {nil, monitors} -> + %{state | monitors: monitors} + + {mref, monitors} -> + Process.demonitor(mref, [:flush]) + %{state | monitors: monitors} + end + + :still_monitored -> + state + end + end + + defp rebuild_monitors(state) do + %{name: name, shard_index: shard} = state + local = node() + + # Scan all entries in this shard's tables and re-establish monitors + # Only monitor local pids — remote pids are cleaned up by their owning + # node's DOWN handler (broadcast) or by nodedown. + reg_table = Data.reg_by_pid_table(name, shard) + + monitors = + try do + :ets.foldl( + fn {{pid, _cluster, _key}, _meta, _time, entry_node}, acc -> + if entry_node == local and not Map.has_key?(acc, pid) do + mref = Process.monitor(pid) + Map.put(acc, pid, mref) + else + acc + end + end, + %{}, + reg_table ) rescue ArgumentError -> %{} @@ -2768,93 +5099,136 @@ defmodule Group.Replica do %{state | monitors: monitors} end - defp cluster_member?(name, cluster) do - node() in Data.cluster_nodes(name, cluster) + defp replay_local_journal(state) do + state.name + |> Data.local_replica_unapplied(state.shard_index) + |> Enum.each(fn {stream_id, seq, mutations} -> + if current_local_stream?(state, stream_id) do + apply_registry_claim_mutations(state, stream_id, seq, mutations) + Enum.each(mutations, &replay_local_mutation(state, &1)) + end + + :ok = Data.mark_local_replica_applied(state.name, state.shard_index, stream_id, seq) + end) + + :ok = + Data.prune_replica_oplog( + state.name, + state.shard_index, + state.replicated_oplog_max_entries + ) + + state end - # Additive merge: inserts new entries and resolves conflicts, but does not - # delete local entries missing from the incoming snapshot. This is safe because - # Erlang dist uses TCP — either all replicate_* messages arrive in order (no - # stale entries) or the connection dies and nodedown purges everything before - # cluster_state can arrive. There is no case where stale entries survive into - # the merge. - defp merge_remote_cluster_data(state, cluster, reg_data, pg_data, events \\ []) do - %{name: name, shard_index: shard, num_shards: num_shards} = state + defp rebuild_registry_projections(state) do + claim_keys = + state.name + |> Data.reg_claim_by_key_table(state.shard_index) + |> :ets.select([ + {{{:"$1", :"$2", :_, :_, :_}, :_, :_, :_, :_}, [], [{{:"$1", :"$2"}}]} + ]) - # Registry merge uses Enum.reduce to thread state + events, because - # resolve_conflict modifies state.monitors (demonitor evicted local pids) - # and may produce an event. - {state, events} = - Enum.reduce(reg_data, {state, events}, fn {key, pid, meta, time}, {acc_state, acc_events} -> - if shard_index_for(cluster, key, num_shards) != shard do - {acc_state, acc_events} - else - case Data.registry_lookup(name, shard, cluster, key) do - nil -> - Data.registry_insert(name, shard, cluster, key, pid, meta, time, node(pid)) - event = build_event(name, :registered, key, pid, meta, %{cluster: cluster}) - {acc_state, [event | acc_events]} - - {^pid, _meta, existing_time, _node} -> - # Same pid (bounceback or metadata update) — apply if newer - if time > existing_time do - Data.registry_insert(name, shard, cluster, key, pid, meta, time, node(pid)) - end + visible_keys = + state.name + |> Data.reg_by_key_table(state.shard_index) + |> :ets.select([ + {{{:"$1", :"$2"}, :_, :_, :_, :_}, [], [{{:"$1", :"$2"}}]} + ]) + + reconcile_registry_keys( + state, + Enum.uniq(claim_keys ++ visible_keys), + :journal_replay, + [] + ) + end + + defp current_local_stream?(state, stream_id) do + cluster = WireProtocol.stream_cluster(stream_id) + + WireProtocol.stream_name(stream_id) == state.name and + WireProtocol.stream_origin(stream_id) == node() and + WireProtocol.stream_generation(stream_id) == Data.generation(state.name) and + WireProtocol.stream_shard(stream_id) == state.shard_index and + WireProtocol.stream_epoch(stream_id) == Data.local_cluster_epoch(state.name, cluster) + end + + defp apply_registry_claim_mutations(state, stream_id, seq, mutations) do + Enum.each(mutations, fn + {:register, _cluster, key, pid, meta, time, _entry_node} -> + Data.put_registry_claim( + state.name, + state.shard_index, + stream_id, + seq, + key, + pid, + meta, + time + ) - {acc_state, acc_events} - - {existing_pid, existing_meta, existing_time, existing_node} - when existing_node == node() -> - # Local entry vs incoming remote — use full conflict resolution - # (runs the configured resolver and re-broadcasts the winner) - {new_state, event} = - resolve_conflict( - acc_state, - cluster, - key, - {existing_pid, existing_meta, existing_time}, - {pid, meta, time} - ) + {:unregister, _cluster, key, pid, _meta, _reason} -> + Data.delete_registry_claim(state.name, state.shard_index, stream_id, seq, key, pid) - acc_events = if event, do: [event | acc_events], else: acc_events - {new_state, acc_events} + _pg_mutation -> + :ok + end) - {existing_pid, _meta, existing_time, _node} when time > existing_time -> - # Both remote — keep the more recent one - if existing_pid != pid do - Data.registry_delete(name, shard, cluster, key, existing_pid) - end + :ok + end - Data.registry_insert(name, shard, cluster, key, pid, meta, time, node(pid)) - {acc_state, acc_events} + defp replay_local_mutation(state, {:register, cluster, key, pid, meta, time, entry_node}) do + Data.registry_insert( + state.name, + state.shard_index, + cluster, + key, + pid, + meta, + time, + entry_node + ) + end - _ -> - {acc_state, acc_events} - end - end - end) + defp replay_local_mutation(state, {:unregister, cluster, key, pid, meta, reason}) do + Data.registry_delete_matching_many( + state.name, + state.shard_index, + [{pid, cluster, key, meta, reason}] + ) - events = - Enum.reduce(pg_data, events, fn {key, pid, meta, time}, acc_events -> - if shard_index_for(cluster, key, num_shards) != shard do - acc_events - else - case Data.pg_lookup(name, shard, cluster, key, pid) do - nil -> - Data.pg_insert(name, shard, cluster, key, pid, meta, time, node(pid)) - [build_event(name, :joined, key, pid, meta, %{cluster: cluster}) | acc_events] + :ok + end + + defp replay_local_mutation( + state, + {:join, cluster, key, pid, meta, time, _reason, entry_node} + ) do + Data.pg_insert( + state.name, + state.shard_index, + cluster, + key, + pid, + meta, + time, + entry_node + ) + end - {_meta, existing_time, _node} when time > existing_time -> - Data.pg_insert(name, shard, cluster, key, pid, meta, time, node(pid)) - acc_events + defp replay_local_mutation(state, {:leave, cluster, key, pid, meta, reason}) do + Data.pg_delete_matching_many( + state.name, + state.shard_index, + [{pid, cluster, key, meta, reason}] + ) - _ -> - acc_events - end - end - end) + :ok + end - {state, events} + defp cluster_member?(name, cluster) do + node() in Data.cluster_nodes(name, cluster) end defp resolve_replicated_registry_conflict( @@ -2882,6 +5256,7 @@ defmodule Group.Replica do cond do winner_pid == remote_pid -> + exit_local_conflict_loser(local_pid, key, remote_meta) time = System.system_time() event = @@ -2893,7 +5268,10 @@ defmodule Group.Replica do { Map.put(entries, entry, {initial, {remote_pid, remote_meta, time, node(remote_pid)}}), [event | events], - broadcasts, + [ + {:unregister, cluster, key, local_pid, local_meta, :resolve_conflict} + | broadcasts + ], MapSet.put(maybe_demonitor_pids, local_pid) } @@ -2908,6 +5286,8 @@ defmodule Group.Replica do } true -> + exit_local_conflict_loser(local_pid, key, nil) + event = build_event(state.name, :unregistered, key, local_pid, local_meta, %{ reason: :resolve_conflict, @@ -2923,174 +5303,472 @@ defmodule Group.Replica do end end - defp resolve_conflict( - state, - cluster, - key, - {local_pid, local_meta, local_time}, - {remote_pid, remote_meta, remote_time} - ) do - %{name: name, shard_index: shard} = state + defp reconcile_registry_projection(state, cluster, key, reason, events) do + claims = Data.registry_claims(state.name, state.shard_index, cluster, key) + state = remember_pending_registry_reprojections(state, cluster, key, claims) + winner = select_registry_claim_winner(state, cluster, key, claims) - winner_pid = - resolve_conflict_winner( - state, - cluster, - key, - {local_pid, local_meta, local_time}, - {remote_pid, remote_meta, remote_time} - ) + {state, retirement} = retire_local_registry_losers(state, cluster, key, claims, winner) - cond do - winner_pid == remote_pid -> - # Remote wins — replace local entry - Data.registry_delete(name, shard, cluster, key, local_pid) - state = maybe_demonitor_pid(state, name, shard, local_pid) - time = System.system_time() + case retirement do + :authority_changed -> + reconcile_registry_projection(state, cluster, key, reason, events) - Data.registry_insert( - name, - shard, + retired? -> + winner = + if retired? do + state.name + |> Data.registry_claims(state.shard_index, cluster, key) + |> then(&select_registry_claim_winner(state, cluster, key, &1)) + else + winner + end + + current = Data.registry_lookup(state.name, state.shard_index, cluster, key) + projection_reason = if retired?, do: :resolve_conflict, else: reason + + project_registry_winner( + state, cluster, key, - remote_pid, - remote_meta, - time, - node(remote_pid) + current, + winner, + projection_reason, + events ) + end + end - # Dispatch lifecycle events so monitors see the eviction. - # The :registered event for remote_pid will arrive via the winner's - # re-broadcast registry op, so we only dispatch :unregistered here. - event = - build_event(name, :unregistered, key, local_pid, local_meta, %{ - reason: :resolve_conflict, - cluster: cluster - }) + defp reconcile_registry_snapshot_batch(state, cluster, stream_id, claims, event_buffer) do + source_node = WireProtocol.stream_origin(stream_id) + generation = WireProtocol.stream_generation(stream_id) + epoch = WireProtocol.stream_epoch(stream_id) + claim_table = Data.reg_claim_by_key_table(state.name, state.shard_index) + projection_table = Data.reg_by_key_table(state.name, state.shard_index) - {state, event} + classified = + Enum.map(claims, fn {key, _pid, _meta, _time} = claim -> + claim_key = {cluster, key, source_node, generation, epoch} - winner_pid == local_pid -> - # Local wins — re-broadcast to override remote - time = System.system_time() + { + claim, + Data.registry_claim_uncontended_in_table?(claim_table, claim_key) + } + end) - Data.registry_insert( - name, - shard, - cluster, - key, - local_pid, - local_meta, - time, - node(local_pid) - ) + entries = + Enum.map(claims, fn {key, pid, meta, time} -> + {cluster, key, pid, meta, time, source_node} + end) - state = - enqueue_replicated_registry_broadcast( - state, - {:register, cluster, key, local_pid, local_meta, time, node(local_pid)} - ) + if Enum.all?(classified, &elem(&1, 1)) and + Data.registry_insert_new_many(state.name, state.shard_index, entries) do + event_buffer = + Enum.reduce(claims, event_buffer, fn {key, pid, meta, _time}, buffer -> + event = build_event(state.name, :registered, key, pid, meta, %{cluster: cluster}) + Snapshot.buffer_event(event, buffer) + end) - {state, nil} + {state, event_buffer} + else + reconcile_registry_snapshot_batch_rows( + state, + cluster, + source_node, + projection_table, + classified, + event_buffer + ) + end + end - true -> - # Neither wins — remove both - Data.registry_delete(name, shard, cluster, key, local_pid) - state = maybe_demonitor_pid(state, name, shard, local_pid) + defp reconcile_registry_snapshot_batch_rows( + state, + cluster, + source_node, + projection_table, + classified, + event_buffer + ) do + {state, event_buffer, inserts} = + Enum.reduce(classified, {state, event_buffer, []}, fn + {{key, pid, meta, time}, uncontended?}, {acc, buffer, inserts} -> + current = Data.registry_lookup_in_table(projection_table, cluster, key) + + case {uncontended?, current} do + {true, nil} -> + event = build_event(acc.name, :registered, key, pid, meta, %{cluster: cluster}) + + { + acc, + Snapshot.buffer_event(event, buffer), + [{cluster, key, pid, meta, time, source_node} | inserts] + } + + {true, {^pid, old_meta, old_time, ^source_node}} -> + if old_meta == meta and old_time == time do + {acc, buffer, inserts} + else + event = + build_event(acc.name, :registered, key, pid, meta, %{ + previous_meta: old_meta, + cluster: cluster + }) + + { + acc, + Snapshot.buffer_event(event, buffer), + [{cluster, key, pid, meta, time, source_node} | inserts] + } + end + + _ -> + :ok = + Data.registry_insert_many(acc.name, acc.shard_index, Enum.reverse(inserts)) + + {acc, events} = + reconcile_registry_projection(acc, cluster, key, :reconcile, []) + + {acc, Snapshot.buffer_events(Enum.reverse(events), buffer), []} + end + end) + + :ok = Data.registry_insert_many(state.name, state.shard_index, Enum.reverse(inserts)) + {state, event_buffer} + end + + defp reconcile_registry_keys(state, keys, reason, events) do + Enum.reduce(keys, {state, events}, fn {cluster, key}, {acc, inner_events} -> + reconcile_registry_projection(acc, cluster, key, reason, inner_events) + end) + end + + defp remember_pending_registry_reprojections(state, _cluster, _key, []), do: state + defp remember_pending_registry_reprojections(state, _cluster, _key, [_claim]), do: state + + defp remember_pending_registry_reprojections(state, cluster, key, claims) do + Enum.reduce(claims, state, fn + {_pid, _meta, _time, origin, generation, epoch, _seq}, acc when origin != node() -> + local_cluster_active? = + is_nil(cluster) or not is_nil(Data.local_cluster_epoch(acc.name, cluster)) + + waiting_for_lane_view? = + local_cluster_active? and generation == Data.remote_generation(acc.name, origin) and + epoch == Data.remote_cluster_epoch(acc.name, origin, cluster) and + not replica_view_current?(acc, origin) + + if waiting_for_lane_view? do + pending = + Map.update( + acc.pending_registry_reprojections, + origin, + MapSet.new([{cluster, key}]), + &MapSet.put(&1, {cluster, key}) + ) + + %{acc | pending_registry_reprojections: pending} + else + acc + end + + _local_claim, acc -> + acc + end) + end + + defp reproject_pending_registry_keys(state, remote_node) do + case Map.pop(state.pending_registry_reprojections, remote_node) do + {nil, _pending} -> + state + + {keys, pending} -> + state = %{state | pending_registry_reprojections: pending} + {state, events} = reconcile_registry_keys(state, keys, :reconcile, []) + notify_monitors(state.name, events) + state + end + end + + defp discard_pending_registry_reprojections(state, remote_node) do + %{ + state + | pending_registry_reprojections: + Map.delete(state.pending_registry_reprojections, remote_node) + } + end + + defp select_registry_claim_winner(_state, _cluster, _key, []), do: nil + defp select_registry_claim_winner(_state, _cluster, _key, [claim]), do: claim + + defp select_registry_claim_winner(%{name: name} = state, cluster, key, claims) do + resolver = Map.get(Group.get_config(name), :resolve_registry_conflict) + + claims + |> Enum.filter(®istry_claim_current?(state, cluster, &1)) + |> Enum.max_by( + fn {pid, meta, time, _origin, _generation, _epoch, _seq} -> + registry_conflict_order_key(name, key, {pid, meta, time}, resolver) + end, + fn -> nil end + ) + end + defp retire_local_registry_losers(state, cluster, key, claims, winner) do + winner_pid = if winner, do: elem(winner, 0), else: nil + + local_losers = + Enum.filter(claims, fn {pid, _meta, _time, origin_node, _generation, _epoch, _seq} -> + origin_node == node() and pid != winner_pid + end) + |> Enum.filter(®istry_claim_current?(state, cluster, &1)) + + cond do + local_losers == [] -> + {state, false} + + registry_winner_authoritative?(state, cluster, winner) -> state = - enqueue_replicated_registry_broadcast( - state, - {:unregister, cluster, key, local_pid, local_meta, :resolve_conflict} - ) + Enum.reduce(local_losers, state, fn + {pid, meta, _time, _origin_node, _generation, _epoch, _seq}, acc -> + op = {:unregister, cluster, key, pid, meta, :resolve_conflict} + record = append_local_replica_record(acc, op) + acc = finish_local_replica_record(acc, record, :registry) + winner_meta = if winner, do: elem(winner, 1), else: nil + exit_local_conflict_loser(pid, key, winner_meta) + acc + end) - event = - build_event(name, :unregistered, key, local_pid, local_meta, %{ - reason: :resolve_conflict, - cluster: cluster - }) + {state, true} - {state, event} + true -> + # Authority changed after winner selection. Nothing irreversible has + # happened yet; select again against the now-current authority. + {state, :authority_changed} end end - defp resolve_conflict_winner( - %{name: name}, + defp registry_claim_current?( + state, + cluster, + {_pid, _meta, _time, origin, generation, epoch, _seq} + ) do + local_cluster_active? = + is_nil(cluster) or not is_nil(Data.local_cluster_epoch(state.name, cluster)) + + local_cluster_active? and + if origin == node() do + generation == Data.generation(state.name) and + epoch == Data.local_cluster_epoch(state.name, cluster) + else + replica_view_current?(state, origin) and + generation == Data.remote_generation(state.name, origin) and + epoch == Data.remote_cluster_epoch(state.name, origin, cluster) + end + end + + defp registry_winner_authoritative?(_state, _cluster, nil), do: true + + defp registry_winner_authoritative?( + _state, + _cluster, + {_pid, _meta, _time, origin, _generation, _epoch, _seq} + ) + when origin == node(), + do: true + + defp registry_winner_authoritative?( + state, + cluster, + {_pid, _meta, _time, origin, generation, epoch, _seq} + ) do + Data.remote_registry_claim_authoritative?( + state.name, + state.shard_index, + origin, + generation, + cluster, + epoch + ) + end + + defp project_registry_winner(state, _cluster, _key, nil, nil, _reason, events), + do: {state, events} + + defp project_registry_winner( + state, cluster, key, - {local_pid, local_meta, local_time}, - {remote_pid, remote_meta, remote_time} + nil, + {pid, meta, time, origin_node, _generation, _epoch, _seq}, + _reason, + events ) do - config = Group.get_config(name) + Data.registry_insert( + state.name, + state.shard_index, + cluster, + key, + pid, + meta, + time, + origin_node + ) - case Map.get(config, :resolve_registry_conflict) do - nil -> - default_resolve_conflict( - name, - cluster, - key, - {local_pid, local_meta, local_time}, - {remote_pid, remote_meta, remote_time} - ) + event = build_event(state.name, :registered, key, pid, meta, %{cluster: cluster}) + {state, [event | events]} + end - {mod, func, extra_args} -> - apply(mod, func, [ - name, - key, - {local_pid, local_meta, local_time}, - {remote_pid, remote_meta, remote_time} | extra_args - ]) + defp project_registry_winner( + state, + cluster, + key, + {pid, old_meta, old_time, old_node}, + {pid, meta, time, origin_node, _generation, _epoch, _seq}, + _reason, + events + ) do + if old_meta == meta and old_time == time and old_node == origin_node do + {state, events} + else + Data.registry_insert( + state.name, + state.shard_index, + cluster, + key, + pid, + meta, + time, + origin_node + ) + + event = + build_event(state.name, :registered, key, pid, meta, %{ + previous_meta: old_meta, + cluster: cluster + }) + + {state, [event | events]} end end - defp default_resolve_conflict( - _name, - _cluster, + defp project_registry_winner( + state, + cluster, key, - {pid1, meta1, time1}, - {pid2, meta2, time2} + {old_pid, old_meta, _old_time, old_node}, + nil, + reason, + events ) do - # Tiebreaker must be deterministic regardless of which node is resolving. - # Using `>=` would pick the remote on BOTH nodes when timestamps are equal, - # causing mutual kill (both processes die, key becomes unregistered). - # Erlang pids have a total order (by node name then id), so pid comparison - # gives a consistent tiebreaker across all nodes. - {winner_pid, winner_meta, loser_pid} = - if time2 > time1 or (time2 == time1 and pid2 > pid1) do - {pid2, meta2, pid1} + Data.registry_delete(state.name, state.shard_index, cluster, key, old_pid) + + state = + if old_node == node() do + maybe_demonitor_pid(state, state.name, state.shard_index, old_pid) else - {pid1, meta1, pid2} + state end - Logger.error(fn -> - "#{inspect(__MODULE__)}: registry conflict detected: key=#{inspect(key)}, " <> - "pid1=#{inspect(pid1)}, pid2=#{inspect(pid2)}, picking #{inspect(winner_pid)} as winner" - end) + event = + build_event(state.name, :unregistered, key, old_pid, old_meta, %{ + reason: reason, + cluster: cluster + }) - Process.exit(loser_pid, {:group_registry_conflict, key, winner_meta}) - winner_pid + {state, [event | events]} end - # Gather local data for all shared clusters in ONE table scan (instead of C scans) - # and send per-cluster cluster_state messages. One O(N) scan vs C × O(N) scans. - defp send_cluster_states(state, clusters, target_node) do - %{name: name, shard_index: shard} = state - {reg_by_cluster, pg_by_cluster} = Data.local_data_by_cluster(name, shard, clusters) + defp project_registry_winner( + state, + cluster, + key, + {old_pid, old_meta, _old_time, old_node}, + {pid, meta, time, origin_node, _generation, _epoch, _seq}, + reason, + events + ) do + Data.registry_delete(state.name, state.shard_index, cluster, key, old_pid) - log_verbose(state, fn -> - "#{log_prefix_shard(state)} sending cluster_states to #{target_node} (#{length(clusters)} clusters)" - end) + state = + if old_node == node() do + maybe_demonitor_pid(state, state.name, state.shard_index, old_pid) + else + state + end + + Data.registry_insert( + state.name, + state.shard_index, + cluster, + key, + pid, + meta, + time, + origin_node + ) + + unregistered = + build_event(state.name, :unregistered, key, old_pid, old_meta, %{ + reason: reason, + cluster: cluster + }) + + registered = build_event(state.name, :registered, key, pid, meta, %{cluster: cluster}) + {state, [registered, unregistered | events]} + end + + defp resolve_conflict_winner( + %{name: name}, + _cluster, + key, + {local_pid, local_meta, local_time}, + {remote_pid, remote_meta, remote_time} + ) do + config = Group.get_config(name) + resolver = Map.get(config, :resolve_registry_conflict) - for cluster <- clusters do - reg_data = Map.get(reg_by_cluster, cluster, []) - pg_data = Map.get(pg_by_cluster, cluster, []) + local = {local_pid, local_meta, local_time} + remote = {remote_pid, remote_meta, remote_time} - if reg_data != [] or pg_data != [] do - send_to_peer(state, target_node, {:cluster_state, cluster, reg_data, pg_data}) + winner = + if registry_conflict_order_key(name, key, remote, resolver) > + registry_conflict_order_key(name, key, local, resolver) do + remote_pid + else + local_pid end + + if is_nil(resolver) do + log_default_registry_conflict(key, local_pid, remote_pid, winner) end + + winner + end + + defp registry_conflict_order_key(_name, _key, {pid, _meta, time}, nil), do: {time, pid} + + defp registry_conflict_order_key(name, key, {pid, _meta, _time} = claim, { + mod, + func, + extra_args + }) do + {apply(mod, func, [name, key, claim | extra_args]), pid} + end + + defp log_default_registry_conflict(key, pid1, pid2, winner_pid) do + Logger.error(fn -> + "#{inspect(__MODULE__)}: registry conflict detected: key=#{inspect(key)}, " <> + "pid1=#{inspect(pid1)}, pid2=#{inspect(pid2)}, picking #{inspect(winner_pid)} as winner" + end) end + defp exit_local_conflict_loser(pid, key, winner_meta) when node(pid) == node() do + Process.exit(pid, {:group_registry_conflict, key, winner_meta}) + :ok + end + + defp exit_local_conflict_loser(_pid, _key, _winner_meta), do: :ok + defp compute_shared_clusters(my_clusters, remote_clusters) do my_set = MapSet.new(my_clusters) remote_set = MapSet.new(remote_clusters) @@ -3102,7 +5780,6 @@ defmodule Group.Replica do # Remote disconnects remove only data owned by the departing node. node_guard = if target == :all, do: [], else: [{:==, :"$5", target}] reg_table = Data.reg_by_key_table(name, shard) - reg_pid_table = Data.reg_by_pid_table(name, shard) purged_reg = :ets.select(reg_table, [ @@ -3112,12 +5789,10 @@ defmodule Group.Replica do |> Enum.map(fn {key, pid, meta, time} -> {cluster, key, pid, meta, time} end) for {^cluster, key, pid, _meta, _time} <- purged_reg do - :ets.delete(reg_table, {cluster, key}) - :ets.delete(reg_pid_table, {pid, cluster, key}) + Data.registry_delete(name, shard, cluster, key, pid) end pg_table = Data.pg_by_key_table(name, shard) - pg_pid_table = Data.pg_by_pid_table(name, shard) purged_pg = :ets.select(pg_table, [ @@ -3127,8 +5802,7 @@ defmodule Group.Replica do |> Enum.map(fn {key, pid, meta, time} -> {cluster, key, pid, meta, time} end) for {^cluster, key, pid, _meta, _time} <- purged_pg do - :ets.delete(pg_table, {cluster, key, pid}) - :ets.delete(pg_pid_table, {pid, cluster, key}) + Data.pg_delete(name, shard, cluster, key, pid) end {purged_reg, purged_pg} @@ -3204,6 +5878,23 @@ defmodule Group.Replica do :ok end + defp notify_snapshot_events(name, table) do + {events, _count} = + Snapshot.fold_events(table, {[], 0}, fn event, {events, count} -> + events = [event | events] + count = count + 1 + + if count >= @snapshot_event_batch_size do + notify_monitors(name, events) + {[], 0} + else + {events, count} + end + end) + + notify_monitors(name, events) + end + defp matching_subscribers(name, cluster, key, cache) do {all_subscribers, cache} = get_cached_subscribers(name, cluster, :all, cache) {exact_subscribers, cache} = get_cached_subscribers(name, cluster, {:exact, key}, cache) diff --git a/lib/group/replica/data.ex b/lib/group/replica/data.ex index edfd9a0..8964999 100644 --- a/lib/group/replica/data.ex +++ b/lib/group/replica/data.ex @@ -2,6 +2,8 @@ defmodule Group.Replica.Data do @moduledoc false use GenServer + alias Group.Replica.WireProtocol + _archdoc = """ GenServer that owns ETS tables for all shards. @@ -10,8 +12,9 @@ defmodule Group.Replica.Data do ## ETS Table Layout - Each shard owns 4 tables. There are also 3 shared tables per Group instance: - 2 for cluster membership and 1 for local named-cluster TTL leases. + Each shard has materialized registry/PG indexes, authoritative registry-claim + indexes, and replica stream/oplog/cursor tables. Shared tables hold cluster + membership, local named-cluster TTL leases, generations, and cluster epochs. ### reg_by_key — `:set`, keyed by `{cluster, key}` @@ -53,6 +56,29 @@ defmodule Group.Replica.Data do `entries_by_pid` can select all entries for a pid as a contiguous range scan. Also used by `maybe_demonitor` to check if a pid has any remaining entries (select with limit 1). + ### reg_claim_by_key / reg_claim_by_pid — authoritative registry claims + + {{cluster, key, origin, generation, epoch}, pid, meta, time, sequence} + {{pid, cluster, key, origin, generation, epoch}, meta, time, sequence} + + The visible `reg_by_key` row is only a deterministic projection. These tables retain one + independently versioned claim per origin so a losing-but-live claim is not forgotten. + Only its owner stream can delete it; conflict resolution then recomputes the visible winner. + + ### replica_stream_meta / replica_oplog / replica_oplog_order / replica_cursor + + Local streams are keyed by `{group, origin, generation, shard, cluster, epoch}`. + `replica_stream_meta` records each stream's head, retained floor, and applied journal + position. `replica_oplog` stores `{stream, sequence}` mutation records while + `replica_oplog_order` gives them one shard-wide append order for bounded pruning. + `replica_cursor` records only the highest contiguous sequence applied from each remote + stream. During exact replacement it temporarily stores + `{:snapshot_installing, snapshot_sequence}`; startup repair treats that marker as an + interrupted transaction, purges the partial origin slice, and removes the cursor so the + sender retransmits. Cursor absence is also the durable retirement marker: every peer, + generation, epoch, and local-cluster purge clears cursors before deleting rows. A gap below + the retained floor is repaired by exact per-origin snapshot replacement. + ### cluster_nodes — `:bag`, keyed by cluster name {cluster, node} @@ -71,9 +97,11 @@ defmodule Group.Replica.Data do targeted deletes from both tables — O(clusters for node) instead of O(total entries). Both tables are shared across all shards. Used for the default cluster (nil) and named - clusters. The nil cluster is maintained by the peer_connect protocol — nodes are added on - peer discovery and removed on nodedown/shard death. `Group.nodes/1` reads nil cluster - from cluster_nodes. + clusters. Peer-connect messages are discovery hints; shard 0's generation-fenced exact + authority installs membership. Exact authority and both membership indexes are replaced + in one Data GenServer turn, closing the local-connect/remote-install race without a scan + outside the remote authority's cluster set. `nodedown`, shard death, or peer-lease expiry + removes it. `Group.nodes/1` reads the nil cluster from cluster_nodes. ### cluster_leases — `:set`, keyed by cluster name @@ -91,6 +119,28 @@ defmodule Group.Replica.Data do Keeping leases separate avoids adding policy state to the hot cluster-membership lookups used by `Group.connect/3`, peer discovery, and replication fanout. + ### replication_meta and epoch tables + + `replication_meta` holds the local origin generation, last exact, complete applied, and + highest observed authority revisions, a persisted `{generation, revision}` authority hint, + per-shard installed remote views, journal metadata, and one append counter per shard. + `local_cluster_epochs` and `closed_local_cluster_epochs` fence local named-cluster + lifetimes; `remote_cluster_epochs` is the node-wide authority installed by shard 0. + The three revision roles are separate so a partial control burst cannot be promoted to + authoritative membership. Contiguous incremental controls compare-and-install against the + current generation, applied revision, observed revision, and hint in this GenServer turn; + a concurrent heartbeat makes the whole update stale. A newer hint atomically fences every + lane view, but cannot be created after exact authority has been retired; only a later exact + hello can reintroduce the peer. Irreversible registry conflict retirement is + revalidated through this GenServer, serializing the decision with node-wide generation, + epoch, observed-revision, installed-lane, and local-cluster changes. + Local cluster activation also projects self and already-authoritative remote routes in + this serialized turn. Deactivation removes local admission and queues explicit old-epoch + cleanup on every shard before replying, so a caller exit cannot strand rows or a close + barrier; a shard that restarts before handling the message repairs from the marker. + Final peer-route cleanup rechecks that both exact authority and its hint are absent, so a + delayed retirement caller cannot erase a rediscovered generation's routes. + ## Match Spec Patterns All match specs use `{:==, :"$N", value}` guards to filter on runtime values (e.g. node @@ -101,10 +151,8 @@ defmodule Group.Replica.Data do ## Bulk Operations & Their Costs - `purge_node/3`: Full table scan via `ets.select` filtering by node, then individual - deletes. O(table size) for the scan, but this only runs on nodedown — rare path. - - - `local_data_by_cluster/3`: Full table scan filtering by `node() == local_node`, - grouped by cluster. Only runs during discovery/sync protocol. + deletes. O(table size) for the scan, but this only runs on nodedown, remote shard death, + or peer-lease expiry — rare paths. - `registry_count`, `pg_count`, `pg_count_by_prefix`, `local_registry_count`, `local_pg_count`, `local_registry_present?`, `local_pg_present?`: Uses @@ -124,8 +172,11 @@ defmodule Group.Replica.Data do local processes that registered before the crash would be orphaned — nobody would monitor them, and their ETS entries would persist forever if they later died. - Remote data doesn't need this protection — the discovery protocol re-syncs everything - from remote nodes on restart. Only local process entries need the ETS scan. + Only locally owned member processes are monitored. Remote owner death arrives as a + sequenced delete from that owner; nodedown or peer-lease expiry removes the complete + remote origin if it cannot report the delete. Periodic anti-entropy then rebuilds a + returning origin from retained deltas or an exact snapshot. No node monitors or exits + another node's member processes. The `state.monitors` map also deduplicates: a pid registered under multiple keys in the same shard gets one monitor, not one per key. @@ -139,8 +190,8 @@ defmodule Group.Replica.Data do All tables are `:public` with `read_concurrency: true`. Reads happen directly from any process (the Replica GenServer, Group API callers, etc.). Writes are serialized through the Replica GenServer for each shard, ensuring consistent paired updates to both the - by_key and by_pid tables. The Data GenServer itself only owns the tables (for crash - survival via rest_for_one) — it handles no messages after init. + by_key and by_pid tables. The Data GenServer owns the tables (for shard-crash survival + via rest_for_one) and serializes cross-shard generation, epoch, and cluster-node changes. """ def start_link(opts) do @@ -151,6 +202,852 @@ defmodule Group.Replica.Data do def data_name(name), do: :"#{name}_data" + # ===================================================================== + # Replica generations, epochs, journal, and cursors + # ===================================================================== + + def generation(name) do + :ets.lookup_element(replication_meta_table(name), :generation, 2) + end + + def local_cluster_epoch_revision(name) do + :ets.lookup_element(replication_meta_table(name), :cluster_epoch_revision, 2) + end + + def local_cluster_epoch(name, nil), do: generation(name) + + def local_cluster_epoch(name, cluster) do + case :ets.lookup(local_cluster_epochs_table(name), cluster) do + [{^cluster, epoch}] -> epoch + [] -> nil + end + end + + def local_cluster_epochs(name) do + [{nil, generation(name)} | :ets.tab2list(local_cluster_epochs_table(name))] + end + + def local_replica_authority(name) do + GenServer.call(data_name(name), :local_replica_authority, :infinity) + end + + def closed_local_cluster_epoch(name, cluster) do + case :ets.lookup(closed_local_cluster_epochs_table(name), cluster) do + [{^cluster, epoch, _pending_shards}] -> epoch + [] -> nil + end + end + + def closed_local_clusters(name) do + closed_local_cluster_epochs_table(name) + |> :ets.tab2list() + |> Enum.map(&elem(&1, 0)) + end + + def closed_local_cluster_epochs(name) do + closed_local_cluster_epochs_table(name) + |> :ets.tab2list() + |> Enum.map(fn {cluster, epoch, _pending_shards} -> {cluster, epoch} end) + end + + def closed_local_cluster_pending?(name, cluster, epoch, shard) do + case :ets.lookup(closed_local_cluster_epochs_table(name), cluster) do + [{^cluster, ^epoch, pending_shards}] -> MapSet.member?(pending_shards, shard) + _ -> false + end + end + + def await_closed_local_clusters(name, clusters, timeout) + when is_list(clusters) and is_integer(timeout) and timeout >= 0 do + started_at = System.monotonic_time(:millisecond) + await_closed_local_clusters(name, clusters, timeout, started_at) + end + + def remote_generation(name, remote_node) do + case :ets.lookup(replication_meta_table(name), {:remote_generation, remote_node}) do + [{{:remote_generation, ^remote_node}, generation}] -> generation + [] -> nil + end + end + + def remote_cluster_epoch(name, remote_node, nil), do: remote_generation(name, remote_node) + + def remote_cluster_epoch(name, remote_node, cluster) do + case :ets.lookup(remote_cluster_epochs_table(name), {remote_node, cluster}) do + [{{^remote_node, ^cluster}, epoch}] -> epoch + [] -> nil + end + end + + def remote_cluster_epoch_revision(name, remote_node) do + case :ets.lookup(replication_meta_table(name), {:remote_epoch_revision, remote_node}) do + [{{:remote_epoch_revision, ^remote_node}, revision}] -> revision + [] -> nil + end + end + + def remote_cluster_epoch_exact_revision(name, remote_node) do + case :ets.lookup(replication_meta_table(name), {:remote_epoch_exact, remote_node}) do + [{{:remote_epoch_exact, ^remote_node}, revision}] -> revision + [] -> nil + end + end + + def remote_cluster_epoch_observed_revision(name, remote_node) do + case :ets.lookup(replication_meta_table(name), {:remote_epoch_observed, remote_node}) do + [{{:remote_epoch_observed, ^remote_node}, revision}] -> revision + [] -> nil + end + end + + def remote_replica_authority_hint(name, remote_node) do + case :ets.lookup(replication_meta_table(name), {:remote_authority_hint, remote_node}) do + [{{:remote_authority_hint, ^remote_node}, generation, revision}] -> + {generation, revision} + + [] -> + nil + end + end + + @doc false + def remote_authority_install_count(name, remote_node) do + case :ets.lookup(replication_meta_table(name), {:remote_authority_installs, remote_node}) do + [{{:remote_authority_installs, ^remote_node}, count}] -> count + [] -> 0 + end + end + + def remote_view_generation(name, shard, remote_node) do + case :ets.lookup(replication_meta_table(name), {:remote_view_info, shard, remote_node}) do + [{{:remote_view_info, ^shard, ^remote_node}, generation, _revision, _observed}] -> + generation + + [] -> + nil + end + end + + def remote_view_cluster_epoch_revision(name, shard, remote_node) do + case :ets.lookup(replication_meta_table(name), {:remote_view_info, shard, remote_node}) do + [{{:remote_view_info, ^shard, ^remote_node}, _generation, revision, _observed}] -> + revision + + [] -> + nil + end + end + + def remote_view_observed_revision(name, shard, remote_node) do + case :ets.lookup(replication_meta_table(name), {:remote_view_info, shard, remote_node}) do + [{{:remote_view_info, ^shard, ^remote_node}, _generation, _revision, observed}] -> + observed + + [] -> + nil + end + end + + def put_remote_replica_info(name, shard, remote_node, generation, epoch_revision, epochs) do + GenServer.call( + data_name(name), + {:put_remote_replica_info, shard, remote_node, generation, epoch_revision, epochs}, + :infinity + ) + end + + def put_remote_view_info(name, shard, remote_node, generation, authoritative, observed) do + GenServer.call( + data_name(name), + {:put_remote_view_info, shard, remote_node, generation, authoritative, observed}, + :infinity + ) + end + + def put_remote_cluster_epochs( + name, + shard, + remote_node, + generation, + expected_revision, + revision, + epochs + ) do + GenServer.call( + data_name(name), + {:put_remote_cluster_epochs, shard, remote_node, generation, expected_revision, revision, + epochs}, + :infinity + ) + end + + def observe_remote_cluster_epoch_revision(name, remote_node, revision) do + GenServer.call( + data_name(name), + {:observe_remote_cluster_epoch_revision, remote_node, revision}, + :infinity + ) + end + + def observe_remote_replica_hint(name, remote_node, generation, revision) do + GenServer.call( + data_name(name), + {:observe_remote_replica_hint, remote_node, generation, revision}, + :infinity + ) + end + + def remote_registry_claim_authoritative?( + name, + shard, + remote_node, + generation, + cluster, + epoch + ) do + GenServer.call( + data_name(name), + {:remote_registry_claim_authoritative, shard, remote_node, generation, cluster, epoch}, + :infinity + ) + end + + def close_remote_cluster_epochs( + name, + shard, + remote_node, + generation, + expected_revision, + revision, + epochs + ) do + GenServer.call( + data_name(name), + {:close_remote_cluster_epochs, shard, remote_node, generation, expected_revision, revision, + epochs}, + :infinity + ) + end + + def delete_remote_replica_info(name, shard, remote_node) do + GenServer.call( + data_name(name), + {:delete_remote_replica_info, shard, remote_node}, + :infinity + ) + end + + def expire_remote_replica_lane(name, shard, remote_node) do + GenServer.call( + data_name(name), + {:expire_remote_replica_lane, shard, remote_node}, + :infinity + ) + end + + def activate_local_clusters(name, clusters) do + GenServer.call(data_name(name), {:activate_local_clusters, clusters}, :infinity) + end + + def activate_local_clusters_durable(name, clusters) do + GenServer.call(data_name(name), {:activate_local_clusters_durable, clusters}, :infinity) + end + + def deactivate_local_clusters(name, clusters) do + GenServer.call(data_name(name), {:deactivate_local_clusters, clusters}, :infinity) + end + + def deactivate_local_clusters_durable(name, clusters) do + GenServer.call(data_name(name), {:deactivate_local_clusters_durable, clusters}, :infinity) + end + + def mark_closed_cluster_shard(name, cluster_epochs, shard) do + GenServer.call( + data_name(name), + {:mark_closed_cluster_shard, cluster_epochs, shard}, + :infinity + ) + end + + def local_stream_id(name, shard, cluster) do + case local_cluster_epoch(name, cluster) do + nil -> + nil + + epoch -> + Group.Replica.WireProtocol.stream_id( + name, + node(), + generation(name), + shard, + cluster, + epoch + ) + end + end + + def append_replica_record(name, shard, stream_id, mutations) when is_list(mutations) do + stream_table = replica_stream_meta_table(name, shard) + + head = + :ets.update_counter( + stream_table, + stream_id, + {2, 1}, + {stream_id, 0, 1, 0} + ) + + append_id = + :ets.update_counter( + replication_meta_table(name), + {:append_counter, shard}, + {2, 1}, + {{:append_counter, shard}, 0} + ) + + :ets.insert(replica_oplog_table(name, shard), {{stream_id, head}, append_id, mutations}) + :ets.insert(replica_oplog_order_table(name, shard), {append_id, stream_id, head}) + {head, mutations} + end + + def mark_local_replica_applied(name, shard, stream_id, seq) do + :ets.update_element(replica_stream_meta_table(name, shard), stream_id, {4, seq}) + :ok + end + + def local_replica_unapplied(name, shard) do + replica_stream_meta_table(name, shard) + |> :ets.tab2list() + |> Enum.flat_map(fn {stream_id, head, _floor, applied} -> + if applied < head do + replica_records(name, shard, stream_id, applied + 1, head - applied) + |> Enum.map(fn {seq, mutations} -> {stream_id, seq, mutations} end) + else + [] + end + end) + end + + @doc false + def repair_local_replica_journal(name, shard) do + stream_table = replica_stream_meta_table(name, shard) + oplog_table = replica_oplog_table(name, shard) + order_table = replica_oplog_order_table(name, shard) + + stream_table + |> :ets.tab2list() + |> Enum.each(fn {stream_id, head, floor, applied} -> + if current_local_stream?(name, shard, stream_id) do + present = + oplog_table + |> :ets.select([ + {{{stream_id, :"$1"}, :_, :_}, [], [:"$1"]} + ]) + |> MapSet.new() + + repaired_floor = + floor + |> missing_applied_sequences(applied, present) + |> case do + [] -> floor + missing -> Enum.max(missing) + 1 + end + + repaired_head = contiguous_unapplied_head(applied, head, present) + + if repaired_head < head do + :ets.select_delete(oplog_table, [ + {{{stream_id, :"$1"}, :_, :_}, [{:>, :"$1", repaired_head}], [true]} + ]) + end + + repaired_floor = min(repaired_floor, repaired_head + 1) + repaired_applied = min(applied, repaired_head) + + :ets.insert( + stream_table, + {stream_id, repaired_head, repaired_floor, repaired_applied} + ) + else + drop_local_stream( + name, + shard, + WireProtocol.stream_cluster(stream_id), + WireProtocol.stream_epoch(stream_id) + ) + end + end) + + :ets.delete_all_objects(order_table) + + oplog_table + |> :ets.tab2list() + |> Enum.each(fn {{stream_id, seq}, append_id, _mutations} -> + :ets.insert(order_table, {append_id, stream_id, seq}) + end) + + :ok + end + + @doc false + def repair_shard_indexes(name, shard) do + repair_interrupted_snapshot_installs(name, shard) + repair_primary_replica_rows(name, shard) + :ok + end + + def replica_stream_heads(name, shard) do + :ets.tab2list(replica_stream_meta_table(name, shard)) + |> Enum.map(fn {stream_id, head, floor, _applied} -> {stream_id, floor, head} end) + end + + defp missing_applied_sequences(floor, applied, _present) when floor > applied, do: [] + + defp missing_applied_sequences(floor, applied, present) do + Enum.reject(floor..applied, &MapSet.member?(present, &1)) + end + + defp contiguous_unapplied_head(applied, head, _present) when applied >= head, do: head + + defp contiguous_unapplied_head(applied, head, present) do + Enum.reduce_while((applied + 1)..head, applied, fn seq, _last -> + if MapSet.member?(present, seq), do: {:cont, seq}, else: {:halt, seq - 1} + end) + end + + defp current_local_stream?(name, shard, stream_id) do + cluster = WireProtocol.stream_cluster(stream_id) + + WireProtocol.stream_name(stream_id) == name and + WireProtocol.stream_origin(stream_id) == node() and + WireProtocol.stream_generation(stream_id) == generation(name) and + WireProtocol.stream_shard(stream_id) == shard and + WireProtocol.stream_epoch(stream_id) == local_cluster_epoch(name, cluster) + end + + defp repair_interrupted_snapshot_installs(name, shard) do + streams = + :ets.select(replica_cursor_table(name, shard), [ + {{:"$1", {:snapshot_installing, :_}}, [], [:"$1"]} + ]) + + {streams, malformed} = + Enum.split_with(streams, fn stream_id -> + WireProtocol.valid_stream_id?(stream_id) and + WireProtocol.stream_name(stream_id) == name and + WireProtocol.stream_shard(stream_id) == shard and + WireProtocol.stream_origin(stream_id) != node() + end) + + Enum.each(malformed, &:ets.delete(replica_cursor_table(name, shard), &1)) + + if streams != [] do + _affected_keys = purge_registry_claims_for_streams(name, shard, streams) + + streams + |> Enum.group_by(&WireProtocol.stream_origin/1, &WireProtocol.stream_cluster/1) + |> Enum.each(fn {origin, clusters} -> + delete_pg_for_origin_clusters(name, shard, Enum.uniq(clusters), origin) + end) + + Enum.each(streams, &:ets.delete(replica_cursor_table(name, shard), &1)) + end + + :ok + end + + defp await_closed_local_clusters(name, clusters, timeout, started_at) do + pending? = + Enum.any?(clusters, fn cluster -> + not is_nil(closed_local_cluster_epoch(name, cluster)) + end) + + elapsed = System.monotonic_time(:millisecond) - started_at + + cond do + not pending? -> + max(timeout - elapsed, 0) + + elapsed >= timeout -> + exit( + {:timeout, + {GenServer, :call, + [data_name(name), {:await_closed_local_clusters, clusters}, timeout]}} + ) + + true -> + receive do + after + min(10, timeout - elapsed) -> + await_closed_local_clusters(name, clusters, timeout, started_at) + end + end + end + + # A shard can crash between writes to its materialized rows and receive + # cursor, or while retiring an epoch across multiple ETS tables. Recover from + # the primary tables themselves: stale claims carry their complete stream + # authority, while a remote PG row is retained only when the current stream + # has a cursor (including the sequence-zero admission marker). This pass also + # replaces the old multi-million-element cluster list with one fixed-table + # traversal and O(number of inactive clusters) accumulator memory. Reverse + # indexes are rebuilt during the same primary-table pass, avoiding both a + # second traversal and a complete `tab2list/1` heap copy per index. + defp repair_primary_replica_rows(name, shard) do + reg_reverse = reg_by_pid_table(name, shard) + claim_reverse = reg_claim_by_pid_table(name, shard) + pg_reverse = pg_by_pid_table(name, shard) + :ets.delete_all_objects(reg_reverse) + :ets.delete_all_objects(claim_reverse) + :ets.delete_all_objects(pg_reverse) + + inactive_clusters = MapSet.new() + + inactive_clusters = + repair_ets_table( + reg_by_key_table(name, shard), + inactive_clusters, + fn {{cluster, key}, pid, meta, time, entry_node}, inactive -> + if active_local_cluster?(name, cluster) do + :ets.insert(reg_reverse, {{pid, cluster, key}, meta, time, entry_node}) + inactive + else + :ets.delete(reg_by_key_table(name, shard), {cluster, key}) + remember_inactive_cluster(name, inactive, cluster) + end + end + ) + + inactive_clusters = + repair_ets_table( + reg_claim_by_key_table(name, shard), + inactive_clusters, + fn {{cluster, key, origin, claim_generation, epoch}, pid, meta, time, seq}, inactive -> + if active_local_cluster?(name, cluster) and + valid_claim_authority?( + name, + shard, + cluster, + origin, + claim_generation, + epoch + ) do + :ets.insert( + claim_reverse, + {{pid, cluster, key, origin, claim_generation, epoch}, meta, time, seq} + ) + + inactive + else + :ets.delete( + reg_claim_by_key_table(name, shard), + {cluster, key, origin, claim_generation, epoch} + ) + + remember_inactive_cluster(name, inactive, cluster) + end + end + ) + + inactive_clusters = + repair_ets_table( + pg_by_key_table(name, shard), + inactive_clusters, + fn {{cluster, key, pid}, meta, time, entry_node}, inactive -> + valid? = + active_local_cluster?(name, cluster) and node(pid) == entry_node and + (entry_node == node() or + valid_remote_pg_authority?(name, shard, cluster, entry_node)) + + if valid? do + :ets.insert(pg_reverse, {{pid, cluster, key}, meta, time, entry_node}) + inactive + else + :ets.delete(pg_by_key_table(name, shard), {cluster, key, pid}) + remember_inactive_cluster(name, inactive, cluster) + end + end + ) + + inactive_clusters = + repair_ets_table( + replica_cursor_table(name, shard), + inactive_clusters, + fn {stream_id, _cursor}, inactive -> + cluster = + if WireProtocol.valid_stream_id?(stream_id), + do: WireProtocol.stream_cluster(stream_id) + + if valid_remote_cursor_authority?(name, shard, stream_id) do + inactive + else + :ets.delete(replica_cursor_table(name, shard), stream_id) + remember_inactive_cluster(name, inactive, cluster) + end + end + ) + + inactive_clusters = MapSet.to_list(inactive_clusters) + if inactive_clusters != [], do: remove_clusters(name, inactive_clusters) + :ok + end + + defp repair_ets_table(table, acc, fun) do + :ets.safe_fixtable(table, true) + + try do + repair_ets_table(table, :ets.first(table), acc, fun) + after + :ets.safe_fixtable(table, false) + end + end + + defp repair_ets_table(_table, :"$end_of_table", acc, _fun), do: acc + + defp repair_ets_table(table, key, acc, fun) do + next_key = :ets.next(table, key) + + acc = + case :ets.lookup(table, key) do + [object] -> fun.(object, acc) + [] -> acc + end + + repair_ets_table(table, next_key, acc, fun) + end + + defp active_local_cluster?(_name, nil), do: true + defp active_local_cluster?(name, cluster), do: not is_nil(local_cluster_epoch(name, cluster)) + + defp remember_inactive_cluster(_name, inactive, nil), do: inactive + + defp remember_inactive_cluster(name, inactive, cluster) do + if active_local_cluster?(name, cluster), do: inactive, else: MapSet.put(inactive, cluster) + end + + defp valid_claim_authority?(name, _shard, cluster, origin, claim_generation, epoch) + when origin == node() do + claim_generation == generation(name) and epoch == local_cluster_epoch(name, cluster) + end + + defp valid_claim_authority?(name, shard, cluster, origin, claim_generation, epoch) do + if claim_generation == remote_generation(name, origin) and + epoch == remote_cluster_epoch(name, origin, cluster) do + stream_id = + WireProtocol.stream_id(name, origin, claim_generation, shard, cluster, epoch) + + :ets.member(replica_cursor_table(name, shard), stream_id) + else + false + end + end + + defp valid_remote_pg_authority?(name, shard, cluster, origin) do + remote_generation = remote_generation(name, origin) + remote_epoch = remote_cluster_epoch(name, origin, cluster) + + if WireProtocol.valid_generation?(remote_generation) and not is_nil(remote_epoch) do + stream_id = + WireProtocol.stream_id(name, origin, remote_generation, shard, cluster, remote_epoch) + + :ets.member(replica_cursor_table(name, shard), stream_id) + else + false + end + end + + defp valid_remote_cursor_authority?(name, shard, stream_id) do + WireProtocol.valid_stream_id?(stream_id) and + WireProtocol.stream_name(stream_id) == name and + WireProtocol.stream_shard(stream_id) == shard and + WireProtocol.stream_origin(stream_id) != node() and + active_local_cluster?(name, WireProtocol.stream_cluster(stream_id)) and + WireProtocol.stream_generation(stream_id) == + remote_generation(name, WireProtocol.stream_origin(stream_id)) and + WireProtocol.stream_epoch(stream_id) == + remote_cluster_epoch( + name, + WireProtocol.stream_origin(stream_id), + WireProtocol.stream_cluster(stream_id) + ) + end + + def replica_stream_head(name, shard, stream_id) do + case :ets.lookup(replica_stream_meta_table(name, shard), stream_id) do + [{^stream_id, head, floor, applied}] -> {floor, head, applied} + [] -> {1, 0, 0} + end + end + + def replica_records(_name, _shard, _stream_id, _from_seq, limit) when limit <= 0, do: [] + + def replica_records(name, shard, stream_id, from_seq, limit) do + table = replica_oplog_table(name, shard) + + table + |> :ets.select( + [ + {{{stream_id, :"$1"}, :_, :"$2"}, [{:>=, :"$1", from_seq}], [{{:"$1", :"$2"}}]} + ], + limit + ) + |> case do + :"$end_of_table" -> [] + {records, _continuation} -> records + end + end + + def prune_replica_oplog(name, shard, max_entries) do + order_table = replica_oplog_order_table(name, shard) + do_prune_replica_oplog(name, shard, order_table, :ets.info(order_table, :size) - max_entries) + end + + defp do_prune_replica_oplog(_name, _shard, _order_table, remaining) when remaining <= 0, + do: :ok + + defp do_prune_replica_oplog(name, shard, order_table, remaining) do + case :ets.first(order_table) do + :"$end_of_table" -> + :ok + + append_id -> + [{^append_id, stream_id, seq}] = :ets.lookup(order_table, append_id) + {_floor, _head, applied} = replica_stream_head(name, shard, stream_id) + + if seq <= applied do + :ets.delete(order_table, append_id) + :ets.delete(replica_oplog_table(name, shard), {stream_id, seq}) + + case :ets.lookup(replica_stream_meta_table(name, shard), stream_id) do + [{^stream_id, head, floor, local_applied}] -> + :ets.insert( + replica_stream_meta_table(name, shard), + {stream_id, head, max(floor, seq + 1), local_applied} + ) + + [] -> + :ok + end + + do_prune_replica_oplog(name, shard, order_table, remaining - 1) + else + :ok + end + end + end + + def replica_cursor(name, shard, stream_id) do + case :ets.lookup(replica_cursor_table(name, shard), stream_id) do + [{^stream_id, seq}] when is_integer(seq) -> seq + [{^stream_id, {:snapshot_installing, _snapshot_seq}}] -> 0 + [] -> 0 + end + end + + def replica_cursor_streams_for_origin_cluster(name, shard, origin_node, cluster) do + :ets.select(replica_cursor_table(name, shard), [ + {{{name, origin_node, :"$1", shard, cluster, :"$2"}, :_}, [], + [{{name, origin_node, :"$1", shard, cluster, :"$2"}}]} + ]) + end + + def replica_cursor_streams_for_origin(name, shard, origin_node) do + :ets.select(replica_cursor_table(name, shard), [ + {{{name, origin_node, :"$1", shard, :"$2", :"$3"}, :_}, [], + [{{name, origin_node, :"$1", shard, :"$2", :"$3"}}]} + ]) + end + + def put_replica_cursor(name, shard, stream_id, seq) do + :ets.insert(replica_cursor_table(name, shard), {stream_id, seq}) + :ok + end + + def ensure_replica_cursor(name, shard, stream_id) do + :ets.insert_new(replica_cursor_table(name, shard), {stream_id, 0}) + :ok + end + + def begin_replica_snapshot_install(name, shard, stream_id, snapshot_seq) do + :ets.insert( + replica_cursor_table(name, shard), + {stream_id, {:snapshot_installing, snapshot_seq}} + ) + + :ok + end + + def delete_replica_cursors_for_origin(name, shard, origin_node) do + :ets.select_delete(replica_cursor_table(name, shard), [ + {{{name, origin_node, :_, shard, :_, :_}, :_}, [], [true]} + ]) + + :ok + end + + def delete_replica_cursors_for_clusters(_name, _shard, []), do: :ok + + def delete_replica_cursors_for_clusters(name, shard, clusters) do + match_specs = + Enum.map(clusters, fn cluster -> + {{{name, :_, :_, shard, cluster, :_}, :_}, [], [true]} + end) + + :ets.select_delete(replica_cursor_table(name, shard), match_specs) + :ok + end + + def delete_replica_cursor(name, shard, stream_id) do + :ets.delete(replica_cursor_table(name, shard), stream_id) + :ok + end + + @doc false + def retained_replica_origins(name, shard) do + # Accepted replica data is always fenced by this persisted lane view. Every + # retirement path destroys data and cursors before deleting the view, so a + # shard crash cannot leave valid data without this restart index. Scanning + # it is O(known peers * shards), never O(registry + PG cardinality). + match_specs = [ + {{{:remote_view_info, shard, :"$1"}, :_, :_, :_}, [], [:"$1"]}, + # A hint is the durable fence left before the observing lane records its + # in-memory lease deadline. Every restarting lane must recognize it so a + # crash in that window cannot strand the peer forever. + {{{:remote_authority_hint, :"$1"}, :_, :_}, [], [:"$1"]} + ] + + match_specs = + if shard == 0 do + [{{{:remote_generation, :"$1"}, :_}, [], [:"$1"]} | match_specs] + else + match_specs + end + + replication_meta_table(name) + |> :ets.select(match_specs) + |> Enum.reject(&(&1 == node())) + |> Enum.uniq() + end + + def drop_local_stream(name, shard, cluster, epoch) do + stream_id = + Group.Replica.WireProtocol.stream_id(name, node(), generation(name), shard, cluster, epoch) + + append_rows = + :ets.select(replica_oplog_table(name, shard), [ + {{{stream_id, :"$1"}, :"$2", :_}, [], [{{:"$1", :"$2"}}]} + ]) + + Enum.each(append_rows, fn {seq, append_id} -> + :ets.delete(replica_oplog_table(name, shard), {stream_id, seq}) + :ets.delete(replica_oplog_order_table(name, shard), append_id) + end) + + :ets.delete(replica_stream_meta_table(name, shard), stream_id) + + :ok + end + # ===================================================================== # Registry operations # ===================================================================== @@ -186,6 +1083,32 @@ defmodule Group.Replica.Data do :ok end + @doc false + def registry_insert_new_many(_name, _shard, []), do: true + + def registry_insert_new_many(name, shard, entries) do + table = reg_by_key_table(name, shard) + table_pid = reg_by_pid_table(name, shard) + + objects = + Enum.map(entries, fn {cluster, key, pid, meta, time, node} -> + {{cluster, key}, pid, meta, time, node} + end) + + if :ets.insert_new(table, objects) do + :ets.insert( + table_pid, + Enum.map(entries, fn {cluster, key, pid, meta, time, node} -> + {{pid, cluster, key}, meta, time, node} + end) + ) + + true + else + false + end + end + def registry_delete(name, shard, cluster, key, pid) do table = reg_by_key_table(name, shard) :ets.delete(table, {cluster, key}) @@ -219,33 +1142,418 @@ defmodule Group.Replica.Data do end def registry_lookup(name, shard, cluster, key) do + registry_lookup_in_table(reg_by_key_table(name, shard), cluster, key) + end + + @doc false + def registry_lookup_in_table(table, cluster, key) do + case :ets.lookup(table, {cluster, key}) do + [{{^cluster, ^key}, pid, meta, time, node}] -> + {pid, meta, time, node} + + [] -> + nil + end + end + + def registry_lookup_by_pid(name, shard, pid) do + table = reg_by_pid_table(name, shard) + + :ets.select(table, [ + {{{pid, :"$1", :"$2"}, :"$3", :"$4", :"$5"}, [], [{{:"$1", :"$2", :"$3", :"$4", :"$5"}}]} + ]) + end + + def registry_lookup_by_prefix(name, shard, cluster, prefix) do table = reg_by_key_table(name, shard) + prefix_end = next_binary_prefix(prefix) + + :ets.select(table, [ + {{{cluster, :"$1"}, :"$2", :"$3", :_, :_}, + [{:andalso, {:>=, :"$1", prefix}, {:<, :"$1", prefix_end}}], [{{:"$2", :"$3"}}]} + ]) + end + + # ===================================================================== + # Authoritative registry claims + # ===================================================================== + + def put_registry_claim(name, shard, stream_id, seq, key, pid, meta, time) do + cluster = Group.Replica.WireProtocol.stream_cluster(stream_id) + origin_node = Group.Replica.WireProtocol.stream_origin(stream_id) + generation = Group.Replica.WireProtocol.stream_generation(stream_id) + epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) + claim_key = {cluster, key, origin_node, generation, epoch} + by_key = reg_claim_by_key_table(name, shard) + + case :ets.lookup(by_key, claim_key) do + [{^claim_key, _old_pid, _old_meta, _old_time, old_seq}] when old_seq >= seq -> + :ok + + [{^claim_key, old_pid, _old_meta, _old_time, _old_seq}] -> + :ets.delete( + reg_claim_by_pid_table(name, shard), + {old_pid, cluster, key, origin_node, generation, epoch} + ) + + insert_registry_claim(name, shard, claim_key, seq, pid, meta, time) + + [] -> + insert_registry_claim(name, shard, claim_key, seq, pid, meta, time) + end + end + + defp insert_registry_claim(name, shard, claim_key, seq, pid, meta, time) do + {cluster, key, origin_node, generation, epoch} = claim_key + :ets.insert(reg_claim_by_key_table(name, shard), {claim_key, pid, meta, time, seq}) + + :ets.insert( + reg_claim_by_pid_table(name, shard), + {{pid, cluster, key, origin_node, generation, epoch}, meta, time, seq} + ) + + :ok + end + + @doc false + def insert_exact_registry_claims_many(_name, _shard, _stream_id, _seq, []), do: :ok + + def insert_exact_registry_claims_many(name, shard, stream_id, seq, claims) + when is_list(claims) do + cluster = Group.Replica.WireProtocol.stream_cluster(stream_id) + origin_node = Group.Replica.WireProtocol.stream_origin(stream_id) + generation = Group.Replica.WireProtocol.stream_generation(stream_id) + epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) + + :ets.insert( + reg_claim_by_key_table(name, shard), + Enum.map(claims, fn {key, pid, meta, time} -> + {{cluster, key, origin_node, generation, epoch}, pid, meta, time, seq} + end) + ) + + :ets.insert( + reg_claim_by_pid_table(name, shard), + Enum.map(claims, fn {key, pid, meta, time} -> + {{pid, cluster, key, origin_node, generation, epoch}, meta, time, seq} + end) + ) + + :ok + end + + def delete_registry_claim(name, shard, stream_id, seq, key, pid) do + cluster = Group.Replica.WireProtocol.stream_cluster(stream_id) + origin_node = Group.Replica.WireProtocol.stream_origin(stream_id) + generation = Group.Replica.WireProtocol.stream_generation(stream_id) + epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) + claim_key = {cluster, key, origin_node, generation, epoch} + + case :ets.lookup(reg_claim_by_key_table(name, shard), claim_key) do + [{^claim_key, ^pid, _meta, _time, old_seq}] when old_seq <= seq -> + delete_registry_claim_indexes( + name, + shard, + cluster, + key, + origin_node, + generation, + epoch, + pid + ) + + :ok + + _ -> + :ok + end + end + + def registry_claims(name, shard, cluster, key) do + :ets.select(reg_claim_by_key_table(name, shard), [ + {{{cluster, key, :"$1", :"$2", :"$3"}, :"$4", :"$5", :"$6", :"$7"}, [], + [{{:"$4", :"$5", :"$6", :"$1", :"$2", :"$3", :"$7"}}]} + ]) + end + + @doc false + def registry_claim_uncontended_in_table?( + table, + {cluster, key, _origin_node, _generation, _epoch} = claim_key + ) do + not same_registry_claim_key?(:ets.prev(table, claim_key), cluster, key) and + not same_registry_claim_key?(:ets.next(table, claim_key), cluster, key) + end + + defp same_registry_claim_key?({cluster, key, _origin, _generation, _epoch}, cluster, key), + do: true + + defp same_registry_claim_key?(_claim_key, _cluster, _key), do: false + + def registry_claims_for_stream(name, shard, stream_id) do + fold_registry_claims_for_stream(name, shard, stream_id, [], fn row, rows -> [row | rows] end) + |> Enum.reverse() + end + + def fold_registry_claims_for_stream(name, shard, stream_id, acc, fun) + when is_function(fun, 2) do + {table, match_spec} = registry_claim_stream_selection(name, shard, stream_id) + + fold_select_batches_fixed(table, match_spec, acc, fn rows, inner -> + Enum.reduce(rows, inner, fun) + end) + end + + def reduce_registry_claim_batches_for_stream(name, shard, stream_id, acc, fun) + when is_function(fun, 2) do + {table, match_spec} = registry_claim_stream_selection(name, shard, stream_id) + fold_select_batches(table, match_spec, acc, fun) + end + + defp registry_claim_stream_selection(name, shard, stream_id) do + cluster = Group.Replica.WireProtocol.stream_cluster(stream_id) + origin_node = Group.Replica.WireProtocol.stream_origin(stream_id) + generation = Group.Replica.WireProtocol.stream_generation(stream_id) + epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) + + { + reg_claim_by_key_table(name, shard), + {{{cluster, :"$1", origin_node, generation, epoch}, :"$2", :"$3", :"$4", :_}, [], + [{{:"$1", :"$2", :"$3", :"$4"}}]} + } + end + + def replace_registry_claims_for_stream(name, shard, stream_id, snapshot_seq, claims) do + cluster = Group.Replica.WireProtocol.stream_cluster(stream_id) + origin_node = Group.Replica.WireProtocol.stream_origin(stream_id) + generation = Group.Replica.WireProtocol.stream_generation(stream_id) + epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) + existing = registry_claims_for_stream(name, shard, stream_id) + + Enum.each(existing, fn {key, pid, _meta, _time} -> + delete_registry_claim_indexes( + name, + shard, + cluster, + key, + origin_node, + generation, + epoch, + pid + ) + end) + + Enum.each(claims, fn {key, pid, meta, time} -> + put_registry_claim(name, shard, stream_id, snapshot_seq, key, pid, meta, time) + end) + + Enum.uniq(Enum.map(existing, &elem(&1, 0)) ++ Enum.map(claims, &elem(&1, 0))) + end + + def replace_registry_claims_for_stream_from_staging( + name, + shard, + stream_id, + snapshot_seq, + staging_table, + chunk_count, + acc, + fun + ) + when is_function(fun, 2) do + replace_registry_claims_for_stream_from_staging( + name, + shard, + stream_id, + snapshot_seq, + staging_table, + chunk_count, + acc, + fun, + fn claims, inner -> + Enum.reduce(claims, inner, fn {key, _pid, _meta, _time}, batch_inner -> + fun.(key, batch_inner) + end) + end + ) + end + + def replace_registry_claims_for_stream_from_staging( + name, + shard, + stream_id, + snapshot_seq, + staging_table, + _chunk_count, + acc, + removed_fun, + installed_batch_fun + ) + when is_function(removed_fun, 2) and is_function(installed_batch_fun, 2) do + cluster = Group.Replica.WireProtocol.stream_cluster(stream_id) + origin_node = Group.Replica.WireProtocol.stream_origin(stream_id) + generation = Group.Replica.WireProtocol.stream_generation(stream_id) + epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) + + acc = + fold_registry_claims_for_stream(name, shard, stream_id, acc, fn + {key, pid, _meta, _time}, inner -> + delete_registry_claim_indexes( + name, + shard, + cluster, + key, + origin_node, + generation, + epoch, + pid + ) + + if Group.Replica.Snapshot.member_registry?(staging_table, key) do + inner + else + removed_fun.(key, inner) + end + end) + + Group.Replica.Snapshot.reduce_registry_batches( + staging_table, + acc, + fn claims, inner -> + :ok = insert_exact_registry_claims_many(name, shard, stream_id, snapshot_seq, claims) + installed_batch_fun.(claims, inner) + end + ) + end + + def purge_registry_claims_for_origin(name, shard, origin_node) do + claims = + :ets.select(reg_claim_by_key_table(name, shard), [ + {{{:"$1", :"$2", origin_node, :"$3", :"$4"}, :"$5", :"$6", :"$7", :_}, [], + [{{:"$1", :"$2", :"$5", :"$6", :"$7", :"$3", :"$4"}}]} + ]) + + Enum.each(claims, fn {cluster, key, pid, _meta, _time, generation, epoch} -> + delete_registry_claim_indexes( + name, + shard, + cluster, + key, + origin_node, + generation, + epoch, + pid + ) + end) + + Enum.uniq(Enum.map(claims, fn {cluster, key, _, _, _, _, _} -> {cluster, key} end)) + end + + def purge_registry_claims_for_streams(_name, _shard, []), do: [] + + def purge_registry_claims_for_streams(name, shard, stream_ids) do + # Select only matching claims into memory. Epoch churn is rare, but the + # complete claim table may contain millions of unrelated rows. + match_specs = + Enum.map(stream_ids, fn stream_id -> + cluster = Group.Replica.WireProtocol.stream_cluster(stream_id) + origin = Group.Replica.WireProtocol.stream_origin(stream_id) + generation = Group.Replica.WireProtocol.stream_generation(stream_id) + epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) + + {{{cluster, :"$1", origin, generation, epoch}, :"$2", :"$3", :"$4", :"$5"}, [], [:"$_"]} + end) + + claims = :ets.select(reg_claim_by_key_table(name, shard), match_specs) + + Enum.each(claims, fn {{cluster, key, origin, generation, epoch}, pid, _meta, _time, _seq} -> + delete_registry_claim_indexes( + name, + shard, + cluster, + key, + origin, + generation, + epoch, + pid + ) + end) + + claims + |> Enum.map(fn {{cluster, key, _origin, _generation, _epoch}, _pid, _meta, _time, _seq} -> + {cluster, key} + end) + |> Enum.uniq() + end + + def purge_registry_claims_for_cluster(name, shard, cluster, origin_node \\ :all) + + def purge_registry_claims_for_cluster(name, shard, cluster, :all) do + claims = + :ets.select(reg_claim_by_key_table(name, shard), [ + {{{cluster, :"$1", :"$2", :"$3", :"$4"}, :"$5", :"$6", :"$7", :_}, [], + [{{:"$1", :"$5", :"$6", :"$7", :"$2", :"$3", :"$4"}}]} + ]) + + delete_registry_claim_rows(name, shard, cluster, claims) + end + + def purge_registry_claims_for_cluster(name, shard, cluster, origin_node) do + claims = + :ets.select(reg_claim_by_key_table(name, shard), [ + {{{cluster, :"$1", origin_node, :"$2", :"$3"}, :"$4", :"$5", :"$6", :_}, [], + [{{:"$1", :"$4", :"$5", :"$6", origin_node, :"$2", :"$3"}}]} + ]) + + delete_registry_claim_rows(name, shard, cluster, claims) + end - case :ets.lookup(table, {cluster, key}) do - [{{^cluster, ^key}, pid, meta, time, node}] -> - {pid, meta, time, node} + defp delete_registry_claim_rows(name, shard, cluster, claims) do + Enum.each(claims, fn {key, pid, _meta, _time, claim_origin, generation, epoch} -> + delete_registry_claim_indexes( + name, + shard, + cluster, + key, + claim_origin, + generation, + epoch, + pid + ) + end) - [] -> - nil - end + Enum.uniq(Enum.map(claims, &elem(&1, 0))) end - def registry_lookup_by_pid(name, shard, pid) do - table = reg_by_pid_table(name, shard) + defp delete_registry_claim_indexes( + name, + shard, + cluster, + key, + origin, + generation, + epoch, + pid + ) do + :ets.delete(reg_claim_by_key_table(name, shard), {cluster, key, origin, generation, epoch}) + + :ets.delete( + reg_claim_by_pid_table(name, shard), + {pid, cluster, key, origin, generation, epoch} + ) - :ets.select(table, [ - {{{pid, :"$1", :"$2"}, :"$3", :"$4", :"$5"}, [], [{{:"$1", :"$2", :"$3", :"$4", :"$5"}}]} - ]) + :ok end - def registry_lookup_by_prefix(name, shard, cluster, prefix) do - table = reg_by_key_table(name, shard) - prefix_end = next_binary_prefix(prefix) + def local_registry_claims_by_pids(name, shard, pids) do + local_node = node() - :ets.select(table, [ - {{{cluster, :"$1"}, :"$2", :"$3", :_, :_}, - [{:andalso, {:>=, :"$1", prefix}, {:<, :"$1", prefix_end}}], [{{:"$2", :"$3"}}]} - ]) + Enum.flat_map(Enum.uniq(pids), fn pid -> + :ets.select(reg_claim_by_pid_table(name, shard), [ + {{{pid, :"$1", :"$2", local_node, :"$3", :"$4"}, :"$5", :"$6", :_}, [], + [{{pid, :"$1", :"$2", :"$5", :"$3", :"$4"}}]} + ]) + end) end # ===================================================================== @@ -423,6 +1731,17 @@ defmodule Group.Replica.Data do process DOWN cleanup. Returns lean `{pid, cluster, key, meta}` tuples for dispatch/event building. """ + def entries_for_pids(_name, _shard, []), do: {[], []} + + def entries_for_pids(name, shard, pids) do + pids = Enum.uniq(pids) + + { + select_entries_for_pids(reg_by_pid_table(name, shard), pids), + select_entries_for_pids(pg_by_pid_table(name, shard), pids) + } + end + def delete_all_for_pids(_name, _shard, []), do: {[], []} def delete_all_for_pids(name, shard, pids) do @@ -594,31 +1913,90 @@ defmodule Group.Replica.Data do ) end - def local_data_by_cluster(name, shard, clusters) do - cluster_set = MapSet.new(clusters) - local_node = node() + def pg_entries_for_origin(name, shard, cluster, origin_node) do + fold_pg_entries_for_origin(name, shard, cluster, origin_node, [], fn row, rows -> + [row | rows] + end) + |> Enum.reverse() + end - reg_table = reg_by_key_table(name, shard) + def fold_pg_entries_for_origin(name, shard, cluster, origin_node, acc, fun) + when is_function(fun, 2) do + {table, match_spec} = pg_origin_selection(name, shard, cluster, origin_node) - reg_by_cluster = - :ets.select(reg_table, [ - {{{:"$1", :"$2"}, :"$3", :"$4", :"$5", :"$6"}, [{:==, :"$6", local_node}], + fold_select_batches_fixed(table, match_spec, acc, fn rows, inner -> + Enum.reduce(rows, inner, fun) + end) + end + + def reduce_pg_entry_batches_for_origin(name, shard, cluster, origin_node, acc, fun) + when is_function(fun, 2) do + {table, match_spec} = pg_origin_selection(name, shard, cluster, origin_node) + fold_select_batches(table, match_spec, acc, fun) + end + + defp pg_origin_selection(name, shard, cluster, origin_node) do + { + pg_by_key_table(name, shard), + {{{cluster, :"$1", :"$2"}, :"$3", :"$4", origin_node}, [], [{{:"$1", :"$2", :"$3", :"$4"}}]} + } + end + + def delete_pg_for_origin_cluster(name, shard, cluster, origin_node) do + entries = pg_entries_for_origin(name, shard, cluster, origin_node) + + Enum.each(entries, fn {key, pid, _meta, _time} -> + pg_delete(name, shard, cluster, key, pid) + end) + + Enum.map(entries, fn {key, pid, meta, time} -> {cluster, key, pid, meta, time} end) + end + + def delete_pg_for_origin_clusters(_name, _shard, [], _origin_node), do: [] + + def delete_pg_for_origin_clusters(name, shard, clusters, origin_node) do + cluster_set = MapSet.new(clusters) + + entries = + :ets.select(pg_by_key_table(name, shard), [ + {{{:"$1", :"$2", :"$3"}, :"$4", :"$5", origin_node}, [], [{{:"$1", :"$2", :"$3", :"$4", :"$5"}}]} ]) - |> Enum.filter(fn {cluster, _, _, _, _} -> MapSet.member?(cluster_set, cluster) end) - |> Enum.group_by(&elem(&1, 0), fn {_, key, pid, meta, time} -> {key, pid, meta, time} end) + |> Enum.filter(fn {cluster, _key, _pid, _meta, _time} -> + MapSet.member?(cluster_set, cluster) + end) - pg_table = pg_by_key_table(name, shard) + Enum.each(entries, fn {cluster, key, pid, _meta, _time} -> + pg_delete(name, shard, cluster, key, pid) + end) - pg_by_cluster = - :ets.select(pg_table, [ - {{{:"$1", :"$2", :"$3"}, :"$4", :"$5", :"$6"}, [{:==, :"$6", local_node}], - [{{:"$1", :"$2", :"$3", :"$4", :"$5"}}]} + entries + end + + def delete_registry_keys(name, shard, cluster, keys) do + Enum.flat_map(keys, fn key -> + case registry_lookup(name, shard, cluster, key) do + {pid, meta, time, _entry_node} -> + registry_delete(name, shard, cluster, key, pid) + [{cluster, key, pid, meta, time}] + + nil -> + [] + end + end) + end + + def delete_pg_cluster(name, shard, cluster) do + entries = + :ets.select(pg_by_key_table(name, shard), [ + {{{cluster, :"$1", :"$2"}, :"$3", :"$4", :_}, [], [{{:"$1", :"$2", :"$3", :"$4"}}]} ]) - |> Enum.filter(fn {cluster, _, _, _, _} -> MapSet.member?(cluster_set, cluster) end) - |> Enum.group_by(&elem(&1, 0), fn {_, key, pid, meta, time} -> {key, pid, meta, time} end) - {reg_by_cluster, pg_by_cluster} + Enum.each(entries, fn {key, pid, _meta, _time} -> + pg_delete(name, shard, cluster, key, pid) + end) + + Enum.map(entries, fn {key, pid, meta, time} -> {cluster, key, pid, meta, time} end) end def purge_node(name, shard, dead_node) do @@ -786,6 +2164,10 @@ defmodule Group.Replica.Data do GenServer.call(data_name(name), {:remove_cluster_node, clusters, node}, :infinity) end + def remove_clusters(name, clusters) when is_list(clusters) do + GenServer.call(data_name(name), {:remove_clusters, clusters}, :infinity) + end + def all_clusters(name) do table = cluster_nodes_table(name) :ets.select(table, [{{:"$1", :_}, [], [:"$1"]}]) |> Enum.uniq() @@ -796,6 +2178,10 @@ defmodule Group.Replica.Data do :ets.lookup(table, node()) |> Enum.map(&elem(&1, 1)) end + def clusters_for_node(name, target_node) do + :ets.lookup(node_clusters_table(name), target_node) |> Enum.map(&elem(&1, 1)) + end + def purge_cluster_node(name, dead_node) do GenServer.call(data_name(name), {:purge_cluster_node, dead_node}, :infinity) end @@ -841,51 +2227,617 @@ defmodule Group.Replica.Data do def reg_by_key_table(name, shard), do: :"#{name}_s#{shard}_reg_by_key" def reg_by_pid_table(name, shard), do: :"#{name}_s#{shard}_reg_by_pid" + def reg_claim_by_key_table(name, shard), do: :"#{name}_s#{shard}_reg_claim_by_key" + def reg_claim_by_pid_table(name, shard), do: :"#{name}_s#{shard}_reg_claim_by_pid" def pg_by_key_table(name, shard), do: :"#{name}_s#{shard}_pg_by_key" def pg_by_pid_table(name, shard), do: :"#{name}_s#{shard}_pg_by_pid" def cluster_nodes_table(name), do: :"#{name}_cluster_nodes" def node_clusters_table(name), do: :"#{name}_node_clusters" def cluster_leases_table(name), do: :"#{name}_cluster_leases" + def replication_meta_table(name), do: :"#{name}_replication_meta" + def local_cluster_epochs_table(name), do: :"#{name}_local_cluster_epochs" + def closed_local_cluster_epochs_table(name), do: :"#{name}_closed_local_cluster_epochs" + def remote_cluster_epochs_table(name), do: :"#{name}_remote_cluster_epochs" + def replica_stream_meta_table(name, shard), do: :"#{name}_s#{shard}_replica_stream_meta" + def replica_oplog_table(name, shard), do: :"#{name}_s#{shard}_replica_oplog" + def replica_oplog_order_table(name, shard), do: :"#{name}_s#{shard}_replica_oplog_order" + def replica_cursor_table(name, shard), do: :"#{name}_s#{shard}_replica_cursor" # ===================================================================== # GenServer callbacks # ===================================================================== + defp replace_remote_cluster_projection(name, remote_node, remote_epochs) do + local_clusters = + local_cluster_epochs_table(name) + |> :ets.select([{{:"$1", :_}, [], [:"$1"]}]) + |> MapSet.new() + |> MapSet.put(nil) + + shared_clusters = + remote_epochs + |> Map.keys() + |> Enum.filter(&MapSet.member?(local_clusters, &1)) + |> MapSet.new() + + previous_clusters = MapSet.new(clusters_for_node(name, remote_node)) + + previous_clusters + |> MapSet.difference(shared_clusters) + |> Enum.each(fn cluster -> + :ets.delete_object(cluster_nodes_table(name), {cluster, remote_node}) + :ets.delete_object(node_clusters_table(name), {remote_node, cluster}) + end) + + rows = MapSet.to_list(shared_clusters) + :ets.insert(cluster_nodes_table(name), Enum.map(rows, &{&1, remote_node})) + :ets.insert(node_clusters_table(name), Enum.map(rows, &{remote_node, &1})) + :ok + end + + defp insert_cluster_nodes(name, clusters, target_node) do + :ets.insert(cluster_nodes_table(name), Enum.map(clusters, &{&1, target_node})) + :ets.insert(node_clusters_table(name), Enum.map(clusters, &{target_node, &1})) + :ok + end + + defp delete_cluster_nodes(name, clusters, target_node) do + Enum.each(clusters, fn cluster -> + :ets.delete_object(cluster_nodes_table(name), {cluster, target_node}) + :ets.delete_object(node_clusters_table(name), {target_node, cluster}) + end) + + :ok + end + + defp delete_cluster_routes(name, clusters) do + Enum.each(clusters, fn cluster -> + nodes = cluster_nodes(name, cluster) + :ets.delete(cluster_nodes_table(name), cluster) + + Enum.each(nodes, fn cluster_node -> + :ets.delete_object(node_clusters_table(name), {cluster_node, cluster}) + end) + end) + + :ok + end + + defp delete_peer_routes(name, remote_node) do + # Scan the forward index directly so this also repairs a one-sided row left + # by an interrupted or older dual-index mutation. + :ets.select_delete(cluster_nodes_table(name), [ + {{:_, remote_node}, [], [true]} + ]) + + :ets.delete(node_clusters_table(name), remote_node) + :ok + end + + defp project_activated_local_clusters(name, clusters) do + :ok = insert_cluster_nodes(name, clusters, node()) + + name + |> cluster_nodes(nil) + |> Enum.reject(&(&1 == node())) + |> Enum.each(fn remote_node -> + shared = + Enum.filter(clusters, fn cluster -> + not is_nil(remote_cluster_epoch(name, remote_node, cluster)) + end) + + :ok = insert_cluster_nodes(name, shared, remote_node) + end) + + :ok + end + + defp cast_cluster_lifecycle(name, shards, request) do + Enum.each(shards, fn shard -> + :ok = Group.Replica.local_cast(Group.Replica.shard_name(name, shard), request) + end) + end + + defp activate_local_clusters(state, clusters, durable?) do + if clusters != [] do + :ets.update_counter( + replication_meta_table(state.name), + :cluster_epoch_revision, + {2, 1}, + {:cluster_epoch_revision, 0} + ) + end + + epochs = + Enum.map(clusters, fn cluster -> + epoch = + case :ets.lookup(local_cluster_epochs_table(state.name), cluster) do + [{^cluster, existing}] -> existing + [] -> make_ref() + end + + :ets.insert(local_cluster_epochs_table(state.name), {cluster, epoch}) + :ets.delete(closed_local_cluster_epochs_table(state.name), cluster) + {cluster, epoch} + end) + + if durable?, do: project_activated_local_clusters(state.name, clusters) + + {epochs, state} + end + + defp deactivate_local_clusters(state, clusters, durable?) do + if clusters != [] do + :ets.update_counter( + replication_meta_table(state.name), + :cluster_epoch_revision, + {2, 1}, + {:cluster_epoch_revision, 0} + ) + end + + epochs = + Enum.map(clusters, fn cluster -> + epoch = local_cluster_epoch(state.name, cluster) + :ets.delete(local_cluster_epochs_table(state.name), cluster) + + if epoch do + pending_shards = MapSet.new(0..(state.num_shards - 1)) + + :ets.insert( + closed_local_cluster_epochs_table(state.name), + {cluster, epoch, pending_shards} + ) + end + + {cluster, epoch} + end) + + if durable? do + :ok = delete_cluster_nodes(state.name, clusters, node()) + + cast_cluster_lifecycle( + state.name, + 0..(state.num_shards - 1), + {:cluster_disconnect, clusters, epochs} + ) + end + + {epochs, state} + end + @impl true def handle_call({:add_cluster_node, clusters, node}, _from, state) do - :ets.insert(cluster_nodes_table(state.name), Enum.map(clusters, &{&1, node})) - :ets.insert(node_clusters_table(state.name), Enum.map(clusters, &{node, &1})) + :ok = insert_cluster_nodes(state.name, clusters, node) {:reply, :ok, state} end def handle_call({:remove_cluster_node, clusters, node}, _from, state) do - Enum.each(clusters, fn cluster -> - :ets.delete_object(cluster_nodes_table(state.name), {cluster, node}) - :ets.delete_object(node_clusters_table(state.name), {node, cluster}) - end) + :ok = delete_cluster_nodes(state.name, clusters, node) + {:reply, :ok, state} + end + + def handle_call({:remove_clusters, clusters}, _from, state) do + # Startup repair discovers inactive clusters outside the Data process. A + # reconnect may install a new epoch before this serialized cleanup runs; + # recheck authority here so stale repair work cannot erase the new routes. + inactive_clusters = + Enum.filter(clusters, &is_nil(local_cluster_epoch(state.name, &1))) + + :ok = delete_cluster_routes(state.name, inactive_clusters) {:reply, :ok, state} end def handle_call({:purge_cluster_node, dead_node}, _from, state) do - # Scan the forward index directly so this also repairs a one-sided row left - # by an interrupted or older dual-index mutation. - :ets.select_delete(cluster_nodes_table(state.name), [ - {{:_, dead_node}, [], [true]} - ]) + # Nodedown/lease callers may resume after a newer exact authority has + # already reinstalled this peer. Recheck the serialized authority fence so + # stale cleanup cannot erase routes belonging to the new incarnation. + if is_nil(remote_generation(state.name, dead_node)) and + is_nil(remote_replica_authority_hint(state.name, dead_node)) do + :ok = delete_peer_routes(state.name, dead_node) + end + + {:reply, :ok, state} + end + + def handle_call({:activate_local_clusters, clusters}, _from, state) do + {epochs, state} = activate_local_clusters(state, clusters, false) + {:reply, epochs, state} + end + + def handle_call({:activate_local_clusters_durable, clusters}, _from, state) do + {epochs, state} = activate_local_clusters(state, clusters, true) + {:reply, epochs, state} + end + + def handle_call(:local_replica_authority, _from, state) do + generation = generation(state.name) + revision = local_cluster_epoch_revision(state.name) + epochs = [{nil, generation} | :ets.tab2list(local_cluster_epochs_table(state.name))] + {:reply, {generation, revision, epochs}, state} + end + + def handle_call({:deactivate_local_clusters, clusters}, _from, state) do + {epochs, state} = deactivate_local_clusters(state, clusters, false) + {:reply, epochs, state} + end + + def handle_call({:deactivate_local_clusters_durable, clusters}, _from, state) do + {epochs, state} = deactivate_local_clusters(state, clusters, true) + {:reply, epochs, state} + end + + def handle_call({:mark_closed_cluster_shard, cluster_epochs, shard}, _from, state) do + completed = + Enum.reduce(cluster_epochs, [], fn {cluster, request_epoch}, acc -> + case :ets.lookup(closed_local_cluster_epochs_table(state.name), cluster) do + [{^cluster, ^request_epoch, pending_shards}] -> + pending_shards = MapSet.delete(pending_shards, shard) + + if MapSet.size(pending_shards) == 0 do + # The close marker is the durable recovery obligation. Remove + # every route before deleting it so a caller and the final shard + # can both disappear immediately after this acknowledgement + # without leaving an unowned cluster membership behind. + :ok = delete_cluster_routes(state.name, [cluster]) + :ets.delete(closed_local_cluster_epochs_table(state.name), cluster) + [cluster | acc] + else + :ets.insert( + closed_local_cluster_epochs_table(state.name), + {cluster, request_epoch, pending_shards} + ) + + acc + end + + _stale_or_completed -> + acc + end + end) + + {:reply, Enum.reverse(completed), state} + end + + def handle_call( + {:put_remote_replica_info, shard, remote_node, generation, epoch_revision, epochs}, + _from, + state + ) do + # The epoch snapshot is node-wide authority, not shard-local replica data. + # Only shard 0 sends it, and Data serializes the one exact replacement for + # every local replica lane. Keeping the argument in the API makes the + # control-owner invariant explicit and catches accidental reintroduction of + # one full copy per shard. + 0 = shard + seen_generation = remote_generation(state.name, remote_node) + + stale? = + stale_remote_authority_install?( + state.name, + remote_node, + generation, + epoch_revision + ) + + if stale? do + {:reply, :stale, state} + else + current_epochs = Map.new(epochs) + + stale_epochs = + if seen_generation == generation do + for {{^remote_node, cluster}, epoch} <- + :ets.match_object( + remote_cluster_epochs_table(state.name), + {{remote_node, :_}, :_} + ), + not is_nil(cluster), + Map.get(current_epochs, cluster) != epoch, + do: {cluster, epoch} + else + [] + end + + # A hello is a complete epoch snapshot. The replica handler fences older + # revisions before this call, so replace the shared view rather than merely + # adding rows; otherwise a dropped close control could leave a cluster epoch + # permanently valid after the heartbeat-driven repair. + :ets.select_delete(remote_cluster_epochs_table(state.name), [ + {{{remote_node, :_}, :_}, [], [true]} + ]) + + :ets.insert( + replication_meta_table(state.name), + {{:remote_generation, remote_node}, generation} + ) + + :ets.insert( + replication_meta_table(state.name), + {{:remote_epoch_revision, remote_node}, epoch_revision} + ) + + :ets.insert( + replication_meta_table(state.name), + {{:remote_epoch_exact, remote_node}, epoch_revision} + ) + + :ets.insert( + replication_meta_table(state.name), + {{:remote_epoch_observed, remote_node}, epoch_revision} + ) + + put_remote_authority_hint(state.name, remote_node, generation, epoch_revision) + + replace_remote_cluster_projection(state.name, remote_node, current_epochs) + + :ets.update_counter( + replication_meta_table(state.name), + {:remote_authority_installs, remote_node}, + {2, 1}, + {{:remote_authority_installs, remote_node}, 0} + ) + + rows = + for {cluster, epoch} <- epochs, + not is_nil(cluster), + do: {{remote_node, cluster}, epoch} + + :ets.insert(remote_cluster_epochs_table(state.name), rows) + + {:reply, {seen_generation, stale_epochs}, state} + end + end + + def handle_call( + {:put_remote_view_info, shard, remote_node, generation, authoritative, observed}, + _from, + state + ) do + if remote_generation(state.name, remote_node) == generation and + remote_cluster_epoch_exact_revision(state.name, remote_node) == authoritative and + remote_cluster_epoch_revision(state.name, remote_node) == observed and + remote_cluster_epoch_observed_revision(state.name, remote_node) == observed and + remote_replica_authority_hint(state.name, remote_node) == {generation, observed} do + :ets.insert( + replication_meta_table(state.name), + {{:remote_view_info, shard, remote_node}, generation, authoritative, observed} + ) + + {:reply, :ok, state} + else + {:reply, :stale, state} + end + end + + def handle_call( + {:put_remote_cluster_epochs, shard, remote_node, generation, expected_revision, revision, + epochs}, + _from, + state + ) do + 0 = shard + + if incremental_authority_installable?( + state.name, + remote_node, + generation, + expected_revision, + revision + ) do + observe_remote_cluster_revision(state.name, remote_node, revision, state.num_shards) + + stale_epochs = + Enum.flat_map(epochs, fn {cluster, epoch} -> + case remote_cluster_epoch(state.name, remote_node, cluster) do + old_epoch when not is_nil(old_epoch) and old_epoch != epoch -> [{cluster, old_epoch}] + _ -> [] + end + end) + + rows = + for {cluster, epoch} <- epochs, + not is_nil(cluster), + do: {{remote_node, cluster}, epoch} + + :ets.insert(remote_cluster_epochs_table(state.name), rows) + + :ets.insert( + replication_meta_table(state.name), + {{:remote_epoch_revision, remote_node}, revision} + ) + + {:reply, {:ok, stale_epochs}, state} + else + {:reply, :stale, state} + end + end + + def handle_call( + {:remote_registry_claim_authoritative, shard, remote_node, generation, cluster, epoch}, + _from, + state + ) do + exact = remote_cluster_epoch_exact_revision(state.name, remote_node) + observed = remote_cluster_epoch_observed_revision(state.name, remote_node) + + current_view? = + case :ets.lookup( + replication_meta_table(state.name), + {:remote_view_info, shard, remote_node} + ) do + [{{:remote_view_info, ^shard, ^remote_node}, ^generation, ^exact, ^observed}] -> true + _ -> false + end + + authoritative? = + current_view? and remote_generation(state.name, remote_node) == generation and + remote_cluster_epoch_revision(state.name, remote_node) == observed and + remote_replica_authority_hint(state.name, remote_node) == {generation, observed} and + remote_cluster_epoch(state.name, remote_node, cluster) == epoch and + (is_nil(cluster) or active_local_cluster?(state.name, cluster)) + + {:reply, authoritative?, state} + end + + def handle_call( + {:observe_remote_cluster_epoch_revision, remote_node, revision}, + _from, + state + ) do + :ok = observe_remote_cluster_revision(state.name, remote_node, revision, state.num_shards) + {:reply, :ok, state} + end + + def handle_call( + {:observe_remote_replica_hint, remote_node, generation, revision}, + _from, + state + ) do + known_generation = remote_generation(state.name, remote_node) + + {hint_generation, hint_revision} = + remote_replica_authority_hint(state.name, remote_node) || + {known_generation, remote_cluster_epoch_observed_revision(state.name, remote_node)} + + fenced? = + cond do + hint_generation == generation and + (is_nil(hint_revision) or revision > hint_revision) -> + put_remote_authority_hint(state.name, remote_node, generation, revision) + + if known_generation == generation do + :ok = + observe_remote_cluster_revision( + state.name, + remote_node, + revision, + state.num_shards + ) + end + + true + + not is_nil(hint_generation) and + WireProtocol.generation_newer?(generation, hint_generation) -> + # This one shared row is the cross-lane admission fence. Replica lanes + # include it in replica_view_current?/2, so no lane can admit the prior + # generation after this insert even while the per-lane breadcrumbs + # below are being updated. Existing materialized projections may remain + # visible until exact-authority repair or bounded lease retirement. + put_remote_authority_hint(state.name, remote_node, generation, revision) + + # Preserve each lane's old generation as the later exact install's + # purge key, but make its authority revisions impossible to match. + # This fences every shard in the same Data turn even when only one + # sideband lane observes the restarted origin first. + state.name + |> replication_meta_table() + |> :ets.match_object({{:remote_view_info, :_, remote_node}, :_, :_, :_}) + |> Enum.each(fn {key, lane_generation, _exact, _observed} -> + :ets.insert( + replication_meta_table(state.name), + {key, lane_generation, nil, nil} + ) + end) + + true + + true -> + false + end + + {:reply, fenced?, state} + end + + def handle_call( + {:close_remote_cluster_epochs, shard, remote_node, generation, expected_revision, + revision, epochs}, + _from, + state + ) do + 0 = shard + + if incremental_authority_installable?( + state.name, + remote_node, + generation, + expected_revision, + revision + ) do + observe_remote_cluster_revision(state.name, remote_node, revision, state.num_shards) + + closed = + Enum.filter(epochs, fn {cluster, epoch} -> + remote_cluster_epoch(state.name, remote_node, cluster) == epoch + end) + + Enum.each(epochs, fn {cluster, epoch} -> + case :ets.lookup(remote_cluster_epochs_table(state.name), {remote_node, cluster}) do + [{{^remote_node, ^cluster}, ^epoch}] -> + :ets.delete(remote_cluster_epochs_table(state.name), {remote_node, cluster}) + + _ -> + :ok + end + end) + + :ets.insert( + replication_meta_table(state.name), + {{:remote_epoch_revision, remote_node}, revision} + ) + + {:reply, {:ok, closed}, state} + else + {:reply, :stale, state} + end + end + + def handle_call({:delete_remote_replica_info, shard, remote_node}, _from, state) do + :ets.delete( + replication_meta_table(state.name), + {:remote_view_info, shard, remote_node} + ) + + if shard == 0 do + delete_remote_authority(state.name, remote_node) + end - :ets.delete(node_clusters_table(state.name), dead_node) {:reply, :ok, state} end + def handle_call({:expire_remote_replica_lane, shard, remote_node}, _from, state) do + :ets.delete( + replication_meta_table(state.name), + {:remote_view_info, shard, remote_node} + ) + + remaining_lanes = + :ets.select_count(replication_meta_table(state.name), [ + {{{:remote_view_info, :_, remote_node}, :_, :_, :_}, [], [true]} + ]) + + result = + if remaining_lanes == 0 do + delete_remote_authority(state.name, remote_node) + :node_retired + else + :lane_retired + end + + {:reply, result, state} + end + @impl true def init({name, num_shards}) do # ETS performance options: # - read_concurrency: splits table into read-optimized segments (less lock contention) # - decentralized_counters: reduces contention on table size counter (OTP 23+) - # Note: write_concurrency is intentionally omitted — the sharded GenServer already - # serializes writes per shard, so ETS write locking is never contended. Adding - # write_concurrency adds overhead (~30-40% on serial benchmarks) without benefit. + # Per-shard tables omit write_concurrency because each shard GenServer serializes + # their writes. The shared replication metadata table is different: every shard + # atomically advances its own {:append_counter, shard} object there, so it needs + # concurrent writes without weakening ETS's single-object atomicity. set_opts = [ :set, :public, @@ -894,6 +2846,8 @@ defmodule Group.Replica.Data do decentralized_counters: true ] + shared_meta_opts = Keyword.put(set_opts, :write_concurrency, :auto) + ordered_set_opts = [ :ordered_set, :public, @@ -913,17 +2867,163 @@ defmodule Group.Replica.Data do for shard <- 0..(num_shards - 1) do :ets.new(reg_by_key_table(name, shard), set_opts) :ets.new(reg_by_pid_table(name, shard), ordered_set_opts) + :ets.new(reg_claim_by_key_table(name, shard), ordered_set_opts) + :ets.new(reg_claim_by_pid_table(name, shard), ordered_set_opts) :ets.new(pg_by_key_table(name, shard), ordered_set_opts) :ets.new(pg_by_pid_table(name, shard), ordered_set_opts) + :ets.new(replica_stream_meta_table(name, shard), set_opts) + :ets.new(replica_oplog_table(name, shard), ordered_set_opts) + :ets.new(replica_oplog_order_table(name, shard), ordered_set_opts) + :ets.new(replica_cursor_table(name, shard), set_opts) end :ets.new(cluster_nodes_table(name), bag_opts) :ets.new(node_clusters_table(name), bag_opts) :ets.new(cluster_leases_table(name), set_opts) + :ets.new(replication_meta_table(name), shared_meta_opts) + :ets.new(local_cluster_epochs_table(name), set_opts) + :ets.new(closed_local_cluster_epochs_table(name), set_opts) + :ets.new(remote_cluster_epochs_table(name), set_opts) + :ets.insert(replication_meta_table(name), {:generation, WireProtocol.new_generation()}) + :ets.insert(replication_meta_table(name), {:cluster_epoch_revision, 0}) {:ok, %{name: name, num_shards: num_shards}} end + defp observe_remote_cluster_revision(name, remote_node, revision, _num_shards) do + key = {:remote_epoch_observed, remote_node} + + case :ets.lookup(replication_meta_table(name), key) do + [{^key, current}] when current >= revision -> :ok + _ -> :ets.insert(replication_meta_table(name), {key, revision}) + end + + case remote_generation(name, remote_node) do + generation when not is_nil(generation) -> + put_remote_authority_hint(name, remote_node, generation, revision) + + nil -> + :ok + end + + :ok + end + + defp put_remote_authority_hint(name, remote_node, generation, revision) do + key = {:remote_authority_hint, remote_node} + + install? = + case remote_replica_authority_hint(name, remote_node) do + nil -> + true + + {^generation, current_revision} -> + revision > current_revision + + {current_generation, _current_revision} -> + WireProtocol.generation_newer?(generation, current_generation) + end + + if install? do + :ets.insert(replication_meta_table(name), {key, generation, revision}) + end + + :ok + end + + defp stale_remote_authority_install?(name, remote_node, generation, revision) do + known_generation = remote_generation(name, remote_node) + authoritative_revision = remote_cluster_epoch_revision(name, remote_node) + observed_revision = remote_cluster_epoch_observed_revision(name, remote_node) + + hinted_stale? = + case remote_replica_authority_hint(name, remote_node) do + nil -> + false + + {^generation, hinted_revision} -> + revision < hinted_revision + + {hinted_generation, _hinted_revision} -> + not WireProtocol.generation_newer?(generation, hinted_generation) + end + + known_stale? = + not is_nil(known_generation) and known_generation != generation and + not WireProtocol.generation_newer?(generation, known_generation) + + revision_stale? = + known_generation == generation and + Enum.any?([authoritative_revision, observed_revision], fn + current when is_integer(current) -> revision < current + _ -> false + end) + + hinted_stale? or known_stale? or revision_stale? + end + + defp incremental_authority_installable?( + name, + remote_node, + generation, + expected_revision, + revision + ) do + is_integer(expected_revision) and is_integer(revision) and revision > expected_revision and + remote_generation(name, remote_node) == generation and + remote_cluster_epoch_revision(name, remote_node) == expected_revision and + remote_cluster_epoch_observed_revision(name, remote_node) == expected_revision and + remote_replica_authority_hint(name, remote_node) == {generation, expected_revision} + end + + defp delete_remote_authority(name, remote_node) do + # Fence every public-ETS reader first. The remaining mutations execute in + # this same Data callback, so a caller cannot observe completion between + # them; if Data itself dies, it takes every owned table with it. Removing + # the generation before routing therefore closes the read interleaving + # where a sibling lane could still accept data during retirement. + :ets.delete(replication_meta_table(name), {:remote_generation, remote_node}) + :ok = delete_peer_routes(name, remote_node) + :ets.delete(replication_meta_table(name), {:remote_epoch_revision, remote_node}) + :ets.delete(replication_meta_table(name), {:remote_epoch_exact, remote_node}) + :ets.delete(replication_meta_table(name), {:remote_epoch_observed, remote_node}) + :ets.delete(replication_meta_table(name), {:remote_authority_hint, remote_node}) + :ets.delete(replication_meta_table(name), {:remote_authority_installs, remote_node}) + + :ets.select_delete(remote_cluster_epochs_table(name), [ + {{{remote_node, :_}, :_}, [], [true]} + ]) + + :ok + end + + defp fold_select_batches(table, match_spec, acc, fun) do + case :ets.select(table, [match_spec], 4_096) do + :"$end_of_table" -> acc + {matches, continuation} -> fold_select_batches(continuation, fun.(matches, acc), fun) + end + end + + defp fold_select_batches_fixed(table, match_spec, acc, fun) do + # Some consumers delete the selected origin slice while folding it. Keep + # the traversal fixed so deleting the current batch cannot move unseen + # ordered-set keys past the continuation and leave a permanent stale row. + :ets.safe_fixtable(table, true) + + try do + fold_select_batches(table, match_spec, acc, fun) + after + :ets.safe_fixtable(table, false) + end + end + + defp fold_select_batches(continuation, acc, fun) do + case :ets.select(continuation) do + :"$end_of_table" -> acc + {matches, next} -> fold_select_batches(next, fun.(matches, acc), fun) + end + end + defp select(table, match_spec, :infinity), do: :ets.select(table, match_spec) defp select(_table, _match_spec, 0), do: [] diff --git a/lib/group/replica/snapshot.ex b/lib/group/replica/snapshot.ex new file mode 100644 index 0000000..95935d2 --- /dev/null +++ b/lib/group/replica/snapshot.ex @@ -0,0 +1,257 @@ +defmodule Group.Replica.Snapshot do + @moduledoc false + + @max_frame_counter 18_446_744_073_709_551_615 + + def stream_envelope_bytes(stream_id, snapshot_seq) do + :erlang.external_size( + {:snapshot_chunk, Group.Replica.WireProtocol.version(), stream_id, snapshot_seq, + @max_frame_counter, [], []} + ) + 10 + end + + def new_stream(target_bytes, envelope_bytes, start_index, emit) + when is_integer(target_bytes) and target_bytes > 0 and is_integer(envelope_bytes) and + envelope_bytes >= 0 and is_integer(start_index) and start_index > 0 and + is_function(emit, 3) do + %{ + payload_target: max(target_bytes - envelope_bytes, 1), + start_index: start_index, + emit: emit, + registry: [], + pg: [], + bytes: 0, + current_count: 0, + chunk_count: 0, + registry_count: 0, + pg_count: 0 + } + end + + def stream_registry_many(rows, stream) when is_list(rows) do + Enum.reduce(rows, stream, &stream_row(&2, :registry, &1)) + end + + def stream_pg_many(rows, stream) when is_list(rows) do + Enum.reduce(rows, stream, &stream_row(&2, :pg, &1)) + end + + def finish_stream(%{current_count: 0, chunk_count: 0} = stream), + do: flush_stream_chunk(stream) + + def finish_stream(%{current_count: 0} = stream), do: stream + def finish_stream(stream), do: flush_stream_chunk(stream) + + def new_staging_table do + :ets.new(__MODULE__, [:set, :private]) + end + + def new_event_table do + :ets.new(__MODULE__, [:ordered_set, :private]) + end + + def delete_staging_table(table) do + try do + :ets.delete(table) + rescue + ArgumentError -> true + end + + :ok + end + + def clear_staging_table(table) do + try do + :ets.delete_all_objects(table) + rescue + ArgumentError -> true + end + + :ok + end + + def stage_rows(table, chunk_index, registry_rows, pg_rows) do + row_objects = + Enum.map(registry_rows, &staging_object(:registry, chunk_index, &1)) ++ + Enum.map(pg_rows, &staging_object(:pg, chunk_index, &1)) + + objects = [{{:chunk, chunk_index}, length(registry_rows), length(pg_rows)} | row_objects] + size_before = :ets.info(table, :size) + + if :ets.insert_new(table, objects) and + :ets.info(table, :size) - size_before == length(objects) do + :ok + else + {:error, :duplicate_row} + end + end + + def chunk_matches?(table, chunk_index, registry_rows, pg_rows) do + case :ets.lookup(table, {:chunk, chunk_index}) do + [{{:chunk, ^chunk_index}, registry_count, pg_count}] -> + registry_count == length(registry_rows) and pg_count == length(pg_rows) and + Enum.all?(registry_rows, &staged_registry_row?(table, chunk_index, &1)) and + Enum.all?(pg_rows, &staged_pg_row?(table, chunk_index, &1)) + + [] -> + false + end + end + + def fold_registry(table, _chunk_count, acc, fun) when is_function(fun, 2) do + reduce_registry_batches(table, acc, fn rows, inner -> + Enum.reduce(rows, inner, fun) + end) + end + + def reduce_registry_batches(table, acc, fun) when is_function(fun, 2) do + fold_staged_row_batches( + table, + [ + {{{:registry, :"$1"}, :"$2", :"$3", :"$4", :_}, [], [{{:"$1", :"$2", :"$3", :"$4"}}]} + ], + acc, + fun + ) + end + + def fold_pg(table, _chunk_count, acc, fun) when is_function(fun, 2) do + reduce_pg_batches(table, acc, fn rows, inner -> + Enum.reduce(rows, inner, fun) + end) + end + + def reduce_pg_batches(table, acc, fun) when is_function(fun, 2) do + fold_staged_row_batches( + table, + [ + {{{:pg, :"$1", :"$2"}, :"$3", :"$4", :_}, [], [{{:"$1", :"$2", :"$3", :"$4"}}]} + ], + acc, + fun + ) + end + + def new_event_buffer(table) do + %{table: table, chunk_index: 0, events: [], count: 0} + end + + def buffer_events(events, buffer) when is_list(events) do + Enum.reduce(events, buffer, &buffer_event/2) + end + + def buffer_event(event, buffer) do + buffer = + if buffer.count == 512 do + flush_event_buffer(buffer) + else + buffer + end + + %{buffer | events: [event | buffer.events], count: buffer.count + 1} + end + + def finish_event_buffer(%{count: 0} = buffer), do: buffer + def finish_event_buffer(buffer), do: flush_event_buffer(buffer) + + def fold_events(table, acc, fun) when is_function(fun, 2) do + :ets.foldl( + fn {{:events, _chunk_index}, events}, inner -> + Enum.reduce(events, inner, fun) + end, + acc, + table + ) + end + + def member_pg?(table, key, pid), do: :ets.member(table, {:pg, key, pid}) + def member_registry?(table, key), do: :ets.member(table, {:registry, key}) + + defp staging_object(:registry, chunk_index, {key, pid, meta, time}), + do: {{:registry, key}, pid, meta, time, chunk_index} + + defp staging_object(:pg, chunk_index, {key, pid, meta, time}), + do: {{:pg, key, pid}, meta, time, chunk_index} + + defp staged_registry_row?(table, chunk_index, {key, pid, meta, time}) do + :ets.lookup(table, {:registry, key}) == + [{{:registry, key}, pid, meta, time, chunk_index}] + end + + defp staged_pg_row?(table, chunk_index, {key, pid, meta, time}) do + :ets.lookup(table, {:pg, key, pid}) == [{{:pg, key, pid}, meta, time, chunk_index}] + end + + defp fold_staged_row_batches(table, match_spec, acc, fun) do + case :ets.select(table, match_spec, 4_096) do + :"$end_of_table" -> acc + {rows, continuation} -> fold_staged_row_batches(continuation, fun.(rows, acc), fun) + end + end + + defp fold_staged_row_batches(continuation, acc, fun) do + case :ets.select(continuation) do + :"$end_of_table" -> acc + {rows, next} -> fold_staged_row_batches(next, fun.(rows, acc), fun) + end + end + + defp stream_row(stream, domain, row) do + row_bytes = :erlang.external_size(row) + + stream = + if stream.current_count > 0 and stream.bytes + row_bytes > stream.payload_target do + flush_stream_chunk(stream) + else + stream + end + + case domain do + :registry -> + %{ + stream + | registry: [row | stream.registry], + bytes: stream.bytes + row_bytes, + current_count: stream.current_count + 1, + registry_count: stream.registry_count + 1 + } + + :pg -> + %{ + stream + | pg: [row | stream.pg], + bytes: stream.bytes + row_bytes, + current_count: stream.current_count + 1, + pg_count: stream.pg_count + 1 + } + end + end + + defp flush_stream_chunk(stream) do + chunk_index = stream.chunk_count + 1 + + if chunk_index >= stream.start_index do + :ok = + stream.emit.( + Enum.reverse(stream.registry), + Enum.reverse(stream.pg), + chunk_index + ) + end + + %{ + stream + | registry: [], + pg: [], + bytes: 0, + current_count: 0, + chunk_count: chunk_index + } + end + + defp flush_event_buffer(buffer) do + chunk_index = buffer.chunk_index + 1 + true = :ets.insert_new(buffer.table, {{:events, chunk_index}, Enum.reverse(buffer.events)}) + %{buffer | chunk_index: chunk_index, events: [], count: 0} + end +end diff --git a/lib/group/replica/wire_protocol.ex b/lib/group/replica/wire_protocol.ex new file mode 100644 index 0000000..8219350 --- /dev/null +++ b/lib/group/replica/wire_protocol.ex @@ -0,0 +1,58 @@ +defmodule Group.Replica.WireProtocol do + @moduledoc false + + @version 3 + + def version, do: @version + + # The counter orders Group incarnations created within one BEAM. The ref + # keeps the identity globally unique across BEAM restarts; Erlang + # distribution supplies nodedown before a restarted VM can install a new + # authority, so ordering is only required within one VM lifetime. + def new_generation do + {System.unique_integer([:monotonic, :positive]), make_ref()} + end + + def valid_generation?({counter, identity}), + do: is_integer(counter) and counter > 0 and is_reference(identity) + + def valid_generation?(_generation), do: false + + def generation_newer?({new_counter, _new_identity}, {old_counter, _old_identity}) + when is_integer(new_counter) and is_integer(old_counter), + do: new_counter > old_counter + + def generation_newer?(_new_generation, _old_generation), do: false + + def stream_id(name, origin_node, origin_generation, shard, cluster, cluster_epoch) do + {name, origin_node, origin_generation, shard, cluster, cluster_epoch} + end + + def valid_stream_id?({name, origin_node, origin_generation, shard, cluster, cluster_epoch}) do + is_atom(name) and is_atom(origin_node) and valid_generation?(origin_generation) and + is_integer(shard) and shard >= 0 and + ((is_nil(cluster) and cluster_epoch == origin_generation) or + (is_binary(cluster) and is_reference(cluster_epoch))) + end + + def valid_stream_id?(_stream_id), do: false + + def stream_name({name, _origin_node, _generation, _shard, _cluster, _epoch}), do: name + + def stream_origin({_name, origin_node, _generation, _shard, _cluster, _epoch}), + do: origin_node + + def stream_generation({_name, _origin_node, generation, _shard, _cluster, _epoch}), + do: generation + + def stream_shard({_name, _origin_node, _generation, shard, _cluster, _epoch}), do: shard + + def stream_cluster({_name, _origin_node, _generation, _shard, cluster, _epoch}), do: cluster + + def stream_epoch({_name, _origin_node, _generation, _shard, _cluster, epoch}), do: epoch + + def op_cluster({:register, cluster, _key, _pid, _meta, _time, _node}), do: cluster + def op_cluster({:unregister, cluster, _key, _pid, _meta, _reason}), do: cluster + def op_cluster({:join, cluster, _key, _pid, _meta, _time, _reason, _node}), do: cluster + def op_cluster({:leave, cluster, _key, _pid, _meta, _reason}), do: cluster +end diff --git a/lib/group/supervisor.ex b/lib/group/supervisor.ex index e4e15b2..3c66484 100644 --- a/lib/group/supervisor.ex +++ b/lib/group/supervisor.ex @@ -42,6 +42,30 @@ defmodule Group.Supervisor do replicated_pg_receiver_local_request_quota = positive_integer_opt(opts, :replicated_pg_receiver_local_request_quota, 8) + replica_transport = + opts + |> Keyword.get(:replica_transport, Group.Transport.DistErl) + |> Group.Transport.normalize() + |> Group.Transport.validate!() + + replicated_oplog_max_entries = + positive_integer_opt(opts, :replicated_oplog_max_entries, 65_536) + + replicated_snapshot_chunk_target_bytes = + positive_integer_opt(opts, :replicated_snapshot_chunk_target_bytes, 1_048_576) + + replicated_anti_entropy_interval = + positive_integer_opt(opts, :replicated_anti_entropy_interval, 1_000) + + replicated_peer_lease_timeout = + positive_integer_opt(opts, :replicated_peer_lease_timeout, 15_000) + + if replicated_peer_lease_timeout <= replicated_anti_entropy_interval do + raise ArgumentError, + ":replicated_peer_lease_timeout must be greater than " <> + ":replicated_anti_entropy_interval" + end + # persistent_term config — must be set before children start (Replica reads it) config = %{ num_shards: num_shards, @@ -54,7 +78,12 @@ defmodule Group.Supervisor do replicated_sender_flush_interval: replicated_sender_flush_interval, busy_dist_retry_attempts: busy_dist_retry_attempts, busy_dist_retry_interval: busy_dist_retry_interval, - replicated_pg_receiver_local_request_quota: replicated_pg_receiver_local_request_quota + replicated_pg_receiver_local_request_quota: replicated_pg_receiver_local_request_quota, + replica_transport: replica_transport, + replicated_oplog_max_entries: replicated_oplog_max_entries, + replicated_snapshot_chunk_target_bytes: replicated_snapshot_chunk_target_bytes, + replicated_anti_entropy_interval: replicated_anti_entropy_interval, + replicated_peer_lease_timeout: replicated_peer_lease_timeout } config = if extract_meta, do: Map.put(config, :extract_meta, extract_meta), else: config @@ -66,18 +95,33 @@ defmodule Group.Supervisor do :persistent_term.put({Group, name}, config) - children = [ - {Group.Replica.Data, name: name, num_shards: num_shards}, - { - Group.PeerReconnect, - name: name, - busy_dist_retry_attempts: busy_dist_retry_attempts, - busy_dist_retry_interval: busy_dist_retry_interval - }, - {Group.Replica.Supervisor, name: name, num_shards: num_shards}, - {Registry, keys: :duplicate, name: Group.registry_name(name)}, - {Group.ClusterLease, name: name, num_shards: num_shards} - ] + transport_children = + case replica_transport do + {module, transport_opts} -> + if function_exported?(module, :child_spec, 1) do + case module.child_spec([name: name, num_shards: num_shards] ++ transport_opts) do + :ignore -> [] + child_spec -> [child_spec] + end + else + [] + end + end + + children = + transport_children ++ + [ + {Group.Replica.Data, name: name, num_shards: num_shards}, + { + Group.PeerReconnect, + name: name, + busy_dist_retry_attempts: busy_dist_retry_attempts, + busy_dist_retry_interval: busy_dist_retry_interval + }, + {Group.Replica.Supervisor, name: name, num_shards: num_shards}, + {Registry, keys: :duplicate, name: Group.registry_name(name)}, + {Group.ClusterLease, name: name, num_shards: num_shards} + ] Supervisor.init(children, strategy: :rest_for_one) end diff --git a/lib/group/transport.ex b/lib/group/transport.ex new file mode 100644 index 0000000..dd1164e --- /dev/null +++ b/lib/group/transport.ex @@ -0,0 +1,139 @@ +defmodule Group.Transport do + @moduledoc """ + Transport contract for Group replica data. + + Implementations must return promptly and must never wait for socket or remote + mailbox backpressure. This applies to `outgoing/5` and the optional lifecycle + callbacks. Returning `:busy` or `:disconnected` is safe: replica anti-entropy + will retransmit the missing state. + + Erlang distribution remains Group's control plane and supplies the stable + node identity used here. A sideband adapter can use its `descriptor/2` in the + control hello to exchange endpoints and pass incoming messages to + `incoming/4` or `incoming_batch/4`. `source_node` is routing metadata supplied + by the adapter; Group uses it to validate stream origins and member pids. + + Adapters do not need to preserve ordering. Group serializes writes per shard + and sequences each origin/generation/shard/cluster/epoch stream; receivers + discard duplicates and request gaps. Per-shard ordered delivery avoids repair + traffic and is therefore the preferred fast path. + + A sideband implementation can delegate `outgoing/5` to + `Group.Transport.Outbox.push/5`. That adds one local send only for + the configured sideband transport; the default distribution adapter retains + its direct remote `:erlang.send_nosuspend/3` path. + """ + + @type message :: term() + @type outgoing_result :: :ok | :busy | :disconnected + + @callback id() :: term() + @callback descriptor(group :: atom(), opts :: keyword()) :: term() + @doc """ + Called when Group has an outgoing replica message for another node. + + This callback must return promptly and must never wait for transport or + remote backpressure. `:ok` means the transport took responsibility for the + message, not that the remote shard received it. + """ + @callback outgoing( + group :: atom(), + target_node :: node(), + shard :: non_neg_integer(), + message(), + opts :: keyword() + ) :: outgoing_result() + + @callback child_spec(keyword()) :: Supervisor.child_spec() | :ignore + + @doc """ + Reports that one shard lane can address a peer through this transport. + + A transport sharing one node-level connection across lanes should retain the + route until `peer_down/4` has retired every shard previously reported up. + """ + @callback peer_up( + group :: atom(), + node(), + shard :: non_neg_integer(), + descriptor :: term(), + opts :: keyword() + ) :: :ok + + @doc "Reports that one shard lane no longer has live peer authority." + @callback peer_down( + group :: atom(), + node(), + shard :: non_neg_integer(), + opts :: keyword() + ) :: :ok + + @optional_callbacks child_spec: 1, peer_up: 5, peer_down: 4 + + @doc """ + Passes an incoming replica message to the corresponding local shard. + + This is a local mailbox operation. `source_node` identifies the peer whose + replica lane supplied the message. Stream generation, epoch, group, shard, + origin, and member-pid ownership are validated by the replica. + + Returns `:disconnected` and drops the message if that shard is not currently + registered, for example while its supervisor is restarting. + """ + def incoming(group, source_node, shard, message) + when is_atom(group) and is_atom(source_node) and is_integer(shard) and shard >= 0 do + case Process.whereis(Group.Replica.shard_name(group, shard)) do + pid when is_pid(pid) -> + send(pid, {:group_replica_frame, source_node, message}) + :ok + + nil -> + :disconnected + end + end + + @doc """ + Passes a complete incoming batch to the corresponding local shard. + + A finite-message transport may segment the encoded batch on the wire, but it + must reassemble every segment before calling this function. Group never + observes or applies a partial batch. `source_node` has the same routing + meaning as in `incoming/4`. + + Like `incoming/4`, this returns `:disconnected` if the destination shard is + unavailable. + """ + def incoming_batch(group, source_node, shard, messages) + when is_atom(group) and is_atom(source_node) and is_integer(shard) and shard >= 0 and + is_list(messages) do + case Process.whereis(Group.Replica.shard_name(group, shard)) do + pid when is_pid(pid) -> + send(pid, {:group_replica_batch, source_node, messages}) + :ok + + nil -> + :disconnected + end + end + + def normalize(module) when is_atom(module), do: {module, []} + def normalize({module, opts}) when is_atom(module) and is_list(opts), do: {module, opts} + + def normalize(other) do + raise ArgumentError, + "expected :replica_transport to be a module or {module, opts}, got: #{inspect(other)}" + end + + def validate!({module, _opts} = transport) do + Code.ensure_loaded!(module) + + for {function, arity} <- [id: 0, descriptor: 2, outgoing: 5] do + unless function_exported?(module, function, arity) do + raise ArgumentError, + "replica transport #{inspect(module)} must implement #{function}/#{arity}" + end + end + + transport + end +end diff --git a/lib/group/transport/dist_erl.ex b/lib/group/transport/dist_erl.ex new file mode 100644 index 0000000..9812fb5 --- /dev/null +++ b/lib/group/transport/dist_erl.ex @@ -0,0 +1,32 @@ +defmodule Group.Transport.DistErl do + @moduledoc """ + Default nonblocking replica transport over Erlang distribution. + + Messages are sent directly to the matching remote shard with + `:erlang.send_nosuspend/3` and `:noconnect`, so the caller never waits for a + busy distribution socket and never initiates a connection. A busy or absent + link returns `:busy`; Group drops that message and repairs it through periodic + anti-entropy. + """ + + @behaviour Group.Transport + + alias Group.Replica + + @impl true + def id, do: :erlang_distribution + + @impl true + def descriptor(_group, _opts), do: :erlang_distribution + + @impl true + def outgoing(group, target_node, shard, replica_message, _opts) do + destination = {Replica.shard_name(group, shard), target_node} + message = {:group_replica_frame, node(), replica_message} + + case :erlang.send_nosuspend(destination, message, [:noconnect]) do + true -> :ok + false -> :busy + end + end +end diff --git a/lib/group/transport/outbox.ex b/lib/group/transport/outbox.ex new file mode 100644 index 0000000..4bd8d55 --- /dev/null +++ b/lib/group/transport/outbox.ex @@ -0,0 +1,402 @@ +defmodule Group.Transport.Outbox do + @moduledoc """ + Lossy per-shard outboxes for sideband replica transports. + + This module is an implementation helper, not a replacement for + `Group.Transport`. Distribution can continue sending directly with + `:erlang.send_nosuspend/3`. A sideband adapter delegates `outgoing/5` to + `push/5`, which performs only a local `send/2` to the matching shard + outbox. + + Each outbox batches messages by target node outside the Group shard. Expired + messages and batches rejected by the backend are deliberately dropped; + anti-entropy repairs them. Backends may perform bounded blocking work in + `send_batch/4` because they run in the outbox rather than a Group process. + + A backend using this helper implements: + + @behaviour Group.Transport.Outbox + + def init_outbox(group, shard, opts), do: {:ok, backend_state} + + def send_batch(target_node, messages, deadline, backend_state) do + # Return promptly once `deadline` has passed. It is safe to drop. + {:ok, backend_state} + end + + The backend must pass only complete logical messages to + `Group.Transport.incoming_batch/4`. + + ## Options + + * `:outbox_batch_size` - maximum logical messages collected per flush, + default `64` + * `:outbox_batch_bytes` - approximate external-term bytes collected per + flush, default `1_048_576` + * `:outbox_flush_interval` - maximum batching delay in milliseconds, + default `1` + * `:outbox_deadline` - maximum useful residence time for an outgoing message + in milliseconds, default `100` + * `:outbox_max_messages` - maximum accepted messages across the local + mailbox, pending batch, and backend send, default `1_024` + + Admission bounds mailbox growth by message count. A backend must also put a + finite bound on every socket enqueue or write it performs. Exact snapshot + messages are independently bounded by `:replicated_snapshot_chunk_target_bytes`. + Other logical messages or a whole batch may still exceed `:outbox_batch_bytes`; + a transport with a smaller finite frame size must segment and completely + reassemble those batches before local delivery. + """ + + @type message :: Group.Transport.message() + @type outgoing_result :: Group.Transport.outgoing_result() + @type backend_state :: term() + + @callback init_outbox(group :: atom(), shard :: non_neg_integer(), opts :: keyword()) :: + {:ok, backend_state()} + + @callback send_batch( + target_node :: node(), + messages :: [message()], + deadline :: integer(), + backend_state() + ) :: {outgoing_result(), backend_state()} + + @default_deadline 100 + + @doc """ + Returns a supervisor child specification for one outbox per Group shard. + + `:name`, `:num_shards`, and `:backend` are required. All options are passed + unchanged to `backend.init_outbox/3`. + """ + def child_spec(opts) do + group = Keyword.fetch!(opts, :name) + + %{ + id: {__MODULE__, group}, + start: {Group.Transport.Outbox.Supervisor, :start_link, [opts]}, + type: :supervisor, + restart: :permanent, + shutdown: :infinity + } + end + + @doc """ + Pushes a replica message into its shard's local outbox. + + This operation is local and nonblocking and performs no backend or socket + work. `:ok` only means the outbox was available; the message may later expire + or be dropped under transport pressure. A concurrently terminating outbox + can also lose an accepted message, which anti-entropy repairs. + """ + def push(group, target_node, shard, message, opts) + when is_atom(group) and is_atom(target_node) and is_integer(shard) and shard >= 0 and + is_list(opts) do + expires_at = monotonic_ms() + deadline(opts) + + case :persistent_term.get(admission_key(group, shard), nil) do + {pid, admission, max_messages} when is_pid(pid) -> + if Process.whereis(name(group, shard)) == pid do + if :atomics.add_get(admission, 1, 1) <= max_messages do + send(pid, {:group_replica_outbox_push, target_node, expires_at, message}) + :ok + else + :atomics.sub(admission, 1, 1) + :busy + end + else + :disconnected + end + + nil -> + :disconnected + end + end + + @doc false + def name(group, shard), do: :"#{group}_replica_transport_outbox_#{shard}" + + @doc false + def admission_key(group, shard), do: {__MODULE__, group, shard, :admission} + + @doc false + def monotonic_ms, do: System.monotonic_time(:millisecond) + + @doc false + def validate_options!(opts) when is_list(opts) do + deadline(opts) + positive_opt(opts, :outbox_batch_size, 64) + positive_opt(opts, :outbox_batch_bytes, 1_048_576) + positive_opt(opts, :outbox_max_messages, 1_024) + non_negative_opt(opts, :outbox_flush_interval, 1) + :ok + end + + defp deadline(opts) do + case Keyword.get(opts, :outbox_deadline, @default_deadline) do + value when is_integer(value) and value > 0 -> + value + + other -> + raise ArgumentError, "expected :outbox_deadline to be positive, got: #{inspect(other)}" + end + end + + defp positive_opt(opts, key, default) do + case Keyword.get(opts, key, default) do + value when is_integer(value) and value > 0 -> + value + + other -> + raise ArgumentError, "expected #{inspect(key)} to be positive, got: #{inspect(other)}" + end + end + + defp non_negative_opt(opts, key, default) do + case Keyword.get(opts, key, default) do + value when is_integer(value) and value >= 0 -> + value + + other -> + raise ArgumentError, + "expected #{inspect(key)} to be non-negative, got: #{inspect(other)}" + end + end +end + +defmodule Group.Transport.Outbox.Supervisor do + @moduledoc false + use Supervisor + + def start_link(opts), do: Supervisor.start_link(__MODULE__, opts) + + @impl true + def init(opts) do + :ok = Group.Transport.Outbox.validate_options!(opts) + group = Keyword.fetch!(opts, :name) + num_shards = Keyword.fetch!(opts, :num_shards) + backend = Keyword.fetch!(opts, :backend) + + Code.ensure_loaded!(backend) + + for {function, arity} <- [init_outbox: 3, send_batch: 4] do + unless function_exported?(backend, function, arity) do + raise ArgumentError, + "outbox backend #{inspect(backend)} must implement #{function}/#{arity}" + end + end + + children = + for shard <- 0..(num_shards - 1) do + %{ + id: {Group.Transport.Outbox.Worker, group, shard}, + start: {Group.Transport.Outbox.Worker, :start_link, [opts, shard]}, + restart: :permanent, + shutdown: 5_000 + } + end + + Supervisor.init(children, strategy: :one_for_one) + end +end + +defmodule Group.Transport.Outbox.Worker do + @moduledoc false + use GenServer + + alias Group.Transport.Outbox + + @default_batch_size 64 + @default_batch_bytes 1_048_576 + @default_flush_interval 1 + @default_max_messages 1_024 + + def start_link(opts, shard) do + group = Keyword.fetch!(opts, :name) + GenServer.start_link(__MODULE__, {opts, shard}, name: Outbox.name(group, shard)) + end + + @impl true + def init({opts, shard}) do + :ok = Outbox.validate_options!(opts) + group = Keyword.fetch!(opts, :name) + backend = Keyword.fetch!(opts, :backend) + {:ok, backend_state} = backend.init_outbox(group, shard, opts) + admission = :atomics.new(1, signed: false) + max_messages = positive_opt(opts, :outbox_max_messages, @default_max_messages) + :persistent_term.put(Outbox.admission_key(group, shard), {self(), admission, max_messages}) + + {:ok, + %{ + group: group, + shard: shard, + backend: backend, + backend_state: backend_state, + admission: admission, + batch_size: positive_opt(opts, :outbox_batch_size, @default_batch_size), + batch_bytes: positive_opt(opts, :outbox_batch_bytes, @default_batch_bytes), + flush_interval: non_negative_opt(opts, :outbox_flush_interval, @default_flush_interval), + pending: [], + pending_count: 0, + pending_bytes: 0, + flush_ref: nil + }} + end + + @impl true + def terminate(_reason, state) do + key = Outbox.admission_key(state.group, state.shard) + + case :persistent_term.get(key, nil) do + {pid, _admission, _max_messages} when pid == self() -> + :persistent_term.erase(key) + + _ -> + :ok + end + + :ok + end + + @impl true + def handle_info({:group_replica_outbox_push, target_node, deadline, message}, state) + when is_atom(target_node) and is_integer(deadline) do + if deadline <= Outbox.monotonic_ms() do + {:noreply, release_admission(state, 1)} + else + bytes = :erlang.external_size({target_node, message}) + + state = + if state.pending_count > 0 and + (state.pending_count + 1 > state.batch_size or + state.pending_bytes + bytes > state.batch_bytes) do + flush(state) + else + state + end + + state = put_pending(state, target_node, deadline, message, bytes) + + if state.pending_count >= state.batch_size or state.pending_bytes >= state.batch_bytes do + {:noreply, flush(state)} + else + {:noreply, schedule_flush(state)} + end + end + end + + def handle_info( + {:group_replica_outbox_flush, ref}, + %{flush_ref: {ref, _timer_ref}} = state + ) do + {:noreply, flush(%{state | flush_ref: nil})} + end + + def handle_info({:group_replica_outbox_flush, _stale_ref}, state), do: {:noreply, state} + def handle_info(_message, state), do: {:noreply, state} + + defp put_pending(state, target_node, deadline, message, bytes) do + entry = {target_node, deadline, message} + + %{ + state + | pending: [entry | state.pending], + pending_count: state.pending_count + 1, + pending_bytes: state.pending_bytes + bytes + } + end + + defp schedule_flush(%{pending_count: 0} = state), do: state + + defp schedule_flush(%{flush_ref: {_ref, timer_ref}} = state) when is_reference(timer_ref), + do: state + + defp schedule_flush(state) do + ref = make_ref() + + timer_ref = + Process.send_after(self(), {:group_replica_outbox_flush, ref}, state.flush_interval) + + %{state | flush_ref: {ref, timer_ref}} + end + + defp flush(%{pending_count: 0} = state), do: cancel_flush(state) + + defp flush(state) do + released = state.pending_count + state = cancel_flush(state) + now = Outbox.monotonic_ms() + + batches = + state.pending + |> Enum.reverse() + |> Enum.reject(fn {_target_node, deadline, _message} -> deadline <= now end) + |> Enum.group_by(fn {target_node, _deadline, _message} -> target_node end) + + backend_state = + Enum.reduce(batches, state.backend_state, fn {target_node, entries}, backend_state -> + messages = Enum.map(entries, fn {_target_node, _deadline, message} -> message end) + + deadline = + entries + |> Enum.map(fn {_target_node, deadline, _message} -> deadline end) + |> Enum.min() + + if deadline <= Outbox.monotonic_ms() do + backend_state + else + case state.backend.send_batch(target_node, messages, deadline, backend_state) do + {result, next_backend_state} when result in [:ok, :busy, :disconnected] -> + next_backend_state + + other -> + raise "invalid #{inspect(state.backend)}.send_batch/4 return: #{inspect(other)}" + end + end + end) + + state = %{ + state + | backend_state: backend_state, + pending: [], + pending_count: 0, + pending_bytes: 0 + } + + release_admission(state, released) + end + + defp release_admission(state, count) do + :atomics.sub(state.admission, 1, count) + state + end + + defp cancel_flush(%{flush_ref: nil} = state), do: state + + defp cancel_flush(%{flush_ref: {_ref, timer_ref}} = state) do + Process.cancel_timer(timer_ref) + %{state | flush_ref: nil} + end + + defp positive_opt(opts, key, default) do + case Keyword.get(opts, key, default) do + value when is_integer(value) and value > 0 -> + value + + other -> + raise ArgumentError, "expected #{inspect(key)} to be positive, got: #{inspect(other)}" + end + end + + defp non_negative_opt(opts, key, default) do + case Keyword.get(opts, key, default) do + value when is_integer(value) and value >= 0 -> + value + + other -> + raise ArgumentError, + "expected #{inspect(key)} to be non-negative, got: #{inspect(other)}" + end + end +end diff --git a/mix.exs b/mix.exs index c618678..f9b9580 100644 --- a/mix.exs +++ b/mix.exs @@ -10,8 +10,10 @@ defmodule Group.MixProject do version: @version, elixir: "~> 1.19", elixirc_paths: elixirc_paths(Mix.env()), + test_ignore_filters: [~r"^test/jepsen/", ~r"^test/mutation/"], start_permanent: Mix.env() == :prod, deps: deps(), + aliases: aliases(), package: package(), docs: docs(), name: "Group", @@ -32,9 +34,14 @@ defmodule Group.MixProject do ] end + def cli do + [preferred_envs: ["test.soak": :test]] + end + defp deps do [ - {:ex_doc, "~> 0.30", only: :dev, runtime: false} + {:ex_doc, "~> 0.30", only: :dev, runtime: false}, + {:stream_data, "~> 1.4", only: :test} ] end @@ -54,4 +61,19 @@ defmodule Group.MixProject do source_ref: "v#{@version}" ] end + + defp aliases do + [ + test: ["test", "cmd test/jepsen/checker.sh"], + "test.soak": [ + # Run the PR gate in a child VM. test_helper starts distribution, and + # keeping that VM alive for the following `cmd` phases can retain a + # fixed ERL_AFLAGS distribution port and make qualification fail with + # eaddrinuse. + "cmd mix test", + "cmd env GROUP_JEPSEN_SKIP_CHECKER=1 test/jepsen/qualify.sh", + "cmd env GROUP_JEPSEN_SKIP_CHECKER=1 test/jepsen/campaign.sh" + ] + ] + end end diff --git a/mix.lock b/mix.lock index 0c19687..90b786f 100644 --- a/mix.lock +++ b/mix.lock @@ -5,4 +5,5 @@ "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, "makeup_erlang": {:hex, :makeup_erlang, "1.0.3", "4252d5d4098da7415c390e847c814bad3764c94a814a0b4245176215615e1035", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "953297c02582a33411ac6208f2c6e55f0e870df7f80da724ed613f10e6706afd"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, + "stream_data": {:hex, :stream_data, "1.4.0", "026f929db613aabea6208012ae9b8970d3fd5f88b3bdf26831bc536f98c42036", [:mix], [], "hexpm", "2b0ee3a340dcce1c8cf6302a763ee757d1e01c54d6e16d9069062509d68b1dc9"}, } diff --git a/priv/bench/README.md b/priv/bench/README.md index a51e3a8..35ccb5a 100644 --- a/priv/bench/README.md +++ b/priv/bench/README.md @@ -7,7 +7,7 @@ separate BEAM VMs. ## Running ```bash -cd priv/group/priv/bench +cd priv/bench mix deps.get ``` @@ -30,10 +30,53 @@ Uses 3 separate BEAM VMs (coordinator + 2 replicas) as OS processes: The script compiles once, starts both replicas in the background, then launches the coordinator. Replicas are killed automatically on exit. +To isolate the 10,000-cluster lifecycle scenario: + +```bash +./run_distributed.sh --shards 4 \ + --coordinator-expr 'GroupBench.Distributed.run_many_clusters_only(shards: 4)' +``` + +To measure the exact-snapshot fallback independently (one shard models one +busy lane of a much larger sharded deployment): + +```bash +./run_distributed.sh --shards 1 \ + --coordinator-expr 'GroupBench.Distributed.run_snapshot_sync_only(shards: 1, entries: 50000)' +``` + +To exercise one million rows across the recommended 32-shard scale point, +including exact catch-up, hottest-shard restart repair, memory, and permanent +peer eviction: + +```bash +./run_distributed.sh --shards 32 \ + --coordinator-expr 'GroupBench.Distributed.run_scale_recovery_only(shards: 32, entries: 1000000, mode: :registry)' + +./run_distributed.sh --shards 32 \ + --coordinator-expr 'GroupBench.Distributed.run_scale_recovery_only(shards: 32, entries: 1000000, mode: :pg_hotspot)' +``` + +`:registry` spreads a million exact claims across all shards. +`:pg_hotspot` deliberately puts a million memberships on one key/shard, which +is the worst case for snapshot installation and restart rebuilding. + +To isolate the synchronous registry projection rebuild performed by a +restarting shard, seed only the proportional load for one shard. For example, +one million total rows at 32 shards benchmarks 31,250 rows in the restarted +lane without including snapshot or network time: + +```bash +./run_distributed.sh --shards 32 \ + --coordinator-expr 'GroupBench.Distributed.run_registry_init_only(shards: 32, entries: 1000000, samples: 3)' +``` + ## Local Scenarios All local benchmarks run for both the default (nil) cluster and a named cluster (`"game"`) to verify there's no performance difference between the two paths. +Each spawned process cohort is stopped after its measurement and before the +next case; cohort teardown is outside the measured interval. ### 1. Lookup throughput @@ -52,8 +95,10 @@ Slower than lookup because each call copies a 100-element list out of ETS. ### 3. Register throughput (shard scaling) Measures concurrent `Group.register/4` calls — each of 10K spawned processes -registers itself in parallel. Varies shard count (1, 2, 4, schedulers_online) -to show how write throughput scales with sharding. +registers itself in parallel. Uses the library default of 8 shards for the +non-scaling scenarios and a fixed 1, 2, 4, 8, 16, 32, 64 shard sweep. The +fixed sweep keeps results comparable across machines and avoids treating BEAM +scheduler count as a shard-count recommendation. ### 4. Register/unregister cycle @@ -84,7 +129,8 @@ The core distributed measurement. Registers a key on replica1, then spin-polls `Group.lookup` on replica2 until it appears. Repeats 1,000 times. Reports p50/p99/max latency covering the full path: GenServer call on replica1, -Erlang distribution message, GenServer cast on replica2, ETS insert. +write-ahead append, nonblocking replica transport, receiver application, and +ETS projection on replica2. ### 2. Bulk sync (new peer catches up) @@ -92,8 +138,10 @@ Measures how fast a new node catches up to an existing peer's state. Registers N keys on replica1 (1K and 10K), then starts Group on replica2 and polls until all N entries are visible. -Group sends all data in a single `cluster_state` message on peer discovery, so -this is bounded by serialization + network, not per-key round-trips. +Group advertises stream heads on peer discovery. A new peer requests the +missing range; if the bounded oplog no longer contains the prefix, Group sends +an exact per-origin snapshot. The measurement therefore covers the normal +catch-up decision as well as serialization and network transfer. ### 3. Concurrent cross-node writes @@ -115,8 +163,8 @@ compared to the default nil cluster. The critical distributed cleanup path. Registers 1K and 5K processes on replica1, kills them all, then measures how long until replica2 sees zero -entries. Exercises: local DOWN handler → `replicate_unregister` broadcast → -remote ETS cleanup. +entries. Exercises: local DOWN handler → authoritative sequenced unregister +records → nonblocking delta batch → remote ETS cleanup. This scenario catches O(N²) message amplification bugs where remote nodes redundantly monitor pids and re-broadcast cleanup messages. @@ -136,10 +184,42 @@ convergence on replica2 via the `replicate_leave` path. All members hash to the same shard (single key), making this the worst case for shard contention during bulk cleanup. +### 8. Many-cluster lifecycle + +Connects 10K named clusters, registers one process in each, forces peer +re-discovery, then disconnects and verifies cleanup. This exposes control-plane +message amplification and epoch-fence costs. + +The connect phase reports local `Group.connect/2` completion separately from +full remote control convergence. Full convergence checks the reverse cluster +index on both nodes for all 10K named clusters plus the default cluster; seeing +only the last submitted cluster is insufficient because controls may be +reordered or repaired asynchronously. On generation/epoch-aware builds the +barrier also requires every replica shard to hold the source's current control +revision and all 10K remote epoch rows. Registration starts only after this +barrier, so its result does not inherit unfinished connect work. + +Registration reports local completion, the remote count at that handoff, and +the remaining data-convergence tail separately. Re-discovery similarly splits +restart, local reconnect, control convergence, and data convergence; disconnect +splits local completion from remote cleanup. + +### 9. Busy application convergence + +Runs registry and PG churn across 50 clusters and 40K initial pids. Reports +application throughput and then verifies the replicas agree exactly. A fast +wall-clock result is not considered successful unless convergence completes. + +### 10. Local writes under replicated PG pressure + +Floods one receiver shard with remote membership updates while measuring local +register and join calls at increasing concurrency. This checks that bounded +replica turns preserve local control-plane progress. + ## Architecture ``` -priv/group/priv/bench/ +priv/bench/ ├── mix.exs # depends on :group via path: "../../" ├── run_distributed.sh # starts 3 VMs, cleans up on exit ├── README.md @@ -147,7 +227,7 @@ priv/group/priv/bench/ │ ├── group_bench.ex # CLI entry — dispatches local/distributed │ ├── group_bench/ │ │ ├── local.ex # 6 local benchmarks -│ │ ├── distributed.ex # coordinator: connects + drives 7 benchmarks +│ │ ├── distributed.ex # coordinator: connects + drives 10 benchmarks │ │ ├── replica.ex # helpers called by coordinator via :erpc │ │ └── helpers.ex # timing, formatting, percentile math ``` diff --git a/priv/bench/lib/group_bench/distributed.ex b/priv/bench/lib/group_bench/distributed.ex index 68369d2..3fbae03 100644 --- a/priv/bench/lib/group_bench/distributed.ex +++ b/priv/bench/lib/group_bench/distributed.ex @@ -52,6 +52,80 @@ defmodule GroupBench.Distributed do IO.puts("\n Done.\n") end + def run_many_clusters_only(opts \\ []) do + shards = Keyword.get(opts, :shards, 4) + Process.put(:bench_shards, shards) + + header("Distributed Many-Clusters Benchmark") + IO.puts(" coordinator: #{node()}") + IO.puts(" shards: #{shards}") + IO.puts(" schedulers: #{System.schedulers_online()}") + + connect_replicas() + bench_many_clusters(@replicas) + + IO.puts("\n Done.\n") + end + + def run_snapshot_sync_only(opts \\ []) do + shards = Keyword.get(opts, :shards, 1) + entries = Keyword.get(opts, :entries, 50_000) + Process.put(:bench_shards, shards) + + header("Distributed Exact-Snapshot Benchmark") + IO.puts(" coordinator: #{node()}") + IO.puts(" shards: #{shards}") + IO.puts(" entries: #{format_number(entries)}") + IO.puts(" schedulers: #{System.schedulers_online()}") + + connect_replicas() + bench_snapshot_sync(@replicas, entries) + + IO.puts("\n Done.\n") + end + + def run_scale_recovery_only(opts \\ []) do + shards = Keyword.get(opts, :shards, 32) + entries = Keyword.get(opts, :entries, 1_000_000) + mode = Keyword.get(opts, :mode, :registry) + Process.put(:bench_shards, shards) + + unless mode in [:registry, :pg_hotspot] do + raise ArgumentError, "expected :mode to be :registry or :pg_hotspot" + end + + header("Distributed Million-Row Recovery Benchmark") + IO.puts(" coordinator: #{node()}") + IO.puts(" shards: #{shards}") + IO.puts(" entries: #{format_number(entries)}") + IO.puts(" mode: #{mode}") + IO.puts(" schedulers: #{System.schedulers_online()}") + + connect_replicas() + bench_scale_recovery(@replicas, mode, entries) + + IO.puts("\n Done.\n") + end + + def run_registry_init_only(opts \\ []) do + shards = Keyword.get(opts, :shards, 32) + total_entries = Keyword.get(opts, :entries, 1_000_000) + samples = Keyword.get(opts, :samples, 3) + Process.put(:bench_shards, shards) + + header("Registry Projection Shard-Init Benchmark") + IO.puts(" coordinator: #{node()}") + IO.puts(" shards: #{shards}") + IO.puts(" total rows: #{format_number(total_entries)}") + IO.puts(" samples: #{samples}") + IO.puts(" schedulers: #{System.schedulers_online()}") + + connect_replicas() + result = bench_registry_init(hd(@replicas), total_entries, samples) + IO.puts("\n Done.\n") + result + end + # ── Connection ──────────────────────────────────────────────────────── defp connect_replicas do @@ -77,6 +151,222 @@ defmodule GroupBench.Distributed do end end + defp bench_scale_recovery([source, receiver] = replicas, mode, entries) do + header("Exact snapshot, shard restart, and permanent Group loss") + + group_opts = [ + replicated_oplog_max_entries: 64, + replicated_snapshot_chunk_target_bytes: 1_048_576, + replicated_anti_entropy_interval: 250, + replicated_peer_lease_timeout: 5_000 + ] + + stop_groups(replicas) + start_group_on(source, group_opts) + + {seed_us, seed_result} = + :timer.tc(fn -> + case mode do + :registry -> + :erpc.call( + source, + GroupBench.Replica, + :seed_registry_slice, + [@name, entries, "scale/registry/", 10_000], + 1_800_000 + ) + + :pg_hotspot -> + :erpc.call( + source, + GroupBench.Replica, + :seed_pg_hotspot, + [@name, entries, "scale/pg-hotspot", 10_000], + 1_800_000 + ) + end + end) + + source_memory = :erpc.call(source, GroupBench.Replica, :memory_snapshot, [@name]) + + {snapshot_us, _} = + :timer.tc(fn -> + start_group_on(receiver, group_opts) + + poll_until( + fn -> replicated_row_count(receiver, mode) == entries end, + 900_000 + ) + + :ok = + :erpc.call(receiver, GroupBench.Replica, :flush_shards, [@name], 900_000) + end) + + receiver_memory = :erpc.call(receiver, GroupBench.Replica, :memory_snapshot, [@name]) + + restart_shard = + case mode do + :registry -> + source + |> :erpc.call(GroupBench.Replica, :registry_counts_by_shard, [@name]) + |> Enum.max_by(&elem(&1, 1)) + |> elem(0) + + :pg_hotspot -> + seed_result + end + + restart = + :erpc.call( + receiver, + GroupBench.Replica, + :restart_shard, + [@name, restart_shard], + 900_000 + ) + + unless replicated_row_count(receiver, mode) == entries do + raise "row count changed across receiver shard restart" + end + + {eviction_us, _} = + :timer.tc(fn -> + stop_group_on(source) + + poll_until( + fn -> + counts = :erpc.call(receiver, GroupBench.Replica, :replica_row_counts, [@name]) + + counts.registry_rows == 0 and counts.registry_claim_rows == 0 and + counts.pg_rows == 0 and counts.replica_cursor_rows == 0 + end, + 900_000 + ) + end) + + after_eviction_memory = + :erpc.call(receiver, GroupBench.Replica, :memory_snapshot, [@name]) + + unless after_eviction_memory.registry_rows == 0 and after_eviction_memory.pg_rows == 0 and + after_eviction_memory.registry_claim_rows == 0 and + after_eviction_memory.replica_cursor_rows == 0 do + raise "retired source left visible rows, authoritative claims, or receive cursors: " <> + inspect( + Map.take(after_eviction_memory, [ + :registry_rows, + :registry_claim_rows, + :pg_rows, + :replica_cursor_rows + ]) + ) + end + + if mode == :registry and is_pid(seed_result) do + :erpc.call(source, Process, :exit, [seed_result, :kill]) + end + + result = %{ + mode: mode, + entries: entries, + shards: Process.get(:bench_shards), + seed_ms: div(seed_us, 1_000), + snapshot_ms: div(snapshot_us, 1_000), + snapshot_rows_per_second: round(entries * 1_000_000 / max(snapshot_us, 1)), + restart_shard: restart_shard, + restart_ms: div(restart.elapsed_us, 1_000), + eviction_ms: div(eviction_us, 1_000), + source_memory: source_memory, + receiver_memory: receiver_memory, + after_eviction_memory: after_eviction_memory + } + + IO.puts("\n PERF_RESULT #{inspect(result, pretty: true, limit: :infinity)}") + stop_groups(replicas) + result + end + + defp bench_registry_init(node, total_entries, samples) do + header("Synchronous projection rebuild during shard restart") + shards = Process.get(:bench_shards) + rows_per_shard = div(total_entries + shards - 1, shards) + target_shard = 0 + + stop_group_on(node) + + start_group_on(node, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ) + + {seed_us, owner} = + :timer.tc(fn -> + :erpc.call( + node, + GroupBench.Replica, + :seed_registry_shard, + [@name, target_shard, rows_per_shard, "init/registry/", 10_000], + 1_800_000 + ) + end) + + counts = :erpc.call(node, GroupBench.Replica, :registry_counts_by_shard, [@name]) + + unless List.keyfind(counts, target_shard, 0) == {target_shard, rows_per_shard} do + raise "registry init benchmark seeded the wrong target-shard row count" + end + + before_memory = :erpc.call(node, GroupBench.Replica, :memory_snapshot, [@name]) + + restart_us = + Enum.map(1..samples, fn _sample -> + result = + :erpc.call( + node, + GroupBench.Replica, + :restart_shard, + [@name, target_shard], + 900_000 + ) + + unless List.keyfind( + :erpc.call(node, GroupBench.Replica, :registry_counts_by_shard, [@name]), + target_shard, + 0 + ) == {target_shard, rows_per_shard} do + raise "registry row count changed across shard restart" + end + + result.elapsed_us + end) + |> Enum.sort() + + result = %{ + total_entries: total_entries, + shards: shards, + rows_per_shard: rows_per_shard, + samples: samples, + seed_ms: div(seed_us, 1_000), + restart_ms: Enum.map(restart_us, &Float.round(&1 / 1_000, 3)), + restart_min_ms: Float.round(hd(restart_us) / 1_000, 3), + restart_p50_ms: Float.round(Enum.at(restart_us, div(length(restart_us), 2)) / 1_000, 3), + restart_max_ms: Float.round(List.last(restart_us) / 1_000, 3), + table_memory_bytes: before_memory.group_table_bytes + } + + IO.puts("\n PERF_RESULT #{inspect(result, pretty: true, limit: :infinity)}") + :erpc.call(node, Process, :exit, [owner, :kill]) + stop_group_on(node) + result + end + + defp replicated_row_count(node, :registry) do + :erpc.call(node, GroupBench.Replica, :total_registry_count, [@name]) + end + + defp replicated_row_count(node, :pg_hotspot) do + :erpc.call(node, GroupBench.Replica, :total_pg_count, [@name]) + end + # ── Group lifecycle helpers (all MFA) ───────────────────────────────── defp start_group_on(node, opts \\ []) do @@ -193,6 +483,48 @@ defmodule GroupBench.Distributed do end end + defp bench_snapshot_sync([r1, r2] = replicas, key_count) do + header("Exact Snapshot (receiver below oplog floor)") + subheader("#{format_number(key_count)} keys") + + start_group_on(r1, + replicated_oplog_max_entries: 64, + replicated_anti_entropy_interval: 100, + replicated_peer_lease_timeout: 15_000 + ) + + :erpc.call( + r1, + GroupBench.Replica, + :bulk_register, + [@name, key_count, "snapshot-"], + 180_000 + ) + + {sync_us, _} = + :timer.tc(fn -> + start_group_on(r2, + replicated_oplog_max_entries: 64, + replicated_anti_entropy_interval: 100, + replicated_peer_lease_timeout: 15_000 + ) + + poll_until( + fn -> + :erpc.call(r2, GroupBench.Replica, :total_registry_count, [@name]) >= key_count + end, + 120_000 + ) + end) + + rate = if sync_us > 0, do: round(key_count * 1_000_000 / sync_us), else: 0 + + IO.puts(" sync time: #{format_number(div(sync_us, 1000))} ms") + IO.puts(" keys/sec: #{format_number(rate)}") + + stop_groups(replicas) + end + # ── 3. Concurrent cross-node writes ────────────────────────────────── defp bench_concurrent_cross_node([r1, r2] = replicas) do @@ -428,7 +760,7 @@ defmodule GroupBench.Distributed do start_group_on(r2) wait_for_peer_discovery(replicas) - {connect_us, _} = + {local_connect_us, _} = :timer.tc(fn -> t1 = Task.async(fn -> @@ -453,48 +785,84 @@ defmodule GroupBench.Distributed do end) Task.await_many([t1, t2], 120_000) - - # Wait for convergence — both nodes see each other in the last cluster - poll_until( - fn -> - n1 = :erpc.call(r1, Group, :nodes, [@name, "#{prefix}#{num_clusters}"]) - n2 = :erpc.call(r2, Group, :nodes, [@name, "#{prefix}#{num_clusters}"]) - length(n1) >= 1 and length(n2) >= 1 - end, - 60_000 - ) end) - IO.puts(" connect: #{format_number(div(connect_us, 1000))} ms") - IO.puts(" clusters/sec: #{format_number(round(num_clusters * 1_000_000 / connect_us))}") + # Local completion only means both nodes accepted all Group.connect calls. + # Controls may be reordered or repaired asynchronously, so seeing one + # sentinel cluster cannot prove that the other 9,999 have converged. + expected_cluster_count = num_clusters + 1 + + {control_convergence_us, _} = + try do + :timer.tc(fn -> + wait_for_cluster_control_convergence(r1, r2, expected_cluster_count) + end) + rescue + error -> + r1_status = :erpc.call(r1, GroupBench.Replica, :cluster_control_status, [@name, r2]) + r2_status = :erpc.call(r2, GroupBench.Replica, :cluster_control_status, [@name, r1]) + + reraise RuntimeError, + [ + message: + "#{Exception.message(error)}; r1=#{inspect(r1_status)} " <> + "r2=#{inspect(r2_status)}" + ], + __STACKTRACE__ + end + + connect_total_us = local_connect_us + control_convergence_us + + IO.puts(" local calls: #{format_number(div(local_connect_us, 1000))} ms") + IO.puts(" control convergence: #{format_number(div(control_convergence_us, 1000))} ms") + IO.puts(" end-to-end connect: #{format_number(div(connect_total_us, 1000))} ms") + + IO.puts( + " local clusters/sec: #{format_number(round(num_clusters * 1_000_000 / local_connect_us))}" + ) + + IO.puts( + " converged clusters/sec: #{format_number(round(num_clusters * 1_000_000 / connect_total_us))}" + ) # -- 8b. Register 1 key per cluster -- subheader("register across #{format_number(num_clusters)} clusters") - {reg_us, pids} = + {local_reg_us, pids} = :timer.tc(fn -> - pids = - :erpc.call( - r1, - GroupBench.Replica, - :bulk_register_per_cluster, - [@name, num_clusters, prefix], - 120_000 - ) + :erpc.call( + r1, + GroupBench.Replica, + :bulk_register_per_cluster, + [@name, num_clusters, prefix], + 120_000 + ) + end) + + remote_count_at_handoff = + :erpc.call(r2, GroupBench.Replica, :total_registry_count, [@name]) + {reg_convergence_us, _} = + :timer.tc(fn -> poll_until( fn -> :erpc.call(r2, GroupBench.Replica, :total_registry_count, [@name]) >= num_clusters end, 60_000 ) - - pids end) - IO.puts(" register+converge: #{format_number(div(reg_us, 1000))} ms") - IO.puts(" ops/sec: #{format_number(round(num_clusters * 1_000_000 / reg_us))}") + reg_total_us = local_reg_us + reg_convergence_us + + IO.puts(" local registration: #{format_number(div(local_reg_us, 1000))} ms") + IO.puts(" remote at handoff: #{format_number(remote_count_at_handoff)} / 10,000") + IO.puts(" data convergence: #{format_number(div(reg_convergence_us, 1000))} ms") + IO.puts(" register end-to-end: #{format_number(div(reg_total_us, 1000))} ms") + + IO.puts( + " converged ops/sec: #{format_number(round(num_clusters * 1_000_000 / reg_total_us))}" + ) # -- 8c. Peer re-discovery with many clusters -- @@ -504,28 +872,75 @@ defmodule GroupBench.Distributed do stop_group_on(r2) Process.sleep(500) - {rediscovery_us, _} = + {restart_us, _} = :timer.tc(fn -> start_group_on(r2) wait_for_peer_discovery(replicas) + end) + {local_reconnect_us, _} = + :timer.tc(fn -> :erpc.call(r2, GroupBench.Replica, :bulk_connect, [@name, num_clusters, prefix], 120_000) + end) - poll_until( - fn -> - :erpc.call(r2, GroupBench.Replica, :total_registry_count, [@name]) >= num_clusters - end, - 120_000 - ) + {reconnect_control_us, _} = + :timer.tc(fn -> + wait_for_cluster_control_convergence(r1, r2, expected_cluster_count) end) - IO.puts(" re-discovery: #{format_number(div(rediscovery_us, 1000))} ms") + {rediscovery_data_us, _} = + try do + :timer.tc(fn -> + poll_until( + fn -> + :erpc.call(r2, GroupBench.Replica, :total_registry_count, [@name]) >= num_clusters + end, + 120_000 + ) + end) + rescue + error -> + counts = + :erpc.call(r2, GroupBench.Replica, :registry_counts_by_shard, [@name]) + + diagnostics = + :erpc.call(r2, GroupBench.Replica, :replica_process_diagnostics, [@name]) + + r1_revision = + :erpc.call(r1, Group.Replica.Data, :local_cluster_epoch_revision, [@name]) + + r2_remote_revision = + :erpc.call(r2, Group.Replica.Data, :remote_cluster_epoch_revision, [@name, r1]) + + r1_remote_revision = + :erpc.call(r1, Group.Replica.Data, :remote_cluster_epoch_revision, [@name, r2]) + + reraise RuntimeError, + [ + message: + "#{Exception.message(error)}; rediscovery_counts=#{inspect(counts)} " <> + "r1_revision=#{r1_revision} " <> + "r2_remote_revision=#{inspect(r2_remote_revision)} " <> + "r1_remote_revision=#{inspect(r1_remote_revision)} " <> + "replica=#{inspect(diagnostics)}" + ], + __STACKTRACE__ + end + + rediscovery_total_us = + restart_us + local_reconnect_us + reconnect_control_us + rediscovery_data_us + + IO.puts(" restart + discovery: #{format_number(div(restart_us, 1000))} ms") + IO.puts(" local reconnect: #{format_number(div(local_reconnect_us, 1000))} ms") + IO.puts(" control convergence: #{format_number(div(reconnect_control_us, 1000))} ms") + IO.puts(" data convergence: #{format_number(div(rediscovery_data_us, 1000))} ms") + IO.puts(" re-discovery total: #{format_number(div(rediscovery_total_us, 1000))} ms") # -- 8d. Disconnect cleanup -- subheader("disconnect #{format_number(num_clusters)} clusters") - {disconnect_us, _} = + {local_disconnect_us, _} = :timer.tc(fn -> :erpc.call( r1, @@ -534,20 +949,58 @@ defmodule GroupBench.Distributed do [@name, num_clusters, prefix], 120_000 ) + end) + {cleanup_us, _} = + :timer.tc(fn -> # Wait for r2 to see r1's entries cleaned from all clusters - poll_until( - fn -> - :erpc.call(r2, GroupBench.Replica, :total_registry_count, [@name]) == 0 - end, - 60_000 - ) + try do + poll_until( + fn -> + :erpc.call(r2, GroupBench.Replica, :total_registry_count, [@name]) == 0 + end, + 60_000 + ) + rescue + error -> + count = :erpc.call(r2, GroupBench.Replica, :total_registry_count, [@name]) + sample = :erpc.call(r2, GroupBench.Replica, :registry_sample, [@name]) + + local_revision = + :erpc.call(r1, Group.Replica.Data, :local_cluster_epoch_revision, [@name]) + + remote_revision = + :erpc.call(r2, Group.Replica.Data, :remote_cluster_epoch_revision, [@name, r1]) + + observed_revision = + :erpc.call( + r2, + Group.Replica.Data, + :remote_cluster_epoch_observed_revision, + [@name, r1] + ) + + reraise RuntimeError, + [ + message: + "#{Exception.message(error)}; remaining=#{count} " <> + "local_revision=#{local_revision} " <> + "remote_revision=#{inspect(remote_revision)} " <> + "observed_revision=#{inspect(observed_revision)} " <> + "sample=#{inspect(sample)}" + ], + __STACKTRACE__ + end end) - IO.puts(" disconnect+cleanup: #{format_number(div(disconnect_us, 1000))} ms") + disconnect_total_us = local_disconnect_us + cleanup_us + + IO.puts(" local disconnect: #{format_number(div(local_disconnect_us, 1000))} ms") + IO.puts(" remote cleanup: #{format_number(div(cleanup_us, 1000))} ms") + IO.puts(" disconnect total: #{format_number(div(disconnect_total_us, 1000))} ms") IO.puts( - " clusters/sec: #{format_number(round(num_clusters * 1_000_000 / disconnect_us))}" + " converged clusters/sec: #{format_number(round(num_clusters * 1_000_000 / disconnect_total_us))}" ) # Kill leftover processes @@ -555,6 +1008,37 @@ defmodule GroupBench.Distributed do stop_groups(replicas) end + defp wait_for_cluster_control_convergence(r1, r2, expected_cluster_count) do + r1_revision = + :erpc.call(r1, GroupBench.Replica, :cluster_control_revision, [@name]) + + r2_revision = + :erpc.call(r2, GroupBench.Replica, :cluster_control_revision, [@name]) + + poll_until( + fn -> + r1_converged? = + :erpc.call( + r1, + GroupBench.Replica, + :cluster_control_converged?, + [@name, r2, expected_cluster_count, r2_revision] + ) + + r2_converged? = + :erpc.call( + r2, + GroupBench.Replica, + :cluster_control_converged?, + [@name, r1, expected_cluster_count, r1_revision] + ) + + r1_converged? and r2_converged? + end, + 120_000 + ) + end + # ── 9. Busy app simulation ────────────────────────────────────────── defp bench_busy_app([r1, r2] = replicas) do diff --git a/priv/bench/lib/group_bench/local.ex b/priv/bench/lib/group_bench/local.ex index dad6cf1..667bae3 100644 --- a/priv/bench/lib/group_bench/local.ex +++ b/priv/bench/lib/group_bench/local.ex @@ -6,11 +6,14 @@ defmodule GroupBench.Local do import GroupBench.Helpers @name :bench - @default_shards System.schedulers_online() + @default_shards 8 + @shard_counts [1, 2, 4, 8, 16, 32, 64] def run do header("Local Benchmarks") IO.puts(" schedulers_online: #{System.schedulers_online()}") + IO.puts(" default_shards: #{@default_shards}") + IO.puts(" shard_sweep: #{Enum.join(@shard_counts, ", ")}") bench_lookup() bench_members() @@ -36,21 +39,26 @@ defmodule GroupBench.Local do measure_count = 100_000 # Each process registers itself - register_from_spawned_processes(key_count, fn i -> - Group.register(@name, "key-#{i}", %{i: i}, cluster_opts(cluster_opt)) - end) + pids = + register_from_spawned_processes(key_count, fn i -> + Group.register(@name, "key-#{i}", %{i: i}, cluster_opts(cluster_opt)) + end) - # warmup - warmup(1_000, fn -> Group.lookup(@name, "key-1", cluster_opts(cluster_opt)) end) + try do + # warmup + warmup(1_000, fn -> Group.lookup(@name, "key-1", cluster_opts(cluster_opt)) end) - # measure - samples = - collect_samples(measure_count, fn -> - i = :rand.uniform(key_count) - Group.lookup(@name, "key-#{i}", cluster_opts(cluster_opt)) - end) + # measure + samples = + collect_samples(measure_count, fn -> + i = :rand.uniform(key_count) + Group.lookup(@name, "key-#{i}", cluster_opts(cluster_opt)) + end) - report_latency("Group.lookup/3", samples) + report_latency("Group.lookup/3", samples) + after + stop_spawned_processes(pids) + end end) end end @@ -72,20 +80,25 @@ defmodule GroupBench.Local do total = group_count * members_per_group # Each process joins a group - register_from_spawned_processes(total, fn i -> - gi = rem(i - 1, group_count) + 1 - Group.join(@name, "group-#{gi}", %{}, cluster_opts(cluster_opt)) - end) + pids = + register_from_spawned_processes(total, fn i -> + gi = rem(i - 1, group_count) + 1 + Group.join(@name, "group-#{gi}", %{}, cluster_opts(cluster_opt)) + end) - warmup(1_000, fn -> Group.members(@name, "group-1", cluster_opts(cluster_opt)) end) + try do + warmup(1_000, fn -> Group.members(@name, "group-1", cluster_opts(cluster_opt)) end) - samples = - collect_samples(measure_count, fn -> - gi = :rand.uniform(group_count) - Group.members(@name, "group-#{gi}", cluster_opts(cluster_opt)) - end) + samples = + collect_samples(measure_count, fn -> + gi = :rand.uniform(group_count) + Group.members(@name, "group-#{gi}", cluster_opts(cluster_opt)) + end) - report_latency("Group.members/3", samples) + report_latency("Group.members/3", samples) + after + stop_spawned_processes(pids) + end end) end end @@ -96,23 +109,26 @@ defmodule GroupBench.Local do header("3. Register Throughput (shard scaling)") n = 10_000 - shard_counts = Enum.uniq([1, 2, 4, @default_shards]) for {cluster_label, cluster_opt} <- clusters() do subheader("cluster: #{cluster_label}") - for shards <- shard_counts do + for shards <- @shard_counts do with_group([name: @name, shards: shards], fn -> maybe_connect_cluster(cluster_opt) - {wall_us, _} = + {wall_us, pids} = time_us(fn -> register_from_spawned_processes(n, fn i -> Group.register(@name, "reg-#{i}", %{}, cluster_opts(cluster_opt)) end) end) - report_throughput("shards=#{shards}", n, wall_us) + try do + report_throughput("shards=#{shards}", n, wall_us) + after + stop_spawned_processes(pids) + end end) end end @@ -151,23 +167,26 @@ defmodule GroupBench.Local do header("5. Join Throughput (shard scaling)") n = 10_000 - shard_counts = Enum.uniq([1, 2, 4, @default_shards]) for {cluster_label, cluster_opt} <- clusters() do subheader("cluster: #{cluster_label}") - for shards <- shard_counts do + for shards <- @shard_counts do with_group([name: @name, shards: shards], fn -> maybe_connect_cluster(cluster_opt) - {wall_us, _} = + {wall_us, pids} = time_us(fn -> register_from_spawned_processes(n, fn i -> Group.join(@name, "join-group-#{rem(i, 100)}", %{}, cluster_opts(cluster_opt)) end) end) - report_throughput("shards=#{shards}", n, wall_us) + try do + report_throughput("shards=#{shards}", n, wall_us) + after + stop_spawned_processes(pids) + end end) end end @@ -186,18 +205,25 @@ defmodule GroupBench.Local do with_group([name: @name, shards: @default_shards], fn -> maybe_connect_cluster(cluster_opt) :ok = Group.monitor(@name, :all, cluster_opts(cluster_opt)) + drain_stale_group_events() - {wall_us, _} = + {wall_us, pids} = time_us(fn -> - register_from_spawned_processes(n, fn i -> - Group.register(@name, "mon-#{i}", %{}, cluster_opts(cluster_opt)) - end) + pids = + register_from_spawned_processes(n, fn i -> + Group.register(@name, "mon-#{i}", %{}, cluster_opts(cluster_opt)) + end) # drain all N events drain_events(n) + pids end) - report_throughput("events (register → receive)", n, wall_us) + try do + report_throughput("events (register → receive)", n, wall_us) + after + stop_spawned_processes(pids) + end end) end end @@ -243,6 +269,19 @@ defmodule GroupBench.Local do pids end + defp stop_spawned_processes(pids) do + refs = Enum.map(pids, &{&1, Process.monitor(&1)}) + Enum.each(pids, &Process.exit(&1, :kill)) + + Enum.each(refs, fn {pid, ref} -> + receive do + {:DOWN, ^ref, :process, ^pid, _reason} -> :ok + after + 5_000 -> raise "Timed out stopping #{inspect(pid)}" + end + end) + end + defp drain_events(0), do: :ok defp drain_events(remaining) do @@ -254,4 +293,12 @@ defmodule GroupBench.Local do 5_000 -> IO.puts(" WARNING: timed out waiting for events, #{remaining} remaining") end end + + defp drain_stale_group_events do + receive do + {:group, _events, _info} -> drain_stale_group_events() + after + 0 -> :ok + end + end end diff --git a/priv/bench/lib/group_bench/replica.ex b/priv/bench/lib/group_bench/replica.ex index 22df3c0..8ba3afe 100644 --- a/priv/bench/lib/group_bench/replica.ex +++ b/priv/bench/lib/group_bench/replica.ex @@ -73,6 +73,25 @@ defmodule GroupBench.Replica do end) end + @doc false + def registry_counts_by_shard(name) do + for shard <- 0..(Group.get_config(name).num_shards - 1) do + {shard, :ets.info(Group.Replica.Data.reg_by_key_table(name, shard), :size)} + end + end + + def registry_sample(name, limit \\ 10) do + shards = Group.get_config(name).num_shards + + 0..(shards - 1) + |> Enum.flat_map(fn shard -> + Group.Replica.Data.reg_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.take(limit) + end) + |> Enum.take(limit) + end + @doc """ Registers a single key from a spawned process. Returns after registration. """ @@ -138,6 +157,228 @@ defmodule GroupBench.Replica do end) end + @doc false + def seed_registry_slice(name, count, key_prefix, batch_size \\ 10_000) do + owner = spawn(fn -> Process.sleep(:infinity) end) + shards = Group.get_config(name).num_shards + + 1..count + |> Stream.chunk_every(batch_size) + |> Enum.each(fn indexes -> + indexes + |> Enum.group_by(fn index -> + Group.Replica.shard_index_for(nil, "#{key_prefix}#{index}", shards) + end) + |> Task.async_stream( + fn {shard, shard_indexes} -> + stream_id = Group.Replica.Data.local_stream_id(name, shard, nil) + + entries = + Enum.map(shard_indexes, fn index -> + key = "#{key_prefix}#{index}" + + :ok = + Group.Replica.Data.put_registry_claim( + name, + shard, + stream_id, + 1, + key, + owner, + %{}, + index + ) + + {nil, key, owner, %{}, index, node()} + end) + + :ok = Group.Replica.Data.registry_insert_many(name, shard, entries) + end, + ordered: false, + timeout: :infinity + ) + |> Stream.run() + end) + + install_benchmark_stream_heads(name) + owner + end + + @doc false + def seed_registry_shard(name, shard, count, key_prefix, batch_size \\ 10_000) do + owner = spawn(fn -> Process.sleep(:infinity) end) + shards = Group.get_config(name).num_shards + stream_id = Group.Replica.Data.local_stream_id(name, shard, nil) + + 1 + |> Stream.iterate(&(&1 + 1)) + |> Stream.filter(fn index -> + Group.Replica.shard_index_for(nil, "#{key_prefix}#{index}", shards) == shard + end) + |> Stream.take(count) + |> Stream.chunk_every(batch_size) + |> Enum.each(fn indexes -> + entries = + Enum.map(indexes, fn index -> + key = "#{key_prefix}#{index}" + + :ok = + Group.Replica.Data.put_registry_claim( + name, + shard, + stream_id, + 1, + key, + owner, + %{}, + index + ) + + {nil, key, owner, %{}, index, node()} + end) + + :ok = Group.Replica.Data.registry_insert_many(name, shard, entries) + end) + + install_benchmark_stream_heads(name) + owner + end + + @doc false + def seed_pg_hotspot(name, count, key, batch_size \\ 10_000) do + shard = Group.Replica.shard_index_for(nil, key, Group.get_config(name).num_shards) + + 1..count + |> Stream.chunk_every(batch_size) + |> Enum.each(fn indexes -> + entries = + Enum.map(indexes, fn index -> + pid = :erlang.list_to_pid(String.to_charlist("<0.#{100_000_000 + index}.0>")) + {nil, key, pid, %{}, index, node()} + end) + + :ok = Group.Replica.Data.pg_insert_many(name, shard, entries) + end) + + install_benchmark_stream_heads(name) + shard + end + + @doc false + def restart_shard(name, shard) do + old_pid = Process.whereis(Group.Replica.shard_name(name, shard)) + started = System.monotonic_time(:microsecond) + Process.exit(old_pid, :kill) + new_pid = await_new_shard(name, shard, old_pid, 600_000) + elapsed = System.monotonic_time(:microsecond) - started + %{elapsed_us: elapsed, old_pid: old_pid, new_pid: new_pid} + end + + @doc false + def flush_shards(name) do + Enum.each(0..(Group.get_config(name).num_shards - 1), fn shard -> + :sys.get_state(Group.Replica.shard_name(name, shard), 600_000) + end) + + :ok + end + + @doc false + def memory_snapshot(name) do + shards = Group.get_config(name).num_shards + counts = replica_row_counts(name) + + table_bytes = + 0..(shards - 1) + |> Enum.flat_map(&replica_tables(name, &1)) + |> Enum.uniq() + |> Enum.reduce(0, fn table, total -> + case :ets.info(table, :memory) do + :undefined -> total + words -> total + words * :erlang.system_info(:wordsize) + end + end) + + %{ + total_bytes: :erlang.memory(:total), + process_bytes: :erlang.memory(:processes_used), + ets_bytes: :erlang.memory(:ets), + group_table_bytes: table_bytes, + registry_rows: counts.registry_rows, + registry_claim_rows: counts.registry_claim_rows, + pg_rows: counts.pg_rows, + replica_cursor_rows: counts.replica_cursor_rows + } + end + + @doc false + def replica_row_counts(name) do + %{ + registry_rows: total_registry_count(name), + registry_claim_rows: total_table_rows(name, &Group.Replica.Data.reg_claim_by_key_table/2), + pg_rows: total_pg_count(name), + replica_cursor_rows: total_table_rows(name, &Group.Replica.Data.replica_cursor_table/2) + } + end + + defp total_table_rows(name, table_fun) do + Enum.reduce(0..(Group.get_config(name).num_shards - 1), 0, fn shard, total -> + total + :ets.info(table_fun.(name, shard), :size) + end) + end + + defp install_benchmark_stream_heads(name) do + shards = Group.get_config(name).num_shards + + Enum.each(0..(shards - 1), fn shard -> + stream_id = Group.Replica.Data.local_stream_id(name, shard, nil) + table = Group.Replica.Data.replica_stream_meta_table(name, shard) + :ets.insert(table, {stream_id, 1, 2, 1}) + end) + + :ok + end + + defp await_new_shard(name, shard, old_pid, timeout) do + started = System.monotonic_time(:millisecond) + do_await_new_shard(name, shard, old_pid, timeout, started) + end + + defp do_await_new_shard(name, shard, old_pid, timeout, started) do + case Process.whereis(Group.Replica.shard_name(name, shard)) do + pid when is_pid(pid) and pid != old_pid -> + :sys.get_state(pid, timeout) + pid + + _ -> + if System.monotonic_time(:millisecond) - started >= timeout do + raise "timed out restarting shard #{shard}" + end + + Process.sleep(10) + do_await_new_shard(name, shard, old_pid, timeout, started) + end + end + + defp replica_tables(name, shard) do + data = Group.Replica.Data + + base = [ + data.reg_by_key_table(name, shard), + data.reg_by_pid_table(name, shard), + data.reg_claim_by_key_table(name, shard), + data.reg_claim_by_pid_table(name, shard), + data.pg_by_key_table(name, shard), + data.pg_by_pid_table(name, shard), + data.replica_stream_meta_table(name, shard), + data.replica_oplog_table(name, shard), + data.replica_oplog_order_table(name, shard), + data.replica_cursor_table(name, shard) + ] + + base + end + @doc """ Starts `worker_count` local processes that repeatedly re-join the same key with changing metadata, generating a sustained stream of replicated PG updates. @@ -577,23 +818,78 @@ defmodule GroupBench.Replica do pids = Enum.map(1..n, fn i -> spawn(fn -> - :ok = Group.register(name, "key", %{}, cluster: "#{prefix}#{i}") - send(parent, {:done, self()}) - Process.sleep(:infinity) + try do + :ok = Group.register(name, "key", %{}, cluster: "#{prefix}#{i}") + send(parent, {:done, self()}) + Process.sleep(:infinity) + catch + kind, reason -> + send( + parent, + {:failed, self(), kind, {reason, replica_process_diagnostics(name)}, + __STACKTRACE__} + ) + end end) end) Enum.each(pids, fn pid -> receive do - {:done, ^pid} -> :ok + {:done, ^pid} -> + :ok + + {:failed, ^pid, kind, reason, stacktrace} -> + :erlang.raise(kind, reason, stacktrace) after - 60_000 -> raise "Timed out waiting for bulk_register_per_cluster" + 60_000 -> + awaited_pid = + {pid, + Process.info(pid, [ + :status, + :current_function, + :message_queue_len, + :reductions + ])} + + registry_count = total_registry_count(name) + + raise "Timed out waiting for bulk_register_per_cluster: " <> + "replica=#{inspect(replica_process_diagnostics(name))} " <> + "awaited=#{inspect(awaited_pid)} registry_count=#{registry_count}" end end) pids end + @doc false + def replica_process_diagnostics(name) do + shards = + for shard <- 0..(Group.get_config(name).num_shards - 1) do + shard_pid = Process.whereis(Group.Replica.shard_name(name, shard)) + + {shard, + Process.info(shard_pid, [ + :status, + :current_function, + :message_queue_len, + :reductions + ])} + end + + data_pid = Process.whereis(Group.Replica.Data.data_name(name)) + + data = + Process.info(data_pid, [ + :status, + :current_function, + :message_queue_len, + :reductions + ]) + + %{shards: shards, data: data} + end + @doc """ Returns the number of clusters this node is a member of (via reverse index). """ @@ -601,6 +897,95 @@ defmodule GroupBench.Replica do length(Group.Replica.Data.my_clusters(name)) end + @doc """ + Returns the number of clusters currently associated with `target_node`. + + The many-cluster benchmark uses the reverse index instead of checking one + sentinel cluster: replica control messages may be reordered, so observing + the last submitted cluster does not prove that every earlier cluster has + converged. + """ + def cluster_count_for_node(name, target_node) do + name + |> Group.Replica.Data.node_clusters_table() + |> :ets.lookup(target_node) + |> length() + end + + @doc false + def cluster_control_revision(name) do + if function_exported?(Group.Replica.Data, :local_cluster_epoch_revision, 1) do + apply(Group.Replica.Data, :local_cluster_epoch_revision, [name]) + else + :legacy + end + end + + @doc false + def cluster_control_converged?(name, target_node, expected_cluster_count, source_revision) do + membership_converged? = + cluster_count_for_node(name, target_node) >= expected_cluster_count + + if source_revision == :legacy or + not function_exported?(Group.Replica.Data, :remote_view_generation, 3) do + membership_converged? + else + shards = Group.get_config(name).num_shards + data = Group.Replica.Data + generation = apply(data, :remote_generation, [name, target_node]) + revision = apply(data, :remote_cluster_epoch_revision, [name, target_node]) + epoch_table = apply(data, :remote_cluster_epochs_table, [name]) + + epoch_count = + :ets.select_count(epoch_table, [ + {{{target_node, :_}, :_}, [], [true]} + ]) + + shard_views_converged? = + Enum.all?(0..(shards - 1), fn shard -> + apply(data, :remote_view_generation, [name, shard, target_node]) == generation and + apply(data, :remote_view_cluster_epoch_revision, [name, shard, target_node]) == + source_revision and + apply(data, :remote_view_observed_revision, [name, shard, target_node]) == + source_revision + end) + + membership_converged? and not is_nil(generation) and revision == source_revision and + epoch_count >= expected_cluster_count - 1 and shard_views_converged? + end + end + + @doc false + def cluster_control_status(name, target_node) do + data = Group.Replica.Data + shards = Group.get_config(name).num_shards + epoch_table = data.remote_cluster_epochs_table(name) + + %{ + local_revision: data.local_cluster_epoch_revision(name), + local_epoch_count: :ets.info(data.local_cluster_epochs_table(name), :size), + membership_count: cluster_count_for_node(name, target_node), + remote_generation: data.remote_generation(name, target_node), + remote_revision: data.remote_cluster_epoch_revision(name, target_node), + remote_exact_revision: data.remote_cluster_epoch_exact_revision(name, target_node), + remote_observed: data.remote_cluster_epoch_observed_revision(name, target_node), + remote_epoch_count: + :ets.select_count(epoch_table, [ + {{{target_node, :_}, :_}, [], [true]} + ]), + authority_installs: data.remote_authority_install_count(name, target_node), + views: + for( + shard <- 0..(shards - 1), + do: + {shard, data.remote_view_generation(name, shard, target_node), + data.remote_view_cluster_epoch_revision(name, shard, target_node), + data.remote_view_observed_revision(name, shard, target_node)} + ), + replica: replica_process_diagnostics(name) + } + end + @doc """ Simulates a busy app worker: registers, joins groups, does lookups and dispatches, then some processes die and re-register. Returns final pids. diff --git a/priv/bench/mix.exs b/priv/bench/mix.exs index 71d6a86..fc48c0b 100644 --- a/priv/bench/mix.exs +++ b/priv/bench/mix.exs @@ -17,6 +17,6 @@ defmodule GroupBench.MixProject do end defp deps do - [{:group, path: "../../"}] + [{:group, path: System.get_env("GROUP_BENCH_GROUP_PATH", "../../")}] end end diff --git a/priv/bench/run_distributed.sh b/priv/bench/run_distributed.sh index 3273aef..5f42bd1 100755 --- a/priv/bench/run_distributed.sh +++ b/priv/bench/run_distributed.sh @@ -6,6 +6,7 @@ cd "$(dirname "$0")" COOKIE=bench SHARDS=8 COORDINATOR_EXPR= +BENCH_ERL_AFLAGS="${GROUP_BENCH_ERL_AFLAGS:-}" while [[ $# -gt 0 ]]; do case "$1" in @@ -24,12 +25,12 @@ mix deps.get --check 2>/dev/null || mix deps.get mix compile echo "==> Starting replica1..." -elixir --name replica1@127.0.0.1 --cookie "$COOKIE" \ +ERL_AFLAGS="$BENCH_ERL_AFLAGS" elixir --name replica1@127.0.0.1 --cookie "$COOKIE" \ -S mix run --no-halt -e "GroupBench.Replica.start()" & REPLICA1_PID=$! echo "==> Starting replica2..." -elixir --name replica2@127.0.0.1 --cookie "$COOKIE" \ +ERL_AFLAGS="$BENCH_ERL_AFLAGS" elixir --name replica2@127.0.0.1 --cookie "$COOKIE" \ -S mix run --no-halt -e "GroupBench.Replica.start()" & REPLICA2_PID=$! @@ -44,5 +45,5 @@ trap cleanup EXIT sleep 2 echo "==> Starting coordinator (shards=$SHARDS)..." -elixir --name coordinator@127.0.0.1 --cookie "$COOKIE" \ +ERL_AFLAGS="$BENCH_ERL_AFLAGS" elixir --name coordinator@127.0.0.1 --cookie "$COOKIE" \ -S mix run -e "$COORDINATOR_EXPR" diff --git a/test/README.md b/test/README.md index 58a6d51..b8eea83 100644 --- a/test/README.md +++ b/test/README.md @@ -3,17 +3,76 @@ ## Running tests ```bash -mix test # all tests +mix test # every-PR ExUnit/property/chaos/checker gate +mix test.soak # nightly six-profile Jepsen campaign mix test test/group_test.exs # local only mix test test/distributed_test.exs # distributed only +mix test test/replica_adversarial_test.exs # seeded transport chaos +mix test test/replica_model_property_test.exs # shrinkable model-based histories +test/jepsen/run.sh # one OS-partition/restart Jepsen model test ``` +`mix test` preserves normal Mix test arguments while always running the pure +Jepsen lifecycle-checker qualification after ExUnit. It does not require +Docker. `mix test.soak` first runs that complete PR gate, kills every defined +protocol mutant, runs live positive/negative checker qualification, and then +runs the distribution/TCP/chaos × mixed/permanent Jepsen campaign. The soak +defaults to 20 five-minute fault histories per combination and is intended for +nightly and release qualification rather than individual edits. + ## Test files | File | What it tests | |------|---------------| | `group_test.exs` | Single-node: register/unregister, join/leave, members, monitor/demonitor, named clusters, concurrent operations | -| `distributed_test.exs` | Multi-node: replication, peer discovery, node disconnect cleanup, partition healing, conflict resolution, event ordering, rolling restarts | +| `distributed_test.exs` | Multi-node: replication, peer discovery, node disconnect cleanup, partition healing, conflict resolution, event ordering, rolling restarts, and adversarial replica-transport loss/busy/snapshot recovery | +| `anti_entropy_fault_regression_test.exs` | Three-node regressions for hidden-winner projection, receiver restart eviction, nodedown/lease lane retirement, authority gaps and cross-lane races, in-flight conflict fencing, crash-journal replay, cursorless/interrupted snapshot repair, malformed ingress, and sideband rediscovery | +| `replica_adversarial_test.exs` | Reproducible three-node mixed-operation state machines: drops, busy returns, duplication, reordering, bounded delay, oplog pruning, conflicts, owner death, and named-cluster epoch churn, followed by exact convergence/dead-owner/internal-index checks | +| `replica_model_property_test.exs` | StreamData-generated and shrunk owner histories against an independent lifecycle oracle and scheduler-controlled replica transport | +| `replica_snapshot_test.exs` | Pure single-pass byte-bounded streaming, suffix resume, receive staging, and event batching | +| `replica_snapshot_distributed_test.exs` | Real-node provisional-chunk/terminal-commit loss, reorder, duplicate, conflicting retransmission/manifest, concurrent-source invalidation, supersession, authority fencing, expiry, pooled staging, and shard-crash recovery | + +## Model-based and formal checks + +`replica_model_property_test.exs` runs real Group instances on three peer VMs. +The controlled transport queues each replica message so generated commands can +deliver, duplicate, drop, reorder, or strand it. After the bounded-fault +prefix, the test enables fair delivery and compares every tracked registry and +PG key against an independent application-level lifecycle oracle. It also +requires internal replica indexes to be consistent, every retained owner to be +alive, and registry conflict losers to be dead. Restart, pruning, and named +cluster histories retain independent C-owned state while A recovers, so repair +cannot pass merely by making one origin and one receiver agree. Model groups +use a deliberately tiny snapshot target so pruning recovery traverses the real +multi-chunk assembly path. + +StreamData reports the ExUnit seed and shrinks a failure to its smallest command +history. Local defaults are intentionally quick. Increase the budgets without +changing the generator: + +```bash +GROUP_MODEL_RUNS=1000 GROUP_MODEL_COMMANDS=100 \ + mix test test/replica_model_property_test.exs +``` + +The independent TLA+ model and TLC configuration live in `test/formal/`. +See [`formal/README.md`](formal/README.md) for its checked invariants, finite +model bounds, and run command. + +The Docker-backed Jepsen harness lives in [`jepsen/`](jepsen/). It drives +three independent BEAM containers through concurrent, multi-entry owner +lifecycles, named-cluster epoch churn, directed/full partitions, transport +session resets, and VM restarts. The same workload runs over distribution, +a test-only real sideband TCP lane, and a lossy/duplicating/reordering +transport. After healing, +its independent oracle checks exact public views and the internal registry, +PG, claim, cluster, cursor, oplog, snapshot-staging, and retired-origin +invariants. Its permanent-retirement scenario proves eviction even when a peer +never returns. `test/jepsen/campaign.sh` runs the full profile/scenario matrix; +`test/jepsen/qualify.sh` mutation-tests the implementation and proves that the +live checker rejects injected faults. Chaos/mixed uses a larger repair window +and is invalid unless it observes a multi-record delta run; the other profiles +retain the one-record stress configuration. ## How distribution works @@ -191,8 +250,93 @@ TestCluster.start_group( ) ``` -The resolver uses "most recent wins" — keeps the registration with the higher -timestamp. +The resolver ranks each claim by timestamp. Group chooses the maximum +`{rank, pid}`, so every peer reaches the same winner regardless of claim arrival +order. + +### Replica transport fault injection + +`Group.TestReplicaTransport` implements the production transport behaviour but +can return `:busy`, drop selected message types, duplicate or delay messages, +and capture messages for explicit stale-generation/epoch replay. Its +`{:chaos, opts}` mode is deterministic for a given message, which makes failures +reproducible. + +`Group.ControlledReplicaTransport` is the model-test transport. It queues messages +at the test process without scheduling timers; `Group.ReplicaModelScheduler` +then owns the exact delivery schedule. These roles are separate so the existing +timing-oriented regressions retain their original mechanics while property +failures can be replayed and shrunk exactly. + +The distributed anti-entropy tests cover dropped creates and deletes, cursor +gaps, globally pruned multi-stream oplogs, exact snapshot fallback, malformed +authority, stale message replay, lease expiry on a live VM, and multi-shard +generation recovery. They also restart a suspended data lane after deliberately +losing its cluster-close fence and require the lane to sweep the stale registry +and PG slices from shared authority. Authority topology tests suspend every +receiver shard and inspect the queued protocol: only shard 0 may receive/install +the full epoch snapshot, nonzero shards receive constant-size lane hellos, and +incremental controls arriving on any lane are serialized through shard 0. +Separate tests suspend a +backlogged authority shard while other replica lanes continue converging and +deliver data before authority to prove rejection does not advance the cursor +and the same message applies after authority repair. Concurrent snapshot tests +require every advertised revision to contain exactly that many unique named +epochs, and heartbeat tests prove observed revisions cannot advance the exact +authority marker. Crash-window tests interrupt journal, dual-index, receive +cursor, and named-cluster close updates, then require startup repair to remove +every invisible row and temporary close barrier. A three-node test-only TCP test +disconnects one origin's real socket, prunes its oplog, reconnects it, and +requires snapshot recovery without changing the third node's independent +registry or PG state. + +Authority races also replace authority while conflict resolution is paused, +remove the local shard-zero owner, and deliver `nodedown` while a sibling lane +is suspended. The assertions require that stale remote winners cannot terminate +local owners, that retained conflict claims are reprojected after an authority +gap closes even when their stream cursor is already current, and that each lane +keeps its own restart breadcrumb until its rows and cursor are gone. +The same conflict-gap schedule is held open until lease expiry to prove a source +that never returns leaves no deferred projection key, claim, cursor, or visible +remote winner behind. A separate nodedown regression proves the immediate +retirement path cannot strand that deferred state after removing its lease. +Newer-generation and same-generation heartbeat schedules assert the persisted +hint fences every lane before exact repair, rejects delayed exact/incremental +authority, and reconstructs its retirement deadline after a lane crash. Once +the peer is fully retired, replayed heartbeats and lane hellos must create no +hint, lease, or outbound route. A separate compare-and-install regression races +a newer hint across an older incremental control and requires no epoch row or +lane view to become authoritative. +The inverse ordering is forced independently: a first-time lane hello is +processed while shard zero is suspended, then exact authority must trigger an +immediate shard-local re-probe without first admitting the speculative route. +Exact-authority regression also activates a local cluster alongside a remote +epoch install and requires the authority plus both cluster indexes to become +visible as one operation; the high-volume control test repeats the projection +check over three nodes before admitting replica writes. +Interrupted lifecycle tests suspend the notification shard, stop after the +durable activation/deactivation mutation, and require activation routing to be +complete immediately and close cleanup to finish after the shard resumes. This +proves neither path relies on the API caller remaining alive. + +`replica_transport_outbox_test.exs` proves that a blocked sideband backend +cannot delay the Group-facing local push, messages expire behind that backend, +busy batches are not retried locally, batching preserves per-target order, +invalid deadlines fail at boot, and ingress drops rather than raising while a +destination shard is absent. Admission is bounded across the local mailbox, +pending batch, and backend send. +The real three-node TCP recovery test runs through the same outbox path. + +`Group.TestCluster.assert_replica_consistent/1` checks the +public dual indexes plus exact deterministic registry projection, row/shard +placement, lane authority revisions, registry/PG origin authority, +oplog/order equivalence, and contiguous retained stream ranges. Seeded tests +additionally require every PID retained as authority to still be alive after +convergence. + +The isolated mutation runner in `test/mutation/` disables individual protocol +guards and repair steps only in copied checkouts. See +[`mutation/README.md`](mutation/README.md) for the command and artifact format. ## Typical test patterns diff --git a/test/anti_entropy_fault_regression_test.exs b/test/anti_entropy_fault_regression_test.exs new file mode 100644 index 0000000..bac5c47 --- /dev/null +++ b/test/anti_entropy_fault_regression_test.exs @@ -0,0 +1,4053 @@ +defmodule Group.AntiEntropyFaultRegressionTest do + use ExUnit.Case, async: false + + @moduletag :capture_log + @moduletag timeout: 120_000 + + alias Group.TestCluster + + setup_all do + peers = TestCluster.start_peers(3, schedulers: 4) + on_exit(fn -> TestCluster.stop_peers(peers) end) + [{_, node_a}, {_, node_b}, {_, node_c}] = peers + {:ok, peers: peers, node_a: node_a, node_b: node_b, node_c: node_c} + end + + test "a receiver shard restart purges a permanently disappeared Group", context do + name = unique_name(:receiver_restart_permanent_loss) + + opts = [ + name: name, + shards: 2, + replicated_sender_buffer_size: 1, + replicated_pg_receiver_buffer_size: 1, + replicated_registry_receiver_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 150 + ] + + start_group_on_peers(context.peers, opts) + + registry_key = "receiver-restart/registry" + pg_key = "receiver-restart/pg" + + owner = + TestCluster.spawn_register_and_join( + context.node_a, + name, + registry_key, + %{owner: :a}, + pg_key, + %{owner: :a} + ) + + for receiver <- [context.node_b, context.node_c] do + TestCluster.assert_eventually(fn -> + match?( + {^owner, %{owner: :a}}, + TestCluster.rpc!(receiver, Group, :lookup, [name, registry_key]) + ) and + match?( + [{^owner, %{owner: :a}}], + TestCluster.rpc!(receiver, Group, :members, [name, pg_key]) + ) + end) + end + + old_receiver = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + :ok = TestCluster.rpc!(context.node_b, :sys, :suspend, [old_receiver]) + + # A's BEAM intentionally remains connected. Only its Group disappears, so + # B must learn retirement from the lease it misses while its shard is down. + supervisor = TestCluster.rpc!(context.node_a, Process, :whereis, [:"#{name}_group_sup"]) + :ok = TestCluster.rpc!(context.node_a, Supervisor, :stop, [supervisor, :normal, 5_000]) + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!(context.node_c, Group, :lookup, [name, registry_key]) == nil and + TestCluster.rpc!(context.node_c, Group, :members, [name, pg_key]) == [] and + context.node_a not in TestCluster.rpc!(context.node_c, Group, :nodes, [name]) + end, + timeout: 5_000 + ) + + Process.sleep(250) + true = TestCluster.rpc!(context.node_b, Process, :exit, [old_receiver, :kill]) + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, 0) + ]) do + pid when is_pid(pid) -> pid != old_receiver + _ -> false + end + end) + + # No peer is allowed to resurrect A or supply a later delete. Correctness + # therefore depends entirely on restart-time authority reconciliation. + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!(context.node_b, Group, :lookup, [name, registry_key]) == nil and + TestCluster.rpc!(context.node_b, Group, :members, [name, pg_key]) == [] and + context.node_a not in TestCluster.rpc!(context.node_b, Group, :nodes, [name]) + end, + timeout: 1_000, + interval: 25 + ) + + assert :ok = + TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_origin_purged, [ + name, + context.node_a + ]) + + assert :ok = + TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end + + test "a sibling restart cannot lose the retirement lease after shard zero expires a peer", + context do + name = unique_name(:sibling_restart_during_peer_expiry) + + opts = [ + name: name, + shards: 2, + replicated_sender_buffer_size: 1, + replicated_pg_receiver_buffer_size: 1, + replicated_registry_receiver_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 150 + ] + + start_group_on_peers(context.peers, opts) + + key = + 1..1_000 + |> Enum.map(&"sibling-expiry/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(nil, &1, 2) == 1)) + + owner = + TestCluster.spawn_register_and_join( + context.node_a, + name, + key, + %{owner: :a}, + key, + %{owner: :a} + ) + + for receiver <- [context.node_b, context.node_c] do + TestCluster.assert_eventually(fn -> + match?({^owner, %{owner: :a}}, TestCluster.rpc!(receiver, Group, :lookup, [name, key])) and + match?( + [{^owner, %{owner: :a}}], + TestCluster.rpc!(receiver, Group, :members, [name, key]) + ) + end) + end + + old_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + :ok = TestCluster.rpc!(context.node_b, :sys, :suspend, [old_lane]) + + supervisor = TestCluster.rpc!(context.node_a, Process, :whereis, [:"#{name}_group_sup"]) + :ok = TestCluster.rpc!(context.node_a, Supervisor, :stop, [supervisor, :normal, 5_000]) + + # Shard zero retires only its lane while the suspended sibling remains a + # live restart breadcrumb. Shared authority must survive until that final + # lane performs its own bounded retirement. + TestCluster.assert_eventually( + fn -> + is_nil( + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 0, + context.node_a + ]) + ) + end, + timeout: 5_000 + ) + + refute is_nil( + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_generation, [ + name, + context.node_a + ]) + ) + + # The lane view is its restart breadcrumb. Shard zero must not erase it on + # behalf of a lane that has not purged its own retained rows and cursor. + refute is_nil( + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_view_generation, + [name, 1, context.node_a] + ) + ) + + assert {^owner, %{owner: :a}} = + TestCluster.rpc!(context.node_b, Group, :lookup, [name, key]) + + true = TestCluster.rpc!(context.node_b, Process, :exit, [old_lane, :kill]) + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, 1) + ]) do + pid when is_pid(pid) -> pid != old_lane + _ -> false + end + end) + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!(context.node_b, Group, :lookup, [name, key]) == nil and + TestCluster.rpc!(context.node_b, Group, :members, [name, key]) == [] + end, + timeout: 1_000, + interval: 25 + ) + + # Restart repair can remove invalid rows before the reconstructed lease + # retires its final constant-size view breadcrumb. + TestCluster.assert_eventually( + fn -> + is_nil( + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_view_generation, + [name, 1, context.node_a] + ) + ) + end, + timeout: 1_000, + interval: 25 + ) + + assert :ok = + TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_origin_purged, [ + name, + context.node_a + ]) + end + + test "nodedown on shard zero preserves a suspended sibling's restart breadcrumb", context do + name = unique_name(:sibling_restart_during_nodedown) + + opts = [ + name: name, + shards: 2, + replicated_sender_buffer_size: 1, + replicated_pg_receiver_buffer_size: 1, + replicated_registry_receiver_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 150 + ] + + start_group_on_peers(context.peers, opts) + + key = + 1..1_000 + |> Enum.map(&"sibling-nodedown/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(nil, &1, 2) == 1)) + + owner = + TestCluster.spawn_register_and_join( + context.node_a, + name, + key, + %{owner: :a}, + key, + %{owner: :a} + ) + + TestCluster.assert_eventually(fn -> + match?({^owner, _}, TestCluster.rpc!(context.node_b, Group, :lookup, [name, key])) and + match?([{^owner, _}], TestCluster.rpc!(context.node_b, Group, :members, [name, key])) + end) + + old_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + :ok = TestCluster.rpc!(context.node_b, :sys, :suspend, [old_lane]) + + on_exit(fn -> + TestCluster.reconnect_nodes(context.node_b, context.node_a) + end) + + TestCluster.disconnect_nodes(context.node_b, context.node_a) + + TestCluster.assert_eventually(fn -> + is_nil( + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_generation, [ + name, + context.node_a + ]) + ) + end) + + # Shard zero owns node-wide authority, but it must not erase another lane's + # durable view before that lane has purged its own rows and cursor. + refute is_nil( + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_view_generation, + [name, 1, context.node_a] + ) + ) + + assert {^owner, %{owner: :a}} = + TestCluster.rpc!(context.node_b, Group, :lookup, [name, key]) + + true = TestCluster.rpc!(context.node_b, Process, :exit, [old_lane, :kill]) + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, 1) + ]) do + pid when is_pid(pid) -> pid != old_lane + _ -> false + end + end) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(context.node_b, Group, :lookup, [name, key]) == nil and + TestCluster.rpc!(context.node_b, Group, :members, [name, key]) == [] + end) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :registry_claims, [ + name, + 1, + nil, + key + ]) == [] + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :pg_entries_for_origin, [ + name, + 1, + nil, + context.node_a + ]) == [] + + assert TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :replica_cursor_streams_for_origin, + [name, 1, context.node_a] + ) == [] + + assert :ok = + TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end + + test "shard zero restart retains the obligation to repair partially observed authority", + context do + name = unique_name(:authority_dirty_restart) + cluster = "authority-dirty/restart" + + opts = [ + name: name, + shards: 1, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + exact_before = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_exact_revision, + [name, context.node_a] + ) + + :ok = TestCluster.rpc!(context.node_a, Group, :connect, [name, cluster]) + + TestCluster.assert_eventually(fn -> + observed = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_observed_revision, + [name, context.node_a] + ) + + exact = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_exact_revision, + [name, context.node_a] + ) + + is_integer(observed) and observed > exact_before and exact == exact_before + end) + + source = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + :ok = TestCluster.rpc!(context.node_a, :sys, :suspend, [source]) + + on_exit(fn -> + _ = TestCluster.resume_shard_if_alive(context.node_a, name, 0) + end) + + old_receiver = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + true = TestCluster.rpc!(context.node_b, Process, :exit, [old_receiver, :kill]) + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, 0) + ]) do + pid when is_pid(pid) -> pid != old_receiver + _ -> false + end + end) + + receiver = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + state = TestCluster.rpc!(context.node_b, :sys, :get_state, [receiver]) + assert Map.has_key?(state.cluster_control_dirty, context.node_a) + + :ok = TestCluster.rpc!(context.node_a, :sys, :resume, [source]) + + TestCluster.assert_eventually( + fn -> + exact = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_exact_revision, + [name, context.node_a] + ) + + observed = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_observed_revision, + [name, context.node_a] + ) + + exact == observed and exact > exact_before + end, + timeout: 5_000 + ) + end + + test "malformed replica frames are rejected without restarting the receiving shard", context do + name = unique_name(:malformed_replica_frames) + + opts = [ + name: name, + shards: 1, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 5_000 + ] + + start_group_on_peers(context.peers, opts) + + receiver = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + stream_id = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [name, 0, nil]) + + version = Group.Replica.WireProtocol.version() + + frames = [ + {:heads, version, [:not_a_head]}, + {:delta_batch, version, [:not_a_delta_run]}, + {:need, version, :not_a_stream, 1}, + {:needs, version, [:not_a_need]}, + {:snapshot_chunk, version, :not_a_stream, 1, 1, [], []}, + {:snapshot_commit, version, :not_a_stream, 1, 1, 0, 0}, + {:delta_batch, version, + [ + {stream_id, 1, + [{1, [{:register, nil, "malformed/pid", :not_a_pid, %{}, 0, context.node_a}]}], 1} + ]} + ] + + for frame <- frames do + assert :ok = + TestCluster.rpc!(context.node_b, Group.Transport, :incoming, [ + name, + context.node_a, + 0, + frame + ]) + + Process.sleep(25) + + assert TestCluster.rpc!(context.node_b, Process, :alive?, [receiver]), + "receiver restarted after malformed frame: #{inspect(frame)}" + end + + assert receiver == + TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, 0) + ]) + + assert nil == + TestCluster.rpc!(context.node_b, Group, :lookup, [name, "malformed/pid"]) + + assert :ok = + TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end + + test "a delta cannot place registry or PG rows on the wrong shard", context do + name = unique_name(:wrong_shard_delta) + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_pg_receiver_buffer_size: 1, + replicated_registry_receiver_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + wrong_key = + 1..1_000 + |> Enum.map(&"wrong-shard/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(nil, &1, 2) == 1)) + + source_key = + 1..1_000 + |> Enum.map(&"source-shard/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(nil, &1, 2) == 0)) + + :ok = + TestCluster.rpc!(context.node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + owner = TestCluster.spawn_register(context.node_a, name, source_key, %{owner: :a}) + + stream_id = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [name, 0, nil]) + + frame = + {:delta_batch, Group.Replica.WireProtocol.version(), + [ + {stream_id, 1, + [ + {1, + [ + {:register, nil, wrong_key, owner, %{owner: :a}, 1, context.node_a}, + {:join, nil, wrong_key, owner, %{owner: :a}, 1, :join, context.node_a} + ]} + ], 1} + ]} + + assert :ok = + TestCluster.rpc!(context.node_b, Group.Transport, :incoming, [ + name, + context.node_a, + 0, + frame + ]) + + TestCluster.flush_shards(context.node_b, name) + + assert nil == TestCluster.rpc!(context.node_b, Group, :lookup, [name, wrong_key]) + assert [] == TestCluster.rpc!(context.node_b, Group, :members, [name, wrong_key]) + + assert 0 == + TestCluster.rpc!(context.node_b, Group.Replica.Data, :replica_cursor, [ + name, + 0, + stream_id + ]) + + assert :ok = + TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end + + test "full snapshot capture never runs on the replica shard", context do + name = unique_name(:snapshot_capture_isolation) + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_oplog_max_entries: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + for index <- 1..4 do + owner = TestCluster.spawn_register(context.node_a, name, "snapshot/isolation/#{index}", %{}) + true = TestCluster.rpc!(context.node_a, Process, :exit, [owner, :kill]) + end + + TestCluster.flush_shards(context.node_a, name) + + stream_id = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [name, 0, nil]) + + {floor, _head, _applied} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_stream_head, [ + name, + 0, + stream_id + ]) + + assert floor > 1 + + shard = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + tracer = TestCluster.spawn_trace_forwarder(context.node_a, self()) + + assert 1 == + TestCluster.rpc!(context.node_a, :erlang, :trace_pattern, [ + {Group.Replica.Data, :reduce_registry_claim_batches_for_stream, 5}, + true, + [:local] + ]) + + assert 1 == + TestCluster.rpc!(context.node_a, :erlang, :trace, [ + shard, + true, + [:call, :set_on_spawn, {:tracer, tracer}] + ]) + + on_exit(fn -> + TestCluster.rpc!(context.node_a, :erlang, :trace, [shard, false, [:all]]) + + TestCluster.rpc!(context.node_a, :erlang, :trace_pattern, [ + {Group.Replica.Data, :reduce_registry_claim_batches_for_stream, 5}, + false, + [:local] + ]) + end) + + assert :ok = + TestCluster.rpc!(context.node_a, Group.Transport, :incoming, [ + name, + context.node_b, + 0, + {:needs, Group.Replica.WireProtocol.version(), [{stream_id, 1}]} + ]) + + assert_receive {:forwarded_trace, + {:trace, capture_pid, :call, + {Group.Replica.Data, :reduce_registry_claim_batches_for_stream, + [^name, 0, ^stream_id, _capture, _fun]}}}, + 1_000 + + refute capture_pid == shard + assert TestCluster.rpc!(context.node_a, Process, :alive?, [shard]) + end + + test "local unregister promotes a retained remote claim after its delta is pruned", context do + %{name: name, pid_a: pid_a, pid_b: pid_b, stream_b: stream_b} = + establish_hidden_remote_claim(context, :unregister) + + {floor, head, _applied} = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :replica_stream_head, [ + name, + 0, + stream_b + ]) + + assert floor > 1 + + assert TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_cursor, [ + name, + 0, + stream_b + ]) == head + + assert :ok = TestCluster.unregister_owner(pid_a) + TestCluster.flush_shards(context.node_a, name) + + assert TestCluster.rpc!(context.node_a, Group.Replica.Data, :registry_claims, [ + name, + 0, + nil, + "hidden/unregister" + ]) + |> Enum.map(&elem(&1, 0)) == [pid_b] + + TestCluster.assert_eventually( + fn -> + match?( + {^pid_b, %{rank: 1}}, + TestCluster.rpc!(context.node_a, Group, :lookup, [name, "hidden/unregister"]) + ) + end, + timeout: 500, + interval: 25 + ) + + for node <- [context.node_a, context.node_b, context.node_c] do + assert :ok = TestCluster.rpc!(node, TestCluster, :assert_replica_consistent, [name]) + end + end + + test "journal replay promotes a retained remote claim before marking the delete applied", + context do + %{name: name, pid_a: pid_a, pid_b: pid_b} = + establish_hidden_remote_claim(context, :journal) + + stream_a = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [name, 0, nil]) + + {seq, _mutations} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :append_replica_record, [ + name, + 0, + stream_a, + [{:unregister, nil, "hidden/journal", pid_a, %{rank: 2}, :injected_crash}] + ]) + + assert [{^stream_a, ^seq, _}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_unapplied, [ + name, + 0 + ]) + + old_shard = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + true = TestCluster.rpc!(context.node_a, Process, :exit, [old_shard, :kill]) + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(context.node_a, Process, :whereis, [ + Group.Replica.shard_name(name, 0) + ]) do + pid when is_pid(pid) -> pid != old_shard + nil -> false + end + end) + + TestCluster.flush_shards(context.node_a, name) + + claims = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :registry_claims, [ + name, + 0, + nil, + "hidden/journal" + ]) + + journal_state = %{ + claims: claims, + head: + TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_stream_head, [ + name, + 0, + stream_a + ]), + unapplied: + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_unapplied, [ + name, + 0 + ]) + } + + assert Enum.map(claims, &elem(&1, 0)) == [pid_b], inspect(journal_state) + + assert {^pid_b, %{rank: 1}} = + TestCluster.rpc!(context.node_a, Group, :lookup, [name, "hidden/journal"]) + + assert [] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_unapplied, [ + name, + 0 + ]) + + assert :ok = + TestCluster.rpc!(context.node_a, TestCluster, :assert_replica_consistent, [name]) + end + + test "local process DOWN promotes a retained remote claim", context do + %{name: name, pid_a: pid_a, pid_b: pid_b} = + establish_hidden_remote_claim(context, :process_down) + + true = TestCluster.rpc!(context.node_a, Process, :exit, [pid_a, :kill]) + TestCluster.flush_shards(context.node_a, name) + + assert [^pid_b] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :registry_claims, [ + name, + 0, + nil, + "hidden/process_down" + ]) + |> Enum.map(&elem(&1, 0)) + + assert {^pid_b, %{rank: 1}} = + TestCluster.rpc!(context.node_a, Group, :lookup, [name, "hidden/process_down"]) + + assert :ok = + TestCluster.rpc!(context.node_a, TestCluster, :assert_replica_consistent, [name]) + end + + test "local cluster disconnect removes a claimless legacy registry row", context do + name = unique_name(:claimless_disconnect) + cluster = "legacy-slice" + + opts = [ + name: name, + shards: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + {:ok, _pid} = TestCluster.start_group(context.node_a, opts) + :ok = TestCluster.rpc!(context.node_a, Group, :connect, [name, cluster]) + + owner = + TestCluster.spawn_register_in_cluster( + context.node_a, + name, + "legacy/claimless", + %{legacy: true}, + cluster + ) + + assert ["legacy/claimless"] = + TestCluster.rpc!( + context.node_a, + Group.Replica.Data, + :purge_registry_claims_for_cluster, + [name, 0, cluster] + ) + + assert {^owner, %{legacy: true}} = + TestCluster.rpc!(context.node_a, Group, :lookup, [ + name, + "legacy/claimless", + [cluster: cluster] + ]) + + :ok = TestCluster.rpc!(context.node_a, Group, :disconnect, [name, cluster]) + + assert nil == + TestCluster.rpc!(context.node_a, Group, :lookup, [ + name, + "legacy/claimless", + [cluster: cluster] + ]) + end + + test "expiring one sideband lane keeps the shared node route while other lanes are live", + context do + name = unique_name(:sideband_lane) + + opts = [ + name: name, + shards: 3, + replica_transport: + {Group.TestTCPTransport, + [connect_timeout: 250, send_timeout: 250, reconnect_interval: 10]}, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!(context.node_b, Group.TestTCPTransport, :connected?, [ + name, + context.node_a + ]) + end, + timeout: 10_000 + ) + + source_lanes = + for shard <- 0..2 do + lane = + TestCluster.rpc!(context.node_a, Process, :whereis, [ + Group.Replica.shard_name(name, shard) + ]) + + :ok = TestCluster.rpc!(context.node_a, :sys, :suspend, [lane]) + lane + end + + on_exit(fn -> + Enum.each(source_lanes, fn lane -> + TestCluster.rpc!(context.node_a, TestCluster, :resume_if_alive, [lane]) + end) + end) + + :ok = + TestCluster.rpc!(context.node_b, TestCluster, :expire_replica_lane, [ + name, + 0, + context.node_a + ]) + + Process.sleep(100) + + assert TestCluster.rpc!(context.node_b, Group.TestTCPTransport, :connected?, [ + name, + context.node_a + ]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_generation, [ + name, + context.node_a + ]) == + TestCluster.rpc!(context.node_a, Group.Replica.Data, :generation, [name]) + + for live_shard <- [1, 2] do + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + live_shard, + context.node_a + ]) != nil + end + + status = TestCluster.rpc!(context.node_b, Group.TestTCPTransport, :status, [name]) + assert context.node_a in status.peers + + for shard <- [1, 2] do + :ok = + TestCluster.rpc!(context.node_b, TestCluster, :expire_replica_lane, [ + name, + shard, + context.node_a + ]) + end + + TestCluster.assert_eventually(fn -> + not TestCluster.rpc!(context.node_b, Group.TestTCPTransport, :connected?, [ + name, + context.node_a + ]) and + context.node_a not in TestCluster.rpc!(context.node_b, Group.TestTCPTransport, :status, [ + name + ]).peers + end) + end + + test "a restarted sideband manager rediscovers live peers without nodeup", context do + name = unique_name(:sideband_manager_restart) + + opts = [ + name: name, + shards: 2, + replica_transport: + {Group.TestTCPTransport, + [connect_timeout: 250, send_timeout: 250, reconnect_interval: 10]}, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000 + ] + + start_group_on_peers(context.peers, opts) + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!(context.node_b, Group.TestTCPTransport, :connected?, [ + name, + context.node_a + ]) + end, + timeout: 10_000 + ) + + seed = TestCluster.spawn_register(context.node_a, name, "manager-restart/seed", %{}) + + TestCluster.assert_eventually(fn -> + match?( + {^seed, %{}}, + TestCluster.rpc!(context.node_b, Group, :lookup, [name, "manager-restart/seed"]) + ) + end) + + manager_name = :"#{name}_replica_tcp_transport" + old_manager = TestCluster.rpc!(context.node_b, Process, :whereis, [manager_name]) + true = TestCluster.rpc!(context.node_b, Process, :exit, [old_manager, :kill]) + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(context.node_b, Process, :whereis, [manager_name]) do + manager when is_pid(manager) -> manager != old_manager + _ -> false + end + end) + + # Group and every replica lane stayed alive; there is deliberately no + # nodeup or authority-generation change to trigger rediscovery. + fresh = TestCluster.spawn_register(context.node_a, name, "manager-restart/fresh", %{}) + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!(context.node_b, Group.TestTCPTransport, :connected?, [ + name, + context.node_a + ]) and + match?( + {^fresh, %{}}, + TestCluster.rpc!(context.node_b, Group, :lookup, [name, "manager-restart/fresh"]) + ) + end, + timeout: 5_000, + interval: 25 + ) + + for node <- [context.node_a, context.node_b, context.node_c] do + assert :ok = TestCluster.rpc!(node, TestCluster, :assert_replica_consistent, [name]) + end + end + + test "three-origin conflict resolution cannot retire every owner", context do + name = unique_name(:three_origin_conflict) + key = "three-origin/conflict" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + resolve_registry_conflict: {Group.CyclicConflictResolver, :resolve, []}, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + nodes = [context.node_a, context.node_b, context.node_c] + + for node <- nodes do + :ok = + TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [ + name, + {:capture_drop, [:delta_batch]} + ]) + end + + owner_a = TestCluster.spawn_register(context.node_a, name, key, %{rank: :a}) + owner_b = TestCluster.spawn_register(context.node_b, name, key, %{rank: :b}) + owner_c = TestCluster.spawn_register(context.node_c, name, key, %{rank: :c}) + TestCluster.flush_shards(context.node_a, name) + TestCluster.flush_shards(context.node_b, name) + TestCluster.flush_shards(context.node_c, name) + + # Deliver one edge of the cyclic preference to each owner. With the old + # pairwise callback A loses to C, B loses to A, and C loses to B before any + # node has observed all three claims. + for {source, target} <- [ + {context.node_c, context.node_a}, + {context.node_a, context.node_b}, + {context.node_b, context.node_c} + ] do + {_target, 0, frame} = + source + |> TestCluster.rpc!(Group.TestReplicaTransport, :captured, [name]) + |> Enum.find(fn {captured_target, shard, frame} -> + captured_target == target and shard == 0 and elem(frame, 0) == :delta_batch + end) + + :ok = TestCluster.rpc!(target, Group.Transport, :incoming, [name, source, 0, frame]) + end + + for node <- nodes, do: TestCluster.flush_shards(node, name) + + for node <- nodes do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :pass]) + end + + TestCluster.assert_eventually(fn -> + alive = + Enum.filter([owner_a, owner_b, owner_c], fn owner -> + TestCluster.rpc!(node(owner), Process, :alive?, [owner]) + end) + + case alive do + [winner] -> + Enum.all?(nodes, fn receiver -> + match?({^winner, _}, TestCluster.rpc!(receiver, Group, :lookup, [name, key])) + end) + + _ -> + false + end + end) + + for node <- nodes do + assert :ok = TestCluster.rpc!(node, TestCluster, :assert_replica_consistent, [name]) + end + end + + test "an exact shard-zero hello repairs a missing local authority view", context do + name = unique_name(:control_view_repair) + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + {generation, revision, epochs} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_authority, [name]) + + assert generation == + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 0, + context.node_a + ]) + + :ok = + TestCluster.rpc!(context.node_b, TestCluster, :delete_remote_view_info, [ + name, + 0, + context.node_a + ]) + + assert nil == + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 0, + context.node_a + ]) + + assert generation == + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_generation, [ + name, + context.node_a + ]) + + source = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + target = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + send( + target, + {:replica_hello, source, Group.Replica.WireProtocol.version(), generation, revision, epochs, + Group.TestReplicaTransport.id(), Group.TestReplicaTransport.descriptor(name, [])} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target]) + + assert generation == + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 0, + context.node_a + ]) + + assert revision == + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_view_cluster_epoch_revision, + [name, 0, context.node_a] + ) + end + + test "shared authority cannot admit a delta before its receiving lane is installed", context do + name = unique_name(:lane_view_fence) + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + key = + 1..1_000 + |> Enum.map(&"lane-view-fence/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(nil, &1, 2) == 1)) + + :ok = + TestCluster.rpc!(context.node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:capture_drop, [:delta_batch]} + ]) + + pid = TestCluster.spawn_register(context.node_a, name, key, %{lane: 1}) + TestCluster.flush_shards(context.node_a, name) + + {stream_id, frame} = + TestCluster.rpc!(context.node_a, Group.TestReplicaTransport, :captured, [name]) + |> Enum.find_value(fn + {target, 1, {:delta_batch, _version, [{stream_id, _first_seq, _records, _head}]} = frame} + when target == context.node_b -> + {stream_id, frame} + + _other -> + nil + end) + + generation = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_generation, [ + name, + context.node_a + ]) + + assert generation == Group.Replica.WireProtocol.stream_generation(stream_id) + + :ok = + TestCluster.rpc!(context.node_b, TestCluster, :delete_remote_view_info, [ + name, + 1, + context.node_a + ]) + + assert nil == + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 1, + context.node_a + ]) + + cursor_before = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :replica_cursor, [ + name, + 1, + stream_id + ]) + + :ok = + TestCluster.rpc!(context.node_b, Group.Transport, :incoming, [ + name, + context.node_a, + 1, + frame + ]) + + TestCluster.flush_shards(context.node_b, name) + + assert cursor_before == + TestCluster.rpc!(context.node_b, Group.Replica.Data, :replica_cursor, [ + name, + 1, + stream_id + ]) + + assert TestCluster.rpc!(context.node_b, Group, :lookup, [name, key]) == nil + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :registry_claims, [ + name, + 1, + nil, + key + ]) == [] + + # A matching lane hello installs the per-shard fence. The exact same + # previously rejected frame is then admissible and advances contiguously. + source_lane = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + target_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + revision = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + send( + target_lane, + {:replica_lane_hello, source_lane, Group.Replica.WireProtocol.version(), generation, + revision, Group.TestReplicaTransport.id(), Group.TestReplicaTransport.descriptor(name, [])} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + + assert generation == + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 1, + context.node_a + ]) + + :ok = + TestCluster.rpc!(context.node_b, Group.Transport, :incoming, [ + name, + context.node_a, + 1, + frame + ]) + + TestCluster.flush_shards(context.node_b, name) + + assert match?( + {^pid, %{lane: 1}}, + TestCluster.rpc!(context.node_b, Group, :lookup, [name, key]) + ) + + assert :ok = + TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end + + test "a missing local authority owner does not crash a healthy replica lane", context do + name = unique_name(:missing_local_authority_owner) + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + source_lane = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + target_control = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + target_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + replica_supervisor = + TestCluster.rpc!(context.node_b, Process, :whereis, [:"#{name}_replica_sup"]) + + :ok = TestCluster.rpc!(context.node_b, :sys, :suspend, [replica_supervisor]) + + on_exit(fn -> + _ = TestCluster.rpc!(context.node_b, TestCluster, :resume_if_alive, [replica_supervisor]) + end) + + monitor = Process.monitor(target_control) + true = TestCluster.rpc!(context.node_b, Process, :exit, [target_control, :kill]) + assert_receive {:DOWN, ^monitor, :process, ^target_control, :killed}, 5_000 + + assert TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, 0) + ]) == nil + + {generation, revision, [{nil, base_epoch}]} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_authority, [name]) + + send( + target_lane, + {:replica_cluster_open, source_lane, generation, revision, [{nil, base_epoch}]} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + assert TestCluster.rpc!(context.node_b, Process, :alive?, [target_lane]) + + :ok = TestCluster.rpc!(context.node_b, :sys, :resume, [replica_supervisor]) + end + + test "incremental authority from different lanes cannot race across a revision gap", context do + name = unique_name(:cross_lane_authority_gap) + cluster = "cross-lane-authority-gap" + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + :ok = TestCluster.rpc!(context.node_b, Group, :connect, [name, cluster]) + + [{^cluster, old_epoch}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :activate_local_clusters, [ + name, + [cluster] + ]) + + old_revision = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + [{^cluster, ^old_epoch}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :deactivate_local_clusters, [ + name, + [cluster] + ]) + + [{^cluster, current_epoch}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :activate_local_clusters, [ + name, + [cluster] + ]) + + generation = TestCluster.rpc!(context.node_a, Group.Replica.Data, :generation, [name]) + + current_revision = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + assert current_revision > old_revision + + source_control = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + source_lane = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + target_control = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + target_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + data_owner = + TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.Data.data_name(name) + ]) + + :ok = TestCluster.rpc!(context.node_a, :sys, :suspend, [source_control]) + :ok = TestCluster.rpc!(context.node_b, :sys, :suspend, [data_owner]) + + on_exit(fn -> + _ = + TestCluster.rpc!(context.node_a, TestCluster, :resume_if_alive, [source_control]) + + _ = TestCluster.rpc!(context.node_b, TestCluster, :resume_if_alive, [data_owner]) + end) + + # Shard 0 sees revision 3 first, detects that revisions 1 and 2 are absent, + # and blocks while recording the gap in the shared Data owner. + send( + target_control, + {:replica_cluster_open, source_control, generation, current_revision, + [{cluster, current_epoch}]} + ) + + TestCluster.assert_eventually(fn -> + {:messages, messages} = + TestCluster.rpc!(context.node_b, Process, :info, [data_owner, :messages]) + + Enum.any?(messages, fn + {:"$gen_call", _from, {:observe_remote_cluster_epoch_revision, source, revision}} -> + source == context.node_a and revision == current_revision + + _ -> + false + end) + end) + + # Before the shared gap fence is visible, another lane inspects revision 0 + # and queues an older revision-1 update behind it. A lane must not be able + # to install that stale epoch after shard 0 records the newer gap. + send( + target_lane, + {:replica_cluster_open, source_lane, generation, old_revision, [{cluster, old_epoch}]} + ) + + TestCluster.assert_eventually(fn -> + {:messages, messages} = + TestCluster.rpc!(context.node_b, Process, :info, [target_control, :messages]) + + Enum.any?(messages, fn + {:replica_cluster_open, ^source_lane, ^generation, ^old_revision, + [{^cluster, ^old_epoch}]} -> + true + + _ -> + false + end) + end) + + :ok = TestCluster.rpc!(context.node_b, :sys, :resume, [data_owner]) + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + context.node_a, + cluster + ]) == nil + + assert TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_observed_revision, + [name, context.node_a] + ) == current_revision + + refute TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_view_observed_revision, + [name, 1, context.node_a] + ) == current_revision + + {^generation, ^current_revision, authority} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_authority, [name]) + + send( + target_control, + {:replica_hello, source_control, Group.Replica.WireProtocol.version(), generation, + current_revision, authority, Group.TestReplicaTransport.id(), + Group.TestReplicaTransport.descriptor(name, [])} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + :ok = TestCluster.rpc!(context.node_a, :sys, :resume, [source_control]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + context.node_a, + cluster + ]) == current_epoch + + assert :ok = TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end + + test "authority replacement fences an in-flight stale registry conflict", context do + name = unique_name(:inflight_stale_conflict) + cluster = "inflight-stale-conflict" + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + resolve_registry_conflict: {Group.PausingConflictResolver, :resolve, [self()]}, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + TestCluster.assert_eventually(fn -> + generation = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_generation, [ + name, + context.node_a + ]) + + observed = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_observed_revision, + [name, context.node_a] + ) + + not is_nil(generation) and + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 1, + context.node_a + ]) == generation and + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_view_observed_revision, + [name, 1, context.node_a] + ) == observed + end) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group, :connect, [name, cluster]) + end + + TestCluster.assert_eventually(fn -> + not is_nil( + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + context.node_a, + cluster + ]) + ) + end) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :drop]) + end + + key = + 1..1_000 + |> Enum.map(&"inflight-stale-conflict/key/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(cluster, &1, 2) == 1)) + + local_owner = + TestCluster.spawn_register_in_cluster( + context.node_b, + name, + key, + %{rank: 1}, + cluster + ) + + remote_owner = + TestCluster.spawn_register_in_cluster( + context.node_a, + name, + key, + %{rank: 2, pause: true}, + cluster + ) + + generation = TestCluster.rpc!(context.node_a, Group.Replica.Data, :generation, [name]) + + epoch = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch, [ + name, + cluster + ]) + + stream_id = + Group.Replica.WireProtocol.stream_id( + name, + context.node_a, + generation, + 1, + cluster, + epoch + ) + + [{^key, ^remote_owner, remote_meta, remote_time}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :registry_claims_for_stream, [ + name, + 1, + stream_id + ]) + + {_floor, head, applied} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_stream_head, [ + name, + 1, + stream_id + ]) + + assert head == applied + + record = + {head, + [ + {:register, cluster, key, remote_owner, remote_meta, remote_time, context.node_a} + ]} + + :ok = + TestCluster.rpc!(context.node_b, Group.Transport, :incoming, [ + name, + context.node_a, + 1, + {:delta_batch, Group.Replica.WireProtocol.version(), [{stream_id, head, [record], head}]} + ]) + + assert_receive {:conflict_resolver_waiting, resolver, ref, ^key, ^remote_owner}, 5_000 + + on_exit(fn -> send(resolver, {:continue_conflict_resolution, ref}) end) + + [{^cluster, ^epoch}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :deactivate_local_clusters, [ + name, + [cluster] + ]) + + close_revision = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + source_control = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + target_control = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + send( + target_control, + {:replica_cluster_close, source_control, generation, close_revision, [{cluster, epoch}]} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + context.node_a, + cluster + ]) == nil + + send(resolver, {:continue_conflict_resolution, ref}) + TestCluster.flush_shards(context.node_b, name) + + assert TestCluster.rpc!(context.node_b, Process, :alive?, [local_owner]) + + assert match?( + {^local_owner, %{rank: 1}}, + TestCluster.rpc!(context.node_b, Group, :lookup, [ + name, + key, + [cluster: cluster] + ]) + ) + + assert :ok = TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end + + test "restoring lane authority reprojects a conflict retained during the authority gap", + context do + name = unique_name(:authority_restore_reprojects_conflict) + authority_bump_cluster = "authority-restore-bump" + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + resolve_registry_conflict: {Group.PausingConflictResolver, :resolve, [self()]}, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + TestCluster.assert_eventually(fn -> + generation = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_generation, [ + name, + context.node_a + ]) + + observed = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_observed_revision, + [name, context.node_a] + ) + + source_generation = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :remote_generation, [ + name, + context.node_b + ]) + + not is_nil(generation) and not is_nil(source_generation) and + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 1, + context.node_a + ]) == generation and + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_view_observed_revision, + [name, 1, context.node_a] + ) == observed + end) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :drop]) + end + + key = + 1..1_000 + |> Enum.map(&"authority-restore-conflict/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(nil, &1, 2) == 1)) + + local_owner = TestCluster.spawn_register(context.node_b, name, key, %{rank: 1}) + + remote_owner = + TestCluster.spawn_register(context.node_a, name, key, %{rank: 2, pause: true}) + + stream_id = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [name, 1, nil]) + + [{^key, ^remote_owner, remote_meta, remote_time}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :registry_claims_for_stream, [ + name, + 1, + stream_id + ]) + + {_floor, head, applied} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_stream_head, [ + name, + 1, + stream_id + ]) + + assert head == applied + + record = + {head, + [ + {:register, nil, key, remote_owner, remote_meta, remote_time, context.node_a} + ]} + + :ok = + TestCluster.rpc!(context.node_b, Group.Transport, :incoming, [ + name, + context.node_a, + 1, + {:delta_batch, Group.Replica.WireProtocol.version(), [{stream_id, head, [record], head}]} + ]) + + assert_receive {:conflict_resolver_waiting, resolver, ref, ^key, ^remote_owner}, 5_000 + on_exit(fn -> send(resolver, {:continue_conflict_resolution, ref}) end) + + :ok = TestCluster.rpc!(context.node_a, Group, :connect, [name, authority_bump_cluster]) + + generation = TestCluster.rpc!(context.node_a, Group.Replica.Data, :generation, [name]) + + bumped_revision = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + bump_epoch = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch, [ + name, + authority_bump_cluster + ]) + + source_control = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + target_control = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + send( + target_control, + {:replica_cluster_open, source_control, generation, bumped_revision, + [{authority_bump_cluster, bump_epoch}]} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + + assert TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_observed_revision, + [name, context.node_a] + ) == bumped_revision + + refute TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_view_observed_revision, + [name, 1, context.node_a] + ) == bumped_revision + + send(resolver, {:continue_conflict_resolution, ref}) + + assert_receive {:conflict_resolver_waiting, ^resolver, repair_ref, ^key, ^remote_owner}, 5_000 + send(resolver, {:continue_conflict_resolution, repair_ref}) + + TestCluster.flush_shards(context.node_b, name) + + assert match?( + {^remote_owner, %{rank: 2, pause: true}}, + TestCluster.rpc!(context.node_b, Group, :lookup, [name, key]) + ) + + refute TestCluster.rpc!(context.node_b, Process, :alive?, [local_owner]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :replica_cursor, [ + name, + 1, + stream_id + ]) == head + + consistency = + Task.async(fn -> + TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end) + + assert_receive {:conflict_resolver_waiting, consistency_resolver, consistency_ref, ^key, + ^remote_owner}, + 5_000 + + send(consistency_resolver, {:continue_conflict_resolution, consistency_ref}) + assert :ok = Task.await(consistency, 5_000) + end + + test "lease expiry discards conflict reprojection state when the source never returns", + context do + name = unique_name(:authority_gap_expiry) + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + resolve_registry_conflict: {Group.PausingConflictResolver, :resolve, [self()]}, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 500 + ] + + start_group_on_peers(context.peers, opts) + + TestCluster.assert_eventually(fn -> + not is_nil( + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_generation, [ + name, + context.node_a + ]) + ) + end) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :drop]) + end + + source_control = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + :ok = TestCluster.rpc!(context.node_a, :sys, :suspend, [source_control]) + + on_exit(fn -> + TestCluster.rpc!(context.node_a, TestCluster, :resume_if_alive, [source_control]) + end) + + key = + 1..1_000 + |> Enum.map(&"authority-gap-expiry/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(nil, &1, 2) == 1)) + + local_owner = TestCluster.spawn_register(context.node_b, name, key, %{rank: 1}) + remote_owner = TestCluster.spawn_register(context.node_a, name, key, %{rank: 2, pause: true}) + + stream_id = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [name, 1, nil]) + + [{^key, ^remote_owner, remote_meta, remote_time}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :registry_claims_for_stream, [ + name, + 1, + stream_id + ]) + + {_floor, head, applied} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_stream_head, [ + name, + 1, + stream_id + ]) + + assert head == applied + + :ok = + TestCluster.rpc!(context.node_b, Group.Transport, :incoming, [ + name, + context.node_a, + 1, + {:delta_batch, Group.Replica.WireProtocol.version(), + [ + {stream_id, head, + [ + {head, + [ + {:register, nil, key, remote_owner, remote_meta, remote_time, context.node_a} + ]} + ], head} + ]} + ]) + + assert_receive {:conflict_resolver_waiting, resolver, ref, ^key, ^remote_owner}, 5_000 + on_exit(fn -> send(resolver, {:continue_conflict_resolution, ref}) end) + + initial_revision = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_observed_revision, + [ + name, + context.node_a + ] + ) + + [{"authority-gap-expiry-1", _epoch1}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :activate_local_clusters, [ + name, + ["authority-gap-expiry-1"] + ]) + + [{"authority-gap-expiry-2", epoch2}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :activate_local_clusters, [ + name, + ["authority-gap-expiry-2"] + ]) + + gap_revision = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + assert gap_revision >= initial_revision + 2 + + generation = TestCluster.rpc!(context.node_a, Group.Replica.Data, :generation, [name]) + + target_control = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + send( + target_control, + {:replica_cluster_open, source_control, generation, gap_revision, + [{"authority-gap-expiry-2", epoch2}]} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + + assert TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_observed_revision, + [name, context.node_a] + ) == gap_revision + + send(resolver, {:continue_conflict_resolution, ref}) + + target_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + TestCluster.assert_eventually(fn -> + state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + + state.pending_registry_reprojections + |> Map.get(context.node_a, MapSet.new()) + |> MapSet.member?({nil, key}) + end) + + TestCluster.assert_eventually( + fn -> + state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + + not Map.has_key?(state.pending_registry_reprojections, context.node_a) and + TestCluster.rpc!(context.node_b, Group.Replica.Data, :registry_claims, [ + name, + 1, + nil, + key + ]) + |> Enum.all?(&(elem(&1, 3) != context.node_a)) and + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :replica_cursor_streams_for_origin, + [name, 1, context.node_a] + ) == [] + end, + timeout: 10_000 + ) + + assert TestCluster.rpc!(context.node_b, Process, :alive?, [local_owner]) + + assert match?( + {^local_owner, %{rank: 1}}, + TestCluster.rpc!(context.node_b, Group, :lookup, [name, key]) + ) + end + + test "a newer heartbeat fences old-epoch data before exact authority arrives", context do + name = unique_name(:heartbeat_authority_fence) + cluster = "heartbeat-authority-fence" + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + resolve_registry_conflict: {Group.ModelConflictResolver, :resolve, []}, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group, :connect, [name, cluster]) + end + + TestCluster.assert_eventually(fn -> + not is_nil( + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + context.node_a, + cluster + ]) + ) + end) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :drop]) + end + + key = + 1..1_000 + |> Enum.map(&"heartbeat-authority-fence/key/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(cluster, &1, 2) == 1)) + + local_owner = + TestCluster.spawn_register_in_cluster(context.node_b, name, key, %{rank: 1}, cluster) + + stale_owner = + TestCluster.spawn_register_in_cluster(context.node_a, name, key, %{rank: 2}, cluster) + + generation = TestCluster.rpc!(context.node_a, Group.Replica.Data, :generation, [name]) + + old_epoch = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch, [ + name, + cluster + ]) + + old_stream = + Group.Replica.WireProtocol.stream_id( + name, + context.node_a, + generation, + 1, + cluster, + old_epoch + ) + + [{^key, ^stale_owner, stale_meta, stale_time}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :registry_claims_for_stream, [ + name, + 1, + old_stream + ]) + + {_floor, old_head, old_head} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_stream_head, [ + name, + 1, + old_stream + ]) + + source_control = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + source_lane = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + target_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + :ok = TestCluster.rpc!(context.node_a, :sys, :suspend, [source_control]) + + on_exit(fn -> + TestCluster.rpc!(context.node_a, TestCluster, :resume_if_alive, [source_control]) + end) + + # Change only the source's durable authority. The incremental close/open + # controls are intentionally absent, while the matching data lane exposes + # the newer revision through its normal heartbeat. + [{^cluster, ^old_epoch}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :deactivate_local_clusters, [ + name, + [cluster] + ]) + + [{^cluster, new_epoch}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :activate_local_clusters, [ + name, + [cluster] + ]) + + refute new_epoch == old_epoch + + new_revision = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + send( + target_lane, + {:replica_heartbeat, source_lane, Group.Replica.WireProtocol.version(), generation, + new_revision, Group.TestReplicaTransport.id(), + Group.TestReplicaTransport.descriptor(name, [])} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + + assert TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_observed_revision, + [name, context.node_a] + ) == new_revision + + old_record = + {old_head, + [ + {:register, cluster, key, stale_owner, stale_meta, stale_time, context.node_a} + ]} + + :ok = + TestCluster.rpc!(context.node_b, Group.Transport, :incoming, [ + name, + context.node_a, + 1, + {:delta_batch, Group.Replica.WireProtocol.version(), + [{old_stream, old_head, [old_record], old_head}]} + ]) + + TestCluster.flush_shards(context.node_b, name) + + assert TestCluster.rpc!(context.node_b, Process, :alive?, [local_owner]) + + assert match?( + {^local_owner, %{rank: 1}}, + TestCluster.rpc!(context.node_b, Group, :lookup, [ + name, + key, + [cluster: cluster] + ]) + ) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :replica_cursor, [ + name, + 1, + old_stream + ]) == 0 + end + + test "a newer-generation heartbeat fences prior-generation data before exact authority", + context do + name = unique_name(:heartbeat_generation_fence) + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + resolve_registry_conflict: {Group.ModelConflictResolver, :resolve, []}, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :drop]) + end + + key = + 1..1_000 + |> Enum.map(&"heartbeat-generation-fence/key/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(nil, &1, 2) == 1)) + + local_owner = TestCluster.spawn_register(context.node_b, name, key, %{rank: 1}) + stale_owner = TestCluster.spawn_register(context.node_a, name, key, %{rank: 2}) + stale_cluster = "heartbeat-generation-fence/stale-cluster" + + [{^stale_cluster, stale_epoch}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :activate_local_clusters, [ + name, + [stale_cluster] + ]) + + old_stream = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [name, 1, nil]) + + old_generation = Group.Replica.WireProtocol.stream_generation(old_stream) + + {^old_generation, old_revision, old_authority} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_authority, [name]) + + [{^key, ^stale_owner, stale_meta, stale_time}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :registry_claims_for_stream, [ + name, + 1, + old_stream + ]) + + {_floor, old_head, old_head} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_stream_head, [ + name, + 1, + old_stream + ]) + + source_control = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + source_lane = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + target_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + target_control = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + :ok = TestCluster.rpc!(context.node_a, :sys, :suspend, [source_control]) + + on_exit(fn -> + TestCluster.rpc!(context.node_a, TestCluster, :resume_if_alive, [source_control]) + end) + + new_generation = + TestCluster.rpc!(context.node_a, Group.Replica.WireProtocol, :new_generation, []) + + assert Group.Replica.WireProtocol.generation_newer?(new_generation, old_generation) + + send( + target_lane, + {:replica_heartbeat, source_lane, Group.Replica.WireProtocol.version(), new_generation, 0, + Group.TestReplicaTransport.id(), Group.TestReplicaTransport.descriptor(name, [])} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + + refute TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_view_cluster_epoch_revision, + [ + name, + 1, + context.node_a + ] + ) == + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_exact_revision, + [name, context.node_a] + ) + + send( + target_control, + {:replica_cluster_open, source_control, old_generation, old_revision, + [{stale_cluster, stale_epoch}]} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + context.node_a, + stale_cluster + ]) == nil + + assert :stale = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :put_remote_replica_info, + [ + name, + 0, + context.node_a, + old_generation, + old_revision, + old_authority + ] + ) + + assert :stale = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :put_remote_view_info, + [name, 0, context.node_a, old_generation, old_revision, old_revision] + ) + + # The newer generation is only a hint until its exact authority arrives, + # but it must still fence a delayed exact hello from the prior generation. + send( + target_control, + {:replica_hello, source_control, Group.Replica.WireProtocol.version(), old_generation, + old_revision, old_authority, Group.TestReplicaTransport.id(), + Group.TestReplicaTransport.descriptor(name, [])} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + + refute TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_view_cluster_epoch_revision, + [name, 0, context.node_a] + ) == + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_exact_revision, + [name, context.node_a] + ) + + old_record = + {old_head, + [ + {:register, nil, key, stale_owner, stale_meta, stale_time, context.node_a} + ]} + + :ok = + TestCluster.rpc!(context.node_b, Group.Transport, :incoming, [ + name, + context.node_a, + 1, + {:delta_batch, Group.Replica.WireProtocol.version(), + [{old_stream, old_head, [old_record], old_head}]} + ]) + + TestCluster.flush_shards(context.node_b, name) + + assert TestCluster.rpc!(context.node_b, Process, :alive?, [local_owner]) + + assert match?( + {^local_owner, %{rank: 1}}, + TestCluster.rpc!(context.node_b, Group, :lookup, [name, key]) + ) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :replica_cursor, [ + name, + 1, + old_stream + ]) == 0 + end + + test "a newer lane hello cannot reinstall a view from stale exact authority", context do + name = unique_name(:lane_hello_exact_fence) + bump_cluster = "lane-hello-exact-fence/bump" + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :drop]) + end + + key = + 1..1_000 + |> Enum.map(&"lane-hello-exact-fence/member/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(nil, &1, 2) == 1)) + + owner = TestCluster.spawn_join(context.node_a, name, key, %{owner: :a}) + + stream = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [name, 1, nil]) + + generation = Group.Replica.WireProtocol.stream_generation(stream) + + {_floor, head, head} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_stream_head, [ + name, + 1, + stream + ]) + + [{^head, mutations}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_records, [ + name, + 1, + stream, + head, + 1 + ]) + + source_control = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + source_lane = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + target_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + :ok = TestCluster.rpc!(context.node_a, :sys, :suspend, [source_control]) + + on_exit(fn -> + TestCluster.rpc!(context.node_a, TestCluster, :resume_if_alive, [source_control]) + end) + + [{^bump_cluster, _epoch}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :activate_local_clusters, [ + name, + [bump_cluster] + ]) + + revision = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + send( + target_lane, + {:replica_lane_hello, source_lane, Group.Replica.WireProtocol.version(), generation, + revision, Group.TestReplicaTransport.id(), Group.TestReplicaTransport.descriptor(name, [])} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + + refute TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_view_observed_revision, + [name, 1, context.node_a] + ) == revision + + :ok = + TestCluster.rpc!(context.node_b, Group.Transport, :incoming, [ + name, + context.node_a, + 1, + {:delta_batch, Group.Replica.WireProtocol.version(), + [{stream, head, [{head, mutations}], head}]} + ]) + + TestCluster.flush_shards(context.node_b, name) + + assert TestCluster.rpc!(context.node_b, Group, :members, [name, key]) == [] + assert TestCluster.rpc!(context.node_a, Process, :alive?, [owner]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :replica_cursor, [ + name, + 1, + stream + ]) == 0 + end + + test "a delayed incremental control cannot roll observed authority backward", context do + name = unique_name(:incremental_authority_rollback) + cluster = "incremental-authority-rollback" + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + resolve_registry_conflict: {Group.ModelConflictResolver, :resolve, []}, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :drop]) + end + + :ok = TestCluster.rpc!(context.node_b, Group, :connect, [name, cluster]) + + [{^cluster, old_epoch}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :activate_local_clusters, [ + name, + [cluster] + ]) + + old_revision = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + [{^cluster, ^old_epoch}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :deactivate_local_clusters, [ + name, + [cluster] + ]) + + [{^cluster, current_epoch}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :activate_local_clusters, [ + name, + [cluster] + ]) + + refute current_epoch == old_epoch + + :ok = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :add_cluster_node, [ + name, + [cluster], + context.node_a + ]) + + generation = TestCluster.rpc!(context.node_a, Group.Replica.Data, :generation, [name]) + + current_revision = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + assert current_revision > old_revision + + source_control = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + source_data_lane = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + target_control = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + target_data_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + # Keep the exact-authority request caused by the intentional revision gap + # queued at its source until this test has exercised the delayed control and + # stale data frame. The Group control plane intentionally bypasses the test + # replica transport, so transport :drop alone cannot create this window. + :ok = TestCluster.rpc!(context.node_a, :sys, :suspend, [source_control]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + context.node_a, + cluster + ]) == nil + + initial_observed_revision = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_observed_revision, + [name, context.node_a] + ) + + assert is_integer(initial_observed_revision) + assert initial_observed_revision < old_revision + + send( + target_data_lane, + {:replica_cluster_open, source_data_lane, generation, current_revision, + [{cluster, current_epoch}]} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_data_lane]) + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + context.node_a, + cluster + ]) == nil + + assert TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_observed_revision, + [name, context.node_a] + ) == current_revision + + # The old open was delayed behind the newer open. Accepting it can make an + # obsolete stream authoritative long enough to retire a valid local owner; + # the later exact hello can purge the stale claim but cannot resurrect that + # killed owner. + send( + target_control, + {:replica_cluster_open, source_control, generation, old_revision, [{cluster, old_epoch}]} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + context.node_a, + cluster + ]) == nil + + key = + 1..1_000 + |> Enum.map(&"incremental-authority-rollback/key/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(cluster, &1, 2) == 0)) + + local_owner = + TestCluster.spawn_register_in_cluster( + context.node_b, + name, + key, + %{rank: 1}, + cluster + ) + + stale_owner = TestCluster.spawn_monitor_forwarder(context.node_a, name, :all, self()) + assert_receive {:monitor_ready, ^stale_owner}, 5_000 + + stale_stream = + Group.Replica.WireProtocol.stream_id( + name, + context.node_a, + generation, + 0, + cluster, + old_epoch + ) + + stale_record = + {1, + [ + {:register, cluster, key, stale_owner, %{rank: 2}, System.system_time(), context.node_a} + ]} + + :ok = + TestCluster.rpc!(context.node_b, Group.Transport, :incoming, [ + name, + context.node_a, + 0, + {:delta_batch, Group.Replica.WireProtocol.version(), + [{stale_stream, 1, [stale_record], 1}]} + ]) + + TestCluster.flush_shards(context.node_b, name) + + {^generation, ^current_revision, current_authority} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_authority, [name]) + + send( + target_control, + {:replica_hello, source_control, Group.Replica.WireProtocol.version(), generation, + current_revision, current_authority, Group.TestReplicaTransport.id(), + Group.TestReplicaTransport.descriptor(name, [])} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + + :ok = TestCluster.rpc!(context.node_a, :sys, :resume, [source_control]) + + assert TestCluster.rpc!(context.node_b, Process, :alive?, [local_owner]) + + assert match?( + {^local_owner, %{rank: 1}}, + TestCluster.rpc!(context.node_b, Group, :lookup, [ + name, + key, + [cluster: cluster] + ]) + ) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + context.node_a, + cluster + ]) == current_epoch + + assert :ok = TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end + + test "a receiver restart forgets cursors for a locally deactivated cluster", context do + name = unique_name(:inactive_cluster_cursor_restart) + cluster = "inactive-cluster-cursor-restart" + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group, :connect, [name, cluster]) + end + + TestCluster.assert_eventually( + fn -> + Enum.all?([context.node_a, context.node_b, context.node_c], fn node -> + length(TestCluster.rpc!(node, Group, :nodes, [name, cluster])) == 3 + end) + end, + timeout: 30_000, + interval: 50 + ) + + key = + 1..1_000 + |> Enum.map(&"inactive-cluster-cursor/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(cluster, &1, 2) == 1)) + + registry_owner = + TestCluster.spawn_register_in_cluster( + context.node_a, + name, + key, + %{kind: :registry}, + cluster + ) + + pg_owner = + TestCluster.spawn_join_in_cluster( + context.node_a, + name, + key, + %{kind: :pg}, + cluster + ) + + TestCluster.assert_eventually(fn -> + match?( + {^registry_owner, %{kind: :registry}}, + TestCluster.rpc!(context.node_b, Group, :lookup, [name, key, [cluster: cluster]]) + ) and + match?( + [{^pg_owner, %{kind: :pg}}], + TestCluster.rpc!(context.node_b, Group, :members, [name, key, [cluster: cluster]]) + ) + end) + + stream_id = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [ + name, + 1, + cluster + ]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :replica_cursor, [ + name, + 1, + stream_id + ]) > 0 + + # This is the durable first half of Group.disconnect. Kill the lane before + # its normal disconnect request can erase receive cursors and rows. + [{^cluster, _local_epoch}] = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :deactivate_local_clusters, [ + name, + [cluster] + ]) + + old_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + monitor = Process.monitor(old_lane) + true = TestCluster.rpc!(context.node_b, Process, :exit, [old_lane, :kill]) + assert_receive {:DOWN, ^monitor, :process, ^old_lane, :killed}, 5_000 + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, 1) + ]) do + pid when is_pid(pid) -> pid != old_lane + _ -> false + end + end) + + new_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + # A registered GenServer name only proves the replacement process exists; + # init still has to finish its retained-ETS repair before direct ETS reads + # can assert the post-restart state. + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [new_lane]) + + assert TestCluster.rpc!(context.node_b, :ets, :lookup, [ + Group.Replica.Data.replica_cursor_table(name, 1), + stream_id + ]) == [] + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :registry_lookup, [ + name, + 1, + cluster, + key + ]) == nil + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :pg_lookup, [ + name, + 1, + cluster, + key, + pg_owner + ]) == nil + + assert :ok = TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end + + test "a receiver restart removes cursorless rows whose named-stream authority closed", + context do + name = unique_name(:cursorless_closed_stream_restart) + cluster = "cursorless-restart" + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 5_000 + ] + + start_group_on_peers(context.peers, opts) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group, :connect, [name, cluster]) + end + + TestCluster.assert_eventually( + fn -> + Enum.all?([context.node_a, context.node_b, context.node_c], fn node -> + length(TestCluster.rpc!(node, Group, :nodes, [name, cluster])) == 3 + end) + end, + timeout: 30_000, + interval: 50 + ) + + key = + 1..1_000 + |> Enum.map(&"cursorless-restart/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(cluster, &1, 2) == 1)) + + owner = + TestCluster.spawn_register_in_cluster( + context.node_a, + name, + key, + %{kind: :registry}, + cluster + ) + + member = + TestCluster.spawn_join_in_cluster( + context.node_a, + name, + key, + %{kind: :pg}, + cluster + ) + + for receiver <- [context.node_b, context.node_c] do + TestCluster.assert_eventually(fn -> + match?( + {^owner, %{kind: :registry}}, + TestCluster.rpc!(receiver, Group, :lookup, [name, key, [cluster: cluster]]) + ) and + match?( + [{^member, %{kind: :pg}}], + TestCluster.rpc!(receiver, Group, :members, [name, key, [cluster: cluster]]) + ) + end) + end + + stream_id = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [ + name, + 1, + cluster + ]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :replica_cursor, [ + name, + 1, + stream_id + ]) > 0 + + # Model either receiver crash window: rows were materialized before the + # cursor was recorded, or authority cleanup deleted the cursor before all + # rows. Restart repair must never depend solely on that missing breadcrumb. + :ok = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :delete_replica_cursor, [ + name, + 1, + stream_id + ]) + + replica_supervisor = + TestCluster.rpc!(context.node_b, Process, :whereis, [:"#{name}_replica_sup"]) + + old_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + :ok = TestCluster.rpc!(context.node_b, :sys, :suspend, [replica_supervisor]) + + on_exit(fn -> + TestCluster.rpc!(context.node_b, TestCluster, :resume_if_alive, [replica_supervisor]) + end) + + lane_monitor = Process.monitor(old_lane) + true = TestCluster.rpc!(context.node_b, Process, :exit, [old_lane, :kill]) + assert_receive {:DOWN, ^lane_monitor, :process, ^old_lane, :killed}, 5_000 + + :ok = TestCluster.rpc!(context.node_a, Group, :disconnect, [name, cluster]) + + exact_revision = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_exact_revision, + [name, context.node_a] + ) == exact_revision and + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + context.node_a, + cluster + ]) == nil + end, + timeout: 5_000, + interval: 25 + ) + + :ok = TestCluster.rpc!(context.node_b, :sys, :resume, [replica_supervisor]) + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, 1) + ]) do + pid when is_pid(pid) -> pid != old_lane + _ -> false + end + end) + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!(context.node_b, Group, :lookup, [name, key, [cluster: cluster]]) == nil and + TestCluster.rpc!(context.node_b, Group, :members, [name, key, [cluster: cluster]]) == [] + end, + timeout: 2_000, + interval: 25 + ) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :registry_claims, [ + name, + 1, + cluster, + key + ]) == [] + + assert :ok = + TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end + + test "a receiver restart rejects cursorless rows under otherwise current authority", context do + name = unique_name(:cursorless_current_authority_restart) + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + key = "cursorless-current-authority/key" + registry_owner = TestCluster.spawn_register(context.node_a, name, key, %{kind: :registry}) + pg_owner = TestCluster.spawn_join(context.node_a, name, key, %{kind: :pg}) + + for receiver <- [context.node_b, context.node_c] do + TestCluster.assert_eventually(fn -> + match?( + {^registry_owner, %{kind: :registry}}, + TestCluster.rpc!(receiver, Group, :lookup, [name, key]) + ) and + match?( + [{^pg_owner, %{kind: :pg}}], + TestCluster.rpc!(receiver, Group, :members, [name, key]) + ) + end) + end + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :drop]) + end + + stream_id = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [name, 0, nil]) + + :ok = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :delete_replica_cursor, [ + name, + 0, + stream_id + ]) + + old_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + monitor = Process.monitor(old_lane) + true = TestCluster.rpc!(context.node_b, Process, :exit, [old_lane, :kill]) + assert_receive {:DOWN, ^monitor, :process, ^old_lane, :killed}, 5_000 + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, 0) + ]) do + pid when is_pid(pid) -> pid != old_lane + _ -> false + end + end) + + new_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [new_lane]) + + assert TestCluster.rpc!(context.node_b, Group, :lookup, [name, key]) == nil + assert TestCluster.rpc!(context.node_b, Group, :members, [name, key]) == [] + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :registry_claims, [ + name, + 0, + nil, + key + ]) == [] + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :pass]) + end + + {floor, head, _applied} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_stream_head, [ + name, + 0, + stream_id + ]) + + :ok = + TestCluster.rpc!(context.node_b, Group.Transport, :incoming, [ + name, + context.node_a, + 0, + {:heads, Group.Replica.WireProtocol.version(), [{stream_id, floor, head}]} + ]) + + TestCluster.assert_eventually( + fn -> + match?( + {^registry_owner, %{kind: :registry}}, + TestCluster.rpc!(context.node_b, Group, :lookup, [name, key]) + ) and + match?( + [{^pg_owner, %{kind: :pg}}], + TestCluster.rpc!(context.node_b, Group, :members, [name, key]) + ) + end, + timeout: 10_000, + interval: 25 + ) + + assert :ok = TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end + + test "a receiver restart never projects a partially installed exact snapshot", context do + name = unique_name(:partial_snapshot_install_restart) + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + replicated_oplog_max_entries: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :drop]) + end + + registry_key = + 1..1_000 + |> Enum.map(&"partial-install/registry/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(nil, &1, 2) == 1)) + + pg_key = + 1..1_000 + |> Enum.map(&"partial-install/pg/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(nil, &1, 2) == 1)) + + registry_owner = + TestCluster.spawn_register(context.node_a, name, registry_key, %{kind: :registry}) + + pg_owner = TestCluster.spawn_join(context.node_a, name, pg_key, %{kind: :pg}) + TestCluster.flush_shards(context.node_a, name) + + stream_id = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [name, 1, nil]) + + {floor, head, applied} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_stream_head, [ + name, + 1, + stream_id + ]) + + assert floor > 1 + assert applied == head + + # Model the exact crash boundary in both row domains: the receiver has + # written only a subset of the exact image after recording its sequence-0 + # admission marker, but dies before publishing the snapshot cursor. These + # are the same public ETS writes used by the commit path. + :ok = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :begin_replica_snapshot_install, [ + name, + 1, + stream_id, + head + ]) + + :ok = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :put_registry_claim, [ + name, + 1, + stream_id, + head, + registry_key, + registry_owner, + %{kind: :registry}, + System.system_time() + ]) + + :ok = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :pg_insert, [ + name, + 1, + nil, + pg_key, + pg_owner, + %{kind: :pg}, + System.system_time(), + context.node_a + ]) + + replica_supervisor = + TestCluster.rpc!(context.node_b, Process, :whereis, [:"#{name}_replica_sup"]) + + old_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + :ok = TestCluster.rpc!(context.node_b, :sys, :suspend, [replica_supervisor]) + + on_exit(fn -> + TestCluster.rpc!(context.node_b, TestCluster, :resume_if_alive, [replica_supervisor]) + end) + + monitor = Process.monitor(old_lane) + true = TestCluster.rpc!(context.node_b, Process, :exit, [old_lane, :kill]) + assert_receive {:DOWN, ^monitor, :process, ^old_lane, :killed}, 5_000 + + :ok = TestCluster.rpc!(context.node_b, :sys, :resume, [replica_supervisor]) + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, 1) + ]) do + pid when is_pid(pid) -> pid != old_lane + _ -> false + end + end) + + assert TestCluster.rpc!(context.node_b, Group, :lookup, [name, registry_key]) == nil + assert TestCluster.rpc!(context.node_b, Group, :members, [name, pg_key]) == [] + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :registry_claims, [ + name, + 1, + nil, + registry_key + ]) == [] + + cursor_table = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :replica_cursor_table, [name, 1]) + + refute TestCluster.rpc!(context.node_b, :ets, :member, [cursor_table, stream_id]) + assert :ok = TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end + + test "a delayed prior-generation hello cannot roll authority backward", context do + name = unique_name(:generation_rollback) + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + {old_generation, old_revision, old_epochs} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_authority, [name]) + + old_control = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + supervisor = TestCluster.rpc!(context.node_a, Process, :whereis, [:"#{name}_group_sup"]) + :ok = TestCluster.rpc!(context.node_a, Supervisor, :stop, [supervisor, :normal, 5_000]) + {:ok, _pid} = TestCluster.start_group(context.node_a, opts) + + new_generation = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :generation, [name]) + + refute new_generation == old_generation + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_generation, [ + name, + context.node_a + ]) == new_generation + end) + + target = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + send( + target, + {:replica_hello, old_control, Group.Replica.WireProtocol.version(), old_generation, + old_revision, old_epochs, Group.TestReplicaTransport.id(), + Group.TestReplicaTransport.descriptor(name, [])} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_generation, [ + name, + context.node_a + ]) == new_generation + end + + test "an unresolved authority hint is retired and cannot recreate a retired peer", context do + name = unique_name(:delayed_hint_retirement) + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 150 + ] + + start_group_on_peers(context.peers, opts) + + old_generation = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :generation, [name]) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_generation, [ + name, + context.node_a + ]) == old_generation + end) + + source_control = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + source_lane = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + target_control = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + target_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + for source <- [source_control, source_lane] do + :ok = TestCluster.rpc!(context.node_a, :sys, :suspend, [source]) + end + + on_exit(fn -> + for source <- [source_control, source_lane] do + TestCluster.rpc!(context.node_a, TestCluster, :resume_if_alive, [source]) + end + end) + + new_generation = + TestCluster.rpc!(context.node_a, Group.Replica.WireProtocol, :new_generation, []) + + assert Group.Replica.WireProtocol.generation_newer?(new_generation, old_generation) + + send( + target_lane, + {:replica_heartbeat, source_lane, Group.Replica.WireProtocol.version(), new_generation, 0, + Group.TestReplicaTransport.id(), Group.TestReplicaTransport.descriptor(name, [])} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + + assert TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_replica_authority_hint, + [name, context.node_a] + ) == {new_generation, 0} + + TestCluster.assert_eventually( + fn -> + hint = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_replica_authority_hint, + [name, context.node_a] + ) + + control_state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + lane_state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + + is_nil(hint) and + not Map.has_key?(control_state.cluster_control_dirty, context.node_a) and + not Map.has_key?(lane_state.peer_last_seen, context.node_a) + end, + timeout: 3_000, + interval: 25 + ) + + # A dirty notification can have been sent locally before the final lane + # retired but reach shard zero afterward. The next sweep must forget that + # now-authority-less repair obligation instead of retaining one map entry + # forever for the departed node. + send(target_control, {:replica_authority_dirty_local, context.node_a}) + control_state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + assert Map.has_key?(control_state.cluster_control_dirty, context.node_a) + + send(target_control, {:group_replica_anti_entropy, control_state.anti_entropy_ref}) + control_state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + refute Map.has_key?(control_state.cluster_control_dirty, context.node_a) + + # Once lease retirement removes exact authority, even a delayed heartbeat + # from that incarnation is only a discovery prompt. It cannot recreate a + # hint, route, or new lease without the dist-Erlang exact hello. + send( + target_lane, + {:replica_heartbeat, source_lane, Group.Replica.WireProtocol.version(), new_generation, 0, + Group.TestReplicaTransport.id(), Group.TestReplicaTransport.descriptor(name, [])} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + + assert TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_replica_authority_hint, + [name, context.node_a] + ) == nil + + refute Map.has_key?( + TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]).peer_last_seen, + context.node_a + ) + + send( + target_lane, + {:replica_lane_hello, source_lane, Group.Replica.WireProtocol.version(), new_generation, 0, + Group.TestReplicaTransport.id(), Group.TestReplicaTransport.descriptor(name, [])} + ) + + lane_state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + + assert TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_replica_authority_hint, + [name, context.node_a] + ) == nil + + refute Map.has_key?(lane_state.remote_shards, context.node_a) + refute Map.has_key?(lane_state.peer_last_seen, context.node_a) + end + + test "restart authority repair runs before stale claims can retire a local owner", context do + name = unique_name(:repair_before_projection) + cluster = "repair-before-projection" + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + resolve_registry_conflict: {Group.ModelConflictResolver, :resolve, []}, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 5_000 + ] + + start_group_on_peers(context.peers, opts) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group, :connect, [name, cluster]) + end + + TestCluster.assert_eventually(fn -> + Enum.all?([context.node_a, context.node_b, context.node_c], fn node -> + length(TestCluster.rpc!(node, Group, :nodes, [name, cluster])) == 3 + end) + end) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :drop]) + end + + key = + 1..1_000 + |> Enum.map(&"repair-before-projection/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(cluster, &1, 2) == 1)) + + local_owner = + TestCluster.spawn_register_in_cluster( + context.node_b, + name, + key, + %{rank: 1}, + cluster + ) + + stale_remote_owner = + TestCluster.spawn_register_in_cluster( + context.node_a, + name, + key, + %{rank: 2}, + cluster + ) + + TestCluster.flush_shards(context.node_a, name) + + stale_stream = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [ + name, + 1, + cluster + ]) + + :ok = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :put_registry_claim, [ + name, + 1, + stale_stream, + 1, + key, + stale_remote_owner, + %{rank: 2}, + System.system_time() + ]) + + assert match?( + {^local_owner, %{rank: 1}}, + TestCluster.rpc!(context.node_b, Group, :lookup, [name, key, [cluster: cluster]]) + ) + + replica_supervisor = + TestCluster.rpc!(context.node_b, Process, :whereis, [:"#{name}_replica_sup"]) + + old_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + :ok = TestCluster.rpc!(context.node_b, :sys, :suspend, [replica_supervisor]) + + on_exit(fn -> + TestCluster.rpc!(context.node_b, TestCluster, :resume_if_alive, [replica_supervisor]) + end) + + monitor = Process.monitor(old_lane) + true = TestCluster.rpc!(context.node_b, Process, :exit, [old_lane, :kill]) + assert_receive {:DOWN, ^monitor, :process, ^old_lane, :killed}, 5_000 + + :ok = TestCluster.rpc!(context.node_a, Group, :disconnect, [name, cluster]) + + exact_revision = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_exact_revision, + [name, context.node_a] + ) == exact_revision and + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + context.node_a, + cluster + ]) == nil + end) + + :ok = TestCluster.rpc!(context.node_b, :sys, :resume, [replica_supervisor]) + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, 1) + ]) do + pid when is_pid(pid) -> pid != old_lane + _ -> false + end + end) + + assert TestCluster.rpc!(context.node_b, Process, :alive?, [local_owner]) + + assert match?( + {^local_owner, %{rank: 1}}, + TestCluster.rpc!(context.node_b, Group, :lookup, [name, key, [cluster: cluster]]) + ) + + claims = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :registry_claims, [ + name, + 1, + cluster, + key + ]) + + assert Enum.map(claims, &elem(&1, 0)) == [local_owner] + assert :ok = TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end + + test "nodedown discards deferred registry reprojection state", context do + name = unique_name(:nodedown_deferred_projection) + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + target_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + key = "nodedown/deferred-projection" + + :ok = + TestCluster.rpc!(context.node_b, TestCluster, :put_pending_registry_reprojection, [ + target_lane, + context.node_a, + nil, + key + ]) + + send(target_lane, {:nodedown, context.node_a}) + + TestCluster.assert_eventually(fn -> + state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + not Map.has_key?(state.pending_registry_reprojections, context.node_a) + end) + + assert TestCluster.rpc!(context.node_b, Process, :alive?, [target_lane]) + end + + test "exact authority installation atomically projects shared clusters", context do + name = unique_name(:atomic_authority_projection) + cluster = "authority/install-race" + + opts = [ + name: name, + shards: 2, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + TestCluster.assert_eventually(fn -> + Enum.sort(TestCluster.rpc!(context.node_b, Group, :nodes, [name])) == + Enum.sort([context.node_a, context.node_c]) + end) + + local_epoch = make_ref() + + local_epochs_table = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :local_cluster_epochs_table, [name]) + + true = + TestCluster.rpc!(context.node_b, :ets, :insert, [ + local_epochs_table, + {cluster, local_epoch} + ]) + + :ok = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :add_cluster_node, [ + name, + [cluster], + context.node_b + ]) + + generation = TestCluster.rpc!(context.node_a, Group.Replica.Data, :generation, [name]) + revision = 1 + remote_epoch = make_ref() + epochs = [{nil, generation}, {cluster, remote_epoch}] + + {_old_generation, _stale_epochs} = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :put_remote_replica_info, [ + name, + 0, + context.node_a, + generation, + revision, + epochs + ]) + + assert Enum.sort(TestCluster.rpc!(context.node_b, Group, :nodes, [name, cluster])) == + Enum.sort([context.node_a, context.node_b]) + end + + test "local cluster activation survives its caller before shard notification", context do + name = unique_name(:interrupted_cluster_activation) + cluster = "cluster/activation-interrupted" + + opts = [ + name: name, + shards: 2, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 5_000 + ] + + start_group_on_peers(context.peers, opts) + :ok = TestCluster.rpc!(context.node_a, Group, :connect, [name, cluster]) + + TestCluster.assert_eventually(fn -> + not is_nil( + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + context.node_a, + cluster + ]) + ) + end) + + control = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + :ok = TestCluster.rpc!(context.node_b, :sys, :suspend, [control]) + + try do + # This is the durable first step of Group.connect/3. Simulate its caller + # disappearing while the queued shard notification cannot run. + [_epoch] = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :activate_local_clusters_durable, [ + name, + [cluster] + ]) + + assert Enum.sort(TestCluster.rpc!(context.node_b, Group, :nodes, [name, cluster])) == + Enum.sort([context.node_a, context.node_b]) + after + :ok = TestCluster.rpc!(context.node_b, TestCluster, :resume_if_alive, [control]) + end + + TestCluster.assert_eventually(fn -> + context.node_b in TestCluster.rpc!(context.node_a, Group, :nodes, [name, cluster]) + end) + end + + test "local cluster deactivation survives its caller before shard cleanup", context do + name = unique_name(:interrupted_cluster_deactivation) + cluster = "cluster/deactivation-interrupted" + reg_key = "deactivation/interrupted/registry" + pg_key = "deactivation/interrupted/pg" + opts = [name: name, shards: 2] + + start_group_on_peers(context.peers, opts) + :ok = TestCluster.rpc!(context.node_b, Group, :connect, [name, cluster]) + + owner = + TestCluster.spawn_register_and_join( + context.node_b, + name, + reg_key, + %{kind: :registry}, + pg_key, + %{kind: :pg}, + cluster: cluster + ) + + assert {^owner, %{kind: :registry}} = + TestCluster.rpc!(context.node_b, Group, :lookup, [ + name, + reg_key, + [cluster: cluster] + ]) + + assert [{^owner, %{kind: :pg}}] = + TestCluster.rpc!(context.node_b, Group, :members, [ + name, + pg_key, + [cluster: cluster] + ]) + + lanes = + for shard <- 0..1 do + lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, shard) + ]) + + :ok = TestCluster.rpc!(context.node_b, :sys, :suspend, [lane]) + lane + end + + try do + # This is the durable first step of Group.disconnect/3. Simulate its + # caller disappearing while no shard can process the queued cleanup. + [_epoch] = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :deactivate_local_clusters_durable, [ + name, + [cluster] + ]) + + refute context.node_b in TestCluster.rpc!(context.node_b, Group, :nodes, [name, cluster]) + after + Enum.each(lanes, fn lane -> + :ok = TestCluster.rpc!(context.node_b, TestCluster, :resume_if_alive, [lane]) + end) + end + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!(context.node_b, Group, :lookup, [ + name, + reg_key, + [cluster: cluster] + ]) == nil and + TestCluster.rpc!(context.node_b, Group, :members, [ + name, + pg_key, + [cluster: cluster] + ]) == [] and + TestCluster.rpc!(context.node_b, Group, :nodes, [name, cluster]) == [] and + TestCluster.rpc!(context.node_b, Group.Replica.Data, :closed_local_clusters, [name]) == + [] + end, + timeout: 5_000 + ) + end + + test "exact authority immediately re-probes a lane whose hello arrived first", context do + name = unique_name(:authority_reprobes_early_lane) + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + {:ok, _pid} = TestCluster.start_group(context.node_b, opts) + {:ok, _pid} = TestCluster.start_group(context.node_c, opts) + + TestCluster.assert_eventually(fn -> + context.node_c in TestCluster.rpc!(context.node_b, Group, :nodes, [name]) + end) + + target_control = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + target_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + :ok = TestCluster.rpc!(context.node_b, :sys, :suspend, [target_control]) + + on_exit(fn -> + TestCluster.rpc!(context.node_b, TestCluster, :resume_if_alive, [target_control]) + end) + + {:ok, _pid} = TestCluster.start_group(context.node_a, opts) + + source_control = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + source_lane = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + {generation, revision, _epochs} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_authority, [name]) + + :ok = TestCluster.rpc!(context.node_a, :sys, :suspend, [source_lane]) + + on_exit(fn -> + TestCluster.rpc!(context.node_a, TestCluster, :resume_if_alive, [source_lane]) + end) + + # Drain any discovery traffic emitted while A was starting, then force the + # lane hello to be processed while shard zero is still suspended. + Process.sleep(100) + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + + send( + target_lane, + {:replica_lane_hello, source_lane, Group.Replica.WireProtocol.version(), generation, + revision, Group.TestReplicaTransport.id(), Group.TestReplicaTransport.descriptor(name, [])} + ) + + target_lane_state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + refute Map.has_key?(target_lane_state.remote_shards, context.node_a) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_generation, [ + name, + context.node_a + ]) == nil + + TestCluster.assert_eventually(fn -> + {:messages, messages} = + TestCluster.rpc!(context.node_b, Process, :info, [target_control, :messages]) + + Enum.any?(messages, fn + {:replica_hello, ^source_control, _version, ^generation, ^revision, _epochs, _transport, + _descriptor} -> + true + + _message -> + false + end) + end) + + :ok = TestCluster.rpc!(context.node_b, :sys, :resume, [target_control]) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 1, + context.node_a + ]) == generation + end) + + # Once authority is exact, a lane can still start after shard-zero's + # best-effort fanout (or lose its lane-local view on restart). Prove that + # the lane hello itself reconstructs that view; merely observing the first + # fanout would not exercise install_current_replica_lane/3. + :ok = + TestCluster.rpc!(context.node_b, TestCluster, :delete_remote_view_info, [ + name, + 1, + context.node_a + ]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 1, + context.node_a + ]) == nil + + send( + target_lane, + {:replica_lane_hello, source_lane, Group.Replica.WireProtocol.version(), generation, + revision, Group.TestReplicaTransport.id(), Group.TestReplicaTransport.descriptor(name, [])} + ) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 1, + context.node_a + ]) == generation + end) + + # The source lane is suspended, so the only way this peer_connect can be + # present is the immediate post-authority re-probe from the target lane. + TestCluster.assert_eventually(fn -> + {:messages, messages} = + TestCluster.rpc!(context.node_a, Process, :info, [source_lane, :messages]) + + Enum.any?(messages, fn + {:peer_connect, ^target_lane, 1, 2, _clusters} -> true + _message -> false + end) + end) + + :ok = TestCluster.rpc!(context.node_a, :sys, :resume, [source_lane]) + + TestCluster.assert_eventually(fn -> + lane_state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + + Map.has_key?(lane_state.remote_shards, context.node_a) and + Map.has_key?(lane_state.peer_last_seen, context.node_a) + end) + + assert :ok = TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end + + defp establish_hidden_remote_claim(context, suffix) do + name = unique_name(suffix) + key = "hidden/#{suffix}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + resolve_registry_conflict: {Group.ModelConflictResolver, :resolve, []}, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 5_000, + replicated_oplog_max_entries: 2 + ] + + start_group_on_peers(context.peers, opts) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :drop]) + end + + pid_a = TestCluster.spawn_register(context.node_a, name, key, %{rank: 2}) + pid_b = TestCluster.spawn_register(context.node_b, name, key, %{rank: 1}) + :ok = TestCluster.rpc!(context.node_b, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + TestCluster.assert_eventually(fn -> + claims = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :registry_claims, [ + name, + 0, + nil, + key + ]) + + match?({^pid_a, %{rank: 2}}, TestCluster.rpc!(context.node_a, Group, :lookup, [name, key])) and + MapSet.new(Enum.map(claims, &elem(&1, 0))) == MapSet.new([pid_a, pid_b]) + end) + + for index <- 1..4 do + churn = + TestCluster.spawn_register(context.node_b, name, "#{key}/churn/#{index}", %{rank: 1}) + + true = TestCluster.rpc!(context.node_b, Process, :exit, [churn, :kill]) + end + + TestCluster.flush_shards(context.node_b, name) + + stream_b = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :local_stream_id, [name, 0, nil]) + + TestCluster.assert_eventually(fn -> + {_floor, head, _applied} = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :replica_stream_head, [ + name, + 0, + stream_b + ]) + + TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_cursor, [ + name, + 0, + stream_b + ]) == head + end) + + %{name: name, pid_a: pid_a, pid_b: pid_b, stream_b: stream_b} + end + + defp start_group_on_peers(peers, opts) do + Enum.each(peers, fn {_peer, node} -> + {:ok, _pid} = TestCluster.start_group(node, opts) + end) + + nodes = Enum.map(peers, &elem(&1, 1)) + + TestCluster.assert_eventually(fn -> + Enum.all?(nodes, fn node -> + Enum.sort(TestCluster.rpc!(node, Group, :nodes, [Keyword.fetch!(opts, :name)])) == + Enum.sort(nodes -- [node]) + end) + end) + end + + defp unique_name(suffix) do + :"ae_fault_#{suffix}_#{System.unique_integer([:positive])}" + end +end diff --git a/test/distributed_test.exs b/test/distributed_test.exs index e62a4d9..0d4e2c0 100644 --- a/test/distributed_test.exs +++ b/test/distributed_test.exs @@ -555,10 +555,10 @@ defmodule Group.DistributedTest do timeout: 10_000 ) - # Custom resolvers own process lifecycle decisions. This resolver only - # picks a registry winner, so neither owner is terminated by Group. - assert TestCluster.rpc!(node_a, Process, :alive?, [pid_a]) + # The resolver selects the winner; each origin is responsible for + # retiring (and terminating) only its own losing owner. assert TestCluster.rpc!(node_b, Process, :alive?, [pid_b]) + refute TestCluster.rpc!(node_a, Process, :alive?, [pid_a]) end end @@ -1025,6 +1025,8 @@ defmodule Group.DistributedTest do cluster = "game" registry_key = "remote/registry" pg_key = "remote/pg" + retained_registry_key = "remote/registry/retained" + retained_pg_key = "remote/pg/retained" start_group_on_peers(peers, name: name, shards: 2) @@ -1045,13 +1047,47 @@ defmodule Group.DistributedTest do pg_pid = TestCluster.spawn_join(node_b, name, pg_key, %{from: :b}, cluster: cluster) + retained_registry_pid = + TestCluster.spawn_register_in_cluster( + node_b, + name, + retained_registry_key, + %{from: :b, retained: true}, + cluster + ) + + retained_pg_pid = + TestCluster.spawn_join( + node_b, + name, + retained_pg_key, + %{from: :b, retained: true}, + cluster: cluster + ) + TestCluster.assert_eventually(fn -> TestCluster.rpc!(node_a, Group, :lookup, [ name, registry_key, [cluster: cluster] ]) != nil and - TestCluster.rpc!(node_a, Group, :members, [name, pg_key, [cluster: cluster]]) != [] + TestCluster.rpc!(node_a, Group, :members, [name, pg_key, [cluster: cluster]]) != [] and + match?( + {^retained_registry_pid, _}, + TestCluster.rpc!(node_a, Group, :lookup, [ + name, + retained_registry_key, + [cluster: cluster] + ]) + ) and + match?( + [{^retained_pg_pid, _}], + TestCluster.rpc!(node_a, Group, :members, [ + name, + retained_pg_key, + [cluster: cluster] + ]) + ) end) assert :ok = TestCluster.rpc!(node_a, Group, :disconnect, [name, cluster]) @@ -1088,6 +1124,25 @@ defmodule Group.DistributedTest do ]) == nil assert TestCluster.rpc!(node_a, Group, :members, [name, pg_key, [cluster: cluster]]) == [] + + TestCluster.assert_eventually(fn -> + match?( + {^retained_registry_pid, %{from: :b, retained: true}}, + TestCluster.rpc!(node_a, Group, :lookup, [ + name, + retained_registry_key, + [cluster: cluster] + ]) + ) and + match?( + [{^retained_pg_pid, %{from: :b, retained: true}}], + TestCluster.rpc!(node_a, Group, :members, [ + name, + retained_pg_key, + [cluster: cluster] + ]) + ) + end) end test "local join does not overtake an earlier remote cluster_disconnect after replicated PG flush" do @@ -1778,21 +1833,30 @@ defmodule Group.DistributedTest do messages = TestCluster.shard_messages(node_b, name, 0) case Enum.filter(messages, fn - {:replicate_registry_batch, _ops} -> true - {:cluster_disconnect, ["game"], _remote_pid} -> true - _ -> false + {:group_replica_frame, _source, {:delta_batch, _version, _runs}} -> + true + + {:replica_cluster_close, _remote_pid, _generation, _revision, [{"game", _epoch}]} -> + true + + _ -> + false end) do [ - {:replicate_registry_batch, ops}, - {:cluster_disconnect, ["game"], _remote_pid} + {:group_replica_frame, _source, {:delta_batch, _version, runs}}, + {:replica_cluster_close, _remote_pid, _generation, _revision, [{"game", _epoch}]} ] -> - Enum.any?(ops, fn - {:register, "game", ^key, reg_pid, %{v: 1}, _time, _entry_node} - when reg_pid == pid -> - true - - _ -> - false + Enum.any?(runs, fn {_stream_id, _first_seq, records, _head} -> + Enum.any?(records, fn {_seq, mutations} -> + Enum.any?(mutations, fn + {:register, "game", ^key, reg_pid, %{v: 1}, _time, _entry_node} + when reg_pid == pid -> + true + + _ -> + false + end) + end) end) _ -> @@ -2290,14 +2354,27 @@ defmodule Group.DistributedTest do # --- Phase 4: verify all data converged on all nodes --- # Check registrations: every node should see every registration - for {_reg_node, cluster, key} <- reg_pids do + for {reg_node, cluster, key} <- reg_pids do TestCluster.assert_eventually( fn -> Enum.all?(nodes, fn check_node -> TestCluster.rpc!(check_node, Group, :lookup, [name, key, [cluster: cluster]]) != nil end) end, - timeout: 10_000 + timeout: 10_000, + diagnostic: fn -> + Map.new(nodes, fn check_node -> + state = + TestCluster.rpc!( + check_node, + Group.TestCluster, + :replica_registry_replication_state, + [name, cluster, key, reg_node] + ) + + {check_node, state} + end) + end ) end @@ -2354,18 +2431,55 @@ defmodule Group.DistributedTest do TestCluster.rpc!(node_a, Group, :disconnect, [name, dropped_cluster]) # A's registrations in org/10 should be gone everywhere - TestCluster.assert_eventually( - fn -> - Enum.all?(nodes, fn check_node -> - TestCluster.rpc!(check_node, Group, :lookup, [ - name, - "user/0_1", - [cluster: dropped_cluster] - ]) == nil - end) - end, - timeout: 10_000 - ) + try do + TestCluster.assert_eventually( + fn -> + Enum.all?(nodes, fn check_node -> + TestCluster.rpc!(check_node, Group, :lookup, [ + name, + "user/0_1", + [cluster: dropped_cluster] + ]) == nil + end) + end, + timeout: 10_000 + ) + rescue + error -> + diagnostics = + for check_node <- nodes do + lookup = + TestCluster.rpc!(check_node, Group, :lookup, [ + name, + "user/0_1", + [cluster: dropped_cluster] + ]) + + protocol = + TestCluster.rpc!( + check_node, + Group.TestCluster, + :replica_protocol_state, + [name] + ) + |> Enum.map(fn %{shard: shard, cursors: cursors} -> + relevant = + Enum.filter(cursors, fn {stream_id, _seq} -> + Group.Replica.WireProtocol.stream_origin(stream_id) == node_a and + Group.Replica.WireProtocol.stream_cluster(stream_id) == dropped_cluster + end) + + {shard, relevant} + end) + + {check_node, lookup, protocol} + end + + flunk( + "cluster disconnect convergence failed: #{Exception.message(error)} " <> + "diagnostics=#{inspect(diagnostics, limit: :infinity)}" + ) + end # B and C still see each other's org/10 data TestCluster.assert_eventually(fn -> @@ -3601,11 +3715,11 @@ defmodule Group.DistributedTest do assert Enum.map(left_events, & &1.key) |> Enum.sort() == Enum.sort(keys) end - test "partition heal (cluster_state merge) batches new entries into one message" do - peers = TestCluster.start_peers(2) + test "partition-heal delta catch-up batches same-stream entries into one message" do + peers = TestCluster.start_peers(3) on_exit(fn -> TestCluster.stop_peers(peers) end) - [{_, node_a}, {_, node_b}] = peers + [{_, node_a}, {_, node_b}, {_, node_c}] = peers num_shards = 4 name = :"batch_heal_#{System.unique_integer([:positive])}" opts = [name: name, shards: num_shards] @@ -3613,7 +3727,18 @@ defmodule Group.DistributedTest do start_group_on_peers(peers, opts) TestCluster.assert_eventually(fn -> - TestCluster.rpc!(node_a, Group, :nodes, [name]) == [node_b] + Enum.sort(TestCluster.rpc!(node_a, Group, :nodes, [name])) == + Enum.sort([node_b, node_c]) + end) + + survivor_key = "heal_batch/survivor" + survivor = TestCluster.spawn_register(node_c, name, survivor_key, %{owner: :c}) + + TestCluster.assert_eventually(fn -> + match?( + {^survivor, %{owner: :c}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, survivor_key]) + ) end) # Partition @@ -3639,7 +3764,7 @@ defmodule Group.DistributedTest do forwarder = TestCluster.spawn_batch_forwarder(node_b, name, :all, self()) assert_receive {:monitor_ready, ^forwarder}, 1000 - # Reconnect — cluster_state exchange merges all 3 entries in one handler turn + # Reconnect — one contiguous delta run applies all 3 entries in one turn. TestCluster.reconnect_nodes(node_a, node_b) # All 3 :registered events should arrive in a single batch @@ -3647,6 +3772,9 @@ defmodule Group.DistributedTest do reg_events = Enum.filter(events, &(&1.type == :registered)) assert length(reg_events) == 3 assert Enum.map(reg_events, & &1.key) |> Enum.sort() == Enum.sort(keys) + + assert {^survivor, %{owner: :c}} = + TestCluster.rpc!(node_b, Group, :lookup, [name, survivor_key]) end test "Group.connect on existing cluster batches incoming data" do @@ -3683,7 +3811,7 @@ defmodule Group.DistributedTest do assert_receive {:monitor_ready, ^forwarder}, 1000 - # A connects — receives cluster_state with all 3 entries from B + # A connects — stream heads trigger sequenced catch-up for all 3 entries. TestCluster.rpc!(node_a, Group, :connect, [name, "game"]) # All 3 :joined events should arrive in a single batch @@ -3834,6 +3962,2004 @@ defmodule Group.DistributedTest do end end + describe "replica transport anti-entropy" do + @tag timeout: 60_000 + test "dropped tails and dropped deletes converge without orphaning registry or PG rows" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_drop_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + reg_key = "anti-entropy/dropped-register" + pg_key = "anti-entropy/dropped-join" + reg_pid = TestCluster.spawn_register(node_a, name, reg_key, %{owner: :a}) + pg_pid = TestCluster.spawn_join(node_a, name, pg_key, %{owner: :a}) + Process.sleep(100) + assert TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key]) == nil + assert TestCluster.rpc!(node_b, Group, :members, [name, pg_key]) == [] + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + TestCluster.assert_eventually(fn -> + match?({^reg_pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key])) and + match?([{^pg_pid, _}], TestCluster.rpc!(node_b, Group, :members, [name, pg_key])) + end) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + TestCluster.rpc!(node_a, Process, :exit, [reg_pid, :kill]) + TestCluster.rpc!(node_a, Process, :exit, [pg_pid, :kill]) + TestCluster.flush_shards(node_a, name) + + assert match?({^reg_pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key])) + assert match?([{^pg_pid, _}], TestCluster.rpc!(node_b, Group, :members, [name, pg_key])) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key]) == nil and + TestCluster.rpc!(node_b, Group, :members, [name, pg_key]) == [] + end) + end + + @tag timeout: 60_000 + test "busy sends recover from the advertised stream head" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_busy_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :busy]) + key = "anti-entropy/busy" + pid = TestCluster.spawn_register(node_a, name, key, %{}) + Process.sleep(100) + assert TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + TestCluster.assert_eventually(fn -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + end + + @tag timeout: 60_000 + test "a pruned gap falls back to an exact origin snapshot and removes stale rows" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_snapshot_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000, + replicated_oplog_max_entries: 2 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + stale_reg_key = "anti-entropy/snapshot/stale-reg" + stale_pg_key = "anti-entropy/snapshot/stale-pg" + stale_reg_pid = TestCluster.spawn_register(node_a, name, stale_reg_key, %{}) + stale_pg_pid = TestCluster.spawn_join(node_a, name, stale_pg_key, %{}) + + TestCluster.assert_eventually(fn -> + match?( + {^stale_reg_pid, _}, + TestCluster.rpc!(node_b, Group, :lookup, [name, stale_reg_key]) + ) and + match?( + [{^stale_pg_pid, _}], + TestCluster.rpc!(node_b, Group, :members, [name, stale_pg_key]) + ) + end) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + TestCluster.rpc!(node_a, Process, :exit, [stale_reg_pid, :kill]) + TestCluster.rpc!(node_a, Process, :exit, [stale_pg_pid, :kill]) + + fresh_keys = for i <- 1..6, do: "anti-entropy/snapshot/fresh-#{i}" + fresh_pids = for key <- fresh_keys, do: TestCluster.spawn_register(node_a, name, key, %{}) + TestCluster.flush_shards(node_a, name) + + [{_stream_id, floor, head}] = + TestCluster.rpc!(node_a, Group.Replica.Data, :replica_stream_heads, [name, 0]) + + assert floor > 3 + assert head >= 10 + + assert match?( + {^stale_reg_pid, _}, + TestCluster.rpc!(node_b, Group, :lookup, [name, stale_reg_key]) + ) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_b, Group, :lookup, [name, stale_reg_key]) == nil and + TestCluster.rpc!(node_b, Group, :members, [name, stale_pg_key]) == [] and + Enum.zip(fresh_keys, fresh_pids) + |> Enum.all?(fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + end) + end + + @tag timeout: 60_000 + test "the control lease removes a stopped Group on a still-connected node and probes recovery" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_lease_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 150 + ] + + start_group_on_peers(peers, opts) + key = "anti-entropy/lease/stale" + stale_pid = TestCluster.spawn_register(node_a, name, key, %{}) + + TestCluster.assert_eventually(fn -> + match?({^stale_pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + + supervisor = TestCluster.rpc!(node_a, Process, :whereis, [:"#{name}_group_sup"]) + :ok = TestCluster.rpc!(node_a, Supervisor, :stop, [supervisor, :normal, 5_000]) + assert TestCluster.rpc!(node_b, Node, :ping, [node_a]) == :pong + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil and + node_a not in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end, + timeout: 5_000 + ) + + {:ok, _pid} = TestCluster.start_group(node_a, opts) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end) + + recovered_key = "anti-entropy/lease/recovered" + recovered_pid = TestCluster.spawn_register(node_a, name, recovered_key, %{}) + + TestCluster.assert_eventually(fn -> + match?( + {^recovered_pid, _}, + TestCluster.rpc!(node_b, Group, :lookup, [name, recovered_key]) + ) + end) + end + + @tag timeout: 60_000 + test "a restarted data lane sweeps a cluster close missed while it was down" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_lane_restart_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 2, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000 + ] + + start_group_on_peers(peers, opts) + :ok = TestCluster.rpc!(node_a, Group, :connect, [name, "game"]) + :ok = TestCluster.rpc!(node_b, Group, :connect, [name, "game"]) + + reg_key = + Enum.find(1..1_000, fn suffix -> + Group.Replica.shard_index_for("game", "lane-reg-#{suffix}", 2) == 1 + end) + |> then(&"lane-reg-#{&1}") + + pg_key = + Enum.find(1..1_000, fn suffix -> + Group.Replica.shard_index_for("game", "lane-pg-#{suffix}", 2) == 1 + end) + |> then(&"lane-pg-#{&1}") + + reg_pid = TestCluster.spawn_register_in_cluster(node_a, name, reg_key, %{}, "game") + pg_pid = TestCluster.spawn_join(node_a, name, pg_key, %{}, cluster: "game") + + TestCluster.assert_eventually(fn -> + match?( + {^reg_pid, _}, + TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key, [cluster: "game"]]) + ) and + match?( + [{^pg_pid, _}], + TestCluster.rpc!(node_b, Group, :members, [name, pg_key, [cluster: "game"]]) + ) + end) + + old_lane = TestCluster.rpc!(node_b, Process, :whereis, [shard_name(name, 1)]) + :ok = TestCluster.rpc!(node_b, :sys, :suspend, [old_lane]) + + :ok = TestCluster.rpc!(node_a, Group, :disconnect, [name, "game"]) + + TestCluster.assert_eventually(fn -> + is_nil( + TestCluster.rpc!(node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + node_a, + "game" + ]) + ) + end) + + control = TestCluster.rpc!(node_b, Process, :whereis, [shard_name(name, 0)]) + _state = TestCluster.rpc!(node_b, :sys, :get_state, [control]) + + messages = TestCluster.rpc!(node_b, Process, :info, [old_lane, :messages]) + + assert {:messages, queued} = messages + + assert Enum.any?(queued, fn + {:replica_cluster_close_control_local, ^node_a, _generation, _revision, + [{"game", _epoch}]} -> + true + + _ -> + false + end) + + true = TestCluster.rpc!(node_b, Process, :exit, [old_lane, :kill]) + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(node_b, Process, :whereis, [shard_name(name, 1)]) do + lane when is_pid(lane) -> lane != old_lane + _ -> false + end + end) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key, [cluster: "game"]]) == nil and + TestCluster.rpc!(node_b, Group, :members, [name, pg_key, [cluster: "game"]]) == [] + end) + + assert :ok = + TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 60_000 + test "full and incremental authority install once through shard zero" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_authority_topology_#{System.unique_integer([:positive])}" + shards = 3 + + opts = [ + name: name, + shards: shards, + replicated_anti_entropy_interval: 600_000, + replicated_peer_lease_timeout: 1_200_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end) + + b_lanes = + for shard <- 0..(shards - 1) do + {shard, TestCluster.rpc!(node_b, Process, :whereis, [shard_name(name, shard)])} + end + + Enum.each(b_lanes, fn {_shard, pid} -> + :ok = TestCluster.rpc!(node_b, :sys, :suspend, [pid]) + end) + + installs_before = + TestCluster.rpc!( + node_b, + Group.Replica.Data, + :remote_authority_install_count, + [name, node_a] + ) + + [_authority_epoch] = + TestCluster.rpc!(node_a, Group.Replica.Data, :activate_local_clusters, [ + name, + ["authority-new"] + ]) + + remote_clusters = TestCluster.rpc!(node_b, Group.Replica.Data, :my_clusters, [name]) + + Enum.each(b_lanes, fn {shard, b_pid} -> + TestCluster.rpc!(node_a, :erlang, :send, [ + shard_name(name, shard), + {:peer_connect_ack, b_pid, shard, shards, remote_clusters} + ]) + end) + + TestCluster.assert_eventually(fn -> + Enum.all?(b_lanes, fn {shard, pid} -> + {:messages, messages} = TestCluster.rpc!(node_b, Process, :info, [pid, :messages]) + + if shard == 0 do + Enum.any?(messages, fn + {:replica_hello, remote_pid, _version, _generation, _revision, _epochs, _transport, + _descriptor} -> + node(remote_pid) == node_a + + _ -> + false + end) + else + Enum.any?(messages, fn + {:replica_lane_hello, remote_pid, _version, _generation, _revision, _transport, + _descriptor} -> + node(remote_pid) == node_a + + _ -> + false + end) + end + end) + end) + + mailboxes = + Map.new(b_lanes, fn {shard, pid} -> + {:messages, messages} = TestCluster.rpc!(node_b, Process, :info, [pid, :messages]) + {shard, messages} + end) + + full_authority_lanes = + for {shard, messages} <- mailboxes, + Enum.any?(messages, fn + {:replica_hello, remote_pid, _version, _generation, _revision, _epochs, _transport, + _descriptor} -> + node(remote_pid) == node_a + + _ -> + false + end), + do: shard + + assert full_authority_lanes == [0] + + assert Enum.any?(mailboxes[0], fn + {:replica_hello, remote_pid, _version, _generation, revision, epochs, _transport, + _descriptor} -> + node(remote_pid) == node_a and + revision == Enum.count(epochs, &(not is_nil(elem(&1, 0)))) + + _ -> + false + end) + + for shard <- 1..(shards - 1) do + assert Enum.any?(mailboxes[shard], fn + {:replica_lane_hello, remote_pid, _version, _generation, _revision, _transport, + _descriptor} -> + node(remote_pid) == node_a + + _ -> + false + end) + end + + [epoch] = + TestCluster.rpc!(node_a, Group.Replica.Data, :activate_local_clusters, [ + name, + ["lane-open"] + ]) + + TestCluster.rpc!(node_a, Group.Replica.Data, :add_cluster_node, [ + name, + ["lane-open"], + node_a + ]) + + assert :ok = + TestCluster.rpc!(node_a, Group.Replica, :local_request, [ + shard_name(name, 2), + {:cluster_connect, ["lane-open"], [epoch]}, + 5_000 + ]) + + TestCluster.assert_eventually(fn -> + {:messages, messages} = + TestCluster.rpc!(node_b, Process, :info, [Map.fetch!(Map.new(b_lanes), 2), :messages]) + + Enum.any?(messages, fn + {:replica_cluster_open, remote_pid, _generation, _revision, [^epoch]} -> + node(remote_pid) == node_a + + _ -> + false + end) + end) + + {:messages, control_messages} = + TestCluster.rpc!(node_b, Process, :info, [Map.fetch!(Map.new(b_lanes), 0), :messages]) + + refute Enum.any?(control_messages, fn + {:replica_cluster_open, remote_pid, _generation, _revision, [^epoch]} -> + node(remote_pid) == node_a + + _ -> + false + end) + + {0, b_control} = List.keyfind!(b_lanes, 0, 0) + :ok = TestCluster.rpc!(node_b, :sys, :resume, [b_control]) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!( + node_b, + Group.Replica.Data, + :remote_authority_install_count, + [name, node_a] + ) == installs_before + 1 + end) + + duplicate_full = + Enum.find(mailboxes[0], fn + {:replica_hello, remote_pid, _version, _generation, _revision, _epochs, _transport, + _descriptor} -> + node(remote_pid) == node_a + + _ -> + false + end) + + {:replica_hello, _remote_pid, _version, _generation, full_revision, _epochs, _transport, + _descriptor} = duplicate_full + + Enum.each(1..128, fn _ -> send(b_control, duplicate_full) end) + drain_ref = make_ref() + send(b_control, {:group_dispatch, [self()], {:authority_duplicates_drained, drain_ref}}) + assert_receive {:authority_duplicates_drained, ^drain_ref}, 5_000 + + assert TestCluster.rpc!( + node_b, + Group.Replica.Data, + :remote_authority_install_count, + [name, node_a] + ) == installs_before + 1 + + Enum.each(b_lanes, fn + {0, _pid} -> :ok + {_shard, pid} -> :ok = TestCluster.rpc!(node_b, :sys, :resume, [pid]) + end) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + node_a, + "lane-open" + ]) == elem(epoch, 1) + end) + + latest_revision = + TestCluster.rpc!(node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + a_lane = TestCluster.rpc!(node_a, Process, :whereis, [shard_name(name, 2)]) + + TestCluster.rpc!(node_b, :erlang, :send, [ + shard_name(name, 2), + {:replica_heartbeat, a_lane, Group.Replica.WireProtocol.version(), + TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]), latest_revision, + Group.Transport.DistErl.id(), Group.Transport.DistErl.descriptor(name, [])} + ]) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_b, Group.Replica.Data, :remote_view_observed_revision, [ + name, + 2, + node_a + ]) == latest_revision + end) + + assert TestCluster.rpc!( + node_b, + Group.Replica.Data, + :remote_cluster_epoch_exact_revision, + [name, node_a] + ) == full_revision + + assert TestCluster.rpc!( + node_b, + Group.Replica.Data, + :remote_view_cluster_epoch_revision, + [name, 2, node_a] + ) == full_revision + end + + @tag timeout: 60_000 + test "a backlogged authority shard cannot block independent replica lanes" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_authority_backlog_#{System.unique_integer([:positive])}" + shards = 3 + + opts = [ + name: name, + shards: shards, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end) + + a_generation = + TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]) + + Enum.each([1, 2], fn shard_index -> + b_lane = TestCluster.rpc!(node_b, Process, :whereis, [shard_name(name, shard_index)]) + + TestCluster.assert_eventually(fn -> + b_lane_state = TestCluster.rpc!(node_b, :sys, :get_state, [b_lane]) + + Map.has_key?(b_lane_state.peer_last_seen, node_a) and + TestCluster.rpc!( + node_b, + Group.Replica.Data, + :remote_view_generation, + [name, shard_index, node_a] + ) == a_generation + end) + end) + + b_control = TestCluster.rpc!(node_b, Process, :whereis, [shard_name(name, 0)]) + a_control = TestCluster.rpc!(node_a, Process, :whereis, [shard_name(name, 0)]) + :ok = TestCluster.rpc!(node_b, :sys, :suspend, [b_control]) + + TestCluster.rpc!(node_a, :erlang, :send, [ + shard_name(name, 0), + {:peer_connect_ack, b_control, 0, shards, [nil]} + ]) + + TestCluster.assert_eventually(fn -> + {:messages, messages} = + TestCluster.rpc!(node_b, Process, :info, [b_control, :messages]) + + Enum.any?(messages, fn + {:replica_hello, ^a_control, _version, _generation, _revision, _epochs, _transport, + _descriptor} -> + true + + _ -> + false + end) + end) + + [reg_key] = keys_for_shard(nil, "authority-backlog/reg", shards, 1, 1) + [pg_key] = keys_for_shard(nil, "authority-backlog/pg", shards, 2, 1) + reg_pid = TestCluster.spawn_register(node_a, name, reg_key, %{lane: 1}) + pg_pid = TestCluster.spawn_join(node_a, name, pg_key, %{lane: 2}) + + TestCluster.assert_eventually(fn -> + match?({^reg_pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key])) and + match?([{^pg_pid, _}], TestCluster.rpc!(node_b, Group, :members, [name, pg_key])) + end) + + :ok = TestCluster.rpc!(node_b, :sys, :resume, [b_control]) + end + + @tag timeout: 60_000 + test "data arriving without authority is rejected without cursor advance and repairs later" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_authority_before_data_#{System.unique_integer([:positive])}" + shards = 2 + + opts = [ + name: name, + shards: shards, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end) + + a_lane = TestCluster.rpc!(node_a, Process, :whereis, [shard_name(name, 1)]) + + TestCluster.assert_eventually(fn -> + lane_state = TestCluster.rpc!(node_a, :sys, :get_state, [a_lane]) + + Map.has_key?(lane_state.peer_last_seen, node_b) and + not is_nil( + TestCluster.rpc!(node_a, Group.Replica.Data, :remote_view_generation, [ + name, + 1, + node_b + ]) + ) + end) + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:capture_drop, [:delta_batch]} + ]) + + [key] = keys_for_shard(nil, "authority-before-data", shards, 1, 1) + pid = TestCluster.spawn_register(node_a, name, key, %{valid: true}) + TestCluster.flush_shards(node_a, name) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) != [] + end) + + {_target, 1, frame} = + node_a + |> TestCluster.rpc!(Group.TestReplicaTransport, :captured, [name]) + |> Enum.find(fn {_target, shard, _frame} -> shard == 1 end) + + stream_id = TestCluster.rpc!(node_a, Group.Replica.Data, :local_stream_id, [name, 1, nil]) + generation = TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]) + + revision = + TestCluster.rpc!(node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + :ok = + TestCluster.rpc!(node_b, Group.Replica.Data, :delete_remote_replica_info, [ + name, + 0, + node_a + ]) + + :ok = + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ + name, + node_a, + 1, + frame + ]) + + TestCluster.flush_shards(node_b, name) + assert TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + + assert TestCluster.rpc!(node_b, Group.Replica.Data, :replica_cursor, [ + name, + 1, + stream_id + ]) == 0 + + TestCluster.rpc!(node_b, :erlang, :send, [ + shard_name(name, 1), + {:replica_heartbeat, a_lane, Group.Replica.WireProtocol.version(), generation, revision, + Group.TestReplicaTransport.id(), Group.TestReplicaTransport.descriptor(name, [])} + ]) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_b, Group.Replica.Data, :remote_generation, [name, node_a]) == + generation + end) + + :ok = + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ + name, + node_a, + 1, + frame + ]) + + TestCluster.assert_eventually(fn -> + match?({^pid, %{valid: true}}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + + assert :ok = + TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 60_000 + test "a delayed cluster close from an older epoch revision cannot undo a reconnect" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_epoch_fence_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + not is_nil( + TestCluster.rpc!(node_a, Group.Replica.Data, :remote_view_generation, [ + name, + 0, + node_b + ]) + ) and + not is_nil( + TestCluster.rpc!(node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 0, + node_a + ]) + ) + end) + + TestCluster.rpc!(node_a, Group, :connect, [name, "game"]) + TestCluster.rpc!(node_b, Group, :connect, [name, "game"]) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name, "game"]) + end) + + generation = TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]) + + old_revision = + TestCluster.rpc!(node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + old_epoch = + TestCluster.rpc!(node_a, Group.Replica.Data, :local_cluster_epoch, [name, "game"]) + + TestCluster.rpc!(node_a, Group, :disconnect, [name, "game"]) + TestCluster.rpc!(node_a, Group, :connect, [name, "game"]) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name, "game"]) + end) + + current_revision = + TestCluster.rpc!(node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + assert current_revision > old_revision + + key = "anti-entropy/epoch-fence/current" + pid = TestCluster.spawn_register_in_cluster(node_a, name, key, %{}, "game") + + TestCluster.assert_eventually(fn -> + match?( + {^pid, _}, + TestCluster.rpc!(node_b, Group, :lookup, [name, key, [cluster: "game"]]) + ) + end) + + remote_pid = TestCluster.rpc!(node_a, Process, :whereis, [shard_name(name, 0)]) + + TestCluster.rpc!(node_a, :erlang, :send, [ + {shard_name(name, 0), node_b}, + {:replica_cluster_close, remote_pid, generation, old_revision, [{"game", old_epoch}]} + ]) + + Process.sleep(100) + TestCluster.flush_shards(node_b, name) + + assert node_a in TestCluster.rpc!(node_b, Group, :nodes, [name, "game"]) + + assert match?( + {^pid, _}, + TestCluster.rpc!(node_b, Group, :lookup, [name, key, [cluster: "game"]]) + ) + end + + @tag timeout: 60_000 + test "duplicate and reordered replica messages are idempotent and emit one lifecycle event" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_duplicate_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + forwarder = TestCluster.spawn_monitor_forwarder(node_b, name, :all, self()) + assert_receive {:monitor_ready, ^forwarder}, 5_000 + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:chaos, [drop_every: 0, duplicate_every: 2, max_delay: 75]} + ]) + + key = "anti-entropy/duplicate" + pid = TestCluster.spawn_register(node_a, name, key, %{version: 1}) + + assert_receive {:got_event, %Group.Event{type: :registered, key: ^key, pid: ^pid}}, 5_000 + + TestCluster.assert_eventually(fn -> + match?({^pid, %{version: 1}}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + + Process.sleep(250) + + refute_received {:got_event, %Group.Event{type: :registered, key: ^key, pid: ^pid}} + + assert :ok = + TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 60_000 + test "captured data from old origin generations and cluster epochs cannot resurrect rows" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_stale_frames_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 150 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:capture_drop, [:delta_batch]} + ]) + + generation_key = "anti-entropy/stale-generation" + old_generation_pid = TestCluster.spawn_register(node_a, name, generation_key, %{old: true}) + TestCluster.flush_shards(node_a, name) + + generation_frames = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) + + assert generation_frames != [] + assert TestCluster.rpc!(node_b, Group, :lookup, [name, generation_key]) == nil + + supervisor = TestCluster.rpc!(node_a, Process, :whereis, [:"#{name}_group_sup"]) + :ok = TestCluster.rpc!(node_a, Supervisor, :stop, [supervisor, :normal, 5_000]) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + TestCluster.assert_eventually( + fn -> node_a not in TestCluster.rpc!(node_b, Group, :nodes, [name]) end, + timeout: 5_000 + ) + + {:ok, _pid} = TestCluster.start_group(node_a, opts) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end) + + Enum.each(generation_frames, fn {_target, shard, frame} -> + :ok = + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ + name, + node_a, + shard, + frame + ]) + end) + + TestCluster.flush_shards(node_b, name) + assert TestCluster.rpc!(node_b, Group, :lookup, [name, generation_key]) == nil + + # The global stream epoch normally equals its generation, so a replayed + # old frame is rejected by both guards. Rewrite only the epoch to the + # current one to prove that the generation fence independently rejects a + # semantically impossible mixed-generation stream instead of advancing + # its cursor or materializing its rows. + current_generation = + TestCluster.rpc!(node_b, Group.Replica.Data, :remote_generation, [name, node_a]) + + mixed_generation_frames = + Enum.map(generation_frames, fn {target, shard, {:delta_batch, version, runs}} -> + runs = + Enum.map(runs, fn + {{stream_name, origin, old_generation, stream_shard, nil, _old_epoch}, first_seq, + records, advertised_head} -> + stream_id = + {stream_name, origin, old_generation, stream_shard, nil, current_generation} + + {stream_id, first_seq, records, advertised_head} + end) + + {target, shard, {:delta_batch, version, runs}} + end) + + Enum.each(mixed_generation_frames, fn {_target, shard, frame} -> + :ok = + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ + name, + node_a, + shard, + frame + ]) + end) + + TestCluster.flush_shards(node_b, name) + assert TestCluster.rpc!(node_b, Group, :lookup, [name, generation_key]) == nil + + new_generation_pid = + TestCluster.spawn_register(node_a, name, generation_key, %{old: false}) + + TestCluster.assert_eventually(fn -> + match?( + {^new_generation_pid, %{old: false}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, generation_key]) + ) + end) + + TestCluster.rpc!(node_a, Group, :connect, [name, "game"]) + TestCluster.rpc!(node_b, Group, :connect, [name, "game"]) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name, "game"]) + end) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :clear_captured, [name]) + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:capture_drop, [:delta_batch]} + ]) + + epoch_key = "anti-entropy/stale-epoch" + + old_epoch_pid = + TestCluster.spawn_register_in_cluster(node_a, name, epoch_key, %{old: true}, "game") + + TestCluster.flush_shards(node_a, name) + epoch_frames = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) + assert epoch_frames != [] + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + TestCluster.rpc!(node_a, Group, :disconnect, [name, "game"]) + TestCluster.rpc!(node_a, Group, :connect, [name, "game"]) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name, "game"]) + end) + + Enum.each(epoch_frames, fn {_target, shard, frame} -> + :ok = + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ + name, + node_a, + shard, + frame + ]) + end) + + TestCluster.flush_shards(node_b, name) + + assert TestCluster.rpc!(node_b, Group, :lookup, [ + name, + epoch_key, + [cluster: "game"] + ]) == nil + + new_epoch_pid = + TestCluster.spawn_register_in_cluster(node_a, name, epoch_key, %{old: false}, "game") + + TestCluster.assert_eventually(fn -> + match?( + {^new_epoch_pid, %{old: false}}, + TestCluster.rpc!(node_b, Group, :lookup, [ + name, + epoch_key, + [cluster: "game"] + ]) + ) + end) + + TestCluster.rpc!(node_a, Process, :exit, [old_generation_pid, :kill]) + TestCluster.rpc!(node_a, Process, :exit, [old_epoch_pid, :kill]) + + assert :ok = + TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 60_000 + test "global shard pruning repairs a cold stream without sending or deleting other origins" do + peers = TestCluster.start_peers(3) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}, {_, node_c}] = peers + name = :"anti_entropy_multistream_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000, + replicated_oplog_max_entries: 4 + ] + + start_group_on_peers(peers, opts) + + for target <- [node_a, node_b, node_c] do + TestCluster.assert_eventually(fn -> + length(TestCluster.rpc!(target, Group, :nodes, [name])) == 2 + end) + + TestCluster.rpc!(target, Group, :connect, [name, ["cold", "hot"]]) + end + + TestCluster.assert_eventually( + fn -> + length(TestCluster.rpc!(node_c, Group, :nodes, [name, "cold"])) == 3 and + length(TestCluster.rpc!(node_c, Group, :nodes, [name, "hot"])) == 3 + end, + timeout: 10_000 + ) + + b_key = "anti-entropy/other-origin" + b_group = "anti-entropy/other-origin-pg" + b_pid = TestCluster.spawn_register(node_b, name, b_key, %{origin: :b}) + b_pg_pid = TestCluster.spawn_join(node_b, name, b_group, %{origin: :b}) + + stale_key = "anti-entropy/cold/stale" + stale_group = "anti-entropy/cold/stale-pg" + + stale_pid = + TestCluster.spawn_register_in_cluster(node_a, name, stale_key, %{origin: :a}, "cold") + + stale_pg_pid = + TestCluster.spawn_join_in_cluster(node_a, name, stale_group, %{origin: :a}, "cold") + + TestCluster.assert_eventually(fn -> + match?({^b_pid, _}, TestCluster.rpc!(node_c, Group, :lookup, [name, b_key])) and + match?([{^b_pg_pid, _}], TestCluster.rpc!(node_c, Group, :members, [name, b_group])) and + match?( + {^stale_pid, _}, + TestCluster.rpc!(node_c, Group, :lookup, [name, stale_key, [cluster: "cold"]]) + ) and + match?( + [{^stale_pg_pid, _}], + TestCluster.rpc!(node_c, Group, :members, [name, stale_group, [cluster: "cold"]]) + ) + end) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + TestCluster.rpc!(node_a, Process, :exit, [stale_pid, :kill]) + TestCluster.rpc!(node_a, Process, :exit, [stale_pg_pid, :kill]) + + fresh = + for i <- 1..12 do + key = "anti-entropy/hot/#{i}" + + {key, TestCluster.spawn_register_in_cluster(node_a, name, key, %{i: i}, "hot")} + end + + TestCluster.flush_shards(node_a, name) + + {_stream_id, floor, _head} = + node_a + |> TestCluster.rpc!(Group.Replica.Data, :replica_stream_heads, [name, 0]) + |> Enum.find(fn {stream_id, _floor, _head} -> + Group.Replica.WireProtocol.stream_cluster(stream_id) == "cold" + end) + + assert floor > 2 + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!(node_c, Group, :lookup, [name, stale_key, [cluster: "cold"]]) == + nil and + TestCluster.rpc!(node_c, Group, :members, [ + name, + stale_group, + [cluster: "cold"] + ]) == [] and + match?({^b_pid, _}, TestCluster.rpc!(node_c, Group, :lookup, [name, b_key])) and + match?( + [{^b_pg_pid, _}], + TestCluster.rpc!(node_c, Group, :members, [name, b_group]) + ) and + Enum.all?(fresh, fn {key, pid} -> + match?( + {^pid, _}, + TestCluster.rpc!(node_c, Group, :lookup, [name, key, [cluster: "hot"]]) + ) + end) + end, + timeout: 10_000 + ) + + assert :ok = + TestCluster.rpc!(node_c, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 60_000 + test "invalid authority and a misattributed frame cannot poison a stream cursor" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_authority_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + stream_id = + TestCluster.rpc!(node_a, Group.Replica.Data, :local_stream_id, [name, 0, nil]) + + foreign_pid = + TestCluster.spawn_register(node_b, name, "anti-entropy/authority/foreign-owner", %{}) + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:capture_drop, [:delta_batch]} + ]) + + key = "anti-entropy/authority/legitimate" + legitimate_pid = TestCluster.spawn_register(node_a, name, key, %{valid: true}) + TestCluster.flush_shards(node_a, name) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) != [] + end) + + [{_target, 0, legitimate_frame} | _] = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) + + invalid_key = "anti-entropy/authority/forged" + + invalid_frame = + {:delta_batch, Group.Replica.WireProtocol.version(), + [ + {stream_id, 1, + [ + {1, + [ + {:register, nil, invalid_key, foreign_pid, %{forged: true}, System.system_time(), + node_b} + ]} + ], 1} + ]} + + :ok = + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ + name, + node_a, + 0, + invalid_frame + ]) + + TestCluster.flush_shards(node_b, name) + assert TestCluster.rpc!(node_b, Group, :lookup, [name, invalid_key]) == nil + + assert TestCluster.rpc!(node_b, Group.Replica.Data, :replica_cursor, [ + name, + 0, + stream_id + ]) == 0 + + :ok = + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ + name, + node_b, + 0, + legitimate_frame + ]) + + TestCluster.flush_shards(node_b, name) + assert TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + + assert TestCluster.rpc!(node_b, Group.Replica.Data, :replica_cursor, [ + name, + 0, + stream_id + ]) == 0 + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + TestCluster.assert_eventually(fn -> + match?( + {^legitimate_pid, %{valid: true}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) + ) + end) + + assert :ok = + TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 60_000 + test "lease expiry purges every shard before accepting a fresh generation" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_multishard_lease_#{System.unique_integer([:positive])}" + shards = 3 + + opts = [ + name: name, + shards: shards, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 2, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 150 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + stale = + for shard <- 0..(shards - 1) do + [reg_key] = keys_for_shard(nil, "anti-entropy/lease/reg/s#{shard}", shards, shard, 1) + [pg_key] = keys_for_shard(nil, "anti-entropy/lease/pg/s#{shard}", shards, shard, 1) + + { + reg_key, + TestCluster.spawn_register(node_a, name, reg_key, %{generation: :old, shard: shard}), + pg_key, + TestCluster.spawn_join(node_a, name, pg_key, %{generation: :old, shard: shard}) + } + end + + TestCluster.assert_eventually( + fn -> + Enum.all?(stale, fn {reg_key, reg_pid, pg_key, pg_pid} -> + match?({^reg_pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key])) and + match?([{^pg_pid, _}], TestCluster.rpc!(node_b, Group, :members, [name, pg_key])) + end) + end, + timeout: 10_000 + ) + + old_generation = TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]) + supervisor = TestCluster.rpc!(node_a, Process, :whereis, [:"#{name}_group_sup"]) + :ok = TestCluster.rpc!(node_a, Supervisor, :stop, [supervisor, :normal, 5_000]) + assert TestCluster.rpc!(node_b, Node, :ping, [node_a]) == :pong + + TestCluster.assert_eventually( + fn -> + node_a not in TestCluster.rpc!(node_b, Group, :nodes, [name]) and + Enum.all?(stale, fn {reg_key, _reg_pid, pg_key, _pg_pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key]) == nil and + TestCluster.rpc!(node_b, Group, :members, [name, pg_key]) == [] + end) + end, + timeout: 5_000 + ) + + assert :ok = + TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + + assert :ok = + TestCluster.rpc!( + node_b, + Group.TestCluster, + :assert_replica_origin_purged, + [name, node_a] + ) + + {:ok, _pid} = TestCluster.start_group(node_a, opts) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end) + + new_generation = TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]) + refute new_generation == old_generation + + fresh = + for shard <- 0..(shards - 1) do + [reg_key] = keys_for_shard(nil, "anti-entropy/fresh/reg/s#{shard}", shards, shard, 1) + [pg_key] = keys_for_shard(nil, "anti-entropy/fresh/pg/s#{shard}", shards, shard, 1) + + { + reg_key, + TestCluster.spawn_register(node_a, name, reg_key, %{generation: :new, shard: shard}), + pg_key, + TestCluster.spawn_join(node_a, name, pg_key, %{generation: :new, shard: shard}) + } + end + + TestCluster.assert_eventually( + fn -> + Enum.all?(fresh, fn {reg_key, reg_pid, pg_key, pg_pid} -> + match?({^reg_pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key])) and + match?([{^pg_pid, _}], TestCluster.rpc!(node_b, Group, :members, [name, pg_key])) + end) + end, + timeout: 10_000 + ) + + assert :ok = + TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 120_000 + test "concurrent many-cluster controls converge every revision before replica writes" do + peers = TestCluster.start_peers(3) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}, {_, node_c}] = peers + nodes = [node_a, node_b, node_c] + name = :"anti_entropy_many_controls_#{System.unique_integer([:positive])}" + clusters = for i <- 1..512, do: "tenant/#{i}" + + opts = [ + name: name, + shards: 4, + replicated_sender_buffer_size: 8, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 2_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + tasks = + for node <- nodes do + Task.async(fn -> TestCluster.connect_many_concurrently(node, name, clusters) end) + end + + assert [:ok, :ok, :ok] = Task.await_many(tasks, 60_000) + + TestCluster.assert_eventually( + fn -> + Enum.all?(nodes, fn node -> + expected = MapSet.new([nil | clusters]) + + actual = + TestCluster.rpc!(node, Group.Replica.Data, :my_clusters, [name]) + |> MapSet.new() + + MapSet.subset?(expected, actual) + end) and + Enum.all?(clusters, fn cluster -> + Enum.all?(nodes, fn node -> + length(TestCluster.rpc!(node, Group, :nodes, [name, cluster])) == 3 + end) + end) + end, + timeout: 30_000, + interval: 100 + ) + + entries = + TestCluster.spawn_register_many_clusters( + node_a, + name, + clusters, + "anti-entropy/many-controls" + ) + + TestCluster.assert_eventually( + fn -> + Enum.all?([node_b, node_c], fn node -> + TestCluster.rpc!(node, Group.TestCluster, :registry_entries_present?, [ + name, + entries + ]) + end) + end, + timeout: 30_000, + interval: 100 + ) + + for node <- [node_a, node_b] do + assert :ok = + TestCluster.rpc!(node, Group.TestCluster, :assert_replica_consistent, [name]) + end + end + + @tag timeout: 60_000 + test "mixed-generation data is rejected even when its epoch matches current authority" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_generation_guard_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end) + + TestCluster.rpc!(node_a, Group, :connect, [name, "game"]) + TestCluster.rpc!(node_b, Group, :connect, [name, "game"]) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name, "game"]) + end) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + key = "anti-entropy/mixed-generation" + meta = %{forged: true} + pid = TestCluster.spawn_register_in_cluster(node_a, name, key, meta, "game") + TestCluster.flush_shards(node_a, name) + + current_generation = + TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]) + + current_epoch = + TestCluster.rpc!(node_a, Group.Replica.Data, :local_cluster_epoch, [name, "game"]) + + forged_generation = + current_generation + |> elem(0) + |> Kernel.-(1) + |> max(1) + |> then(&{&1, make_ref()}) + + stream_id = + Group.Replica.WireProtocol.stream_id( + name, + node_a, + forged_generation, + 0, + "game", + current_epoch + ) + + mutation = {:register, "game", key, pid, meta, System.monotonic_time(), node_a} + + :ok = + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ + name, + node_a, + 0, + {:delta_batch, Group.Replica.WireProtocol.version(), + [{stream_id, 1, [{1, [mutation]}], 1}]} + ]) + + TestCluster.flush_shards(node_b, name) + + assert TestCluster.rpc!(node_b, Group, :lookup, [name, key, [cluster: "game"]]) == nil + + assert TestCluster.rpc!(node_b, Group.Replica.Data, :replica_cursor, [name, 0, stream_id]) == + 0 + end + + @tag timeout: 60_000 + test "an out-of-order delta cannot advance the cursor across a missing sequence" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_gap_guard_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:capture_drop, [:delta_batch]} + ]) + + first_key = "anti-entropy/gap/first" + second_key = "anti-entropy/gap/second" + group_key = "anti-entropy/gap/group" + first_pid = TestCluster.spawn_register(node_a, name, first_key, %{seq: 1}) + second_pid = TestCluster.spawn_register(node_a, name, second_key, %{seq: 2}) + member_pid = TestCluster.spawn_join(node_a, name, group_key, %{seq: 3}) + TestCluster.flush_shards(node_a, name) + + captured = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) + + frames_by_first_seq = + Map.new(captured, fn + {_target, shard, + {:delta_batch, _version, [{_stream_id, first_seq, _records, _head}]} = frame} -> + {first_seq, {shard, frame}} + end) + + {shard, {:delta_batch, _version, [{stream_id, 2, _records, _head}]} = second_frame} = + Map.fetch!(frames_by_first_seq, 2) + + :ok = + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ + name, + node_a, + shard, + second_frame + ]) + + TestCluster.flush_shards(node_b, name) + + assert TestCluster.rpc!(node_b, Group.Replica.Data, :replica_cursor, [ + name, + shard, + stream_id + ]) == 0 + + assert TestCluster.rpc!(node_b, Group, :lookup, [name, second_key]) == nil + + {^shard, first_frame} = Map.fetch!(frames_by_first_seq, 1) + {^shard, third_frame} = Map.fetch!(frames_by_first_seq, 3) + + for frame <- [first_frame, second_frame, third_frame] do + :ok = + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ + name, + node_a, + shard, + frame + ]) + end + + TestCluster.flush_shards(node_b, name) + + assert match?( + {^first_pid, %{seq: 1}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, first_key]) + ) + + assert match?( + {^second_pid, %{seq: 2}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, second_key]) + ) + + assert match?( + [{^member_pid, %{seq: 3}}], + TestCluster.rpc!(node_b, Group, :members, [name, group_key]) + ) + + # Model a receiver crash after all materialized ETS writes but before + # its cursor write. Replaying the accepted prefix must be exactly + # idempotent for both registry claims and PG membership. + :ok = + TestCluster.rpc!(node_b, Group.Replica.Data, :put_replica_cursor, [ + name, + shard, + stream_id, + 0 + ]) + + for frame <- [first_frame, second_frame, third_frame] do + :ok = + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ + name, + node_a, + shard, + frame + ]) + end + + TestCluster.flush_shards(node_b, name) + + assert TestCluster.rpc!(node_b, Group.Replica.Data, :replica_cursor, [ + name, + shard, + stream_id + ]) == 3 + + assert match?( + {^first_pid, %{seq: 1}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, first_key]) + ) + + assert match?( + {^second_pid, %{seq: 2}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, second_key]) + ) + + assert match?( + [{^member_pid, %{seq: 3}}], + TestCluster.rpc!(node_b, Group, :members, [name, group_key]) + ) + + assert :ok = + TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 60_000 + test "a new full authority generation purges nonzero shards before any lane-down signal" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_authority_fanout_#{System.unique_integer([:positive])}" + shards = 2 + + opts = [ + name: name, + shards: shards, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end) + + key = + Enum.find(Stream.iterate(0, &(&1 + 1)), fn suffix -> + :erlang.phash2({nil, "anti-entropy/authority-fanout/#{suffix}"}, shards) == 1 + end) + |> then(&"anti-entropy/authority-fanout/#{&1}") + + pid = TestCluster.spawn_register(node_a, name, key, %{generation: :old}) + + TestCluster.assert_eventually(fn -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + + forwarder = TestCluster.spawn_monitor_forwarder(node_b, name, :all, self()) + assert_receive {:monitor_ready, ^forwarder}, 5_000 + + a_control = TestCluster.rpc!(node_a, Process, :whereis, [shard_name(name, 0)]) + b_control = TestCluster.rpc!(node_b, Process, :whereis, [shard_name(name, 0)]) + b_lane = TestCluster.rpc!(node_b, Process, :whereis, [shard_name(name, 1)]) + + {old_generation_counter, _old_generation_identity} = + TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]) + + # Generation counters are ordered only within the origin BEAM. Construct + # the injected restart generation relative to node A's actual authority; + # using this test runner's independent counter makes the test flaky. + new_generation = {old_generation_counter + 1, make_ref()} + + new_key = + Enum.find(Stream.iterate(0, &(&1 + 1)), fn suffix -> + :erlang.phash2({nil, "anti-entropy/authority-fanout/new/#{suffix}"}, shards) == 1 + end) + |> then(&"anti-entropy/authority-fanout/new/#{&1}") + + stream_id = + Group.Replica.WireProtocol.stream_id( + name, + node_a, + new_generation, + 1, + nil, + new_generation + ) + + new_mutation = + {:register, nil, new_key, pid, %{generation: :new}, System.system_time(), node_a} + + new_frame = + {:delta_batch, Group.Replica.WireProtocol.version(), + [{stream_id, 1, [{1, [new_mutation]}], 1}]} + + :ok = TestCluster.rpc!(node_b, :sys, :suspend, [b_lane]) + + # Queue new-generation lane data before the control shard can enqueue + # its local authority marker. Shared authority will be current by the + # time this frame runs, but shard 1 must still reject it until its own + # old-generation purge has completed. + :ok = + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ + name, + node_a, + 1, + new_frame + ]) + + send( + b_control, + {:replica_hello, a_control, Group.Replica.WireProtocol.version(), new_generation, 0, + [{nil, new_generation}], Group.TestReplicaTransport.id(), + Group.TestReplicaTransport.descriptor(name, [])} + ) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_b, Group.Replica.Data, :remote_generation, [name, node_a]) == + new_generation + end) + + :ok = TestCluster.rpc!(node_b, :sys, :resume, [b_lane]) + TestCluster.flush_shards(node_b, name) + + assert TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + assert TestCluster.rpc!(node_b, Group, :lookup, [name, new_key]) == nil + refute_receive {:got_event, %Group.Event{key: ^new_key}}, 100 + + assert TestCluster.rpc!(node_b, Group.Replica.Data, :replica_cursor, [ + name, + 1, + stream_id + ]) == 0 + + :ok = + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ + name, + node_a, + 1, + new_frame + ]) + + TestCluster.flush_shards(node_b, name) + + assert TestCluster.rpc!(node_b, Group, :lookup, [name, new_key]) == + {pid, %{generation: :new}} + + assert :ok = TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 60_000 + test "three nodes recover a pruned origin over sideband TCP without disturbing another origin" do + peers = TestCluster.start_peers(3) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}, {_, node_c}] = peers + name = :"anti_entropy_sideband_tcp_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 2, + replica_transport: + {Group.TestTCPTransport, + [ + max_queue: 16, + connect_timeout: 250, + send_timeout: 250, + reconnect_interval: 10 + ]}, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000, + replicated_oplog_max_entries: 2 + ] + + start_group_on_peers(peers, opts) + + nodes = [node_a, node_b, node_c] + + TestCluster.assert_eventually( + fn -> + Enum.all?(nodes, fn source -> + Enum.all?(nodes -- [source], fn target -> + TestCluster.rpc!( + source, + Group.TestTCPTransport, + :connected?, + [name, target] + ) + end) + end) + end, + timeout: 10_000 + ) + + stale_key = "sideband/a/stale" + fresh_key = "sideband/a/fresh" + c_key = "sideband/c/independent" + c_group = "sideband/c/group" + + stale_pid = TestCluster.spawn_register(node_a, name, stale_key, %{origin: :a}) + + c_pid = + TestCluster.spawn_register_and_join( + node_c, + name, + c_key, + %{origin: :c}, + c_group, + %{origin: :c} + ) + + TestCluster.assert_eventually(fn -> + Enum.all?(nodes, fn receiver -> + match?( + {^stale_pid, %{origin: :a}}, + TestCluster.rpc!(receiver, Group, :lookup, [name, stale_key]) + ) and + match?( + {^c_pid, %{origin: :c}}, + TestCluster.rpc!(receiver, Group, :lookup, [name, c_key]) + ) and + match?( + [{^c_pid, %{origin: :c}}], + TestCluster.rpc!(receiver, Group, :members, [name, c_group]) + ) + end) + end) + + :ok = + TestCluster.rpc!(node_a, Group.TestTCPTransport, :disconnect_peer, [name, node_b]) + + old_reader = + TestCluster.rpc!(node_b, Group.TestTCPTransport, :status, [name]) + |> get_in([:inbound, node_a]) + + refute TestCluster.rpc!( + node_a, + Group.TestTCPTransport, + :connected?, + [name, node_b] + ) + + true = TestCluster.rpc!(node_a, Process, :exit, [stale_pid, :kill]) + + for i <- 1..6 do + churn_key = "sideband/a/churn/#{i}" + churn_pid = TestCluster.spawn_register(node_a, name, churn_key, %{i: i}) + true = TestCluster.rpc!(node_a, Process, :exit, [churn_pid, :kill]) + end + + fresh_pid = TestCluster.spawn_register(node_a, name, fresh_key, %{origin: :a, fresh: true}) + TestCluster.flush_shards(node_a, name) + + assert match?( + {^stale_pid, %{origin: :a}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, stale_key]) + ) + + assert TestCluster.rpc!(node_b, Group, :lookup, [name, fresh_key]) == nil + + assert match?( + {^c_pid, %{origin: :c}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, c_key]) + ) + + :ok = + TestCluster.rpc!(node_a, Group.TestTCPTransport, :reconnect_peer, [name, node_b]) + + TestCluster.assert_eventually(fn -> + new_reader = + TestCluster.rpc!(node_b, Group.TestTCPTransport, :status, [name]) + |> get_in([:inbound, node_a]) + + is_pid(new_reader) and new_reader != old_reader + end) + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!( + node_a, + Group.TestTCPTransport, + :connected?, + [name, node_b] + ) and + TestCluster.rpc!(node_b, Group, :lookup, [name, stale_key]) == nil and + match?( + {^fresh_pid, %{origin: :a, fresh: true}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, fresh_key]) + ) and + match?( + {^c_pid, %{origin: :c}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, c_key]) + ) and + match?( + [{^c_pid, %{origin: :c}}], + TestCluster.rpc!(node_b, Group, :members, [name, c_group]) + ) + end, + timeout: 10_000 + ) + + for receiver <- nodes do + assert :ok = + TestCluster.rpc!( + receiver, + Group.TestCluster, + :assert_replica_consistent, + [name] + ) + end + end + end + # Helpers for event assertion tests defp flush_events do diff --git a/test/formal/AuthorityHint.cfg b/test/formal/AuthorityHint.cfg new file mode 100644 index 0000000..b9d2036 --- /dev/null +++ b/test/formal/AuthorityHint.cfg @@ -0,0 +1,17 @@ +SPECIFICATION Spec + +CONSTANTS + MaxGeneration = 2 + MaxRevision = 2 + +INVARIANTS + TypeOK + EnabledLaneMatchesExactAuthority + ExactAuthorityOwnsItsRoute + NoLaneWithoutExactAuthority + HintRequiresPriorExactAuthority + AppliedAuthorityRespectsItsFence + +PROPERTIES + UnresolvedHintIsBounded + DelayedCleanupCompletes diff --git a/test/formal/AuthorityHint.tla b/test/formal/AuthorityHint.tla new file mode 100644 index 0000000..5f851b1 --- /dev/null +++ b/test/formal/AuthorityHint.tla @@ -0,0 +1,249 @@ +---------------------------- MODULE AuthorityHint ---------------------------- +EXTENDS Integers, TLC + +(* +Finite model of the shared authority-hint fence. A heartbeat on any replica +lane may observe a newer generation/revision before shard zero receives its +exact hello. The hint must disable every old lane, reject delayed old exact +and lane-view installs, retain a bounded repair/lease obligation, and allow a +stale cleanup caller to finish without deleting a rediscovered peer's route. +*) + +CONSTANTS MaxGeneration, MaxRevision + +ASSUME /\ MaxGeneration >= 2 + /\ MaxRevision >= 1 + +Generations == 1..MaxGeneration +Revisions == 0..MaxRevision + +VARIABLES exactGeneration, + exactRevision, + appliedRevision, + hintedGeneration, + hintedRevision, + viewGeneration, + viewExactRevision, + viewObservedRevision, + laneEnabled, + repairPending, + leasePending, + routeGeneration, + cleanupPending + +vars == + <> + +Init == + /\ exactGeneration = 1 + /\ exactRevision = 0 + /\ appliedRevision = 0 + /\ hintedGeneration = 1 + /\ hintedRevision = 0 + /\ viewGeneration = 1 + /\ viewExactRevision = 0 + /\ viewObservedRevision = 0 + /\ laneEnabled = TRUE + /\ repairPending = FALSE + /\ leasePending = TRUE + /\ routeGeneration = 1 + /\ cleanupPending = FALSE + +Newer(generation, revision, currentGeneration, currentRevision) == + \/ generation > currentGeneration + \/ /\ generation = currentGeneration + /\ revision > currentRevision + +NewerOrEqual(generation, revision, currentGeneration, currentRevision) == + \/ Newer(generation, revision, currentGeneration, currentRevision) + \/ /\ generation = currentGeneration + /\ revision = currentRevision + +ObserveHint(generation, revision) == + /\ exactGeneration > 0 + /\ hintedGeneration > 0 + /\ generation \in Generations + /\ revision \in Revisions + /\ Newer(generation, revision, hintedGeneration, hintedRevision) + /\ hintedGeneration' = generation + /\ hintedRevision' = revision + /\ laneEnabled' = FALSE + /\ repairPending' = TRUE + /\ leasePending' = TRUE + /\ UNCHANGED + <> + +InstallExact(generation, revision) == + /\ generation \in Generations + /\ revision \in Revisions + /\ NewerOrEqual(generation, revision, hintedGeneration, hintedRevision) + /\ \/ exactGeneration = 0 + \/ NewerOrEqual(generation, revision, exactGeneration, exactRevision) + /\ exactGeneration' = generation + /\ exactRevision' = revision + /\ appliedRevision' = revision + /\ hintedGeneration' = generation + /\ hintedRevision' = revision + /\ routeGeneration' = generation + /\ laneEnabled' = FALSE + /\ repairPending' = TRUE + /\ leasePending' = TRUE + /\ UNCHANGED + <> + +(* +Shard zero may install a contiguous incremental authority change without +moving the last exact-snapshot revision. The expected revision is compared to +the shared applied revision and hint in the same serialized transition; a +heartbeat racing it forward therefore disables this action instead of allowing +partial epoch rows to become authoritative. +*) +InstallIncremental(expected, revision) == + /\ exactGeneration > 0 + /\ expected \in Revisions + /\ revision \in Revisions + /\ revision = expected + 1 + /\ appliedRevision = expected + /\ hintedGeneration = exactGeneration + /\ hintedRevision = expected + /\ appliedRevision' = revision + /\ hintedRevision' = revision + /\ laneEnabled' = FALSE + /\ repairPending' = TRUE + /\ leasePending' = TRUE + /\ UNCHANGED + <> + +InstallLaneView == + /\ exactGeneration > 0 + /\ exactGeneration = hintedGeneration + /\ appliedRevision = hintedRevision + /\ viewGeneration' = exactGeneration + /\ viewExactRevision' = exactRevision + /\ viewObservedRevision' = hintedRevision + /\ laneEnabled' = TRUE + /\ repairPending' = FALSE + /\ leasePending' = TRUE + /\ UNCHANGED + <> + +QueueCleanup == + /\ ~cleanupPending + /\ cleanupPending' = TRUE + /\ UNCHANGED + <> + +RunCleanup == + /\ cleanupPending + /\ cleanupPending' = FALSE + /\ routeGeneration' = + IF exactGeneration = 0 /\ hintedGeneration = 0 + THEN 0 + ELSE routeGeneration + /\ UNCHANGED + <> + +RetireUnresolvedHint == + /\ repairPending + /\ leasePending + /\ exactGeneration' = 0 + /\ exactRevision' = 0 + /\ appliedRevision' = 0 + /\ hintedGeneration' = 0 + /\ hintedRevision' = 0 + /\ viewGeneration' = 0 + /\ viewExactRevision' = 0 + /\ viewObservedRevision' = 0 + /\ laneEnabled' = FALSE + /\ repairPending' = FALSE + /\ leasePending' = FALSE + /\ routeGeneration' = 0 + /\ UNCHANGED cleanupPending + +DropPeer == + /\ exactGeneration' = 0 + /\ exactRevision' = 0 + /\ appliedRevision' = 0 + /\ hintedGeneration' = 0 + /\ hintedRevision' = 0 + /\ viewGeneration' = 0 + /\ viewExactRevision' = 0 + /\ viewObservedRevision' = 0 + /\ laneEnabled' = FALSE + /\ repairPending' = FALSE + /\ leasePending' = FALSE + /\ routeGeneration' = 0 + /\ UNCHANGED cleanupPending + +Next == + \/ \E generation \in Generations, revision \in Revisions : + ObserveHint(generation, revision) + \/ \E generation \in Generations, revision \in Revisions : + InstallExact(generation, revision) + \/ \E expected \in Revisions, revision \in Revisions : + InstallIncremental(expected, revision) + \/ InstallLaneView + \/ QueueCleanup + \/ RunCleanup + \/ RetireUnresolvedHint + \/ DropPeer + +TypeOK == + /\ exactGeneration \in 0..MaxGeneration + /\ exactRevision \in Revisions + /\ appliedRevision \in Revisions + /\ hintedGeneration \in 0..MaxGeneration + /\ hintedRevision \in Revisions + /\ viewGeneration \in 0..MaxGeneration + /\ viewExactRevision \in Revisions + /\ viewObservedRevision \in Revisions + /\ laneEnabled \in BOOLEAN + /\ repairPending \in BOOLEAN + /\ leasePending \in BOOLEAN + /\ routeGeneration \in 0..MaxGeneration + /\ cleanupPending \in BOOLEAN + +EnabledLaneMatchesExactAuthority == + laneEnabled => + /\ exactGeneration > 0 + /\ viewGeneration = exactGeneration + /\ viewExactRevision = exactRevision + /\ viewObservedRevision = hintedRevision + /\ hintedGeneration = exactGeneration + /\ hintedRevision = appliedRevision + +ExactAuthorityOwnsItsRoute == + exactGeneration > 0 => routeGeneration = exactGeneration + +NoLaneWithoutExactAuthority == + exactGeneration = 0 => ~laneEnabled + +HintRequiresPriorExactAuthority == + hintedGeneration > 0 => exactGeneration > 0 + +AppliedAuthorityRespectsItsFence == + /\ (exactGeneration > 0 => exactRevision <= appliedRevision) + /\ (hintedGeneration = exactGeneration => appliedRevision <= hintedRevision) + +UnresolvedHintIsBounded == repairPending ~> ~repairPending + +DelayedCleanupCompletes == cleanupPending ~> ~cleanupPending + +Spec == + /\ Init + /\ [][Next]_vars + /\ WF_vars(RetireUnresolvedHint) + /\ WF_vars(RunCleanup) + +============================================================================= diff --git a/test/formal/AuthorityProjection.cfg b/test/formal/AuthorityProjection.cfg new file mode 100644 index 0000000..519f335 --- /dev/null +++ b/test/formal/AuthorityProjection.cfg @@ -0,0 +1,10 @@ +SPECIFICATION Spec + +INVARIANTS + TypeOK + SelfProjectionIsExact + SettledRemoteProjectionIsExact + InactiveRowsAreBeingCleaned + +PROPERTY + CleanupSurvivesCaller diff --git a/test/formal/AuthorityProjection.tla b/test/formal/AuthorityProjection.tla new file mode 100644 index 0000000..bacb93b --- /dev/null +++ b/test/formal/AuthorityProjection.tla @@ -0,0 +1,117 @@ +------------------------ MODULE AuthorityProjection ------------------------ +EXTENDS TLC + +(* +Finite model of the serialized boundary between named-cluster authority and +its routing/materialized state. The API caller may disappear after either +durable lifecycle mutation. Activation must already project every exact remote +authority; deactivation must leave cleanup enabled independently of the caller. +*) + +VARIABLES localActive, + selfRoute, + remoteExact, + remoteRoute, + rows, + cleanupPending, + callerAlive + +vars == + <> + +Init == + /\ localActive = FALSE + /\ selfRoute = FALSE + /\ remoteExact = FALSE + /\ remoteRoute = FALSE + /\ rows = FALSE + /\ cleanupPending = FALSE + /\ callerAlive = TRUE + +InstallRemote(exact) == + /\ remoteExact' = exact + /\ remoteRoute' = localActive /\ exact + /\ UNCHANGED + <> + +Activate == + /\ callerAlive = TRUE + /\ localActive = FALSE + /\ cleanupPending = FALSE + /\ localActive' = TRUE + /\ selfRoute' = TRUE + /\ remoteRoute' = remoteExact + /\ UNCHANGED <> + +Write == + /\ callerAlive = TRUE + /\ localActive = TRUE + /\ rows' = TRUE + /\ UNCHANGED + <> + +Deactivate == + /\ callerAlive = TRUE + /\ localActive = TRUE + /\ localActive' = FALSE + /\ selfRoute' = FALSE + /\ cleanupPending' = TRUE + /\ UNCHANGED <> + +Cleanup == + /\ cleanupPending = TRUE + /\ cleanupPending' = FALSE + /\ remoteRoute' = FALSE + /\ rows' = FALSE + /\ UNCHANGED <> + +CrashCaller == + /\ callerAlive = TRUE + /\ callerAlive' = FALSE + /\ UNCHANGED + <> + +NewCaller == + /\ callerAlive = FALSE + /\ callerAlive' = TRUE + /\ UNCHANGED + <> + +Next == + \/ \E exact \in BOOLEAN : InstallRemote(exact) + \/ Activate + \/ Write + \/ Deactivate + \/ Cleanup + \/ CrashCaller + \/ NewCaller + +TypeOK == + /\ localActive \in BOOLEAN + /\ selfRoute \in BOOLEAN + /\ remoteExact \in BOOLEAN + /\ remoteRoute \in BOOLEAN + /\ rows \in BOOLEAN + /\ cleanupPending \in BOOLEAN + /\ callerAlive \in BOOLEAN + +SelfProjectionIsExact == selfRoute = localActive + +SettledRemoteProjectionIsExact == + cleanupPending = FALSE => remoteRoute = (localActive /\ remoteExact) + +InactiveRowsAreBeingCleaned == + (localActive = FALSE /\ rows = TRUE) => cleanupPending = TRUE + +CleanupSurvivesCaller == cleanupPending = TRUE ~> cleanupPending = FALSE + +Spec == + /\ Init + /\ [][Next]_vars + /\ WF_vars(Cleanup) + +============================================================================= diff --git a/test/formal/GroupAntiEntropy.cfg b/test/formal/GroupAntiEntropy.cfg new file mode 100644 index 0000000..566fae5 --- /dev/null +++ b/test/formal/GroupAntiEntropy.cfg @@ -0,0 +1,17 @@ +SPECIFICATION Spec + +CONSTANTS + Nodes = {n1, n2, n3} + Origins = {n1} + Keys = {k1} + MaxSeq = 2 + OplogBound = 1 + MaxMessages = 1 + +INVARIANTS + TypeOK + BoundedJournal + CurrentReplicaIsAStreamPrefix + +PROPERTY + HealedConvergence diff --git a/test/formal/GroupAntiEntropy.tla b/test/formal/GroupAntiEntropy.tla new file mode 100644 index 0000000..96a3238 --- /dev/null +++ b/test/formal/GroupAntiEntropy.tla @@ -0,0 +1,481 @@ +------------------------- MODULE GroupAntiEntropy ------------------------- +EXTENDS Integers, FiniteSets, TLC + +(* +An abstract model of Group's per-origin anti-entropy stream. + +The model intentionally does not duplicate the Elixir data structures. It +models the protocol contract: exact generation/epoch authority, sequenced +deltas, bounded retained prefixes, exact snapshot fallback, arbitrary finite +loss/reordering/duplication, and fair repair after the network heals. +*) + +CONSTANTS Nodes, Origins, Keys, MaxSeq, OplogBound, MaxMessages + +ASSUME /\ IsFiniteSet(Nodes) + /\ Cardinality(Nodes) >= 2 + /\ Origins \subseteq Nodes + /\ Cardinality(Origins) >= 1 + /\ IsFiniteSet(Keys) + /\ Cardinality(Keys) >= 1 + /\ MaxSeq >= 1 + /\ OplogBound >= 1 + /\ MaxMessages >= 1 + +Seq == 1..MaxSeq +Generations == 0..2 +Revisions == 0..4 +Epochs == 0..4 +BoolMap == [Keys -> BOOLEAN] +EmptyView == [key \in Keys |-> FALSE] +EmptyRecord == [key |-> CHOOSE key \in Keys : TRUE, value |-> FALSE] + +HelloMessages == + [kind : {"hello"}, + from : Nodes, + to : Nodes, + wireGeneration : Generations, + wireRevision : Revisions, + wireEpoch : Epochs, + wireActive : BOOLEAN] + +DeltaMessages == + [kind : {"delta"}, + from : Nodes, + to : Nodes, + wireGeneration : Generations, + wireRevision : Revisions, + wireEpoch : Epochs, + seq : Seq, + key : Keys, + value : BOOLEAN] + +SnapshotMessages == + [kind : {"snapshot"}, + from : Nodes, + to : Nodes, + wireGeneration : Generations, + wireRevision : Revisions, + wireEpoch : Epochs, + seq : 0..MaxSeq, + state : BoolMap] + +Message == HelloMessages \union DeltaMessages \union SnapshotMessages + +VARIABLES phase, + generation, + revision, + epoch, + active, + truth, + head, + floor, + history, + authorityGeneration, + authorityRevision, + authorityEpoch, + authorityActive, + cursor, + replica, + messages + +vars == + <> + +RECURSIVE Replay(_, _) +Replay(records, n) == + IF n = 0 + THEN EmptyView + ELSE [Replay(records, n - 1) EXCEPT + ![records[n].key] = records[n].value] + +CurrentAuthority(receiver, origin) == + /\ authorityGeneration[receiver][origin] = generation[origin] + /\ authorityRevision[receiver][origin] = revision[origin] + /\ authorityEpoch[receiver][origin] = epoch[origin] + /\ authorityActive[receiver][origin] = active[origin] + +PairConverged(receiver, origin) == + IF receiver = origin + THEN TRUE + ELSE + /\ CurrentAuthority(receiver, origin) + /\ IF active[origin] + THEN /\ cursor[receiver][origin] = head[origin] + /\ replica[receiver][origin] = truth[origin] + ELSE /\ cursor[receiver][origin] = 0 + /\ replica[receiver][origin] = EmptyView + +Converged == + \A receiver \in Nodes : + \A origin \in Origins : + PairConverged(receiver, origin) + +Init == + /\ phase = "faulting" + /\ generation = [node \in Nodes |-> 1] + /\ revision = [node \in Nodes |-> 0] + /\ epoch = [node \in Nodes |-> 0] + /\ active = [node \in Nodes |-> FALSE] + /\ truth = [node \in Nodes |-> EmptyView] + /\ head = [node \in Nodes |-> 0] + /\ floor = [node \in Nodes |-> 1] + /\ history = [node \in Nodes |-> + [seq \in Seq |-> EmptyRecord]] + /\ authorityGeneration = + [receiver \in Nodes |-> [origin \in Nodes |-> 0]] + /\ authorityRevision = + [receiver \in Nodes |-> [origin \in Nodes |-> 0]] + /\ authorityEpoch = + [receiver \in Nodes |-> [origin \in Nodes |-> 0]] + /\ authorityActive = + [receiver \in Nodes |-> [origin \in Nodes |-> FALSE]] + /\ cursor = + [receiver \in Nodes |-> [origin \in Nodes |-> 0]] + /\ replica = + [receiver \in Nodes |-> [origin \in Nodes |-> EmptyView]] + /\ messages = {} + +Open(origin) == + /\ phase = "faulting" + /\ ~active[origin] + /\ revision[origin] < 4 + /\ epoch[origin] < 4 + /\ active' = [active EXCEPT ![origin] = TRUE] + /\ revision' = [revision EXCEPT ![origin] = @ + 1] + /\ epoch' = [epoch EXCEPT ![origin] = @ + 1] + /\ truth' = [truth EXCEPT ![origin] = EmptyView] + /\ head' = [head EXCEPT ![origin] = 0] + /\ floor' = [floor EXCEPT ![origin] = 1] + /\ history' = [history EXCEPT + ![origin] = [seq \in Seq |-> EmptyRecord]] + /\ UNCHANGED <> + +Close(origin) == + /\ phase = "faulting" + /\ active[origin] + /\ revision[origin] < 4 + /\ active' = [active EXCEPT ![origin] = FALSE] + /\ revision' = [revision EXCEPT ![origin] = @ + 1] + /\ truth' = [truth EXCEPT ![origin] = EmptyView] + /\ head' = [head EXCEPT ![origin] = 0] + /\ floor' = [floor EXCEPT ![origin] = 1] + /\ history' = [history EXCEPT + ![origin] = [seq \in Seq |-> EmptyRecord]] + /\ UNCHANGED <> + +Restart(origin) == + /\ phase = "faulting" + /\ generation[origin] < 2 + /\ generation' = [generation EXCEPT ![origin] = @ + 1] + /\ revision' = [revision EXCEPT ![origin] = 0] + /\ epoch' = [epoch EXCEPT ![origin] = 0] + /\ active' = [active EXCEPT ![origin] = FALSE] + /\ truth' = [truth EXCEPT ![origin] = EmptyView] + /\ head' = [head EXCEPT ![origin] = 0] + /\ floor' = [floor EXCEPT ![origin] = 1] + /\ history' = [history EXCEPT + ![origin] = [seq \in Seq |-> EmptyRecord]] + /\ UNCHANGED <> + +Mutate(origin, key, value) == + /\ phase = "faulting" + /\ active[origin] + /\ head[origin] < MaxSeq + /\ LET next == head[origin] + 1 + nextFloor == IF next - floor[origin] + 1 > OplogBound + THEN floor[origin] + 1 + ELSE floor[origin] + IN /\ history' = + [history EXCEPT + ![origin][next] = [key |-> key, value |-> value]] + /\ truth' = [truth EXCEPT ![origin][key] = value] + /\ head' = [head EXCEPT ![origin] = next] + /\ floor' = [floor EXCEPT ![origin] = nextFloor] + /\ UNCHANGED <> + +SendHello(origin, receiver) == + /\ origin # receiver + /\ Cardinality(messages) < MaxMessages + /\ messages' = + messages \union + {[kind |-> "hello", + from |-> origin, + to |-> receiver, + wireGeneration |-> generation[origin], + wireRevision |-> revision[origin], + wireEpoch |-> epoch[origin], + wireActive |-> active[origin]]} + /\ UNCHANGED <> + +SendDelta(origin, receiver, seq) == + /\ origin # receiver + /\ active[origin] + /\ seq \in floor[origin]..head[origin] + /\ Cardinality(messages) < MaxMessages + /\ LET record == history[origin][seq] + IN messages' = + messages \union + {[kind |-> "delta", + from |-> origin, + to |-> receiver, + wireGeneration |-> generation[origin], + wireRevision |-> revision[origin], + wireEpoch |-> epoch[origin], + seq |-> seq, + key |-> record.key, + value |-> record.value]} + /\ UNCHANGED <> + +SendSnapshot(origin, receiver) == + /\ origin # receiver + /\ active[origin] + /\ Cardinality(messages) < MaxMessages + /\ messages' = + messages \union + {[kind |-> "snapshot", + from |-> origin, + to |-> receiver, + wireGeneration |-> generation[origin], + wireRevision |-> revision[origin], + wireEpoch |-> epoch[origin], + seq |-> head[origin], + state |-> truth[origin]]} + /\ UNCHANGED <> + +FreshHello(message) == + \/ message.wireGeneration > authorityGeneration[message.to][message.from] + \/ /\ message.wireGeneration = + authorityGeneration[message.to][message.from] + /\ message.wireRevision >= authorityRevision[message.to][message.from] + +DeliverHello(message) == + /\ message.kind = "hello" + /\ LET changed == + \/ message.wireGeneration # + authorityGeneration[message.to][message.from] + \/ message.wireRevision # + authorityRevision[message.to][message.from] + \/ message.wireEpoch # + authorityEpoch[message.to][message.from] + \/ message.wireActive # + authorityActive[message.to][message.from] + install == FreshHello(message) + IN /\ authorityGeneration' = + IF install + THEN [authorityGeneration EXCEPT + ![message.to][message.from] = message.wireGeneration] + ELSE authorityGeneration + /\ authorityRevision' = + IF install + THEN [authorityRevision EXCEPT + ![message.to][message.from] = message.wireRevision] + ELSE authorityRevision + /\ authorityEpoch' = + IF install + THEN [authorityEpoch EXCEPT + ![message.to][message.from] = message.wireEpoch] + ELSE authorityEpoch + /\ authorityActive' = + IF install + THEN [authorityActive EXCEPT + ![message.to][message.from] = message.wireActive] + ELSE authorityActive + /\ cursor' = + IF install /\ (changed \/ ~message.wireActive) + THEN [cursor EXCEPT ![message.to][message.from] = 0] + ELSE cursor + /\ replica' = + IF install /\ (changed \/ ~message.wireActive) + THEN [replica EXCEPT + ![message.to][message.from] = EmptyView] + ELSE replica + /\ UNCHANGED <> + +ValidData(message) == + /\ authorityGeneration[message.to][message.from] = message.wireGeneration + /\ authorityRevision[message.to][message.from] = message.wireRevision + /\ authorityEpoch[message.to][message.from] = message.wireEpoch + /\ authorityActive[message.to][message.from] + +DeliverDelta(message) == + /\ message.kind = "delta" + /\ IF ValidData(message) /\ + message.seq = cursor[message.to][message.from] + 1 + THEN /\ cursor' = + [cursor EXCEPT + ![message.to][message.from] = message.seq] + /\ replica' = + [replica EXCEPT + ![message.to][message.from][message.key] = message.value] + ELSE /\ UNCHANGED cursor + /\ UNCHANGED replica + /\ UNCHANGED <> + +DeliverSnapshot(message) == + /\ message.kind = "snapshot" + /\ IF ValidData(message) /\ + message.seq >= cursor[message.to][message.from] + THEN /\ cursor' = + [cursor EXCEPT + ![message.to][message.from] = message.seq] + /\ replica' = + [replica EXCEPT + ![message.to][message.from] = message.state] + ELSE /\ UNCHANGED cursor + /\ UNCHANGED replica + /\ UNCHANGED <> + +Deliver(message) == + /\ message \in messages + /\ \/ DeliverHello(message) + \/ DeliverDelta(message) + \/ DeliverSnapshot(message) + +Drop(message) == + /\ phase = "faulting" + /\ message \in messages + /\ messages' = messages \ {message} + /\ UNCHANGED <> + +Heal == + /\ phase = "faulting" + /\ phase' = "healed" + /\ UNCHANGED <> + +(* +After healing, Repair represents one fair successful hello/head/need response. +If the retained prefix no longer contains the next sequence, it performs an +exact snapshot replacement. Otherwise it applies exactly the next delta. +*) +Repair(receiver, origin) == + /\ phase = "healed" + /\ receiver # origin + /\ ~PairConverged(receiver, origin) + /\ IF ~CurrentAuthority(receiver, origin) + THEN /\ authorityGeneration' = + [authorityGeneration EXCEPT + ![receiver][origin] = generation[origin]] + /\ authorityRevision' = + [authorityRevision EXCEPT + ![receiver][origin] = revision[origin]] + /\ authorityEpoch' = + [authorityEpoch EXCEPT + ![receiver][origin] = epoch[origin]] + /\ authorityActive' = + [authorityActive EXCEPT + ![receiver][origin] = active[origin]] + /\ cursor' = [cursor EXCEPT ![receiver][origin] = 0] + /\ replica' = + [replica EXCEPT ![receiver][origin] = EmptyView] + ELSE IF ~active[origin] + THEN /\ UNCHANGED <> + /\ cursor' = [cursor EXCEPT ![receiver][origin] = 0] + /\ replica' = + [replica EXCEPT ![receiver][origin] = EmptyView] + ELSE IF cursor[receiver][origin] + 1 < floor[origin] + THEN /\ UNCHANGED <> + /\ cursor' = + [cursor EXCEPT ![receiver][origin] = head[origin]] + /\ replica' = + [replica EXCEPT ![receiver][origin] = truth[origin]] + ELSE LET next == cursor[receiver][origin] + 1 + record == history[origin][next] + IN /\ UNCHANGED <> + /\ cursor' = [cursor EXCEPT ![receiver][origin] = next] + /\ replica' = + [replica EXCEPT + ![receiver][origin][record.key] = record.value] + /\ UNCHANGED <> + +Next == + \/ \E origin \in Origins : Open(origin) + \/ \E origin \in Origins : Close(origin) + \/ \E origin \in Origins : Restart(origin) + \/ \E origin \in Origins, key \in Keys, value \in BOOLEAN : + Mutate(origin, key, value) + \/ \E origin \in Origins, receiver \in Nodes : + SendHello(origin, receiver) + \/ \E origin \in Origins, receiver \in Nodes, seq \in Seq : + SendDelta(origin, receiver, seq) + \/ \E origin \in Origins, receiver \in Nodes : + SendSnapshot(origin, receiver) + \/ \E message \in messages : Deliver(message) + \/ \E message \in messages : Drop(message) + \/ Heal + \/ \E receiver \in Nodes, origin \in Origins : + Repair(receiver, origin) + +TypeOK == + /\ phase \in {"faulting", "healed"} + /\ generation \in [Nodes -> Generations] + /\ revision \in [Nodes -> Revisions] + /\ epoch \in [Nodes -> Epochs] + /\ active \in [Nodes -> BOOLEAN] + /\ truth \in [Nodes -> BoolMap] + /\ head \in [Nodes -> 0..MaxSeq] + /\ floor \in [Nodes -> 1..(MaxSeq + 1)] + /\ history \in [Nodes -> [Seq -> [key : Keys, value : BOOLEAN]]] + /\ authorityGeneration \in [Nodes -> [Nodes -> Generations]] + /\ authorityRevision \in [Nodes -> [Nodes -> Revisions]] + /\ authorityEpoch \in [Nodes -> [Nodes -> Epochs]] + /\ authorityActive \in [Nodes -> [Nodes -> BOOLEAN]] + /\ cursor \in [Nodes -> [Nodes -> 0..MaxSeq]] + /\ replica \in [Nodes -> [Nodes -> BoolMap]] + /\ messages \subseteq Message + +BoundedJournal == + \A origin \in Origins : + /\ floor[origin] <= head[origin] + 1 + /\ head[origin] - floor[origin] + 1 <= OplogBound + +CurrentReplicaIsAStreamPrefix == + \A receiver \in Nodes : + \A origin \in Origins : + IF receiver # origin /\ + CurrentAuthority(receiver, origin) /\ + active[origin] + THEN /\ cursor[receiver][origin] <= head[origin] + /\ replica[receiver][origin] = + Replay(history[origin], cursor[receiver][origin]) + ELSE TRUE + +HealedConvergence == + phase = "healed" ~> Converged + +Spec == + /\ Init + /\ [][Next]_vars + /\ \A receiver \in Nodes : + \A origin \in Origins : + WF_vars(Repair(receiver, origin)) + +============================================================================= diff --git a/test/formal/GroupAntiEntropyExtended.cfg b/test/formal/GroupAntiEntropyExtended.cfg new file mode 100644 index 0000000..25d4ff5 --- /dev/null +++ b/test/formal/GroupAntiEntropyExtended.cfg @@ -0,0 +1,17 @@ +SPECIFICATION Spec + +CONSTANTS + Nodes = {n1, n2, n3} + Origins = {n1} + Keys = {k1, k2} + MaxSeq = 3 + OplogBound = 2 + MaxMessages = 1 + +INVARIANTS + TypeOK + BoundedJournal + CurrentReplicaIsAStreamPrefix + +PROPERTY + HealedConvergence diff --git a/test/formal/PeerEviction.cfg b/test/formal/PeerEviction.cfg new file mode 100644 index 0000000..3a3017a --- /dev/null +++ b/test/formal/PeerEviction.cfg @@ -0,0 +1,15 @@ +SPECIFICATION Spec + +CONSTANTS + Receivers = {r1, r2} + Keys = {k1, k2} + MaxGeneration = 2 + MaxMessages = 2 + +INVARIANTS + TypeOK + VisibleRowsMatchInstalledGeneration + NoRowsWithoutAuthority + +PROPERTY + HealedConvergence diff --git a/test/formal/PeerEviction.tla b/test/formal/PeerEviction.tla new file mode 100644 index 0000000..35b98ad --- /dev/null +++ b/test/formal/PeerEviction.tla @@ -0,0 +1,207 @@ +--------------------------- MODULE PeerEviction --------------------------- +EXTENDS Integers, FiniteSets, TLC + +(* +Finite model of the peer lease and reincarnation boundary. An origin may +disappear permanently or restart at a larger generation while arbitrary old +hello and snapshot messages remain in the network. A receiver may expire its +lease at any point. Once the faulting prefix ends, fair repair must either +install the current generation exactly or erase the permanently absent peer. +*) + +CONSTANTS Receivers, Keys, MaxGeneration, MaxMessages + +ASSUME /\ IsFiniteSet(Receivers) + /\ Cardinality(Receivers) >= 1 + /\ IsFiniteSet(Keys) + /\ Cardinality(Keys) >= 1 + /\ MaxGeneration >= 2 + /\ MaxMessages >= 1 + +Generations == 1..MaxGeneration +Token == Generations \X Keys + +HelloMessage == + [kind : {"hello"}, to : Receivers, wireGeneration : Generations, + wireUp : BOOLEAN] + +SnapshotMessage == + [kind : {"snapshot"}, to : Receivers, wireGeneration : Generations, + rows : SUBSET Token] + +Message == HelloMessage \union SnapshotMessage + +VARIABLES phase, up, generation, truth, authorityGeneration, authorityUp, view, messages + +vars == + <> + +Init == + /\ phase = "faulting" + /\ up = TRUE + /\ generation = 1 + /\ truth = {} + /\ authorityGeneration = [receiver \in Receivers |-> 0] + /\ authorityUp = [receiver \in Receivers |-> FALSE] + /\ view = [receiver \in Receivers |-> {}] + /\ messages = {} + +Write(key, present) == + /\ phase = "faulting" + /\ up + /\ IF present + THEN truth' = truth \union {<>} + ELSE truth' = truth \ {<>} + /\ UNCHANGED + <> + +Crash == + /\ phase = "faulting" + /\ up + /\ up' = FALSE + /\ truth' = {} + /\ UNCHANGED + <> + +Restart == + /\ phase = "faulting" + /\ ~up + /\ generation < MaxGeneration + /\ up' = TRUE + /\ generation' = generation + 1 + /\ truth' = {} + /\ UNCHANGED <> + +SendHello(receiver) == + /\ phase = "faulting" + /\ Cardinality(messages) < MaxMessages + /\ messages' = messages \union + {[kind |-> "hello", to |-> receiver, + wireGeneration |-> generation, wireUp |-> up]} + /\ UNCHANGED + <> + +SendSnapshot(receiver) == + /\ phase = "faulting" + /\ up + /\ Cardinality(messages) < MaxMessages + /\ messages' = messages \union + {[kind |-> "snapshot", to |-> receiver, + wireGeneration |-> generation, rows |-> truth]} + /\ UNCHANGED + <> + +DeliverHello(message) == + /\ message.kind = "hello" + /\ IF message.wireGeneration >= authorityGeneration[message.to] + THEN /\ authorityGeneration' = + [authorityGeneration EXCEPT ![message.to] = message.wireGeneration] + /\ authorityUp' = [authorityUp EXCEPT ![message.to] = message.wireUp] + /\ view' = + IF message.wireGeneration # authorityGeneration[message.to] + \/ ~message.wireUp + THEN [view EXCEPT ![message.to] = {}] + ELSE view + ELSE /\ UNCHANGED authorityGeneration + /\ UNCHANGED authorityUp + /\ UNCHANGED view + /\ UNCHANGED <> + +DeliverSnapshot(message) == + /\ message.kind = "snapshot" + /\ IF message.wireGeneration = authorityGeneration[message.to] + /\ authorityUp[message.to] + THEN view' = [view EXCEPT ![message.to] = message.rows] + ELSE UNCHANGED view + /\ UNCHANGED + <> + +Deliver(message) == + /\ phase = "faulting" + /\ message \in messages + /\ \/ DeliverHello(message) + \/ DeliverSnapshot(message) + +Drop(message) == + /\ phase = "faulting" + /\ message \in messages + /\ messages' = messages \ {message} + /\ UNCHANGED + <> + +ExpireLease(receiver) == + /\ phase = "faulting" + /\ authorityGeneration' = [authorityGeneration EXCEPT ![receiver] = 0] + /\ authorityUp' = [authorityUp EXCEPT ![receiver] = FALSE] + /\ view' = [view EXCEPT ![receiver] = {}] + /\ UNCHANGED <> + +Heal == + /\ phase = "faulting" + /\ phase' = "healed" + /\ UNCHANGED + <> + +Repair(receiver) == + /\ phase = "healed" + /\ IF up + THEN /\ authorityGeneration' = + [authorityGeneration EXCEPT ![receiver] = generation] + /\ authorityUp' = [authorityUp EXCEPT ![receiver] = TRUE] + /\ view' = [view EXCEPT ![receiver] = truth] + ELSE /\ authorityGeneration' = + [authorityGeneration EXCEPT ![receiver] = 0] + /\ authorityUp' = [authorityUp EXCEPT ![receiver] = FALSE] + /\ view' = [view EXCEPT ![receiver] = {}] + /\ UNCHANGED <> + +Next == + \/ \E key \in Keys, present \in BOOLEAN : Write(key, present) + \/ Crash + \/ Restart + \/ \E receiver \in Receivers : SendHello(receiver) + \/ \E receiver \in Receivers : SendSnapshot(receiver) + \/ \E message \in messages : Deliver(message) + \/ \E message \in messages : Drop(message) + \/ \E receiver \in Receivers : ExpireLease(receiver) + \/ Heal + \/ \E receiver \in Receivers : Repair(receiver) + +TypeOK == + /\ phase \in {"faulting", "healed"} + /\ up \in BOOLEAN + /\ generation \in Generations + /\ truth \subseteq Token + /\ authorityGeneration \in [Receivers -> 0..MaxGeneration] + /\ authorityUp \in [Receivers -> BOOLEAN] + /\ view \in [Receivers -> SUBSET Token] + /\ messages \subseteq Message + +VisibleRowsMatchInstalledGeneration == + \A receiver \in Receivers : + \A token \in view[receiver] : token[1] = authorityGeneration[receiver] + +NoRowsWithoutAuthority == + \A receiver \in Receivers : + (authorityGeneration[receiver] = 0 \/ ~authorityUp[receiver]) + => view[receiver] = {} + +Converged == + \A receiver \in Receivers : + IF up + THEN /\ authorityGeneration[receiver] = generation + /\ authorityUp[receiver] + /\ view[receiver] = truth + ELSE /\ authorityGeneration[receiver] = 0 + /\ ~authorityUp[receiver] + /\ view[receiver] = {} + +HealedConvergence == phase = "healed" ~> Converged + +Spec == + /\ Init + /\ [][Next]_vars + /\ \A receiver \in Receivers : WF_vars(Repair(receiver)) + +============================================================================= diff --git a/test/formal/README.md b/test/formal/README.md new file mode 100644 index 0000000..7eae990 --- /dev/null +++ b/test/formal/README.md @@ -0,0 +1,109 @@ +# Group anti-entropy formal model + +`GroupAntiEntropy.tla` is an independent finite-state model of the replica +contract. It covers: + +- generation and named-cluster epoch fencing; +- arbitrary finite frame loss, duplication, and reordering; +- contiguous sequence application; +- bounded oplog pruning; +- exact per-origin snapshot fallback; and +- fair convergence after healing. + +`SnapshotAssembly.tla` separately models the non-atomic wire delivery of an +exact snapshot. It explores independent provisional-chunk and terminal-commit +loss, duplication, and reordering, source invalidation before commit emission, +newer-snapshot supersession, authority epoch changes, staging expiry, and +receiver crashes. Its invariants require visible data and the cursor to remain +at a previously committed exact state until every chunk and a valid terminal +commit for one snapshot are present; stale or mixed partial state can never +become visible. + +`PeerEviction.tla` isolates the lifecycle boundary for a peer which never +returns and for a later process using the same node name with a fresh +generation. During its finite faulty prefix it retains and reorders stale +hello and snapshot messages while leases expire. Authority consists of both a +generation and an active bit, so an inactive hello fences even a same-epoch +snapshot. After healing, fair repair must either install only the current +generation or erase every row and authority reference for the absent peer. + +`AuthorityProjection.tla` models concurrent exact remote installs, local named +cluster activation/deactivation, materialized rows, and a lifecycle caller that +may disappear after the durable mutation. Its safety invariants require local +activation and shared routing to project authority consistently; its liveness +property requires queued close cleanup to finish without the original caller. + +`AuthorityHint.tla` models the cross-lane fence created when a heartbeat or lane +hello observes a newer authority before the exact hello arrives. It checks that +delayed old exact/view installs cannot re-enable a lane, unresolved hints retain +a bounded lease/repair obligation, contiguous incremental authority is installed +only from the currently hinted/applied revision, unknown post-retirement hints +cannot establish authority, and delayed retirement cleanup cannot erase a +rediscovered generation's route. + +The default TLC configuration uses three nodes: one origin and two independent +receivers. The origin has one key, a two-record stream, a one-record oplog, and +the system retains one arbitrary network frame. This forces delta repair, +snapshot fallback, stale-frame fencing, and independent recovery at both +receivers. The retained frame may be redelivered for duplication, while +nondeterministic sequence selection and delivery model out-of-order arrival +without paying the state-space cost of every two-frame set. + +The TLA+ protocol state is deliberately factored per origin: no transition for +one origin reads or writes another origin's stream. Checking multiple origins +in this model therefore forms a Cartesian product of the same state machine +rather than adding an interaction. Concurrent A/C authority, registry conflict +projection, and preservation of C-owned state while A recovers are instead +driven against three real BEAM nodes by `replica_model_property_test.exs`. + +Run it with Java 17 or later and a current `tla2tools.jar`: + +```bash +TLA_JAR=/path/to/tla2tools.jar test/formal/check.sh + +TLA_JAR=/path/to/tla2tools.jar \ + TLA_SPEC="$PWD/test/formal/SnapshotAssembly.tla" \ + TLA_CONFIG="$PWD/test/formal/SnapshotAssembly.cfg" \ + test/formal/check.sh + +# Run all default models +TLA_JAR=/path/to/tla2tools.jar test/formal/check_matrix.sh + +# Also run the larger two-key, three-sequence anti-entropy state space +TLA_JAR=/path/to/tla2tools.jar TLA_EXTENDED=1 test/formal/check_matrix.sh +``` + +`TLC_WORKERS` controls worker concurrency and defaults to 4. `TLA_CONFIG` can +point at an alternate finite configuration. + +TLC proves the listed invariants and liveness property for the configured +finite instance, not for arbitrary unbounded node and key sets. Larger models +should be run periodically by increasing `Nodes`, `Origins`, `Keys`, `MaxSeq`, +`OplogBound`, and `MaxMessages`. `check_matrix.sh` runs the protocol, snapshot +assembly, peer-eviction, authority-projection, and authority-hint models; set +`TLA_EXTENDED=1` for the larger anti-entropy configuration. + +The checked three-node default explores 1,835,826 states, finds 490,236 +distinct states to a depth of 30, and completes in roughly one minute on the +development machine used for the validation run. + +The snapshot-assembly model generates 3,305,473 states, finds 167,936 distinct +states to a depth of 23, and completes in roughly five seconds on the current +development machine. + +The peer-eviction model explores 1,527,116 states, finds 238,120 distinct +states to a depth of 26, and completes in roughly 20 seconds on the development +machine used for validation. + +The authority-projection model explores 71 states, finds 24 distinct states to +a depth of 6, and completes in under a second. Its small state space is +intentional: it exhaustively crosses the two authority directions, lifecycle +caller loss/replacement, writes, and independently fair close cleanup. + +The authority-hint model explores 2,989 states, finds 428 distinct states to a +depth of 9, and completes in roughly one second. It separates the last exact +revision from the complete applied revision and highest persisted hint. + +The extended two-key, three-sequence model explores 127,557,634 states, finds +32,238,304 distinct states to a depth of 34, and completes in roughly two hours +on the development machine used for validation. diff --git a/test/formal/SnapshotAssembly.cfg b/test/formal/SnapshotAssembly.cfg new file mode 100644 index 0000000..6c7706f --- /dev/null +++ b/test/formal/SnapshotAssembly.cfg @@ -0,0 +1,7 @@ +SPECIFICATION Spec + +INVARIANTS + TypeOK + VisibleIsAnExactCommittedSnapshot + StagingBelongsToOneSnapshot + NoCommitMeansNoInstall diff --git a/test/formal/SnapshotAssembly.tla b/test/formal/SnapshotAssembly.tla new file mode 100644 index 0000000..ce75e21 --- /dev/null +++ b/test/formal/SnapshotAssembly.tla @@ -0,0 +1,225 @@ +------------------------- MODULE SnapshotAssembly ------------------------- +EXTENDS Integers, FiniteSets, TLC + +(* +Finite model of the exact-snapshot assembly boundary. Snapshot chunks are +provisional: a separately lossy, duplicable, and reorderable terminal commit +is required before a complete candidate can replace visible state. Two +snapshots share an authority epoch and one belongs to a replacement epoch so +TLC also explores supersession, expiry, receiver crashes, and stale messages. +*) + +Snapshots == {1, 2, 3} +Chunks == {1, 2} +Rows == {"a", "b", "c", "d"} + +SnapshotEpoch(snapshot) == + CASE snapshot = 0 -> 0 + [] snapshot = 1 -> 1 + [] snapshot = 2 -> 1 + [] snapshot = 3 -> 2 + +SnapshotSeq(snapshot) == + CASE snapshot = 0 -> 0 + [] snapshot = 1 -> 1 + [] snapshot = 2 -> 2 + [] snapshot = 3 -> 1 + +SnapshotRows(snapshot) == + CASE snapshot = 1 -> {"a", "b"} + [] snapshot = 2 -> {"c", "d"} + [] snapshot = 3 -> {"a", "d"} + +ChunkRows(snapshot, chunk) == + CASE snapshot = 1 /\ chunk = 1 -> {"a"} + [] snapshot = 1 /\ chunk = 2 -> {"b"} + [] snapshot = 2 /\ chunk = 1 -> {"c"} + [] snapshot = 2 /\ chunk = 2 -> {"d"} + [] snapshot = 3 /\ chunk = 1 -> {"a"} + [] snapshot = 3 /\ chunk = 2 -> {"d"} + +Message == + [kind : {"chunk", "commit"}, snapshot : Snapshots, chunk : {0} \union Chunks] + +VARIABLES authorityEpoch, + cursor, + visible, + stagedSnapshot, + stagedChunks, + stagedRows, + stagedCommitted, + commitAllowed, + messages + +vars == + <> + +Init == + /\ authorityEpoch = 1 + /\ cursor = 0 + /\ visible = {} + /\ stagedSnapshot = 0 + /\ stagedChunks = {} + /\ stagedRows = {} + /\ stagedCommitted = FALSE + /\ commitAllowed = Snapshots + /\ messages = {} + +SendChunk(snapshot, chunk) == + /\ messages' = messages \union + {[kind |-> "chunk", snapshot |-> snapshot, chunk |-> chunk]} + /\ UNCHANGED <> + +SendCommit(snapshot) == + /\ snapshot \in commitAllowed + /\ messages' = messages \union + {[kind |-> "commit", snapshot |-> snapshot, chunk |-> 0]} + /\ UNCHANGED <> + +InvalidateBeforeCommit(snapshot) == + /\ snapshot \in commitAllowed + /\ commitAllowed' = commitAllowed \ {snapshot} + /\ UNCHANGED <> + +Valid(snapshot) == + /\ SnapshotEpoch(snapshot) = authorityEpoch + /\ SnapshotSeq(snapshot) > cursor + +StartsNewAssembly(snapshot) == + /\ Valid(snapshot) + /\ \/ stagedSnapshot = 0 + \/ SnapshotEpoch(stagedSnapshot) # authorityEpoch + \/ SnapshotSeq(snapshot) > SnapshotSeq(stagedSnapshot) + +StartChunk(message) == + /\ message.kind = "chunk" + /\ StartsNewAssembly(message.snapshot) + /\ stagedSnapshot' = message.snapshot + /\ stagedChunks' = {message.chunk} + /\ stagedRows' = ChunkRows(message.snapshot, message.chunk) + /\ stagedCommitted' = FALSE + /\ UNCHANGED <> + +StartCommit(message) == + /\ message.kind = "commit" + /\ StartsNewAssembly(message.snapshot) + /\ stagedSnapshot' = message.snapshot + /\ stagedChunks' = {} + /\ stagedRows' = {} + /\ stagedCommitted' = TRUE + /\ UNCHANGED <> + +ContinueChunk(message) == + /\ message.kind = "chunk" + /\ Valid(message.snapshot) + /\ stagedSnapshot = message.snapshot + /\ LET nextChunks == stagedChunks \union {message.chunk} + nextRows == stagedRows \union ChunkRows(message.snapshot, message.chunk) + IN IF stagedCommitted /\ nextChunks = Chunks + THEN /\ cursor' = SnapshotSeq(message.snapshot) + /\ visible' = SnapshotRows(message.snapshot) + /\ stagedSnapshot' = 0 + /\ stagedChunks' = {} + /\ stagedRows' = {} + /\ stagedCommitted' = FALSE + ELSE /\ UNCHANGED <> + /\ stagedChunks' = nextChunks + /\ stagedRows' = nextRows + /\ UNCHANGED <> + +ContinueCommit(message) == + /\ message.kind = "commit" + /\ Valid(message.snapshot) + /\ stagedSnapshot = message.snapshot + /\ IF stagedChunks = Chunks + THEN /\ cursor' = SnapshotSeq(message.snapshot) + /\ visible' = SnapshotRows(message.snapshot) + /\ stagedSnapshot' = 0 + /\ stagedChunks' = {} + /\ stagedRows' = {} + /\ stagedCommitted' = FALSE + ELSE /\ stagedCommitted' = TRUE + /\ UNCHANGED <> + /\ UNCHANGED <> + +Ignore(message) == + /\ ~StartsNewAssembly(message.snapshot) + /\ ~(/\ Valid(message.snapshot) + /\ stagedSnapshot = message.snapshot) + /\ UNCHANGED vars + +Deliver(message) == + /\ message \in messages + /\ \/ StartChunk(message) + \/ StartCommit(message) + \/ ContinueChunk(message) + \/ ContinueCommit(message) + \/ Ignore(message) + +Drop(message) == + /\ message \in messages + /\ messages' = messages \ {message} + /\ UNCHANGED <> + +InstallNewAuthority == + /\ authorityEpoch = 1 + /\ authorityEpoch' = 2 + /\ cursor' = 0 + /\ visible' = {} + (* Invisible old staging may remain until expiry, but can never commit. *) + /\ UNCHANGED <> + +DiscardStaging == + /\ stagedSnapshot # 0 + /\ stagedSnapshot' = 0 + /\ stagedChunks' = {} + /\ stagedRows' = {} + /\ stagedCommitted' = FALSE + /\ UNCHANGED <> + +Next == + \/ \E snapshot \in Snapshots, chunk \in Chunks : SendChunk(snapshot, chunk) + \/ \E snapshot \in Snapshots : SendCommit(snapshot) + \/ \E snapshot \in Snapshots : InvalidateBeforeCommit(snapshot) + \/ \E message \in messages : Deliver(message) + \/ \E message \in messages : Drop(message) + \/ InstallNewAuthority + \/ DiscardStaging + +TypeOK == + /\ authorityEpoch \in {1, 2} + /\ cursor \in 0..2 + /\ visible \subseteq Rows + /\ stagedSnapshot \in {0} \union Snapshots + /\ stagedChunks \subseteq Chunks + /\ stagedRows \subseteq Rows + /\ stagedCommitted \in BOOLEAN + /\ commitAllowed \subseteq Snapshots + /\ messages \subseteq Message + +VisibleIsAnExactCommittedSnapshot == + \/ /\ authorityEpoch = 1 + /\ \/ /\ cursor = 0 /\ visible = {} + \/ /\ cursor = 1 /\ visible = SnapshotRows(1) + \/ /\ cursor = 2 /\ visible = SnapshotRows(2) + \/ /\ authorityEpoch = 2 + /\ \/ /\ cursor = 0 /\ visible = {} + \/ /\ cursor = 1 /\ visible = SnapshotRows(3) + +StagingBelongsToOneSnapshot == + stagedSnapshot # 0 => + stagedRows = UNION {ChunkRows(stagedSnapshot, chunk) : chunk \in stagedChunks} + +NoCommitMeansNoInstall == + stagedSnapshot # 0 /\ ~stagedCommitted => + SnapshotSeq(stagedSnapshot) > cursor + +Spec == Init /\ [][Next]_vars + +============================================================================= diff --git a/test/formal/check.sh b/test/formal/check.sh new file mode 100755 index 0000000..1f5b1c5 --- /dev/null +++ b/test/formal/check.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ -z "${TLA_JAR:-}" ]]; then + echo "TLA_JAR must point to tla2tools.jar" >&2 + exit 2 +fi + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +metadir="${repo_root}/tmp/tlc" +config="${TLA_CONFIG:-${repo_root}/test/formal/GroupAntiEntropy.cfg}" +spec="${TLA_SPEC:-${repo_root}/test/formal/GroupAntiEntropy.tla}" +mkdir -p "${metadir}" + +exec java -XX:+UseParallelGC -cp "${TLA_JAR}" tlc2.TLC \ + -cleanup \ + -metadir "${metadir}" \ + -workers "${TLC_WORKERS:-4}" \ + -config "${config}" \ + "${spec}" diff --git a/test/formal/check_matrix.sh b/test/formal/check_matrix.sh new file mode 100755 index 0000000..19521e9 --- /dev/null +++ b/test/formal/check_matrix.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +: "${TLA_JAR:?TLA_JAR must point to tla2tools.jar}" + +run_check() { + local spec="$1" + local config="$2" + + TLA_SPEC="${script_dir}/${spec}.tla" \ + TLA_CONFIG="${script_dir}/${config}.cfg" \ + "${script_dir}/check.sh" +} + +run_check GroupAntiEntropy GroupAntiEntropy +run_check SnapshotAssembly SnapshotAssembly +run_check PeerEviction PeerEviction +run_check AuthorityProjection AuthorityProjection +run_check AuthorityHint AuthorityHint + +if [[ "${TLA_EXTENDED:-0}" == "1" ]]; then + run_check GroupAntiEntropy GroupAntiEntropyExtended +fi diff --git a/test/group_test.exs b/test/group_test.exs index d0c1c22..7751fba 100644 --- a/test/group_test.exs +++ b/test/group_test.exs @@ -2,22 +2,6 @@ defmodule GroupTest.ExtractMeta do def strip(meta), do: Map.take(meta, [:public]) end -defmodule GroupTest.ResolveRegistryConflict do - def pick( - _name, - _key, - {local_pid, _local_meta, _local_time}, - {remote_pid, _remote_meta, _remote_time}, - winner - ) do - case winner do - :local -> local_pid - :remote -> remote_pid - :none -> :none - end - end -end - defmodule GroupTest do use ExUnit.Case, async: true @@ -49,6 +33,67 @@ defmodule GroupTest do end end + describe "replica ingress fairness" do + test "an oversized incoming batch yields to an already queued local write", %{name: name} do + key = + 1..1_000 + |> Enum.map(&"ingress-fairness/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(nil, &1, 4) == 0)) + + shard = Process.whereis(Group.Replica.shard_name(name, 0)) + :ok = :sys.suspend(shard) + + on_exit(fn -> + if Process.alive?(shard), do: :erlang.trace(shard, false, [:call]) + :erlang.trace_pattern({Group.Replica, :handle_replica_message, 3}, false, [:local]) + Group.TestCluster.resume_if_alive(shard) + end) + + parent = self() + owner = spawn(fn -> replica_ingress_fairness_owner(parent) end) + + :erlang.trace(shard, true, [:call, {:tracer, owner}]) + :erlang.trace_pattern({Group.Replica, :handle_replica_message, 3}, true, [:local]) + + source_node = :"ingress-source@test" + batch_size = 65 + messages = List.duplicate({:malformed_replica_message, make_ref()}, batch_size) + + assert :ok = Group.Transport.incoming_batch(name, source_node, 0, messages) + + epoch = Group.Replica.Data.local_cluster_epoch(name, nil) + send(owner, {:write, shard, {:register, nil, epoch, key, owner, %{kind: :local}}}) + + wait_until(fn -> + match?( + {:message_queue_len, length} when length >= 2, + Process.info(shard, :message_queue_len) + ) + end) + + :ok = :sys.resume(shard) + + assert_receive {:local_write_finished, ^owner, :ok, calls_before_local}, 5_000 + assert calls_before_local < batch_size + assert match?({^owner, %{kind: :local}}, Group.lookup(name, key)) + end + end + + describe "monitor_generation/1" do + test "notifies long-lived owners when local membership storage exits", %{name: name} do + assert {:ok, generation_pid, monitor_ref} = Group.monitor_generation(name) + + Process.exit(generation_pid, :kill) + + assert_receive {:DOWN, ^monitor_ref, :process, ^generation_pid, :killed} + end + + test "returns not_running for an unknown Group" do + name = :"missing_group_#{System.unique_integer([:positive])}" + assert {:error, :not_running} = Group.monitor_generation(name) + end + end + describe "join/3 and leave/2" do test "joined process appears in members/2", %{name: name} do key = "chat/room/#{System.unique_integer([:positive])}" @@ -183,6 +228,42 @@ defmodule GroupTest do end end + describe "named-cluster mutation fencing" do + test "a shard rejects registry and PG writes after their cluster epoch retires", %{name: name} do + cluster = "retired/#{System.unique_integer([:positive])}" + registry_key = "retired/registry" + pg_key = "retired/pg" + + assert :ok = Group.connect(name, cluster) + old_epoch = Group.Replica.Data.local_cluster_epoch(name, cluster) + assert :ok = Group.disconnect(name, cluster) + assert :ok = Group.connect(name, cluster) + refute Group.Replica.Data.local_cluster_epoch(name, cluster) == old_epoch + + registry_shard = Group.Replica.shard_for(name, cluster, registry_key) + pg_shard = Group.Replica.shard_for(name, cluster, pg_key) + + registry_result = + Group.Replica.local_request( + registry_shard, + {:register, cluster, old_epoch, registry_key, self(), %{stale: true}}, + 5_000 + ) + + pg_result = + Group.Replica.local_request( + pg_shard, + {:join, cluster, old_epoch, pg_key, self(), %{stale: true}}, + 5_000 + ) + + assert registry_result == {:error, :stale_cluster_epoch} + assert pg_result == {:error, :stale_cluster_epoch} + assert Group.lookup(name, registry_key, cluster: cluster) == nil + assert Group.members(name, pg_key, cluster: cluster) == [] + end + end + describe "register/unregister" do test "register makes process discoverable via lookup", %{name: name} do key = "user/#{System.unique_integer([:positive])}" @@ -752,9 +833,20 @@ defmodule GroupTest do disconnect_queued? = Enum.any?(messages, fn - {:group_local_request, _alias, {:cluster_disconnect, [^cluster]}} -> true - {:group_local_request, _caller, _ref, {:cluster_disconnect, [^cluster]}} -> true - _ -> false + {:group_local_request, _alias, {:cluster_disconnect, [^cluster]}} -> + true + + {:group_local_request, _alias, {:cluster_disconnect, [^cluster], _epochs}} -> + true + + {:group_local_request, _caller, _ref, {:cluster_disconnect, [^cluster]}} -> + true + + {:group_local_request, _caller, _ref, {:cluster_disconnect, [^cluster], _epochs}} -> + true + + _ -> + false end) not Group.connected?(name, cluster) and disconnect_queued? @@ -1329,6 +1421,9 @@ defmodule GroupTest do assert :ok = Group.join(name, key, %{}, cluster: cluster) assert Group.members(name, key, cluster: cluster) == [{self(), %{}}] + remote_route = :"disconnect-timeout@remote" + :ok = Group.Replica.Data.add_cluster_node(name, [cluster], remote_route) + shard_zero = Group.Replica.shard_name(name, 0) :ok = :sys.suspend(shard_zero) @@ -1343,297 +1438,118 @@ defmodule GroupTest do :sys.get_state(shard_zero) refute Group.connected?(name, cluster) assert Group.members(name, key, cluster: cluster) == [] - end - end - describe "local request fairness" do - test "local register gets a turn ahead of replicated registry backlog" do - name = - start_single_shard_group( - replicated_registry_receiver_buffer_size: 1, - replicated_registry_receiver_flush_interval: 60_000 - ) - - shard = suspend_only_shard(name) - remote_pid = spawn_forever() - local_key = "fair/register/local/#{System.unique_integer([:positive])}" - backlog_prefix = "fair/register/backlog/#{System.unique_integer([:positive])}" - - on_exit(fn -> - kill_if_alive(remote_pid) + wait_until(fn -> + Group.Replica.Data.closed_local_clusters(name) == [] and + Group.Replica.Data.cluster_nodes(name, cluster) == [] end) - - enqueue_replicated_registry_backlog(shard, backlog_prefix, remote_pid, 1_000) - - caller = - spawn_requester( - fn -> - Group.register(name, local_key, %{local: true}) - end, - :local_register_result - ) - - on_exit(fn -> kill_if_alive(caller) end) - Process.sleep(20) - :ok = :sys.resume(shard) - - assert_receive {:local_register_result, ^caller, :ok}, 1_000 - assert shard_message_queue_len(shard) > 0 - assert Group.lookup(name, local_key) == {caller, %{local: true}} - wait_until(fn -> shard_message_queue_len(shard) == 0 end, 5_000) end - test "local join gets a turn ahead of replicated registry backlog" do - name = - start_single_shard_group( - replicated_registry_receiver_buffer_size: 1, - replicated_registry_receiver_flush_interval: 60_000 - ) - - shard = suspend_only_shard(name) - remote_pid = spawn_forever() - local_key = "fair/join-registry/local/#{System.unique_integer([:positive])}" - backlog_prefix = "fair/join-registry/backlog/#{System.unique_integer([:positive])}" - - on_exit(fn -> - kill_if_alive(remote_pid) - end) - - enqueue_replicated_registry_backlog(shard, backlog_prefix, remote_pid, 1_000) - - caller = - spawn_requester( - fn -> - Group.join(name, local_key, %{local: true}) - end, - :local_join_result - ) - - on_exit(fn -> kill_if_alive(caller) end) - Process.sleep(20) - :ok = :sys.resume(shard) - - assert_receive {:local_join_result, ^caller, :ok}, 1_000 - assert shard_message_queue_len(shard) > 0 - assert Group.members(name, local_key) == [{caller, %{local: true}}] - wait_until(fn -> shard_message_queue_len(shard) == 0 end, 5_000) - end - - test "local join gets a turn ahead of replicated PG backlog" do - name = - start_single_shard_group( - replicated_pg_receiver_buffer_size: 1, - replicated_pg_receiver_flush_interval: 60_000 - ) - - shard = suspend_only_shard(name) - remote_pid = spawn_forever() - local_key = "fair/join/local/#{System.unique_integer([:positive])}" - backlog_prefix = "fair/join/backlog/#{System.unique_integer([:positive])}" - - on_exit(fn -> - kill_if_alive(remote_pid) - end) - - enqueue_replicated_pg_backlog(shard, backlog_prefix, remote_pid, 1_000) - - caller = - spawn_requester( - fn -> - Group.join(name, local_key, %{local: true}) - end, - :local_join_result - ) - - on_exit(fn -> kill_if_alive(caller) end) - Process.sleep(20) - :ok = :sys.resume(shard) - - assert_receive {:local_join_result, ^caller, :ok}, 1_000 - assert shard_message_queue_len(shard) > 0 - assert Group.members(name, local_key) == [{caller, %{local: true}}] - wait_until(fn -> shard_message_queue_len(shard) == 0 end, 5_000) - end - - test "local connect and disconnect each get a turn ahead of replicated PG backlog" do - name = - start_single_shard_group( - replicated_pg_receiver_buffer_size: 1, - replicated_pg_receiver_flush_interval: 60_000 - ) - - shard = suspend_only_shard(name) - remote_pid = spawn_forever() - cluster = "fair/connect/#{System.unique_integer([:positive])}" - connect_prefix = "fair/connect/backlog/#{System.unique_integer([:positive])}" - - on_exit(fn -> - kill_if_alive(remote_pid) - end) - - enqueue_replicated_pg_backlog(shard, connect_prefix, remote_pid, 1_000) - - connect_caller = - spawn_requester( - fn -> - Group.connect(name, cluster) - end, - :local_connect_result - ) - - on_exit(fn -> kill_if_alive(connect_caller) end) - Process.sleep(20) - :ok = :sys.resume(shard) - - assert_receive {:local_connect_result, ^connect_caller, :ok}, 1_000 - assert shard_message_queue_len(shard) > 0 - assert Group.connected?(name, cluster) - wait_until(fn -> shard_message_queue_len(shard) == 0 end, 5_000) + test "reconnect waits for every old-epoch shard cleanup before admitting new writes" do + name = :"test_reconnect_barrier_#{System.unique_integer([:positive])}" + cluster = "reconnect_barrier" + key = "reconnect/barrier/#{System.unique_integer([:positive])}" + start_supervised!({Group, name: name, shards: 2, log: false}) - :ok = :sys.suspend(shard) + assert :ok = Group.connect(name, cluster) + shard_zero = Group.Replica.shard_name(name, 0) + :ok = :sys.suspend(shard_zero) - disconnect_prefix = "fair/disconnect/backlog/#{System.unique_integer([:positive])}" + reconnect_caller = + try do + assert_genserver_call_timeout(fn -> + Group.disconnect(name, cluster, timeout: 10) + end) - enqueue_replicated_pg_backlog(shard, disconnect_prefix, remote_pid, 1_000) + caller = + spawn_requester( + fn -> Group.connect(name, cluster) end, + :reconnect_barrier_result + ) - disconnect_caller = - spawn_requester( - fn -> - Group.disconnect(name, cluster) - end, - :local_disconnect_result - ) + refute_receive {:reconnect_barrier_result, ^caller, _result}, 50 + caller + after + resume_shard_if_alive(shard_zero) + end - on_exit(fn -> kill_if_alive(disconnect_caller) end) - Process.sleep(20) - :ok = :sys.resume(shard) + on_exit(fn -> kill_if_alive(reconnect_caller) end) + assert_receive {:reconnect_barrier_result, ^reconnect_caller, :ok}, 1_000 - assert_receive {:local_disconnect_result, ^disconnect_caller, :ok}, 1_000 - assert shard_message_queue_len(shard) > 0 - refute Group.connected?(name, cluster) - wait_until(fn -> shard_message_queue_len(shard) == 0 end, 5_000) + assert Group.Replica.Data.closed_local_clusters(name) == [] + assert :ok = Group.join(name, key, %{epoch: :new}, cluster: cluster) + assert Group.members(name, key, cluster: cluster) == [{self(), %{epoch: :new}}] + assert :ok = Group.TestCluster.assert_replica_consistent(name) end - test "fairness preserves FIFO within the local request lane" do - name = - start_single_shard_group( - replicated_pg_receiver_buffer_size: 1, - replicated_pg_receiver_flush_interval: 60_000 - ) - - shard = suspend_only_shard(name) - remote_pid = spawn_forever() - backlog_prefix = "fair/fifo/backlog/#{System.unique_integer([:positive])}" - key1 = "fair/fifo/local/#{System.unique_integer([:positive])}/1" - key2 = "fair/fifo/local/#{System.unique_integer([:positive])}/2" - - on_exit(fn -> - kill_if_alive(remote_pid) - end) - - enqueue_replicated_pg_backlog(shard, backlog_prefix, remote_pid, 1_000) - - caller1 = - spawn_requester( - fn -> - Group.join(name, key1, %{order: 1}) - end, - :fifo_result - ) - - Process.sleep(20) - - caller2 = - spawn_requester( - fn -> - Group.join(name, key2, %{order: 2}) - end, - :fifo_result - ) - - on_exit(fn -> - kill_if_alive(caller1) - kill_if_alive(caller2) - end) + test "a delayed duplicate disconnect cannot purge a reconnected epoch" do + name = start_single_shard_group() + cluster = "duplicate-disconnect/#{System.unique_integer([:positive])}" + reg_key = "duplicate-disconnect/registry/#{System.unique_integer([:positive])}" + pg_key = "duplicate-disconnect/pg/#{System.unique_integer([:positive])}" - Process.sleep(20) - :ok = :sys.resume(shard) + :ok = Group.connect(name, cluster) + old_epoch = Group.Replica.Data.local_cluster_epoch(name, cluster) + :ok = Group.disconnect(name, cluster) + :ok = Group.connect(name, cluster) + :ok = Group.register(name, reg_key, %{epoch: :new}, cluster: cluster) + :ok = Group.join(name, pg_key, %{epoch: :new}, cluster: cluster) + + # A concurrent second disconnect that observed the already-closed local + # epoch can queue either an unfenced request or the completed old epoch. + # Model both arriving only after reconnect. + for stale_epoch <- [nil, old_epoch] do + assert :ok = + Group.Replica.local_request( + Group.Replica.shard_name(name, 0), + {:cluster_disconnect, [cluster], [{cluster, stale_epoch}]}, + 5_000 + ) + + assert Group.connected?(name, cluster) + assert Group.lookup(name, reg_key, cluster: cluster) == {self(), %{epoch: :new}} + assert Group.members(name, pg_key, cluster: cluster) == [{self(), %{epoch: :new}}] + end - assert_receive {:fifo_result, ^caller1, :ok}, 1_000 - assert_receive {:fifo_result, ^caller2, :ok}, 1_000 - assert shard_message_queue_len(shard) > 0 - assert Group.members(name, key1) == [{caller1, %{order: 1}}] - assert Group.members(name, key2) == [{caller2, %{order: 2}}] - wait_until(fn -> shard_message_queue_len(shard) == 0 end, 5_000) + assert :ok = Group.TestCluster.assert_replica_consistent(name) end + end - test "configurable local fairness quota drains multiple local requests before older non-local work" do - name = - start_single_shard_group( - replicated_pg_receiver_buffer_size: 1, - replicated_pg_receiver_flush_interval: 60_000, - replicated_pg_receiver_local_request_quota: 2 - ) - + describe "local request fairness" do + test "a control flood yields to a queued local request after bounded work" do + name = start_single_shard_group() shard = suspend_only_shard(name) - remote_pid = spawn_forever() - remote_key1 = "fair/quota/remote/#{System.unique_integer([:positive])}/1" - remote_key2 = "fair/quota/remote/#{System.unique_integer([:positive])}/2" - local_key1 = "fair/quota/local/#{System.unique_integer([:positive])}/1" - local_key2 = "fair/quota/local/#{System.unique_integer([:positive])}/2" - local_key3 = "fair/quota/local/#{System.unique_integer([:positive])}/3" + remote_node = :"missing-control-peer@nohost" + key = "fair/control/local/#{System.unique_integer([:positive])}" - on_exit(fn -> - kill_if_alive(remote_pid) - end) - - send(shard, replicated_pg_join(nil, remote_key1, remote_pid, %{remote: 1}, :join)) - send(shard, replicated_pg_join(nil, remote_key2, remote_pid, %{remote: 2}, :join)) - send(shard, {:group_dispatch, [self()], {:quota_marker, shard}}) - - caller1 = - spawn_requester( - fn -> - Group.join(name, local_key1, %{order: 1}) - end, - :quota_result - ) + send( + shard, + {:group_replica_frame, remote_node, {:heads, Group.Replica.WireProtocol.version(), []}} + ) - caller2 = - spawn_requester( - fn -> - Group.join(name, local_key2, %{order: 2}) - end, - :quota_result - ) + for _ <- 1..10_000 do + send(shard, {:nodedown, remote_node}) + end - caller3 = - spawn_requester( - fn -> - Group.join(name, local_key3, %{order: 3}) - end, - :quota_result - ) + ref = make_ref() + epoch = Group.Replica.Data.generation(name) - on_exit(fn -> - kill_if_alive(caller1) - kill_if_alive(caller2) - kill_if_alive(caller3) - end) + send( + shard, + {:group_local_request, self(), ref, {:register, nil, epoch, key, self(), %{local: true}}} + ) - wait_until(fn -> shard_message_queue_len(shard) >= 6 end, 1_000) + wait_until(fn -> shard_message_queue_len(shard) >= 10_002 end, 2_000) :ok = :sys.resume(shard) - assert_receive {:quota_result, ^caller1, :ok}, 1_000 - assert_receive {:quota_result, ^caller2, :ok}, 1_000 - assert_receive {:quota_result, ^caller3, :ok}, 1_000 - assert_receive {:quota_marker, ^shard}, 1_000 + assert_receive {:group_local_reply, ^ref, :ok}, 100 + assert Group.lookup(name, key) == {self(), %{local: true}} - assert Group.members(name, remote_key1) == [{remote_pid, %{remote: 1}}] - assert Group.members(name, remote_key2) == [{remote_pid, %{remote: 2}}] - assert Group.members(name, local_key1) == [{caller1, %{order: 1}}] - assert Group.members(name, local_key2) == [{caller2, %{order: 2}}] - assert Group.members(name, local_key3) == [{caller3, %{order: 3}}] + # The assertion has already proved bounded yielding. Drop the synthetic + # duplicate-control tail instead of charging every PR for draining it. + old_shard = Process.whereis(shard) + Process.exit(old_shard, :kill) + wait_until(fn -> is_pid(Process.whereis(shard)) and Process.whereis(shard) != old_shard end) end test "local PG batching applies mixed join and leave requests correctly" do @@ -2273,7 +2189,12 @@ defmodule GroupTest do assert_receive {:replicated_registry_buffer_flushed, ^shard_name}, 1_000 :sys.replace_state(shard_pid, fn state -> - %{state | remote_shards: Map.put(state.remote_shards, node(), self())} + %{ + state + | remote_shards: Map.put(state.remote_shards, node(), self()), + peer_last_seen: + Map.put(state.peer_last_seen, node(), System.monotonic_time(:millisecond)) + } end) :erlang.trace(shard_pid, true, [:call]) @@ -2291,10 +2212,16 @@ defmodule GroupTest do {:erlang, :send_nosuspend, [ {^shard_name, ^local_node}, - {:replicate_process_down_batch, _reg_entries, _pg_entries}, + {:group_replica_frame, ^local_node, {:delta_batch, _version, runs}}, [:noconnect] ]}}, 1_000 + + assert Enum.any?(runs, fn {_stream_id, _first_seq, records, _head} -> + Enum.any?(records, fn {_seq, mutations} -> + {:unregister, nil, key, owner, %{}, :killed} in mutations + end) + end) end test "process death batches multiple :left events for same-shard keys", %{name: name} do @@ -2350,507 +2277,689 @@ defmodule GroupTest do end describe "replicated PG receiver buffering" do - test "flushes buffered replicated joins when buffer size is reached" do - name = - start_single_shard_group( - replicated_pg_receiver_buffer_size: 2, - replicated_pg_receiver_flush_interval: 60_000 - ) - + test "legacy unsequenced ingress cannot materialize or delete rows" do + name = start_single_shard_group() shard = Group.Replica.shard_name(name, 0) - key1 = "replicated/size/#{System.unique_integer([:positive])}/1" - key2 = "replicated/size/#{System.unique_integer([:positive])}/2" - pid1 = spawn_forever() - pid2 = spawn_forever() + registry_key = "legacy-ingress/registry/#{System.unique_integer([:positive])}" + pg_key = "legacy-ingress/pg/#{System.unique_integer([:positive])}" + cluster_state_key = "legacy-ingress/cluster-state/#{System.unique_integer([:positive])}" + retained_key = "legacy-ingress/retained/#{System.unique_integer([:positive])}" + owner = spawn_forever() - on_exit(fn -> - kill_if_alive(pid1) - kill_if_alive(pid2) - end) + on_exit(fn -> kill_if_alive(owner) end) - :ok = Group.monitor(name, :all) + send(shard, replicated_register(nil, registry_key, owner, %{legacy: true}, :register)) + send(shard, replicated_pg_join(nil, pg_key, owner, %{legacy: true}, :join)) + send(shard, {:cluster_state, nil, [{cluster_state_key, owner, %{}, 1}], []}) - send(shard, replicated_pg_join(nil, key1, pid1, %{v: 1}, :join)) + :ok = Group.register(name, retained_key, %{retained: true}) - assert Group.members(name, key1) == [] - refute_receive {:group, _events, _info}, 50 + # Pre-AE cluster lifecycle messages were unsequenced and unfenced. A + # delayed copy must not purge current rows while leaving their stream + # cursor advanced. + send(shard, {:cluster_disconnect, [nil], self()}) - send(shard, replicated_pg_join(nil, key2, pid2, %{v: 2}, :join)) + send( + shard, + {:replicate_process_down_batch, + [{self(), nil, retained_key, %{retained: true}, :legacy_delete}], []} + ) - assert_receive {:group, events, _}, 1000 - assert Enum.map(events, & &1.key) == [key1, key2] - assert Enum.map(events, & &1.type) == [:joined, :joined] - assert Group.members(name, key1) == [{pid1, %{v: 1}}] - assert Group.members(name, key2) == [{pid2, %{v: 2}}] + _state = :sys.get_state(shard) + + assert Group.lookup(name, registry_key) == nil + assert Group.members(name, pg_key) == [] + assert Group.lookup(name, cluster_state_key) == nil + assert Group.lookup(name, retained_key) == {self(), %{retained: true}} + assert :ok = Group.TestCluster.assert_replica_consistent(name) end + end - test "barrier messages flush buffered replicated ops in order" do - name = - start_single_shard_group( - replicated_pg_receiver_buffer_size: 32, - replicated_pg_receiver_flush_interval: 60_000 - ) + describe "replica write-ahead journal" do + test "concurrent shards retain independent append order", %{name: name} do + named_cluster = "journal/append-order" + operations_per_shard = 100 + :ok = Group.connect(name, named_cluster) - shard = Group.Replica.shard_name(name, 0) - key = "replicated/barrier/#{System.unique_integer([:positive])}" - pid = spawn_forever() + parent = self() - on_exit(fn -> kill_if_alive(pid) end) + owners = + for shard <- 0..3 do + nil_keys = + keys_for_shard(nil, "journal/append-order/nil/#{shard}", 4, shard, 50) - :ok = Group.monitor(name, :all) + named_keys = + keys_for_shard( + named_cluster, + "journal/append-order/named/#{shard}", + 4, + shard, + 50 + ) - send(shard, replicated_pg_join(nil, key, pid, %{v: 1}, :join)) - send(shard, replicated_pg_join(nil, key, pid, %{v: 2}, :update)) - send(shard, replicated_pg_leave(nil, key, pid, %{v: 2}, :leave)) + spawn(fn -> + nil_keys + |> Enum.zip(named_keys) + |> Enum.each(fn {nil_key, named_key} -> + :ok = Group.register(name, nil_key, %{}) + :ok = Group.register(name, named_key, %{}, cluster: named_cluster) + end) - assert Group.members(name, key) == [] - flush_replicated_pg_barrier(shard) + send(parent, {:append_order_complete, shard, self()}) + Process.sleep(:infinity) + end) + end - assert_receive {:group, events, _}, 1000 + on_exit(fn -> Enum.each(owners, &kill_if_alive/1) end) - assert [ - %Group.Event{ - type: :joined, - key: ^key, - pid: ^pid, - meta: %{v: 1}, - previous_meta: nil - }, - %Group.Event{ - type: :joined, - key: ^key, - pid: ^pid, - meta: %{v: 2}, - previous_meta: %{v: 1} - }, - %Group.Event{type: :left, key: ^key, pid: ^pid, meta: %{v: 2}, reason: :leave} - ] = events - - assert_receive {:replicated_pg_buffer_flushed, ^shard}, 1000 - assert Group.members(name, key) == [] + Enum.with_index(owners) + |> Enum.each(fn {owner, shard} -> + assert_receive {:append_order_complete, ^shard, ^owner}, 10_000 + end) + + metadata = Group.Replica.Data.replication_meta_table(name) + assert :ets.info(metadata, :write_concurrency) == :auto + + for shard <- 0..3 do + assert [{{:append_counter, shard}, operations_per_shard}] == + :ets.lookup(metadata, {:append_counter, shard}) + + order_rows = + name + |> Group.Replica.Data.replica_oplog_order_table(shard) + |> :ets.tab2list() + |> Enum.sort() + + assert Enum.map(order_rows, &elem(&1, 0)) == + Enum.to_list(1..operations_per_shard) + + assert Enum.all?(order_rows, fn {_append_id, stream_id, _seq} -> + Group.Replica.WireProtocol.stream_shard(stream_id) == shard + end) + + oplog_rows = + name + |> Group.Replica.Data.replica_oplog_table(shard) + |> :ets.tab2list() + |> MapSet.new(fn {{stream_id, seq}, append_id, _mutations} -> + {append_id, stream_id, seq} + end) + + assert MapSet.new(order_rows) == oplog_rows + + order_rows + |> Enum.group_by(fn {_append_id, stream_id, _seq} -> stream_id end) + |> Enum.each(fn {_stream_id, rows} -> + assert Enum.map(rows, &elem(&1, 2)) == Enum.to_list(1..50) + end) + end end - test "batchable traffic can flush an overdue buffer without relying on the timer" do - name = - start_single_shard_group( - replicated_pg_receiver_buffer_size: 32, - replicated_pg_receiver_flush_interval: 1_000 - ) + test "a shard restart replays an appended mixed record and later cleans its owner" do + name = start_single_shard_group(replicated_oplog_max_entries: 16) + key = "journal/replay/#{System.unique_integer([:positive])}" + owner = spawn(fn -> Process.sleep(:infinity) end) + on_exit(fn -> kill_if_alive(owner) end) - shard = Group.Replica.shard_name(name, 0) - key1 = "replicated/due/#{System.unique_integer([:positive])}/1" - key2 = "replicated/due/#{System.unique_integer([:positive])}/2" - pid1 = spawn_forever() - pid2 = spawn_forever() + stream_id = Group.Replica.Data.local_stream_id(name, 0, nil) + time = System.system_time() - on_exit(fn -> - kill_if_alive(pid1) - kill_if_alive(pid2) - end) + {seq, _mutations} = + Group.Replica.Data.append_replica_record(name, 0, stream_id, [ + {:register, nil, key, owner, %{kind: :registry}, time, node()}, + {:join, nil, key, owner, %{kind: :pg}, time, :join, node()} + ]) - :ok = Group.monitor(name, :all) + old_shard = Process.whereis(Group.Replica.shard_name(name, 0)) + Process.exit(old_shard, :kill) - send(shard, replicated_pg_join(nil, key1, pid1, %{v: 1}, :join)) - Process.sleep(10) + Group.TestCluster.assert_eventually(fn -> + new_shard = Process.whereis(Group.Replica.shard_name(name, 0)) - :sys.replace_state(shard, fn state -> - %{ - state - | pending_replicated_pg_started_at: System.monotonic_time(:millisecond) - 5_000, - pending_replicated_pg_flush_ref: nil - } + is_pid(new_shard) and new_shard != old_shard and + Group.lookup(name, key) == {owner, %{kind: :registry}} and + Group.members(name, key) == [{owner, %{kind: :pg}}] end) - send(shard, replicated_pg_join(nil, key2, pid2, %{v: 2}, :join)) + assert {_floor, ^seq, ^seq} = + Group.Replica.Data.replica_stream_head(name, 0, stream_id) - assert_receive {:group, events, _}, 1000 - assert Enum.map(events, & &1.key) == [key1, key2] + Process.exit(owner, :kill) - state = :sys.get_state(shard) - assert state.pending_replicated_pg_len == 0 - assert Group.members(name, key1) == [{pid1, %{v: 1}}] - assert Group.members(name, key2) == [{pid2, %{v: 2}}] + Group.TestCluster.assert_eventually(fn -> + Group.lookup(name, key) == nil and Group.members(name, key) == [] + end) end - test "terminate flushes buffered replicated ops before shard restart" do - name = - start_single_shard_group( - replicated_pg_receiver_buffer_size: 32, - replicated_pg_receiver_flush_interval: 60_000 - ) + test "a shard restart repairs an interrupted append tail and interrupted prune floor" do + name = start_single_shard_group(replicated_oplog_max_entries: 16) + stream_id = Group.Replica.Data.local_stream_id(name, 0, nil) + first_key = "journal/crash-window/first/#{System.unique_integer([:positive])}" + second_key = "journal/crash-window/second/#{System.unique_integer([:positive])}" + third_key = "journal/crash-window/third/#{System.unique_integer([:positive])}" - shard = Group.Replica.shard_name(name, 0) - shard_pid = Process.whereis(shard) - key = "replicated/terminate/#{System.unique_integer([:positive])}" - pid = spawn_forever() + :ok = Group.register(name, first_key, %{seq: 1}) + :ok = Group.register(name, second_key, %{seq: 2}) - on_exit(fn -> kill_if_alive(pid) end) + oplog = Group.Replica.Data.replica_oplog_table(name, 0) + order = Group.Replica.Data.replica_oplog_order_table(name, 0) + stream_meta = Group.Replica.Data.replica_stream_meta_table(name, 0) - :ok = Group.monitor(name, :all) + [{{^stream_id, 1}, first_append_id, _mutations}] = :ets.lookup(oplog, {stream_id, 1}) - send(shard, replicated_pg_join(nil, key, pid, %{v: 1}, :join)) - assert Group.members(name, key) == [] + # Pruning removes order -> record -> advances floor. Model a kill after + # the first two writes but before the floor update. + :ets.delete(order, first_append_id) + :ets.delete(oplog, {stream_id, 1}) - ref = Process.monitor(shard_pid) - :ok = GenServer.stop(shard_pid, :shutdown) - assert_receive {:DOWN, ^ref, :process, ^shard_pid, :shutdown}, 1000 - assert_receive {:group, [%Group.Event{type: :joined, key: ^key, pid: ^pid}], _}, 1000 + # Appending advances head -> append counter -> record -> order. Model a + # kill after the head update but before the record exists. + assert 3 = :ets.update_counter(stream_meta, stream_id, {2, 1}) - wait_until(fn -> - case Process.whereis(shard) do - nil -> false - new_pid -> new_pid != shard_pid - end + old_shard = Process.whereis(Group.Replica.shard_name(name, 0)) + Process.exit(old_shard, :kill) + + Group.TestCluster.assert_eventually(fn -> + new_shard = Process.whereis(Group.Replica.shard_name(name, 0)) + is_pid(new_shard) and new_shard != old_shard end) - assert Group.members(name, key) == [{pid, %{v: 1}}] + :sys.get_state(Group.Replica.shard_name(name, 0)) + assert {2, 2, 2} = Group.Replica.Data.replica_stream_head(name, 0, stream_id) + assert :ok = Group.TestCluster.assert_replica_consistent(name) + + :ok = Group.register(name, third_key, %{seq: 3}) + assert {2, 3, 3} = Group.Replica.Data.replica_stream_head(name, 0, stream_id) + + assert Group.lookup(name, first_key) == {self(), %{seq: 1}} + assert Group.lookup(name, second_key) == {self(), %{seq: 2}} + assert Group.lookup(name, third_key) == {self(), %{seq: 3}} + assert :ok = Group.TestCluster.assert_replica_consistent(name) end - end - describe "replicated registry receiver buffering" do - test "barrier messages flush buffered replicated register and unregister ops in order" do - name = - start_single_shard_group( - replicated_registry_receiver_buffer_size: 32, - replicated_registry_receiver_flush_interval: 60_000 - ) + test "a shard restart rebuilds every one-sided materialized index" do + name = start_single_shard_group(replicated_oplog_max_entries: 16) + reg_key = "indexes/crash-window/reg/#{System.unique_integer([:positive])}" + pg_key = "indexes/crash-window/pg/#{System.unique_integer([:positive])}" - shard = Group.Replica.shard_name(name, 0) - key = "replicated-registry/barrier/#{System.unique_integer([:positive])}" - pid = spawn_forever() - time1 = System.system_time() - time2 = time1 + 1 + :ok = Group.register(name, reg_key, %{kind: :registry}) + :ok = Group.join(name, pg_key, %{kind: :pg}) - on_exit(fn -> kill_if_alive(pid) end) + stream_id = Group.Replica.Data.local_stream_id(name, 0, nil) + generation = Group.Replica.WireProtocol.stream_generation(stream_id) + epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) - :ok = Group.monitor(name, :all) + :ets.delete(Group.Replica.Data.reg_by_pid_table(name, 0), {self(), nil, reg_key}) + :ets.delete(Group.Replica.Data.pg_by_pid_table(name, 0), {self(), nil, pg_key}) - send(shard, replicated_register(nil, key, pid, %{v: 1}, :register, time1)) - send(shard, replicated_register(nil, key, pid, %{v: 2}, :update, time2)) - send(shard, replicated_unregister(nil, key, pid, %{v: 2}, :unregister)) + :ets.delete( + Group.Replica.Data.reg_claim_by_pid_table(name, 0), + {self(), nil, reg_key, node(), generation, epoch} + ) - assert Group.lookup(name, key) == nil - flush_replicated_registry_barrier(shard) - - assert_receive {:group, events, _}, 1_000 - - assert [ - %Group.Event{ - type: :registered, - key: ^key, - pid: ^pid, - meta: %{v: 1}, - previous_meta: nil - }, - %Group.Event{ - type: :registered, - key: ^key, - pid: ^pid, - meta: %{v: 2}, - previous_meta: %{v: 1} - }, - %Group.Event{ - type: :unregistered, - key: ^key, - pid: ^pid, - meta: %{v: 2}, - reason: :unregister - } - ] = events - - assert_receive {:replicated_registry_buffer_flushed, ^shard}, 1_000 - assert Group.lookup(name, key) == nil - end + orphan = spawn(fn -> Process.sleep(:infinity) end) + on_exit(fn -> kill_if_alive(orphan) end) - test "batchable registry traffic can flush an overdue buffer without relying on the timer" do - name = - start_single_shard_group( - replicated_registry_receiver_buffer_size: 32, - replicated_registry_receiver_flush_interval: 1_000 - ) + :ets.insert( + Group.Replica.Data.reg_by_pid_table(name, 0), + {{orphan, nil, "indexes/orphan/reg"}, %{}, 0, node()} + ) - shard = Group.Replica.shard_name(name, 0) - key1 = "replicated-registry/due/#{System.unique_integer([:positive])}/1" - key2 = "replicated-registry/due/#{System.unique_integer([:positive])}/2" - pid1 = spawn_forever() - pid2 = spawn_forever() + :ets.insert( + Group.Replica.Data.pg_by_pid_table(name, 0), + {{orphan, nil, "indexes/orphan/pg"}, %{}, 0, node()} + ) - on_exit(fn -> - kill_if_alive(pid1) - kill_if_alive(pid2) - end) + :ets.insert( + Group.Replica.Data.reg_claim_by_pid_table(name, 0), + {{orphan, nil, "indexes/orphan/claim", node(), generation, epoch}, %{}, 0, 1} + ) - :ok = Group.monitor(name, :all) + old_shard = Process.whereis(Group.Replica.shard_name(name, 0)) + Process.exit(old_shard, :kill) - send(shard, replicated_register(nil, key1, pid1, %{v: 1}, :register)) - Process.sleep(10) + Group.TestCluster.assert_eventually(fn -> + new_shard = Process.whereis(Group.Replica.shard_name(name, 0)) + is_pid(new_shard) and new_shard != old_shard + end) - :sys.replace_state(shard, fn state -> - %{ - state - | pending_replicated_registry_started_at: System.monotonic_time(:millisecond) - 5_000, - pending_replicated_registry_flush_ref: nil - } + :sys.get_state(Group.Replica.shard_name(name, 0)) + assert Group.lookup(name, reg_key) == {self(), %{kind: :registry}} + assert Group.members(name, pg_key) == [{self(), %{kind: :pg}}] + assert Group.Replica.Data.registry_lookup_by_pid(name, 0, orphan) == [] + assert Group.Replica.Data.entries_by_pid(name, 0, orphan) == [] + assert :ok = Group.TestCluster.assert_replica_consistent(name) + end + + test "streamed exact replacement deletes every claim across select continuations" do + name = start_single_shard_group(replicated_oplog_max_entries: 16) + stream_id = Group.Replica.Data.local_stream_id(name, 0, nil) + staging = Group.Replica.Snapshot.new_staging_table() + + on_exit(fn -> Group.Replica.Snapshot.delete_staging_table(staging) end) + + Enum.each(1..10_000, fn index -> + :ok = + Group.Replica.Data.put_registry_claim( + name, + 0, + stream_id, + 1, + "snapshot/select-continuation/#{index}", + self(), + %{}, + index + ) end) - send(shard, replicated_register(nil, key2, pid2, %{v: 2}, :register)) + assert :ok = Group.Replica.Snapshot.stage_rows(staging, 1, [], []) - assert_receive {:group, events, _}, 1_000 - assert Enum.map(events, & &1.key) == [key1, key2] + assert :ok = + Group.Replica.Data.replace_registry_claims_for_stream_from_staging( + name, + 0, + stream_id, + 1, + staging, + 1, + :ok, + fn _key, :ok -> :ok end + ) - state = :sys.get_state(shard) - assert state.pending_replicated_registry_len == 0 - assert Group.lookup(name, key1) == {pid1, %{v: 1}} - assert Group.lookup(name, key2) == {pid2, %{v: 2}} + assert Group.Replica.Data.registry_claims_for_stream(name, 0, stream_id) == [] + assert :ets.info(Group.Replica.Data.reg_claim_by_key_table(name, 0), :size) == 0 + assert :ets.info(Group.Replica.Data.reg_claim_by_pid_table(name, 0), :size) == 0 end - test "terminate flushes buffered replicated registry ops before shard restart" do - name = - start_single_shard_group( - replicated_registry_receiver_buffer_size: 32, - replicated_registry_receiver_flush_interval: 60_000 - ) + test "a shard restart completes an interrupted named-cluster close without retained rows" do + name = start_single_shard_group(replicated_oplog_max_entries: 16) + cluster = "close/crash-window/#{System.unique_integer([:positive])}" + reg_key = "close/crash-window/reg/#{System.unique_integer([:positive])}" + pg_key = "close/crash-window/pg/#{System.unique_integer([:positive])}" + remote_route = :"close-crash-window@remote" - shard = Group.Replica.shard_name(name, 0) - shard_pid = Process.whereis(shard) - key = "replicated-registry/terminate/#{System.unique_integer([:positive])}" - pid = spawn_forever() + :ok = Group.connect(name, cluster) + :ok = Group.register(name, reg_key, %{kind: :registry}, cluster: cluster) + :ok = Group.join(name, pg_key, %{kind: :pg}, cluster: cluster) + :ok = Group.Replica.Data.add_cluster_node(name, [cluster], remote_route) - on_exit(fn -> kill_if_alive(pid) end) + stream_id = Group.Replica.Data.local_stream_id(name, 0, cluster) + old_epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) + old_shard = Process.whereis(Group.Replica.shard_name(name, 0)) + :ok = :sys.suspend(old_shard) - :ok = Group.monitor(name, :all) + # Group.disconnect/3 closes authority and routing before its request + # reaches every shard. Hold the durable cleanup message in the shard's + # mailbox, then model a kill in that exact window. + assert [{^cluster, ^old_epoch}] = + Group.Replica.Data.deactivate_local_clusters(name, [cluster]) - send(shard, replicated_register(nil, key, pid, %{v: 1}, :register)) - assert Group.lookup(name, key) == nil + assert Group.Replica.Data.closed_local_clusters(name) == [cluster] - ref = Process.monitor(shard_pid) - :ok = GenServer.stop(shard_pid, :shutdown) - assert_receive {:DOWN, ^ref, :process, ^shard_pid, :shutdown}, 1_000 - assert_receive {:group, [%Group.Event{type: :registered, key: ^key, pid: ^pid}], _}, 1_000 + Process.exit(old_shard, :kill) - wait_until(fn -> - case Process.whereis(shard) do - nil -> false - new_pid -> new_pid != shard_pid - end + Group.TestCluster.assert_eventually(fn -> + new_shard = Process.whereis(Group.Replica.shard_name(name, 0)) + + is_pid(new_shard) and new_shard != old_shard and + Group.lookup(name, reg_key, cluster: cluster) == nil and + Group.members(name, pg_key, cluster: cluster) == [] and + Group.Replica.Data.closed_local_clusters(name) == [] and + Group.Replica.Data.cluster_nodes(name, cluster) == [] end) - assert Group.lookup(name, key) == {pid, %{v: 1}} + assert :ets.lookup(Group.Replica.Data.replica_stream_meta_table(name, 0), stream_id) == [] + assert :ok = Group.TestCluster.assert_replica_consistent(name) end - test "replaces stale remote registry owner and clears the old by-pid entry" do - name = - start_single_shard_group( - replicated_registry_receiver_buffer_size: 32, - replicated_registry_receiver_flush_interval: 60_000 - ) + test "the last close acknowledgement atomically removes all cluster routing" do + name = start_single_shard_group() + cluster = "close/terminal-ack/#{System.unique_integer([:positive])}" + remote_route = :"close-terminal-ack@remote" - shard = Group.Replica.shard_name(name, 0) - key = "replicated-registry/replace/#{System.unique_integer([:positive])}" - old_pid = spawn_forever() - new_pid = spawn_forever() - time1 = System.system_time() - time2 = time1 + 1 + :ok = Group.connect(name, cluster) + :ok = Group.Replica.Data.add_cluster_node(name, [cluster], remote_route) - on_exit(fn -> - kill_if_alive(old_pid) - kill_if_alive(new_pid) - end) + shard = Process.whereis(Group.Replica.shard_name(name, 0)) + :ok = :sys.suspend(shard) - Group.Replica.Data.registry_insert( - name, - 0, - nil, - key, - old_pid, - %{v: 1}, - time1, - :"remote_a@127.0.0.1" - ) + on_exit(fn -> Group.TestCluster.resume_if_alive(shard) end) - send(shard, replicated_register(nil, key, new_pid, %{v: 2}, :register, time2)) - flush_replicated_registry_barrier(shard) + # Model the durable deactivation caller and the final shard both dying + # immediately after the final acknowledgement. Once the close marker is + # gone there is no later recovery hook, so routing must already be gone. + assert [{^cluster, epoch}] = + Group.Replica.Data.deactivate_local_clusters_durable(name, [cluster]) - assert_receive {:replicated_registry_buffer_flushed, ^shard}, 1_000 - assert Group.lookup(name, key) == {new_pid, %{v: 2}} - assert Group.Replica.Data.registry_lookup_by_pid(name, 0, old_pid) == [] + assert is_reference(epoch) - assert [{nil, ^key, %{v: 2}, ^time2, _entry_node}] = - Group.Replica.Data.registry_lookup_by_pid(name, 0, new_pid) - end + assert [] = + Group.Replica.Data.mark_closed_cluster_shard(name, [{cluster, make_ref()}], 0) - test "custom conflict resolver controls the losing registry owner's lifecycle" do - key = "replicated-registry/custom-loser/#{System.unique_integer([:positive])}" + assert Group.Replica.Data.closed_local_clusters(name) == [cluster] - name = - start_single_shard_group( - replicated_registry_receiver_buffer_size: 1, - resolve_registry_conflict: {GroupTest.ResolveRegistryConflict, :pick, [:remote]} - ) + assert [^cluster] = + Group.Replica.Data.mark_closed_cluster_shard(name, [{cluster, epoch}], 0) - parent = self() + assert Group.Replica.Data.closed_local_clusters(name) == [] + assert Group.Replica.Data.cluster_nodes(name, cluster) == [] + assert Group.Replica.Data.clusters_for_node(name, remote_route) == [] + end - local_owner = - spawn(fn -> - :ok = Group.register(name, key, %{owner: :local}) - send(parent, {:custom_conflict_owner_ready, self()}) - Process.sleep(:infinity) - end) + test "delayed restart cleanup cannot remove a reactivated cluster's routes" do + name = start_single_shard_group() + cluster = "close/reactivated-cleanup/#{System.unique_integer([:positive])}" + remote_route = :"close-reactivated-cleanup@remote" - remote_pid = spawn_forever() + :ok = Group.connect(name, cluster) + :ok = Group.Replica.Data.add_cluster_node(name, [cluster], remote_route) - on_exit(fn -> - kill_if_alive(local_owner) - kill_if_alive(remote_pid) - end) + # Model repair_primary_replica_rows/2 observing this cluster while it was + # inactive, followed by a concurrent reconnect completing before repair's + # serialized routing cleanup runs. + :ok = Group.Replica.Data.remove_clusters(name, [cluster]) - assert_receive {:custom_conflict_owner_ready, ^local_owner}, 1_000 - owner_ref = Process.monitor(local_owner) - shard = Group.Replica.shard_name(name, 0) + assert Enum.sort(Group.Replica.Data.cluster_nodes(name, cluster)) == + Enum.sort([node(), remote_route]) - send( - shard, - replicated_register( - nil, - key, - remote_pid, - %{owner: :remote}, - :register, - System.system_time() + assert cluster in Group.Replica.Data.clusters_for_node(name, node()) + assert cluster in Group.Replica.Data.clusters_for_node(name, remote_route) + end + + test "the last expired replica lane atomically removes peer routing" do + name = start_single_shard_group() + remote_node = :"terminal-peer-retirement@remote" + generation = Group.Replica.WireProtocol.new_generation() + + assert {nil, []} = + Group.Replica.Data.put_remote_replica_info( + name, + 0, + remote_node, + generation, + 0, + [{nil, generation}] + ) + + :ok = + Group.Replica.Data.put_remote_view_info( + name, + 0, + remote_node, + generation, + 0, + 0 ) - ) - wait_until(fn -> - Group.lookup(name, key) == {remote_pid, %{owner: :remote}} - end) + assert remote_node in Group.Replica.Data.cluster_nodes(name, nil) + assert [nil] = Group.Replica.Data.clusters_for_node(name, remote_node) + + # Once this call removes the last persisted lane view, no shard restart + # can reconstruct an expiry obligation for the peer. Its routing must be + # gone before the terminal retirement is acknowledged. + assert :node_retired = + Group.Replica.Data.expire_remote_replica_lane(name, 0, remote_node) - refute_receive {:DOWN, ^owner_ref, :process, ^local_owner, _reason}, 50 - assert Process.alive?(local_owner) + assert Group.Replica.Data.cluster_nodes(name, nil) |> Enum.member?(remote_node) == false + assert Group.Replica.Data.clusters_for_node(name, remote_node) == [] + assert Group.Replica.Data.remote_generation(name, remote_node) == nil end - test "batched remote conflict keeps the staged local winner when later unregister arrives" do - key = "replicated-registry/conflict-local/#{System.unique_integer([:positive])}" + test "nodedown removes a retired peer's pending authority repair" do + name = start_single_shard_group() + remote_node = :"retired-authority-repair@remote" + shard = Group.Replica.shard_name(name, 0) - name = - start_single_shard_group( - replicated_registry_receiver_buffer_size: 32, - replicated_registry_receiver_flush_interval: 60_000, - resolve_registry_conflict: {GroupTest.ResolveRegistryConflict, :pick, [:local]} - ) + send(shard, {:replica_authority_dirty_local, remote_node}) - parent = self() + assert %{cluster_control_dirty: %{^remote_node => _timestamp}} = :sys.get_state(shard) - local_owner = - spawn(fn -> - :ok = Group.register(name, key, %{owner: :local}) - send(parent, {:local_registry_owner_ready, self()}) - Process.sleep(:infinity) - end) + send(shard, {:nodedown, remote_node}) - remote_pid = spawn_forever() + refute Map.has_key?(:sys.get_state(shard).cluster_control_dirty, remote_node) + end - on_exit(fn -> - kill_if_alive(local_owner) - kill_if_alive(remote_pid) - end) + test "delayed peer cleanup cannot remove a rediscovered generation's routes" do + name = start_single_shard_group() + remote_node = :"rediscovered-peer-route@remote" + old_generation = Group.Replica.WireProtocol.new_generation() + + assert {nil, []} = + Group.Replica.Data.put_remote_replica_info( + name, + 0, + remote_node, + old_generation, + 0, + [{nil, old_generation}] + ) + + :ok = + Group.Replica.Data.put_remote_view_info( + name, + 0, + remote_node, + old_generation, + 0, + 0 + ) - assert_receive {:local_registry_owner_ready, ^local_owner}, 1_000 - assert Group.lookup(name, key) == {local_owner, %{owner: :local}} + assert :node_retired = + Group.Replica.Data.expire_remote_replica_lane(name, 0, remote_node) - :ok = Group.monitor(name, :all) - shard = Group.Replica.shard_name(name, 0) + new_generation = Group.Replica.WireProtocol.new_generation() - send( - shard, - replicated_register( - nil, - key, - remote_pid, - %{owner: :remote}, - :register, - System.system_time() - ) - ) + assert {nil, []} = + Group.Replica.Data.put_remote_replica_info( + name, + 0, + remote_node, + new_generation, + 0, + [{nil, new_generation}] + ) - send(shard, replicated_unregister(nil, key, remote_pid, %{owner: :remote}, :unregister)) - flush_replicated_registry_barrier(shard) + # Model the old expiry/nodedown caller resuming only after rediscovery. + :ok = Group.Replica.Data.purge_cluster_node(name, remote_node) - assert_receive {:replicated_registry_buffer_flushed, ^shard}, 1_000 - refute_receive {:group, _events, _info}, 50 - assert Group.lookup(name, key) == {local_owner, %{owner: :local}} - assert Group.Replica.Data.registry_lookup_by_pid(name, 0, remote_pid) == [] + assert remote_node in Group.Replica.Data.cluster_nodes(name, nil) + assert [nil] = Group.Replica.Data.clusters_for_node(name, remote_node) + assert Group.Replica.Data.remote_generation(name, remote_node) == new_generation end - test "batched remote conflict removes the staged remote winner when later unregister arrives" do - key = "replicated-registry/conflict-remote/#{System.unique_integer([:positive])}" - - name = - start_single_shard_group( - replicated_registry_receiver_buffer_size: 32, - replicated_registry_receiver_flush_interval: 60_000, - resolve_registry_conflict: {GroupTest.ResolveRegistryConflict, :pick, [:remote]} - ) - - parent = self() + test "a lane restart reconstructs retirement from a persisted authority hint" do + name = :"group_hint_restart_#{System.unique_integer([:positive])}" - local_owner = - spawn(fn -> - :ok = Group.register(name, key, %{owner: :local}) - send(parent, {:remote_registry_owner_ready, self()}) - Process.sleep(:infinity) - end) + start_supervised!( + {Group, + name: name, + shards: 2, + log: false, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 150} + ) - remote_pid = spawn_forever() + remote_node = :"hint-restart-retirement@remote" + old_generation = Group.Replica.WireProtocol.new_generation() + + assert {nil, []} = + Group.Replica.Data.put_remote_replica_info( + name, + 0, + remote_node, + old_generation, + 0, + [{nil, old_generation}] + ) + + generation = Group.Replica.WireProtocol.new_generation() + + assert Group.Replica.Data.observe_remote_replica_hint( + name, + remote_node, + generation, + 0 + ) + + old_lane = Process.whereis(Group.Replica.shard_name(name, 1)) + monitor = Process.monitor(old_lane) + Process.exit(old_lane, :kill) + assert_receive {:DOWN, ^monitor, :process, ^old_lane, :killed}, 5_000 - on_exit(fn -> - kill_if_alive(local_owner) - kill_if_alive(remote_pid) + wait_until(fn -> + case Process.whereis(Group.Replica.shard_name(name, 1)) do + pid when is_pid(pid) -> pid != old_lane + _ -> false + end end) - assert_receive {:remote_registry_owner_ready, ^local_owner}, 1_000 - assert Group.lookup(name, key) == {local_owner, %{owner: :local}} + wait_until( + fn -> + Group.Replica.Data.remote_replica_authority_hint(name, remote_node) == nil + end, + 2_000 + ) - :ok = Group.monitor(name, :all) - shard = Group.Replica.shard_name(name, 0) + refute Map.has_key?( + :sys.get_state(Group.Replica.shard_name(name, 1)).peer_last_seen, + remote_node + ) + end + end - send( - shard, - replicated_register( - nil, - key, - remote_pid, - %{owner: :remote}, - :register, - System.system_time() + describe "replica authority snapshots" do + test "a raced authority hint prevents a partial incremental view install", %{name: name} do + remote_node = :"incremental-authority-race@remote" + generation = Group.Replica.WireProtocol.new_generation() + cluster = "incremental-authority-race/cluster" + epoch = make_ref() + + assert {nil, []} = + Group.Replica.Data.put_remote_replica_info( + name, + 0, + remote_node, + generation, + 0, + [{nil, generation}] + ) + + :ok = + Group.Replica.Data.put_remote_view_info( + name, + 0, + remote_node, + generation, + 0, + 0 ) - ) - send(shard, replicated_unregister(nil, key, remote_pid, %{owner: :remote}, :unregister)) - flush_replicated_registry_barrier(shard) - - assert_receive {:group, events, _}, 1_000 - - assert [ - %Group.Event{ - type: :unregistered, - key: ^key, - pid: ^local_owner, - meta: %{owner: :local}, - reason: :resolve_conflict - }, - %Group.Event{ - type: :unregistered, - key: ^key, - pid: ^remote_pid, - meta: %{owner: :remote}, - reason: :unregister - } - ] = events - - assert_receive {:replicated_registry_buffer_flushed, ^shard}, 1_000 - assert Group.lookup(name, key) == nil - assert Group.Replica.Data.registry_lookup_by_pid(name, 0, local_owner) == [] - assert Group.Replica.Data.registry_lookup_by_pid(name, 0, remote_pid) == [] + assert Group.Replica.Data.observe_remote_replica_hint( + name, + remote_node, + generation, + 2 + ) + + assert :stale = + Group.Replica.Data.put_remote_cluster_epochs( + name, + 0, + remote_node, + generation, + 0, + 1, + [{cluster, epoch}] + ) + + assert Group.Replica.Data.remote_cluster_epoch(name, remote_node, cluster) == nil + + assert :stale = + Group.Replica.Data.put_remote_view_info( + name, + 0, + remote_node, + generation, + 0, + 2 + ) + + refute Group.Replica.Data.remote_registry_claim_authoritative?( + name, + 0, + remote_node, + generation, + nil, + generation + ) + end + + test "a newer-generation hint atomically rejects an old lane view", %{name: name} do + remote_node = :"lane-view-generation-race@remote" + old_generation = Group.Replica.WireProtocol.new_generation() + + assert {nil, []} = + Group.Replica.Data.put_remote_replica_info( + name, + 0, + remote_node, + old_generation, + 0, + [{nil, old_generation}] + ) + + new_generation = Group.Replica.WireProtocol.new_generation() + assert Group.Replica.WireProtocol.generation_newer?(new_generation, old_generation) + + assert Group.Replica.Data.observe_remote_replica_hint( + name, + remote_node, + new_generation, + 0 + ) + + assert :stale = + Group.Replica.Data.put_remote_view_info( + name, + 1, + remote_node, + old_generation, + 0, + 0 + ) + + assert Group.Replica.Data.remote_view_generation(name, 1, remote_node) == nil + end + + test "revision and epoch rows remain coherent during concurrent activation", %{name: name} do + clusters = for i <- 1..1_000, do: "authority/#{i}" + + activators = + clusters + |> Enum.chunk_every(125) + |> Enum.map(fn chunk -> + Task.async(fn -> + Enum.each(chunk, fn cluster -> + [{^cluster, _epoch}] = + Group.Replica.Data.activate_local_clusters(name, [cluster]) + end) + end) + end) + + for _ <- 1..200 do + {generation, revision, epochs} = + Group.Replica.Data.local_replica_authority(name) + + assert {nil, generation} in epochs + assert revision == Enum.count(epochs, &(not is_nil(elem(&1, 0)))) + end + + Task.await_many(activators, 10_000) + + {generation, 1_000, epochs} = Group.Replica.Data.local_replica_authority(name) + assert {nil, generation} in epochs + assert length(epochs) == 1_001 + assert Map.new(epochs) |> map_size() == 1_001 end end @@ -2861,6 +2970,14 @@ defmodule GroupTest do name end + defp keys_for_shard(cluster, prefix, num_shards, shard, count) do + 1 + |> Stream.iterate(&(&1 + 1)) + |> Stream.map(&"#{prefix}/#{&1}") + |> Stream.filter(&(Group.Replica.shard_index_for(cluster, &1, num_shards) == shard)) + |> Enum.take(count) + end + defp suspend_only_shard(name) do shard = Group.Replica.shard_name(name, 0) :ok = :sys.suspend(shard) @@ -2884,34 +3001,10 @@ defmodule GroupTest do [{:join, cluster, key, pid, meta, System.system_time(), reason, node(pid)}]} end - defp replicated_pg_leave(cluster, key, pid, meta, reason) do - {:replicate_pg_batch, [{:leave, cluster, key, pid, meta, reason}]} - end - defp replicated_register(cluster, key, pid, meta, _reason, time \\ System.system_time()) do {:replicate_registry_batch, [{:register, cluster, key, pid, meta, time, node(pid)}]} end - defp replicated_unregister(cluster, key, pid, meta, reason) do - {:replicate_registry_batch, [{:unregister, cluster, key, pid, meta, reason}]} - end - - defp enqueue_replicated_pg_backlog(shard, key_prefix, pid, count) do - for i <- 1..count do - send(shard, replicated_pg_join(nil, "#{key_prefix}/#{i}", pid, %{}, :join)) - end - - :ok - end - - defp enqueue_replicated_registry_backlog(shard, key_prefix, pid, count) do - for i <- 1..count do - send(shard, replicated_register(nil, "#{key_prefix}/#{i}", pid, %{seq: i}, :register)) - end - - :ok - end - defp spawn_requester(fun, tag) do parent = self() @@ -2929,10 +3022,6 @@ defmodule GroupTest do end end - defp flush_replicated_pg_barrier(shard) do - send(shard, {:group_dispatch, [self()], {:replicated_pg_buffer_flushed, shard}}) - end - defp flush_replicated_registry_barrier(shard) do send(shard, {:group_dispatch, [self()], {:replicated_registry_buffer_flushed, shard}}) end @@ -2961,6 +3050,28 @@ defmodule GroupTest do spawn(fn -> Process.sleep(:infinity) end) end + defp replica_ingress_fairness_owner(parent) do + receive do + {:write, shard, request} -> + ref = make_ref() + send(shard, {:group_local_request, self(), ref, request}) + {reply, calls} = receive_local_write_with_trace(shard, ref, 0) + send(parent, {:local_write_finished, self(), reply, calls}) + Process.sleep(:infinity) + end + end + + defp receive_local_write_with_trace(shard, ref, calls) do + receive do + {:trace, ^shard, :call, + {Group.Replica, :handle_replica_message, [_state, _source_node, _message]}} -> + receive_local_write_with_trace(shard, ref, calls + 1) + + {:group_local_reply, ^ref, reply} -> + {reply, calls} + end + end + defp kill_if_alive(pid) do if Process.alive?(pid) do Process.exit(pid, :kill) diff --git a/test/jepsen/.gitignore b/test/jepsen/.gitignore new file mode 100644 index 0000000..acbda9e --- /dev/null +++ b/test/jepsen/.gitignore @@ -0,0 +1,5 @@ +.cache/ +.lein-failures +.nrepl-port +store/ +target/ diff --git a/test/jepsen/Dockerfile.node b/test/jepsen/Dockerfile.node new file mode 100644 index 0000000..5f1107e --- /dev/null +++ b/test/jepsen/Dockerfile.node @@ -0,0 +1,23 @@ +FROM elixir:1.19.5-otp-28-slim + +RUN apt-get update \ + && apt-get install --yes --no-install-recommends ca-certificates iptables procps \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /group + +ENV MIX_ENV=prod + +COPY mix.exs mix.lock ./ +COPY lib ./lib +COPY test/jepsen/node.exs ./test/jepsen/node.exs +COPY test/support/test_tcp_transport.ex ./test/support/test_tcp_transport.ex + +RUN mix local.hex --force \ + && mix deps.get --only prod \ + && mix compile + +COPY test/jepsen/entrypoint.sh /usr/local/bin/group-jepsen-node +RUN chmod +x /usr/local/bin/group-jepsen-node + +ENTRYPOINT ["/usr/local/bin/group-jepsen-node"] diff --git a/test/jepsen/README.md b/test/jepsen/README.md new file mode 100644 index 0000000..b510981 --- /dev/null +++ b/test/jepsen/README.md @@ -0,0 +1,150 @@ +# Group Jepsen model test + +This harness drives real Group instances in three independent BEAM VMs and +checks their quiescent state against a separate process-lifecycle oracle. It +tests failures outside the deterministic in-VM scheduler used by the +StreamData suite. + +Each node has eight independent client drivers, so operations on different +owners can overlap on the same node. Owners can hold multiple registry and PG +entries in the root, `red`, and `blue` cluster epochs. During the bounded-fault +prefix Jepsen: + +- registers, unregisters, joins, leaves, and kills real owner processes; +- closes and recreates named-cluster epochs while old traffic is stranded; +- creates a three-node registry conflict and requires every loser to die; +- partitions the full Erlang mesh, or only selected directed replica lanes; +- exercises isolation, all-way partition, and asymmetric one-way loss; +- resets transport sessions and kills/restarts complete BEAM nodes; and +- uses a 16-entry oplog and 1 KiB snapshot target so repair crosses pruning + and multi-chunk exact-snapshot paths. + +The replica lane is selectable without changing the workload or checker: + +- `distribution` delegates to Group's production Erlang-distribution adapter; +- `tcp` uses Group's hidden test-only TCP adapter while Erlang distribution + remains the control plane; and +- `chaos` is a local per-shard outbox which deterministically drops, + duplicates, delays, and reorders replica messages. + +After faults stop, every surviving node reconnects and the harness takes two +terminal snapshots. The independent checker requires: + +- exact, identical public registry and PG views on every survivor; +- every live owner claim to be visible, and no dead owner token to remain; +- deterministic resolution of registry conflicts with no unexpected owner + deaths; +- the expected peer set, including complete removal of a permanently retired + node; +- consistent registry, PG, cluster, claim, cursor, oplog, and remote-authority + indexes inside every shard; +- no staged partial snapshot and no retained data for a retired origin; +- coverage of delta batches, snapshot fallback, multi-chunk assembly, and + registry conflict termination; +- two identical quiescent observations per node; and +- every acknowledged Group operation below the configured latency ceiling. + +The oracle lives in owner processes outside Group's ETS and replica indexes. +Unexpected deaths and low-volume qualification evidence such as registry +conflict termination are also written to container-local logs which survive a +BEAM restart. A restart changes the boot component of new owner tokens, so a +stale generation, delayed delta, partial snapshot, or orphaned registry/PG row +cannot masquerade as a current owner. Every history explicitly restarts one +node after the deterministic conflict prelude, proving the checker does not +mistake restart-sensitive instrumentation for missing protocol coverage. + +## Requirements + +- Docker with Compose v2 +- Java 21 or newer +- `curl` + +The runner downloads Leiningen 2.12.0 into the ignored `.cache/` directory and +uses Jepsen 0.3.13. It installs nothing globally. + +## Run + +From the repository root: + +```bash +test/jepsen/run.sh +``` + +The default is a 60-second mixed-lifecycle run over three nodes and six client +workers. Normal Jepsen options and Group-specific options can be supplied: + +```bash +test/jepsen/run.sh test \ + --no-ssh \ + --nodes n1,n2,n3 \ + --concurrency 3n \ + --time-limit 300 \ + --test-count 10 \ + --key-count 32 \ + --owner-count 128 \ + --fault-interval 2 \ + --recovery-time 15 \ + --transport tcp \ + --scenario permanent \ + --max-operation-latency-ms 2000 +``` + +`--transport` accepts `distribution`, `tcp`, or `chaos`. `--scenario mixed` +restarts every transiently killed node; `--scenario permanent` finally retires +`n1` and proves that `n2` and `n3` converge after purging all of its public and +internal state. + +Run the sustained matrix over all transport and lifecycle combinations with: + +```bash +test/jepsen/campaign.sh +``` + +Its defaults are 20 five-minute runs for each of six combinations. The +`GROUP_JEPSEN_CAMPAIGN_*` environment variables in the script control duration, +count, concurrency, keys, owners, and recovery time. + +Run mutation qualification plus live positive- and negative-checker tests: + +```bash +test/jepsen/qualify.sh +``` + +This runs every mutation defined by `test/mutation/run.exs`, then verifies that +a healthy live history is accepted and deliberately injected owner-death, +internal-index, and stranded snapshot-cursor corruption are rejected. + +Results and histories are written below `test/jepsen/store/`. Containers are +removed after a run. Set `GROUP_JEPSEN_KEEP_CONTAINERS=1` to retain them and +inspect logs with: + +```bash +docker compose -f test/jepsen/docker-compose.yml logs +``` + +The pure checker qualification tests do not require Docker: + +```bash +test/jepsen/checker.sh +``` + +At the repository root, `mix test` runs this pure checker after the complete +ExUnit, StreamData, and deterministic-chaos suite. `mix test.soak` runs that +same PR gate, the complete mutation/live-checker qualification, and then +`campaign.sh`. Chaos/mixed uses a sender/repair buffer of 32 and requires +evidence that one repaired delta run contained at least two records; all other +profiles keep the single-record stress setting. + +## Scope + +This checker verifies Group's eventual lifecycle contract, not +linearizability. While communication is unavailable, each side may serve its +local view and accept new owners. The requirement begins after the explicitly +bounded fault prefix: surviving peers must then always resolve to the exact +same lifecycle view, while a peer which never returns must be completely +evicted after its lease expires. + +The formal models prove the protocol for finite state spaces; StreamData +generates and shrinks scheduler-controlled histories inside real Group nodes; +this harness tests OS sockets, independent VMs, VM death, and real transport +adapters. None of these layers alone is treated as a proof of the others. diff --git a/test/jepsen/campaign.sh b/test/jepsen/campaign.sh new file mode 100755 index 0000000..179d4cb --- /dev/null +++ b/test/jepsen/campaign.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_dir="$(cd "${script_dir}/../.." && pwd)" + +time_limit="${GROUP_JEPSEN_CAMPAIGN_TIME:-300}" +test_count="${GROUP_JEPSEN_CAMPAIGN_COUNT:-20}" +concurrency="${GROUP_JEPSEN_CAMPAIGN_CONCURRENCY:-4n}" +owner_count="${GROUP_JEPSEN_CAMPAIGN_OWNERS:-128}" +key_count="${GROUP_JEPSEN_CAMPAIGN_KEYS:-32}" +recovery_time="${GROUP_JEPSEN_CAMPAIGN_RECOVERY:-15}" +profile_grace="${GROUP_JEPSEN_CAMPAIGN_PROFILE_GRACE:-90}" +artifact_dir="${GROUP_JEPSEN_CAMPAIGN_ARTIFACT_DIR:-}" + +cd "${repo_dir}" + +if [[ "${GROUP_JEPSEN_SKIP_CHECKER:-0}" != "1" ]]; then + "${script_dir}/checker.sh" +fi + +export GROUP_JEPSEN_SKIP_CHECKER=1 +mkdir -p "${script_dir}/.cache" + +if [[ -z "${artifact_dir}" ]]; then + artifact_dir="$(mktemp -d "${script_dir}/.cache/campaign.XXXXXX")" +else + mkdir -p "${artifact_dir}" +fi + +echo "Jepsen campaign artifacts: ${artifact_dir}" + +for transport in distribution tcp chaos; do + for scenario in mixed permanent; do + log="${artifact_dir}/${transport}-${scenario}.log" + sender_buffer_size=1 + min_delta_run_records=1 + + if [[ "${transport}/${scenario}" == "chaos/mixed" ]]; then + sender_buffer_size=32 + min_delta_run_records=2 + fi + + echo "Starting ${transport}/${scenario}: ${test_count} histories x ${time_limit}s (sender buffer ${sender_buffer_size})" + + # Jepsen's active generator is time-limited, but setup, recovery, checker, + # and bugs in a terminal generator live outside that limit. Keep the + # nightly gate itself bounded as a final defense against hung campaigns. + profile_timeout=$((test_count * (time_limit + recovery_time + profile_grace))) + + if GROUP_JEPSEN_SENDER_BUFFER_SIZE="${sender_buffer_size}" timeout \ + --signal=TERM --kill-after=30 "${profile_timeout}" \ + "${script_dir}/run.sh" test \ + --no-ssh \ + --nodes n1,n2,n3 \ + --concurrency "${concurrency}" \ + --time-limit "${time_limit}" \ + --test-count "${test_count}" \ + --key-count "${key_count}" \ + --owner-count "${owner_count}" \ + --fault-interval 2 \ + --recovery-time "${recovery_time}" \ + --min-delta-run-records "${min_delta_run_records}" \ + --transport "${transport}" \ + --scenario "${scenario}" >"${log}" 2>&1; then + valid_count="$(rg -c "Everything looks good" "${log}" || true)" + + if [[ "${valid_count}" != "${test_count}" ]]; then + echo "Failed ${transport}/${scenario}: expected ${test_count} valid histories, found ${valid_count:-0}" >&2 + tail -200 "${log}" >&2 + exit 1 + fi + + echo "Passed ${transport}/${scenario}: ${valid_count}/${test_count} valid histories" + else + status=$? + echo "Failed ${transport}/${scenario}; tail of ${log}:" >&2 + tail -200 "${log}" >&2 + exit "${status}" + fi + done +done + +echo "Jepsen campaign passed: ${artifact_dir}" diff --git a/test/jepsen/checker.sh b/test/jepsen/checker.sh new file mode 100755 index 0000000..336063f --- /dev/null +++ b/test/jepsen/checker.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +exec "${script_dir}/lein.sh" test diff --git a/test/jepsen/docker-compose.yml b/test/jepsen/docker-compose.yml new file mode 100644 index 0000000..039f07a --- /dev/null +++ b/test/jepsen/docker-compose.yml @@ -0,0 +1,52 @@ +services: + n1: + build: + context: ../.. + dockerfile: test/jepsen/Dockerfile.node + container_name: group-jepsen-n1 + hostname: n1 + cap_add: [NET_ADMIN] + environment: + GROUP_JEPSEN_NODE: n1 + GROUP_JEPSEN_PORT: 9080 + GROUP_JEPSEN_PEERS: group@n1,group@n2,group@n3 + GROUP_JEPSEN_TRANSPORT: ${GROUP_JEPSEN_TRANSPORT:-distribution} + GROUP_JEPSEN_SENDER_BUFFER_SIZE: ${GROUP_JEPSEN_SENDER_BUFFER_SIZE:-1} + ports: ["19081:9080"] + networks: [group] + + n2: + build: + context: ../.. + dockerfile: test/jepsen/Dockerfile.node + container_name: group-jepsen-n2 + hostname: n2 + cap_add: [NET_ADMIN] + environment: + GROUP_JEPSEN_NODE: n2 + GROUP_JEPSEN_PORT: 9080 + GROUP_JEPSEN_PEERS: group@n1,group@n2,group@n3 + GROUP_JEPSEN_TRANSPORT: ${GROUP_JEPSEN_TRANSPORT:-distribution} + GROUP_JEPSEN_SENDER_BUFFER_SIZE: ${GROUP_JEPSEN_SENDER_BUFFER_SIZE:-1} + ports: ["19082:9080"] + networks: [group] + + n3: + build: + context: ../.. + dockerfile: test/jepsen/Dockerfile.node + container_name: group-jepsen-n3 + hostname: n3 + cap_add: [NET_ADMIN] + environment: + GROUP_JEPSEN_NODE: n3 + GROUP_JEPSEN_PORT: 9080 + GROUP_JEPSEN_PEERS: group@n1,group@n2,group@n3 + GROUP_JEPSEN_TRANSPORT: ${GROUP_JEPSEN_TRANSPORT:-distribution} + GROUP_JEPSEN_SENDER_BUFFER_SIZE: ${GROUP_JEPSEN_SENDER_BUFFER_SIZE:-1} + ports: ["19083:9080"] + networks: [group] + +networks: + group: + name: group-jepsen-net diff --git a/test/jepsen/entrypoint.sh b/test/jepsen/entrypoint.sh new file mode 100755 index 0000000..8911254 --- /dev/null +++ b/test/jepsen/entrypoint.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${GROUP_JEPSEN_NODE:?GROUP_JEPSEN_NODE is required}" + +exec elixir \ + --sname group \ + --cookie group_jepsen \ + --erl "-kernel net_ticktime 2" \ + -S mix run --no-compile --no-deps-check test/jepsen/node.exs diff --git a/test/jepsen/lein.sh b/test/jepsen/lein.sh new file mode 100755 index 0000000..5c70bef --- /dev/null +++ b/test/jepsen/lein.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cache_dir="${script_dir}/.cache" +lein="${cache_dir}/lein" + +mkdir -p "${cache_dir}" + +if [[ ! -x "${lein}" ]]; then + curl --fail --silent --show-error --location \ + https://raw.githubusercontent.com/technomancy/leiningen/2.12.0/bin/lein \ + --output "${lein}" + chmod +x "${lein}" +fi + +export LEIN_HOME="${cache_dir}/lein-home" +cd "${script_dir}" + +exec "${lein}" "$@" diff --git a/test/jepsen/node.exs b/test/jepsen/node.exs new file mode 100644 index 0000000..424259d --- /dev/null +++ b/test/jepsen/node.exs @@ -0,0 +1,1464 @@ +Code.require_file("../support/test_tcp_transport.ex", __DIR__) + +defmodule Group.Jepsen.Transport.Stats do + @moduledoc false + use GenServer + + @table :group_jepsen_transport_stats + @gate :group_jepsen_transport_gate + @persistent_event_log "/tmp/group-jepsen-persistent-events" + + def start_link(_opts), do: GenServer.start_link(__MODULE__, :ok, name: __MODULE__) + + def increment(event, amount \\ 1) do + :ets.update_counter(@table, event, {2, amount}, {event, 0}) + :ok + rescue + ArgumentError -> :ok + end + + def increment_persistent(event, amount \\ 1) do + increment(event, amount) + File.write!(@persistent_event_log, "#{event}\t#{amount}\n", [:append]) + :ok + end + + def observe_max(event, value) when is_integer(value) and value >= 0 do + current = :ets.update_counter(@table, event, {2, 0}, {event, 0}) + if value > current, do: :ets.insert(@table, {event, value}) + :ok + rescue + ArgumentError -> :ok + end + + def snapshot do + persisted = persistent_events() + + @table + |> :ets.tab2list() + |> Map.new() + |> Map.merge(persisted, fn _event, current, durable -> max(current, durable) end) + end + + def block(target_node), do: :ets.insert(@gate, {target_node}) + def unblock(target_node), do: :ets.delete(@gate, target_node) + def heal, do: :ets.delete_all_objects(@gate) + def blocked?(target_node), do: :ets.member(@gate, target_node) + + @impl true + def init(:ok) do + _table = :ets.new(@table, [:named_table, :public, :set, write_concurrency: true]) + _gate = :ets.new(@gate, [:named_table, :public, :set, read_concurrency: true]) + {:ok, %{}} + end + + defp persistent_events do + case File.read(@persistent_event_log) do + {:ok, contents} -> + contents + |> String.split("\n", trim: true) + |> Enum.reduce(%{}, fn line, events -> + case String.split(line, "\t", parts: 2) do + [event, amount] -> + Map.update( + events, + String.to_existing_atom(event), + String.to_integer(amount), + &(&1 + String.to_integer(amount)) + ) + + _invalid -> + events + end + end) + + {:error, :enoent} -> + %{} + end + end +end + +defmodule Group.Jepsen.Transport.Common do + @moduledoc false + alias Group.Jepsen.Transport.Stats + + def outgoing(delegate, group, target_node, shard, message, opts) do + record(message) + + if Stats.blocked?(target_node) do + Stats.increment(:logical_drop) + :ok + else + result = delegate.outgoing(group, target_node, shard, message, opts) + Stats.increment(transport_result(result)) + observe_outbox(group, shard) + result + end + end + + def record({:snapshot_chunk, _version, _stream, _seq, _index, _registry, _pg}) do + Stats.increment(:snapshot_chunk) + end + + def record({:snapshot_commit, _version, _stream, _seq, chunk_count, _, _}) do + Stats.increment(:snapshot_commit) + + if chunk_count > 1 do + Stats.increment(:multi_chunk_snapshot) + end + end + + def record({:delta_batch, _version, runs}) do + Stats.increment(:delta_batch) + + peak = + runs + |> Enum.map(fn + {_stream, _first_seq, records, _head} when is_list(records) -> length(records) + _invalid_run -> 0 + end) + |> Enum.max(fn -> 0 end) + + Stats.observe_max(:delta_run_records_peak, peak) + end + + def record(_message), do: Stats.increment(:other_message) + + defp transport_result(:ok), do: :transport_ok + defp transport_result(:busy), do: :transport_busy + defp transport_result(:disconnected), do: :transport_disconnected + + defp observe_outbox(group, shard) do + case Process.whereis(Group.Transport.Outbox.name(group, shard)) do + pid when is_pid(pid) -> + case Process.info(pid, :message_queue_len) do + {:message_queue_len, length} -> Stats.observe_max(:outbox_mailbox_peak, length) + _ -> :ok + end + + nil -> + :ok + end + end +end + +defmodule Group.Jepsen.Transport.Distribution do + @moduledoc false + @behaviour Group.Transport + + alias Group.Jepsen.Transport.{Common, Stats} + alias Group.Transport.DistErl, as: Delegate + + @impl true + def id, do: Delegate.id() + + @impl true + def descriptor(group, opts), do: Delegate.descriptor(group, opts) + + @impl true + def child_spec(opts), do: {Stats, opts} + + @impl true + def outgoing(group, target_node, shard, message, opts) do + Common.outgoing(Delegate, group, target_node, shard, message, opts) + end +end + +defmodule Group.Jepsen.Transport.TCP do + @moduledoc false + @behaviour Group.Transport + + alias Group.Jepsen.Transport.Common + alias Group.TestTCPTransport, as: Delegate + + @impl true + def id, do: Delegate.id() + + @impl true + def descriptor(group, opts), do: Delegate.descriptor(group, opts) + + @impl true + def child_spec(opts) do + %{ + id: {__MODULE__, Keyword.fetch!(opts, :name)}, + start: {Group.Jepsen.Transport.TCP.Supervisor, :start_link, [opts]}, + type: :supervisor + } + end + + @impl true + def outgoing(group, target_node, shard, message, opts) do + Common.outgoing(Delegate, group, target_node, shard, message, opts) + end + + @impl true + def peer_up(group, remote_node, shard, descriptor, opts), + do: Delegate.peer_up(group, remote_node, shard, descriptor, opts) + + @impl true + def peer_down(group, remote_node, shard, opts), + do: Delegate.peer_down(group, remote_node, shard, opts) +end + +defmodule Group.Jepsen.Transport.TCP.Supervisor do + @moduledoc false + use Supervisor + + def start_link(opts), do: Supervisor.start_link(__MODULE__, opts) + + @impl true + def init(opts) do + children = [ + {Group.Jepsen.Transport.Stats, opts}, + Group.TestTCPTransport.child_spec(opts) + ] + + Supervisor.init(children, strategy: :one_for_one) + end +end + +defmodule Group.Jepsen.Transport.Chaos do + @moduledoc false + @behaviour Group.Transport + + alias Group.Jepsen.Transport.{Common, Stats} + + @impl true + def id, do: :group_jepsen_unordered_v1 + + @impl true + def descriptor(_group, _opts), do: :group_jepsen_unordered_v1 + + @impl true + def child_spec(opts) do + %{ + id: {__MODULE__, Keyword.fetch!(opts, :name)}, + start: {Group.Jepsen.Transport.Chaos.Supervisor, :start_link, [opts]}, + type: :supervisor + } + end + + @impl true + def outgoing(group, target_node, shard, message, _opts) do + Common.record(message) + + if Stats.blocked?(target_node) do + Stats.increment(:logical_drop) + :ok + else + case Process.whereis(worker_name(group, shard)) do + pid when is_pid(pid) -> + send(pid, {:outgoing, target_node, message}) + :ok + + nil -> + :disconnected + end + end + end + + def worker_name(group, shard), do: :"#{group}_jepsen_chaos_#{shard}" +end + +defmodule Group.Jepsen.Transport.Chaos.Supervisor do + @moduledoc false + use Supervisor + + def start_link(opts), do: Supervisor.start_link(__MODULE__, opts) + + @impl true + def init(opts) do + group = Keyword.fetch!(opts, :name) + num_shards = Keyword.fetch!(opts, :num_shards) + + workers = + for shard <- 0..(num_shards - 1) do + %{ + id: {Group.Jepsen.Transport.Chaos.Worker, shard}, + start: {Group.Jepsen.Transport.Chaos.Worker, :start_link, [group, shard]} + } + end + + Supervisor.init([{Group.Jepsen.Transport.Stats, opts} | workers], strategy: :one_for_one) + end +end + +defmodule Group.Jepsen.Transport.Chaos.Worker do + @moduledoc false + use GenServer + + alias Group.Jepsen.Transport.{Chaos, Stats} + + def start_link(group, shard) do + GenServer.start_link(__MODULE__, {group, shard}, name: Chaos.worker_name(group, shard)) + end + + @impl true + def init({group, shard}), do: {:ok, %{group: group, shard: shard, counter: 0}} + + @impl true + def handle_info({:outgoing, target_node, message}, state) do + counter = state.counter + 1 + next = %{state | counter: counter} + + if rem(counter, 5) == 0 do + Stats.increment(:chaos_drop) + else + delay = rem(counter * 17, 41) + Process.send_after(self(), {:forward, target_node, message}, delay) + + if rem(counter, 7) == 0 do + Stats.increment(:chaos_duplicate) + Process.send_after(self(), {:forward, target_node, message}, rem(delay + 19, 47)) + end + + if delay > 0, do: Stats.increment(:chaos_delay) + end + + {:noreply, next} + end + + def handle_info({:forward, target_node, replica_message}, state) do + destination = {Group.Replica.shard_name(state.group, state.shard), target_node} + message = {:group_replica_frame, node(), replica_message} + + case :erlang.send_nosuspend(destination, message, [:noconnect]) do + true -> Stats.increment(:chaos_delivered) + false -> Stats.increment(:chaos_busy) + end + + {:noreply, state} + end +end + +defmodule Group.Jepsen.Transport.Control do + @moduledoc false + + alias Group.Jepsen.Transport.Stats + + def profile do + case System.get_env("GROUP_JEPSEN_TRANSPORT", "distribution") do + "distribution" -> :distribution + "tcp" -> :tcp + "chaos" -> :chaos + other -> raise "unknown GROUP_JEPSEN_TRANSPORT #{inspect(other)}" + end + end + + def transport(node_id) do + case profile() do + :distribution -> + Group.Jepsen.Transport.Distribution + + :chaos -> + Group.Jepsen.Transport.Chaos + + :tcp -> + {:ok, advertised_ip} = :inet.getaddr(String.to_charlist(node_id), :inet) + + {Group.Jepsen.Transport.TCP, + [ + ip: {0, 0, 0, 0}, + advertised_ip: advertised_ip, + port: 10_000, + max_queue: 32, + connect_timeout: 100, + send_timeout: 100, + reconnect_interval: 25, + outbox_batch_size: 16, + outbox_batch_bytes: 65_536, + outbox_flush_interval: 1, + outbox_deadline: 100 + ]} + end + end + + def block(target_node) do + Stats.block(target_node) + maybe_disconnect(target_node) + :ok + end + + def unblock(target_node) do + Stats.unblock(target_node) + maybe_reconnect(target_node) + :ok + end + + def heal(peer_nodes) do + Stats.heal() + Enum.each(peer_nodes, &maybe_reconnect/1) + :ok + end + + def reset(target_node) do + maybe_disconnect(target_node) + Process.sleep(10) + maybe_reconnect(target_node) + :ok + end + + defp maybe_disconnect(target_node) do + if profile() == :tcp do + Group.TestTCPTransport.disconnect_peer(:jepsen_group, target_node) + end + catch + :exit, _ -> :ok + end + + defp maybe_reconnect(target_node) do + if profile() == :tcp do + Group.TestTCPTransport.reconnect_peer(:jepsen_group, target_node) + end + catch + :exit, _ -> :ok + end +end + +defmodule Group.Jepsen.ConflictResolver do + @moduledoc false + + def resolve(_name, _key, {_pid, meta, _time}), do: rank(meta) + + defp rank(%{revision: revision, token: token}), do: {revision, token} + defp rank(_meta), do: {-1, ""} +end + +defmodule Group.Jepsen.Owner do + @moduledoc false + use GenServer + + def start(token), do: GenServer.start(__MODULE__, token) + + @impl true + def init(token), do: {:ok, %{token: token, registrations: %{}, memberships: %{}}} + + @impl true + def handle_call({:mutate, :register, cluster, key, revision}, _from, state) do + meta = %{token: state.token, revision: revision} + + case safe_group_call(fn -> + Group.register(:jepsen_group, registry_key(key), meta, cluster_opts(cluster)) + end) do + :ok -> + entry = %{cluster: cluster, key: key, revision: revision} + state = put_in(state.registrations[{cluster, key}], entry) + {:reply, {:ok, snapshot(state)}, state} + + {:error, reason} -> + {:reply, {:error, reason, snapshot(state)}, state} + end + end + + def handle_call({:mutate, :unregister, cluster, key, _revision}, _from, state) do + owner_key = {cluster, key} + + if Map.has_key?(state.registrations, owner_key) do + case safe_group_call(fn -> + Group.unregister(:jepsen_group, registry_key(key), cluster_opts(cluster)) + end) do + :ok -> + state = %{state | registrations: Map.delete(state.registrations, owner_key)} + {:reply, {:ok, snapshot(state)}, state} + + {:error, reason} -> + {:reply, {:error, reason, snapshot(state)}, state} + end + else + {:reply, {:error, :not_owned, snapshot(state)}, state} + end + end + + def handle_call({:mutate, :join, cluster, key, revision}, _from, state) do + meta = %{token: state.token, revision: revision} + + case safe_group_call(fn -> + Group.join(:jepsen_group, pg_key(key), meta, cluster_opts(cluster)) + end) do + :ok -> + entry = %{cluster: cluster, key: key, revision: revision} + state = put_in(state.memberships[{cluster, key}], entry) + {:reply, {:ok, snapshot(state)}, state} + + {:error, reason} -> + {:reply, {:error, reason, snapshot(state)}, state} + end + end + + def handle_call({:mutate, :leave, cluster, key, _revision}, _from, state) do + owner_key = {cluster, key} + + if Map.has_key?(state.memberships, owner_key) do + case safe_group_call(fn -> + Group.leave(:jepsen_group, pg_key(key), cluster_opts(cluster)) + end) do + :ok -> + state = %{state | memberships: Map.delete(state.memberships, owner_key)} + {:reply, {:ok, snapshot(state)}, state} + + {:error, reason} -> + {:reply, {:error, reason, snapshot(state)}, state} + end + else + {:reply, {:error, :not_owned, snapshot(state)}, state} + end + end + + def handle_call({:drop_cluster, cluster}, _from, state) do + registrations = drop_cluster(state.registrations, cluster) + memberships = drop_cluster(state.memberships, cluster) + state = %{state | registrations: registrations, memberships: memberships} + {:reply, :ok, state} + end + + def handle_call(:snapshot, _from, state), do: {:reply, snapshot(state), state} + + defp drop_cluster(entries, cluster) do + entries + |> Enum.reject(fn {{entry_cluster, _key}, _entry} -> entry_cluster == cluster end) + |> Map.new() + end + + defp snapshot(state) do + %{ + token: state.token, + registrations: state.registrations |> Map.values() |> sort_entries(), + memberships: state.memberships |> Map.values() |> sort_entries() + } + end + + defp sort_entries(entries), do: Enum.sort_by(entries, &{&1.cluster || "", &1.key}) + + defp safe_group_call(fun) do + fun.() + rescue + exception -> {:error, {:exception, Exception.message(exception)}} + catch + kind, reason -> {:error, {kind, reason}} + end + + defp cluster_opts(nil), do: [] + defp cluster_opts(cluster), do: [cluster: cluster] + defp registry_key(key), do: "jepsen/registry/#{key}" + defp pg_key(key), do: "jepsen/pg/#{key}" +end + +defmodule Group.Jepsen.Driver do + @moduledoc false + use GenServer + + alias Group.Jepsen.Transport.Stats + + @driver_count 8 + @unexpected_death_log "/tmp/group-jepsen-unexpected-deaths" + + def start_link(opts) do + index = Keyword.fetch!(opts, :index) + GenServer.start_link(__MODULE__, opts, name: name(index)) + end + + def child_specs(opts) do + for index <- 0..(@driver_count - 1) do + %{ + id: {__MODULE__, index}, + start: {__MODULE__, :start_link, [Keyword.put(opts, :index, index)]} + } + end + end + + def mutate(operation, logical_owner, cluster, key, revision) do + started_at = System.monotonic_time(:microsecond) + + response = + GenServer.call( + driver(logical_owner), + {:mutate, operation, logical_owner, cluster, key, revision}, + 10_000 + ) + + Map.put(response, :latency_us, System.monotonic_time(:microsecond) - started_at) + end + + def kill(logical_owner), do: GenServer.call(driver(logical_owner), {:kill, logical_owner}) + + def drop_cluster(cluster) do + Enum.each(names(), &GenServer.call(&1, {:drop_cluster, cluster}, 10_000)) + :ok + end + + def owner_snapshots do + names() + |> Enum.flat_map(fn driver -> + case GenServer.call(driver, :owner_snapshots, 30_000) do + {:ok, snapshots} -> snapshots + {:error, reason} -> raise "owner snapshot refresh failed: #{inspect(reason)}" + end + end) + |> Enum.sort_by(& &1.token) + end + + def unexpected_deaths do + in_memory = Enum.flat_map(names(), &GenServer.call(&1, :unexpected_deaths, 30_000)) + + (in_memory ++ persisted_unexpected_deaths()) + |> Enum.uniq() + |> Enum.sort_by(& &1.token) + end + + @impl true + def init(opts) do + {:ok, + %{ + node_id: Keyword.fetch!(opts, :node_id), + boot_id: Keyword.fetch!(opts, :boot_id), + owners: %{}, + monitors: %{}, + incarnations: %{}, + unexpected_deaths: [] + }} + end + + @impl true + def handle_call({:mutate, operation, logical_owner, cluster, key, revision}, _from, state) do + {pid, state} = owner(state, logical_owner) + + try do + case GenServer.call(pid, {:mutate, operation, cluster, key, revision}, 8_000) do + {:ok, owner_state} -> + {:reply, %{status: :ok, owner: owner_state}, + put_owner_state(state, logical_owner, pid, owner_state)} + + {:error, reason, owner_state} -> + {:reply, %{status: :fail, error: inspect(reason), owner: owner_state}, + put_owner_state(state, logical_owner, pid, owner_state)} + end + catch + :exit, reason -> + {:reply, %{status: :unknown, error: inspect(reason)}, state} + end + end + + def handle_call({:kill, logical_owner}, _from, state) do + case Map.get(state.owners, logical_owner) do + nil -> + {:reply, %{status: :ok, killed: nil}, state} + + {pid, token, monitor_ref, _owner_state} -> + if Process.alive?(pid) do + Process.exit(pid, :kill) + Process.demonitor(monitor_ref, [:flush]) + + {:reply, %{status: :ok, killed: token}, + %{ + state + | owners: Map.delete(state.owners, logical_owner), + monitors: Map.delete(state.monitors, monitor_ref) + }} + else + {:reply, %{status: :unknown, error: "owner already dead"}, state} + end + end + end + + def handle_call({:drop_cluster, cluster}, _from, state) do + Enum.each(state.owners, fn {_logical_owner, {pid, _token, _monitor, _owner_state}} -> + if Process.alive?(pid), do: GenServer.call(pid, {:drop_cluster, cluster}, 10_000) + end) + + owners = + Map.new(state.owners, fn {logical_owner, {pid, token, monitor, _owner_state}} -> + owner_state = if Process.alive?(pid), do: GenServer.call(pid, :snapshot), else: nil + {logical_owner, {pid, token, monitor, owner_state}} + end) + + {:reply, :ok, %{state | owners: owners}} + end + + def handle_call(:owner_snapshots, _from, state) do + result = + Enum.reduce_while(state.owners, {[], %{}}, fn + {logical_owner, {pid, token, monitor_ref, cached}}, {snapshots, acc} -> + case live_owner_snapshot(pid) do + {:ok, owner_state} -> + {:cont, + { + [owner_state | snapshots], + Map.put(acc, logical_owner, {pid, token, monitor_ref, owner_state}) + }} + + {:error, reason} -> + {:halt, {:error, {logical_owner, token, reason, cached}}} + end + end) + + case result do + {:error, reason} -> + {:reply, {:error, reason}, state} + + {owners, refreshed} -> + {:reply, {:ok, Enum.reverse(owners)}, %{state | owners: refreshed}} + end + end + + defp live_owner_snapshot(pid) do + if Process.alive?(pid) do + try do + {:ok, GenServer.call(pid, :snapshot, 10_000)} + catch + :exit, reason -> {:error, reason} + end + else + {:error, :not_alive} + end + end + + def handle_call(:unexpected_deaths, _from, state) do + {:reply, state.unexpected_deaths, state} + end + + @impl true + def handle_info({:DOWN, monitor_ref, :process, _pid, reason}, state) do + case Map.pop(state.monitors, monitor_ref) do + {nil, monitors} -> + {:noreply, %{state | monitors: monitors}} + + {{logical_owner, token}, monitors} -> + owners = + case Map.get(state.owners, logical_owner) do + {_pid, ^token, ^monitor_ref, _owner_state} -> Map.delete(state.owners, logical_owner) + _newer_incarnation -> state.owners + end + + unexpected_deaths = + if match?({:group_registry_conflict, _key, _winner_meta}, reason) do + Stats.increment_persistent(:registry_conflict_death) + state.unexpected_deaths + else + death = %{token: token, reason: inspect(reason)} + :ok = persist_unexpected_death(death) + [death | state.unexpected_deaths] + end + + {:noreply, + %{state | owners: owners, monitors: monitors, unexpected_deaths: unexpected_deaths}} + end + end + + defp owner(state, logical_owner) do + case Map.get(state.owners, logical_owner) do + {pid, _token, _monitor_ref, _owner_state} when is_pid(pid) -> + if Process.alive?(pid), do: {pid, state}, else: start_owner(state, logical_owner) + + nil -> + start_owner(state, logical_owner) + end + end + + defp start_owner(state, logical_owner) do + incarnation = Map.get(state.incarnations, logical_owner, 0) + 1 + token = "#{state.node_id}/#{state.boot_id}/#{logical_owner}/#{incarnation}" + {:ok, pid} = Group.Jepsen.Owner.start(token) + monitor_ref = Process.monitor(pid) + owner_state = %{token: token, registrations: [], memberships: []} + + state = %{ + state + | owners: Map.put(state.owners, logical_owner, {pid, token, monitor_ref, owner_state}), + monitors: Map.put(state.monitors, monitor_ref, {logical_owner, token}), + incarnations: Map.put(state.incarnations, logical_owner, incarnation) + } + + {pid, state} + end + + defp put_owner_state(state, logical_owner, pid, owner_state) do + owners = + case Map.get(state.owners, logical_owner) do + {^pid, token, monitor_ref, _old_owner_state} -> + Map.put(state.owners, logical_owner, {pid, token, monitor_ref, owner_state}) + + _replaced_owner -> + state.owners + end + + %{state | owners: owners} + end + + defp names, do: Enum.map(0..(@driver_count - 1), &name/1) + defp driver(logical_owner), do: name(:erlang.phash2(logical_owner, @driver_count)) + defp name(index), do: :"group_jepsen_driver_#{index}" + + defp persist_unexpected_death(%{token: token, reason: reason}) do + File.write(@unexpected_death_log, token <> "\t" <> reason <> "\n", [:append]) + end + + defp persisted_unexpected_deaths do + case File.read(@unexpected_death_log) do + {:ok, contents} -> + contents + |> String.split("\n", trim: true) + |> Enum.flat_map(fn line -> + case String.split(line, "\t", parts: 2) do + [token, reason] -> [%{token: token, reason: reason}] + _invalid -> [] + end + end) + + {:error, :enoent} -> + [] + + {:error, reason} -> + [%{token: "ORACLE-READ-FAILURE", reason: inspect(reason)}] + end + end +end + +defmodule Group.Jepsen.Driver.Supervisor do + @moduledoc false + use Supervisor + + def start_link(opts), do: Supervisor.start_link(__MODULE__, opts, name: __MODULE__) + + @impl true + def init(opts), + do: Supervisor.init(Group.Jepsen.Driver.child_specs(opts), strategy: :one_for_one) +end + +defmodule Group.Jepsen.Cluster do + @moduledoc false + use GenServer + + def start_link(clusters), do: GenServer.start_link(__MODULE__, clusters, name: __MODULE__) + def connect(cluster), do: GenServer.call(__MODULE__, {:connect, cluster}, 60_000) + def disconnect(cluster), do: GenServer.call(__MODULE__, {:disconnect, cluster}, 60_000) + def connect_all, do: GenServer.call(__MODULE__, :connect_all, 60_000) + + @impl true + def init(clusters) do + :ok = Group.connect(:jepsen_group, clusters) + {:ok, %{clusters: clusters}} + end + + @impl true + def handle_call({:connect, cluster}, _from, state) do + result = Group.connect(:jepsen_group, cluster) + {:reply, result(result), state} + end + + def handle_call({:disconnect, cluster}, _from, state) do + result = Group.disconnect(:jepsen_group, cluster) + if result == :ok, do: Group.Jepsen.Driver.drop_cluster(cluster) + {:reply, result(result), state} + end + + def handle_call(:connect_all, _from, state) do + result = Group.connect(:jepsen_group, state.clusters) + {:reply, result(result), state} + end + + defp result(:ok), do: %{status: :ok} + defp result({:error, reason}), do: %{status: :fail, error: inspect(reason)} +end + +defmodule Group.Jepsen.Invariant do + @moduledoc false + + alias Group.Replica.{Data, WireProtocol} + + def snapshot(retired_nodes) do + config = Group.get_config(:jepsen_group) + shards = 0..(config.num_shards - 1) + maybe_inject_cursor_marker_corruption(shards) + + errors = + check("dual indexes", &assert_dual_indexes/0) ++ + check("registry claims", &assert_registry_claims/0) ++ + check("oplog", &assert_oplogs/0) ++ + check("cursor authority", &assert_cursor_authority/0) ++ + check("retired origins", fn -> assert_retired_origins(retired_nodes) end) + + staging_count = + Enum.reduce(shards, 0, fn shard, total -> + state = :sys.get_state(Group.Replica.shard_name(:jepsen_group, shard)) + total + map_size(state.snapshot_transfers) + end) + + oplog_entries = + Enum.reduce(shards, 0, fn shard, total -> + total + :ets.info(Data.replica_oplog_order_table(:jepsen_group, shard), :size) + end) + + %{ + healthy: errors == [] and staging_count == 0, + errors: errors, + snapshot_staging_count: staging_count, + oplog_entries: oplog_entries, + oplog_max_entries_per_shard: config.replicated_oplog_max_entries, + shard_mailbox_max: + mailbox_max(Enum.map(shards, &Group.Replica.shard_name(:jepsen_group, &1))), + outbox_mailbox_max: + mailbox_max(Enum.map(shards, &Group.Transport.Outbox.name(:jepsen_group, &1))), + total_memory_bytes: :erlang.memory(:total) + } + rescue + exception -> + %{ + healthy: false, + errors: ["invariant snapshot failed: #{Exception.message(exception)}"], + snapshot_staging_count: -1 + } + end + + defp maybe_inject_cursor_marker_corruption(shards) do + if File.exists?("/tmp/group-jepsen-cursor-marker-corruption") do + cursor = + Enum.find_value(shards, fn shard -> + Data.replica_cursor_table(:jepsen_group, shard) + |> :ets.tab2list() + |> case do + [{stream, _cursor} | _] -> {shard, stream} + [] -> nil + end + end) + + case cursor do + {shard, stream} -> + :ets.insert( + Data.replica_cursor_table(:jepsen_group, shard), + {stream, {:snapshot_installing, 1}} + ) + + nil -> + raise "no remote replica cursor available for corruption" + end + end + end + + defp check(label, fun) do + fun.() + [] + rescue + exception -> ["#{label}: #{Exception.message(exception)}"] + catch + kind, reason -> ["#{label}: #{inspect({kind, reason})}"] + end + + defp assert_dual_indexes do + shards(fn shard -> + reg_key = + Data.reg_by_key_table(:jepsen_group, shard) + |> :ets.tab2list() + |> MapSet.new(fn {{cluster, key}, pid, meta, time, origin} -> + {cluster, key, pid, meta, time, origin} + end) + + reg_pid = + Data.reg_by_pid_table(:jepsen_group, shard) + |> :ets.tab2list() + |> MapSet.new(fn {{pid, cluster, key}, meta, time, origin} -> + {cluster, key, pid, meta, time, origin} + end) + + pg_key = + Data.pg_by_key_table(:jepsen_group, shard) + |> :ets.tab2list() + |> MapSet.new(fn {{cluster, key, pid}, meta, time, origin} -> + {cluster, key, pid, meta, time, origin} + end) + + pg_pid = + Data.pg_by_pid_table(:jepsen_group, shard) + |> :ets.tab2list() + |> MapSet.new(fn {{pid, cluster, key}, meta, time, origin} -> + {cluster, key, pid, meta, time, origin} + end) + + assert_equal!(reg_key, reg_pid, "registry dual indexes shard #{shard}") + assert_equal!(pg_key, pg_pid, "PG dual indexes shard #{shard}") + end) + + cluster_nodes = + Data.cluster_nodes_table(:jepsen_group) + |> :ets.tab2list() + |> MapSet.new(fn {cluster, origin} -> {cluster, origin} end) + + node_clusters = + Data.node_clusters_table(:jepsen_group) + |> :ets.tab2list() + |> MapSet.new(fn {origin, cluster} -> {cluster, origin} end) + + assert_equal!(cluster_nodes, node_clusters, "cluster dual indexes") + end + + defp assert_registry_claims do + shards(fn shard -> + by_key = + Data.reg_claim_by_key_table(:jepsen_group, shard) + |> :ets.tab2list() + |> MapSet.new(fn + {{cluster, key, origin, generation, epoch}, pid, meta, time, seq} -> + {cluster, key, pid, meta, time, origin, generation, epoch, seq} + end) + + by_pid = + Data.reg_claim_by_pid_table(:jepsen_group, shard) + |> :ets.tab2list() + |> MapSet.new(fn + {{pid, cluster, key, origin, generation, epoch}, meta, time, seq} -> + {cluster, key, pid, meta, time, origin, generation, epoch, seq} + end) + + assert_equal!(by_key, by_pid, "registry claim indexes shard #{shard}") + + invalid_origin = + Enum.find(by_key, fn {_cluster, _key, pid, _meta, _time, origin, _gen, _epoch, _seq} -> + node(pid) != origin + end) + + if invalid_origin, do: raise("claim with invalid PID origin #{inspect(invalid_origin)}") + + authority = + MapSet.new(by_key, fn {cluster, key, pid, meta, time, origin, _gen, _epoch, _seq} -> + {cluster, key, pid, meta, time, origin} + end) + + visible = + Data.reg_by_key_table(:jepsen_group, shard) + |> :ets.tab2list() + |> MapSet.new(fn {{cluster, key}, pid, meta, time, origin} -> + {cluster, key, pid, meta, time, origin} + end) + + missing = MapSet.difference(visible, authority) + + if MapSet.size(missing) > 0, + do: raise("visible registry rows lack claims #{inspect(missing)}") + end) + end + + defp assert_oplogs do + max_entries = Group.get_config(:jepsen_group).replicated_oplog_max_entries + + shards(fn shard -> + oplog = + Data.replica_oplog_table(:jepsen_group, shard) + |> :ets.tab2list() + |> MapSet.new(fn {{stream, seq}, append_id, _mutations} -> {append_id, stream, seq} end) + + order = + Data.replica_oplog_order_table(:jepsen_group, shard) |> :ets.tab2list() |> MapSet.new() + + assert_equal!(oplog, order, "oplog/order shard #{shard}") + + if MapSet.size(order) > max_entries do + raise "oplog bound exceeded shard #{shard}: #{MapSet.size(order)} > #{max_entries}" + end + + Data.replica_stream_meta_table(:jepsen_group, shard) + |> :ets.tab2list() + |> Enum.each(fn {stream, head, floor, applied} -> + unless floor >= 1 and floor <= head + 1 and applied >= 0 and applied <= head do + raise "invalid stream bounds #{inspect({stream, head, floor, applied})}" + end + + retained = + oplog + |> Enum.filter(fn {_append, row_stream, _seq} -> row_stream == stream end) + |> Enum.map(&elem(&1, 2)) + |> Enum.sort() + + expected = if floor <= head, do: Enum.to_list(floor..head), else: [] + if retained != expected, do: raise("non-contiguous oplog #{inspect(stream)}") + end) + end) + end + + defp assert_cursor_authority do + shards(fn shard -> + Data.replica_cursor_table(:jepsen_group, shard) + |> :ets.tab2list() + |> Enum.each(fn {stream, seq} -> + origin = WireProtocol.stream_origin(stream) + cluster = WireProtocol.stream_cluster(stream) + + valid? = + WireProtocol.stream_name(stream) == :jepsen_group and + WireProtocol.stream_shard(stream) == shard and + origin != node() and + WireProtocol.stream_generation(stream) == + Data.remote_generation(:jepsen_group, origin) and + WireProtocol.stream_epoch(stream) == + Data.remote_cluster_epoch(:jepsen_group, origin, cluster) and is_integer(seq) and + seq >= 0 + + unless valid?, do: raise("cursor lacks current authority #{inspect({stream, seq})}") + end) + end) + end + + defp assert_retired_origins(retired_nodes) do + Enum.each(retired_nodes, fn origin -> + shards(fn shard -> + claims = + Data.reg_claim_by_key_table(:jepsen_group, shard) + |> :ets.tab2list() + |> Enum.filter(fn {{_cluster, _key, row_origin, _gen, _epoch}, _, _, _, _} -> + row_origin == origin + end) + + registry = + Data.reg_by_key_table(:jepsen_group, shard) + |> :ets.tab2list() + |> Enum.filter(fn {_key, _pid, _meta, _time, row_origin} -> row_origin == origin end) + + pg = + Data.pg_by_key_table(:jepsen_group, shard) + |> :ets.tab2list() + |> Enum.filter(fn {_key, _meta, _time, row_origin} -> row_origin == origin end) + + cursors = + Data.replica_cursor_table(:jepsen_group, shard) + |> :ets.tab2list() + |> Enum.filter(fn {stream, _seq} -> WireProtocol.stream_origin(stream) == origin end) + + view = Data.remote_view_generation(:jepsen_group, shard, origin) + + unless claims == [] and registry == [] and pg == [] and cursors == [] and is_nil(view) do + raise "retired origin retained on shard #{shard}: #{inspect(origin)}" + end + end) + + unless is_nil(Data.remote_generation(:jepsen_group, origin)) and + Data.clusters_for_node(:jepsen_group, origin) == [] do + raise "retired origin retained shared authority: #{inspect(origin)}" + end + end) + end + + defp shards(fun) do + num_shards = Group.get_config(:jepsen_group).num_shards + Enum.each(0..(num_shards - 1), fun) + end + + defp assert_equal!(left, right, label) do + if left != right do + raise "#{label}: left-only=#{inspect(MapSet.difference(left, right))} " <> + "right-only=#{inspect(MapSet.difference(right, left))}" + end + end + + defp mailbox_max(names) do + names + |> Enum.map(fn name -> + case Process.whereis(name) do + pid when is_pid(pid) -> + case Process.info(pid, :message_queue_len) do + {:message_queue_len, length} -> length + _ -> 0 + end + + nil -> + 0 + end + end) + |> Enum.max(fn -> 0 end) + end +end + +defmodule Group.Jepsen.Snapshot do + @moduledoc false + + def capture(node_id, boot_id, key_count, clusters, retired_nodes) do + owners = Group.Jepsen.Driver.owner_snapshots() + + registry = + Map.new([nil | clusters], fn cluster -> + values = + Map.new(0..(key_count - 1), fn key -> + value = + case Group.lookup(:jepsen_group, registry_key(key), cluster_opts(cluster)) do + nil -> nil + {_pid, %{token: token}} -> token + {_pid, other} -> "INVALID:#{inspect(other)}" + end + + {key, value} + end) + + {cluster_name(cluster), values} + end) + + pg = + Map.new([nil | clusters], fn cluster -> + values = + Map.new(0..(key_count - 1), fn key -> + tokens = + :jepsen_group + |> Group.members(pg_key(key), cluster_opts(cluster)) + |> Enum.map(fn + {_pid, %{token: token}} -> token + {_pid, other} -> "INVALID:#{inspect(other)}" + end) + |> Enum.sort() + + {key, tokens} + end) + + {cluster_name(cluster), values} + end) + + %{ + status: :ok, + snapshot: %{ + node: node_id, + boot: boot_id, + peers: Group.nodes(:jepsen_group) |> Enum.map(&Atom.to_string/1) |> Enum.sort(), + owners: owners, + unexpected_deaths: Group.Jepsen.Driver.unexpected_deaths(), + transport_events: Group.Jepsen.Transport.Stats.snapshot(), + transport_profile: Group.Jepsen.Transport.Control.profile(), + internal: Group.Jepsen.Invariant.snapshot(retired_nodes), + registry: registry, + pg: pg + } + } + end + + defp cluster_name(nil), do: "root" + defp cluster_name(cluster), do: cluster + defp cluster_opts(nil), do: [] + defp cluster_opts(cluster), do: [cluster: cluster] + defp registry_key(key), do: "jepsen/registry/#{key}" + defp pg_key(key), do: "jepsen/pg/#{key}" +end + +defmodule Group.Jepsen.EDN do + @moduledoc false + + def encode(nil), do: "nil" + def encode(true), do: "true" + def encode(false), do: "false" + def encode(value) when is_integer(value), do: Integer.to_string(value) + def encode(value) when is_binary(value), do: inspect(value) + + def encode(value) when is_atom(value) do + ":" <> (value |> Atom.to_string() |> String.replace("_", "-")) + end + + def encode(value) when is_list(value) do + "[" <> Enum.map_join(value, " ", &encode/1) <> "]" + end + + def encode(%MapSet{} = value) do + "#" <> "{" <> (value |> Enum.sort() |> Enum.map_join(" ", &encode/1)) <> "}" + end + + def encode(value) when is_map(value) do + encoded = + value + |> Enum.map(fn {key, inner} -> {encode(key), encode(inner)} end) + |> Enum.sort_by(&elem(&1, 0)) + |> Enum.map_join(" ", fn {key, inner} -> key <> " " <> inner end) + + "{" <> encoded <> "}" + end +end + +defmodule Group.Jepsen.Wire do + @moduledoc false + + def serve(port, context) do + {:ok, listener} = + :gen_tcp.listen(port, [:binary, packet: 4, active: false, reuseaddr: true]) + + accept(listener, context) + end + + defp accept(listener, context) do + {:ok, socket} = :gen_tcp.accept(listener) + spawn(fn -> connection(socket, context) end) + accept(listener, context) + end + + defp connection(socket, context) do + case :gen_tcp.recv(socket, 0) do + {:ok, payload} -> + response = payload |> command(context) |> Group.Jepsen.EDN.encode() + :ok = :gen_tcp.send(socket, response) + connection(socket, context) + + {:error, _reason} -> + :gen_tcp.close(socket) + end + end + + defp command(payload, context) do + case String.split(payload, "\t") do + ["ping"] -> + %{status: :ok} + + ["ready", expected] -> + expected = String.to_integer(expected) + + if length(Group.nodes(:jepsen_group)) == expected - 1 do + %{status: :ok} + else + %{status: :retry, peers: length(Group.nodes(:jepsen_group))} + end + + ["mutate", operation, logical_owner, cluster, key, revision] -> + Group.Jepsen.Driver.mutate( + String.to_existing_atom(operation), + logical_owner, + parse_cluster(cluster), + String.to_integer(key), + String.to_integer(revision) + ) + + ["kill", logical_owner] -> + Group.Jepsen.Driver.kill(logical_owner) + + ["cluster", "connect", cluster] -> + Group.Jepsen.Cluster.connect(cluster) + + ["cluster", "disconnect", cluster] -> + Group.Jepsen.Cluster.disconnect(cluster) + + ["cluster", "connect-all"] -> + Group.Jepsen.Cluster.connect_all() + + ["transport", "block", target] -> + :ok = Group.Jepsen.Transport.Control.block(String.to_atom(target)) + %{status: :ok} + + ["transport", "unblock", target] -> + :ok = Group.Jepsen.Transport.Control.unblock(String.to_atom(target)) + %{status: :ok} + + ["transport", "heal"] -> + :ok = Group.Jepsen.Transport.Control.heal(context.peers) + %{status: :ok} + + ["transport", "reset", target] -> + :ok = Group.Jepsen.Transport.Control.reset(String.to_atom(target)) + %{status: :ok} + + ["snapshot", key_count, clusters, retired] -> + Group.Jepsen.Snapshot.capture( + context.node_id, + context.boot_id, + String.to_integer(key_count), + parse_list(clusters), + retired |> parse_list() |> Enum.map(&String.to_atom/1) + ) + + ["corrupt", mode] -> + corrupt(mode) + + other -> + %{status: :fail, error: "unknown command #{inspect(other)}"} + end + rescue + exception -> %{status: :unknown, error: Exception.message(exception)} + catch + kind, reason -> %{status: :unknown, error: inspect({kind, reason})} + end + + defp corrupt("unexpected-death") do + File.write!( + "/tmp/group-jepsen-unexpected-deaths", + "oracle-self-test\t:injected\n", + [:append] + ) + + %{status: :ok} + end + + defp corrupt("internal-index") do + table = Group.Replica.Data.reg_by_pid_table(:jepsen_group, 0) + :ets.insert(table, {{self(), nil, "jepsen/registry/corrupt"}, %{}, 0, node()}) + %{status: :ok} + end + + defp corrupt("cursor-marker") do + File.write!("/tmp/group-jepsen-cursor-marker-corruption", "enabled\n") + %{status: :ok} + end + + defp corrupt(other), do: %{status: :fail, error: "unknown corruption #{inspect(other)}"} + defp parse_cluster("root"), do: nil + defp parse_cluster(cluster), do: cluster + defp parse_list(""), do: [] + defp parse_list(value), do: String.split(value, ",", trim: true) +end + +defmodule Group.Jepsen.Main do + @moduledoc false + @clusters ["red", "blue"] + + def run(argv) do + {opts, _rest, []} = + OptionParser.parse(argv, + strict: [node: :string, port: :integer, peers: :string] + ) + + node_id = Keyword.get(opts, :node) || System.fetch_env!("GROUP_JEPSEN_NODE") + + port = + Keyword.get(opts, :port) || + System.get_env("GROUP_JEPSEN_PORT", "9080") |> String.to_integer() + + peers = + (Keyword.get(opts, :peers) || + System.get_env("GROUP_JEPSEN_PEERS", "group@n1,group@n2,group@n3")) + |> String.split(",", trim: true) + |> Enum.map(&String.to_atom/1) + + {:ok, _apps} = Application.ensure_all_started(:group) + + {:ok, group} = + Group.start_link( + name: :jepsen_group, + shards: 4, + log: false, + resolve_registry_conflict: {Group.Jepsen.ConflictResolver, :resolve, []}, + replica_transport: Group.Jepsen.Transport.Control.transport(node_id), + replicated_sender_buffer_size: positive_env!("GROUP_JEPSEN_SENDER_BUFFER_SIZE", 1), + replicated_oplog_max_entries: 16, + replicated_snapshot_chunk_target_bytes: 1_024, + replicated_anti_entropy_interval: 50, + replicated_peer_lease_timeout: 750 + ) + + Process.unlink(group) + boot_id = Base.encode16(:crypto.strong_rand_bytes(8), case: :lower) + + {:ok, _drivers} = + Group.Jepsen.Driver.Supervisor.start_link(node_id: node_id, boot_id: boot_id) + + {:ok, _cluster} = Group.Jepsen.Cluster.start_link(@clusters) + spawn_link(fn -> reconnect_loop(peers) end) + + Group.Jepsen.Wire.serve(port, %{ + node_id: node_id, + boot_id: boot_id, + peers: Enum.reject(peers, &(&1 == node())) + }) + end + + defp reconnect_loop(peers) do + Enum.each(peers, fn peer -> + if peer != node(), do: Node.connect(peer) + end) + + Process.sleep(100) + reconnect_loop(peers) + end + + defp positive_env!(name, default) do + value = System.get_env(name, Integer.to_string(default)) |> String.to_integer() + if value > 0, do: value, else: raise("#{name} must be positive") + end +end + +Group.Jepsen.Main.run(System.argv()) diff --git a/test/jepsen/project.clj b/test/jepsen/project.clj new file mode 100644 index 0000000..05284a5 --- /dev/null +++ b/test/jepsen/project.clj @@ -0,0 +1,8 @@ +(defproject group-jepsen "0.1.0-SNAPSHOT" + :description "Jepsen lifecycle and convergence tests for Group" + :url "https://github.com/phoenixframework/group" + :license {:name "MIT"} + :dependencies [[org.clojure/clojure "1.12.4"] + [jepsen "0.3.13"]] + :main group.jepsen.core + :jvm-opts ["-Xmx4g" "-Djava.awt.headless=true" "-server"]) diff --git a/test/jepsen/qualify.sh b/test/jepsen/qualify.sh new file mode 100755 index 0000000..46d4770 --- /dev/null +++ b/test/jepsen/qualify.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_dir="$(cd "${script_dir}/../.." && pwd)" +artifact_dir="$(mktemp -d "${script_dir}/.cache/qualification.XXXXXX")" + +cd "${repo_dir}" + +mix run test/mutation/run.exs + +export GROUP_JEPSEN_SKIP_CHECKER=1 + +run_jepsen() { + local expectation="$1" + local corruption="$2" + local log="${artifact_dir}/${expectation}-${corruption}.log" + local status=0 + + set +e + "${script_dir}/run.sh" test \ + --no-ssh \ + --nodes n1,n2,n3 \ + --concurrency 2n \ + --time-limit 6 \ + --fault-interval 1 \ + --recovery-time 5 \ + --transport distribution \ + --scenario mixed \ + --corruption "${corruption}" >"${log}" 2>&1 + status=$? + set -e + + if [[ "${expectation}" == "pass" ]] && [[ "${status}" -ne 0 ]]; then + echo "healthy Jepsen baseline failed; see ${log}" >&2 + return 1 + fi + + if [[ "${expectation}" == "fail" ]] && [[ "${status}" -eq 0 ]]; then + echo "Jepsen checker accepted corruption ${corruption}; see ${log}" >&2 + return 1 + fi + + echo "${expectation}: ${corruption} (${log})" +} + +run_jepsen pass none +run_jepsen fail unexpected-death +run_jepsen fail internal-index +run_jepsen fail cursor-marker + +echo "mutation and live checker qualification passed" diff --git a/test/jepsen/run.sh b/test/jepsen/run.sh new file mode 100755 index 0000000..9ee7f33 --- /dev/null +++ b/test/jepsen/run.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +compose_file="${script_dir}/docker-compose.yml" + +if [[ "${GROUP_JEPSEN_SKIP_CHECKER:-0}" != "1" ]]; then + "${script_dir}/checker.sh" +fi + +cleanup() { + if [[ "${GROUP_JEPSEN_KEEP_CONTAINERS:-0}" != "1" ]]; then + docker compose --file "${compose_file}" down --volumes >/dev/null + fi +} + +trap cleanup EXIT + +transport="${GROUP_JEPSEN_TRANSPORT:-distribution}" +args=("$@") + +for ((index = 0; index < ${#args[@]}; index++)); do + if [[ "${args[index]}" == "--transport" ]] && ((index + 1 < ${#args[@]})); then + transport="${args[index + 1]}" + fi +done + +export GROUP_JEPSEN_TRANSPORT="${transport}" + +docker compose --file "${compose_file}" up --detach --build --force-recreate + +if [[ "$#" -eq 0 ]]; then + set -- test \ + --no-ssh \ + --nodes n1,n2,n3 \ + --concurrency 2n \ + --time-limit 60 \ + --transport "${transport}" +fi + +"${script_dir}/lein.sh" run -- "$@" diff --git a/test/jepsen/src/group/jepsen/client.clj b/test/jepsen/src/group/jepsen/client.clj new file mode 100644 index 0000000..e6de357 --- /dev/null +++ b/test/jepsen/src/group/jepsen/client.clj @@ -0,0 +1,132 @@ +(ns group.jepsen.client + (:require [clojure.edn :as edn] + [clojure.string :as str] + [group.jepsen.docker :as docker] + [jepsen.client :as client]) + (:import (java.io DataInputStream DataOutputStream) + (java.net InetSocketAddress Socket) + (java.nio.charset StandardCharsets))) + +(defn request! + ([node fields] (request! node fields 3000)) + ([node fields timeout-ms] + (with-open [socket (Socket.)] + (let [^String host "127.0.0.1"] + (.connect socket (InetSocketAddress. host (int (docker/port node))) 1000)) + (.setSoTimeout socket timeout-ms) + (let [payload (.getBytes (str/join "\t" fields) StandardCharsets/UTF_8) + out (DataOutputStream. (.getOutputStream socket)) + in (DataInputStream. (.getInputStream socket))] + (.writeInt out (alength payload)) + (.write out payload) + (.flush out) + (let [length (.readInt in) + response (byte-array length)] + (.readFully in response) + (edn/read-string (String. response StandardCharsets/UTF_8))))))) + +(defn wait-ready! + ([node expected] (wait-ready! node expected 30000)) + ([node expected timeout-ms] + (let [deadline (+ (System/currentTimeMillis) timeout-ms)] + (loop [] + (let [response (try + (request! node ["ready" (str expected)] 1000) + (catch Exception _ nil))] + (cond + (= :ok (:status response)) true + (< (System/currentTimeMillis) deadline) + (do (Thread/sleep 100) (recur)) + :else + (throw (ex-info "Group node did not become ready" + {:node node, :last-response response})))))))) + +(defn wait-listening! + ([node] (wait-listening! node 15000)) + ([node timeout-ms] + (let [deadline (+ (System/currentTimeMillis) timeout-ms)] + (loop [] + (if (try + (= :ok (:status (request! node ["ping"] 1000))) + (catch Exception _ false)) + true + (if (< (System/currentTimeMillis) deadline) + (do (Thread/sleep 100) (recur)) + (throw (ex-info "Group node did not start listening" {:node node})))))))) + +(defn cluster-field [cluster] + (if (nil? cluster) "root" cluster)) + +(defn command [op] + (let [logical-owner (str (or (get-in op [:value :owner]) (:process op))) + cluster (cluster-field (get-in op [:value :cluster]))] + (case (:f op) + :register + ["mutate" "register" logical-owner cluster + (str (get-in op [:value :key])) + (str (get-in op [:value :revision]))] + + :unregister + ["mutate" "unregister" logical-owner cluster + (str (get-in op [:value :key])) "0"] + + :join + ["mutate" "join" logical-owner cluster + (str (get-in op [:value :key])) + (str (get-in op [:value :revision]))] + + :leave + ["mutate" "leave" logical-owner cluster + (str (get-in op [:value :key])) "0"] + + :kill + ["kill" logical-owner] + + :cluster-connect + ["cluster" "connect" (get-in op [:value :cluster])] + + :cluster-disconnect + ["cluster" "disconnect" (get-in op [:value :cluster])] + + :connect-all + ["cluster" "connect-all"] + + :corrupt + ["corrupt" (name (get-in op [:value :mode]))] + + :snapshot + ["snapshot" + (str (get-in op [:value :key-count])) + (str/join "," (get-in op [:value :clusters])) + (->> (get-in op [:value :retired-nodes]) + (map #(str "group@" (name %))) + (str/join ","))]))) + +(defrecord GroupClient [node] + client/Client + (open! [this _test node] (assoc this :node (name node))) + (setup! [this _test] this) + + (invoke! [_this _test op] + (let [target (name (or (get-in op [:value :target]) node))] + (try + (let [response (request! target (command op) 10000) + value (if (= :snapshot (:f op)) + (:snapshot response) + {:request (:value op), :response response, :node target})] + (case (:status response) + :ok (assoc op :type :ok, :value value) + :fail (assoc op :type :fail, :value value, :error (:error response)) + (assoc op :type :info, :value value, :error (:error response)))) + (catch Exception exception + (assoc op :type :info + :error {:class (str (class exception)) + :message (.getMessage exception)}))))) + + (teardown! [this _test] this) + (close! [_this _test]) + + client/Reusable + (reusable? [_this _test] true)) + +(defn client [], (GroupClient. nil)) diff --git a/test/jepsen/src/group/jepsen/core.clj b/test/jepsen/src/group/jepsen/core.clj new file mode 100644 index 0000000..9b737b1 --- /dev/null +++ b/test/jepsen/src/group/jepsen/core.clj @@ -0,0 +1,203 @@ +(ns group.jepsen.core + (:gen-class) + (:require [group.jepsen.client :as group-client] + [group.jepsen.db :as group-db] + [group.jepsen.model :as model] + [group.jepsen.nemesis :as group-nemesis] + [jepsen.cli :as cli] + [jepsen.generator :as gen] + [jepsen.os :as os] + [jepsen.tests :as tests])) + +(def clusters ["red" "blue"]) + +(defn mutation [revision key-count owner-count] + (let [key (rand-int key-count) + owner (rand-int owner-count) + cluster (rand-nth [nil nil nil "red" "blue"]) + rev #(swap! revision inc)] + (case (long (rand-int 16)) + 0 {:f :kill, :value {:owner owner}} + 1 {:f :unregister, :value {:owner owner, :cluster cluster, :key key}} + 2 {:f :leave, :value {:owner owner, :cluster cluster, :key key}} + 3 {:f :cluster-disconnect, :value {:cluster (rand-nth clusters)}} + 4 {:f :cluster-connect, :value {:cluster (rand-nth clusters)}} + 5 {:f :join, :value {:owner owner, :cluster cluster, :key key, :revision (rev)}} + 6 {:f :join, :value {:owner owner, :cluster cluster, :key key, :revision (rev)}} + 7 {:f :join, :value {:owner owner, :cluster cluster, :key key, :revision (rev)}} + 8 {:f :join, :value {:owner owner, :cluster cluster, :key key, :revision (rev)}} + {:f :register, :value {:owner owner, :cluster cluster, :key key, :revision (rev)}}))) + +(defn fault-cycle [fault-interval] + (cycle [(gen/sleep fault-interval) + {:type :info, :f :replica-partition-start} + (gen/sleep fault-interval) + {:type :info, :f :replica-reset} + (gen/sleep fault-interval) + {:type :info, :f :kill-node} + (gen/sleep fault-interval) + {:type :info, :f :restart-node} + (gen/sleep fault-interval) + {:type :info, :f :replica-partition-stop} + (gen/sleep fault-interval) + {:type :info, :f :partition-start} + (gen/sleep fault-interval) + {:type :info, :f :partition-stop}])) + +(defn targeted [target f value] + {:f f, :value (assoc value :target target)}) + +(defn prelude [] + [(gen/log "Opening a three-way replica partition for deterministic conflict and epoch fencing") + (gen/nemesis {:type :info, :f :replica-partition-start, :value {:shape :all}}) + (gen/sleep 0.25) + (gen/clients + (gen/once + (targeted :n1 :register + {:owner "epoch-old", :cluster "red", :key 0, :revision 1000}))) + (gen/clients + (gen/once (targeted :n1 :cluster-disconnect {:cluster "red"}))) + (gen/clients + (gen/once (targeted :n1 :cluster-connect {:cluster "red"}))) + (gen/clients + (gen/once + (targeted :n1 :register + {:owner "epoch-new", :cluster "red", :key 0, :revision 1001}))) + (gen/log "Creating three independent claims for one root registry key") + (gen/clients + [(targeted :n1 :register {:owner "triple-n1", :cluster nil, :key 0, :revision 2001}) + (targeted :n2 :register {:owner "triple-n2", :cluster nil, :key 0, :revision 2002}) + (targeted :n3 :register {:owner "triple-n3", :cluster nil, :key 0, :revision 2003})]) + (gen/sleep 0.25) + (gen/nemesis {:type :info, :f :replica-partition-stop}) + (gen/sleep 1) + (gen/log "Restarting n2 after conflict resolution to prove durable qualification evidence") + (gen/nemesis {:type :info, :f :kill-node, :value {:node :n2}}) + (gen/nemesis {:type :info, :f :restart-node}) + (gen/sleep 1)]) + +(defn terminal-snapshot [opts] + {:f :snapshot + :value {:key-count (:key-count opts) + :clusters clusters + :retired-nodes (:retired-nodes opts)}}) + +(defn snapshot-round [opts] + (let [read (terminal-snapshot opts) + permanent? (= "permanent" (:scenario opts))] + (gen/clients + (gen/each-thread + (if permanent? + (gen/once read) + (gen/until-ok (repeat read))))))) + +(defn recovery-connect-round [opts] + ;; A dead node can reject connections immediately. An unpaced until-ok loop + ;; then creates an unbounded history after the active workload's time limit + ;; has already elapsed. Bound terminal recovery too: failure to reconnect is + ;; evidence for an invalid history, not a reason for the harness to run + ;; forever. + (gen/clients + (gen/each-thread + (->> (repeat {:f :connect-all, :value {}}) + (gen/stagger 0.1) + gen/until-ok + (gen/time-limit (:recovery-time opts)))))) + +(defn terminal-phases [opts] + (let [permanent? (= "permanent" (:scenario opts)) + corruption (keyword (:corruption opts))] + (cond-> + [(gen/log "Healing every fault and restarting transiently killed nodes") + (gen/nemesis {:type :info, :f :restart-node}) + (gen/nemesis {:type :info, :f :partition-stop}) + (gen/nemesis {:type :info, :f :replica-partition-stop}) + (recovery-connect-round opts) + (gen/sleep (:recovery-time opts))] + permanent? + (conj (gen/log "Retiring n1 permanently and waiting for complete eviction") + (gen/nemesis {:type :info, :f :retire-node, :value {:node :n1}}) + (gen/sleep (:recovery-time opts))) + + (not= :none corruption) + (conj (gen/log "Injecting a checker-qualification corruption") + (gen/clients + (gen/once + (targeted (first (:terminal-nodes opts)) :corrupt {:mode corruption})))) + + true + (conj (gen/log "Collecting first terminal model snapshot") + (snapshot-round opts) + (gen/sleep 1) + (gen/log "Collecting stable terminal model snapshot") + (snapshot-round opts))))) + +(defn workload [opts] + (let [revision (atom 3000) + active (->> (repeatedly #(mutation revision (:key-count opts) (:owner-count opts))) + (gen/stagger 0.005) + (gen/nemesis (fault-cycle (:fault-interval opts))) + (gen/time-limit (:time-limit opts)))] + (apply gen/phases (concat (prelude) [active] (terminal-phases opts))))) + +(defn group-test [opts] + (let [db (group-db/db) + permanent? (= "permanent" (:scenario opts)) + terminal-nodes (if permanent? (vec (rest (:nodes opts))) (:nodes opts)) + retired-nodes (if permanent? [(first (:nodes opts))] []) + opts (assoc opts + :clusters clusters + :terminal-nodes terminal-nodes + :retired-nodes retired-nodes)] + (merge tests/noop-test + opts + {:name (str "group lifecycle convergence (" (:transport opts) "/" + (:scenario opts) ")") + :os os/noop + :db db + :client (group-client/client) + :nemesis (group-nemesis/nemesis db) + :pure-generators true + :generator (workload opts) + :checker (model/checker)}))) + +(def cli-options + [[nil "--key-count NUMBER" "Number of keys in each cluster and data type" + :default 8 + :parse-fn #(Long/parseLong %) + :validate [pos? "Must be positive"]] + [nil "--fault-interval SECONDS" "Seconds between fault transitions" + :default 2 + :parse-fn #(Double/parseDouble %) + :validate [pos? "Must be positive"]] + [nil "--owner-count NUMBER" "Logical owner slots per node" + :default 32 + :parse-fn #(Long/parseLong %) + :validate [pos? "Must be positive"]] + [nil "--recovery-time SECONDS" "Fault-free convergence time before checking" + :default 8 + :parse-fn #(Long/parseLong %) + :validate [pos? "Must be positive"]] + [nil "--transport PROFILE" "Replica transport: distribution, tcp, or chaos" + :default "distribution" + :validate [#{"distribution" "tcp" "chaos"} "Unsupported transport"]] + [nil "--scenario SCENARIO" "Lifecycle scenario: mixed or permanent" + :default "mixed" + :validate [#{"mixed" "permanent"} "Unsupported scenario"]] + [nil "--corruption MODE" "Checker qualification corruption mode" + :default "none" + :validate [#{"none" "unexpected-death" "internal-index" "cursor-marker"} + "Unsupported corruption"]] + [nil "--max-operation-latency-ms MILLIS" "Maximum acknowledged Group call latency" + :default 2000 + :parse-fn #(Long/parseLong %) + :validate [pos? "Must be positive"]] + [nil "--min-delta-run-records NUMBER" "Required peak records in one repaired delta run" + :default 1 + :parse-fn #(Long/parseLong %) + :validate [pos? "Must be positive"]]]) + +(defn -main [& args] + (cli/run! + (cli/single-test-cmd {:test-fn group-test, :opt-spec cli-options}) + args)) diff --git a/test/jepsen/src/group/jepsen/db.clj b/test/jepsen/src/group/jepsen/db.clj new file mode 100644 index 0000000..702b41c --- /dev/null +++ b/test/jepsen/src/group/jepsen/db.clj @@ -0,0 +1,28 @@ +(ns group.jepsen.db + (:require [group.jepsen.client :as group-client] + [group.jepsen.docker :as docker] + [jepsen.db :as db])) + +(defrecord DockerDB [] + db/DB + (setup! [_this test node] + (docker/heal! (:nodes test)) + (docker/restart! node) + (docker/reset-oracle! node) + (group-client/wait-ready! node (count (:nodes test)))) + + (teardown! [_this test _node] + (docker/heal! (:nodes test))) + + db/Kill + (kill! [_this _test node] + (docker/stop! node)) + + (start! [_this _test node] + (docker/start! node) + (group-client/wait-listening! node)) + + db/LogFiles + (log-files [_this _test _node] [])) + +(defn db [], (DockerDB.)) diff --git a/test/jepsen/src/group/jepsen/docker.clj b/test/jepsen/src/group/jepsen/docker.clj new file mode 100644 index 0000000..da9cc6f --- /dev/null +++ b/test/jepsen/src/group/jepsen/docker.clj @@ -0,0 +1,123 @@ +(ns group.jepsen.docker + (:require [clojure.java.shell :as shell] + [clojure.string :as str])) + +(def containers + {"n1" "group-jepsen-n1" + "n2" "group-jepsen-n2" + "n3" "group-jepsen-n3"}) + +(def ports + {"n1" 19081 + "n2" 19082 + "n3" 19083}) + +(def full-chain "GROUP_JEPSEN_FULL") +(def replica-chain "GROUP_JEPSEN_REPLICA") +(def replica-port 10000) + +(defn container [node] + (or (get containers (name node)) + (throw (ex-info "unknown Jepsen node" {:node node})))) + +(defn port [node] + (or (get ports (name node)) + (throw (ex-info "unknown Jepsen node" {:node node})))) + +(defn shell! + [& args] + (let [{:keys [exit out err]} (apply shell/sh args)] + (when-not (zero? exit) + (throw (ex-info "command failed" + {:command args, :exit exit, :out out, :err err}))) + (str/trim out))) + +(defn docker! + [& args] + (apply shell! "docker" args)) + +(defn running? [node] + (= "true" + (try + (docker! "inspect" "--format" "{{.State.Running}}" (container node)) + (catch Exception _ "false")))) + +(defn start! [node] + (docker! "start" (container node))) + +(defn stop! [node] + (when (running? node) + (docker! "stop" "--time" "0" (container node)))) + +(defn restart! [node] + (docker! "restart" "--time" "0" (container node))) + +(defn ip [node] + (docker! "inspect" + "--format" + "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" + (container node))) + +(defn exec-sh! + [node script] + (docker! "exec" (container node) "sh" "-c" script)) + +(defn reset-oracle! [node] + (exec-sh! node + (str "rm -f /tmp/group-jepsen-unexpected-deaths " + "/tmp/group-jepsen-persistent-events " + "/tmp/group-jepsen-cursor-marker-corruption"))) + +(defn ensure-firewall-chain! [node chain] + (exec-sh! + node + (str "iptables -N " chain " 2>/dev/null || true; " + "iptables -C INPUT -j " chain " 2>/dev/null || " + "iptables -I INPUT 1 -j " chain "; " + "iptables -C OUTPUT -j " chain " 2>/dev/null || " + "iptables -I OUTPUT 1 -j " chain))) + +(defn flush-chain! [node chain] + (when (running? node) + (ensure-firewall-chain! node chain) + (exec-sh! node (str "iptables -F " chain)))) + +(defn heal-full! [nodes] + (doseq [node nodes] + (flush-chain! node full-chain))) + +(defn heal-replica! [nodes] + (doseq [node nodes] + (flush-chain! node replica-chain))) + +(defn heal! [nodes] + (heal-full! nodes) + (heal-replica! nodes)) + +(defn isolate! + "Cuts one node off from every other DB node while preserving client traffic." + [nodes isolated] + (heal-full! nodes) + (let [ips (into {} (map (juxt identity ip) nodes))] + (doseq [node nodes + peer nodes + :when (and (not= node peer) + (or (= node isolated) (= peer isolated)))] + (exec-sh! + node + (str "iptables -A " full-chain + " -d " (get ips peer) " -j DROP; " + "iptables -A " full-chain + " -s " (get ips peer) " -j DROP"))))) + +(defn partition-replica! + "Drops only sideband TCP packets for the supplied directed node pairs." + [nodes edges] + (heal-replica! nodes) + (let [ips (into {} (map (juxt identity ip) nodes))] + (doseq [[source target] edges] + (exec-sh! + source + (str "iptables -A " replica-chain + " -p tcp -d " (get ips target) + " --dport " replica-port " -j DROP"))))) diff --git a/test/jepsen/src/group/jepsen/model.clj b/test/jepsen/src/group/jepsen/model.clj new file mode 100644 index 0000000..bc772aa --- /dev/null +++ b/test/jepsen/src/group/jepsen/model.clj @@ -0,0 +1,231 @@ +(ns group.jepsen.model + (:require [clojure.set :as set] + [jepsen.checker :as checker] + [jepsen.history :as history])) + +(def default-required-transport-events + #{:delta-batch :snapshot-chunk :multi-chunk-snapshot + :registry-conflict-death}) + +(defn successful-snapshots [history] + (->> history + (remove history/invoke?) + (filter #(and (= :snapshot (:f %)) (= :ok (:type %)))) + (sort-by :index))) + +(defn snapshots-by-node [history] + (reduce (fn [snapshots op] + (update snapshots (get-in op [:value :node]) (fnil conj []) (:value op))) + {} + (successful-snapshots history))) + +(defn latest-snapshots [history] + (->> (successful-snapshots history) + (reduce (fn [snapshots op] + (assoc snapshots (get-in op [:value :node]) (:value op))) + {}))) + +(defn spaces [test] + (cons "root" (:clusters test))) + +(defn empty-view [test empty-value] + (into {} + (for [cluster (spaces test)] + [cluster (zipmap (range (:key-count test)) (repeat empty-value))]))) + +(defn owner-entries [owner field] + (or (get owner field) [])) + +(defn expected-state [test snapshots] + (let [owners (->> snapshots vals (mapcat :owners) (map (juxt :token identity)) (into {})) + registry-candidates + (reduce (fn [by-key [_ owner]] + (reduce (fn [entries {:keys [cluster key]}] + (update entries [(or cluster "root") key] + (fnil conj #{}) (:token owner))) + by-key + (owner-entries owner :registrations))) + {} + owners) + conflicts (into {} (filter (comp #(< 1 %) count val) registry-candidates)) + registry + (reduce (fn [view [[cluster key] tokens]] + (assoc-in view [cluster key] (first tokens))) + (empty-view test nil) + registry-candidates) + pg + (reduce (fn [view [_ owner]] + (reduce (fn [entries {:keys [cluster key]}] + (update-in entries [(or cluster "root") key] + (fnil conj #{}) (:token owner))) + view + (owner-entries owner :memberships))) + (empty-view test #{}) + owners)] + {:owners owners, :registry registry, :pg pg, :conflicts conflicts})) + +(defn normalize-view [test snapshot] + {:registry + (into {} + (for [cluster (spaces test)] + [cluster + (into {} + (for [key (range (:key-count test))] + [key (get-in snapshot [:registry cluster key])]))])) + :pg + (into {} + (for [cluster (spaces test)] + [cluster + (into {} + (for [key (range (:key-count test))] + [key (set (get-in snapshot [:pg cluster key]))]))]))}) + +(defn stable-internal [snapshot] + (select-keys (:internal snapshot) + [:healthy :errors :snapshot-staging-count :oplog-entries])) + +(defn snapshot-fingerprint [test snapshot] + {:owners (set (:owners snapshot)) + :peers (set (:peers snapshot)) + :unexpected-deaths (set (:unexpected-deaths snapshot)) + :view (normalize-view test snapshot) + :internal (stable-internal snapshot)}) + +(defn operation-latencies [history] + (->> history + (remove history/invoke?) + (keep #(get-in % [:value :response :latency-us])))) + +(defn analyze [test history] + (let [observations (snapshots-by-node history) + snapshots (latest-snapshots history) + required-nodes (set (map name (or (:terminal-nodes test) (:nodes test)))) + relevant-observations (select-keys observations required-nodes) + relevant-snapshots (select-keys snapshots required-nodes) + missing-nodes (set/difference required-nodes (set (keys relevant-snapshots))) + minimum-observations (get test :terminal-snapshots-per-node 2) + insufficient-observations + (into {} + (keep (fn [node] + (let [observation-count (count (get relevant-observations node))] + (when (< observation-count minimum-observations) + [node observation-count])))) + required-nodes) + unstable-observations + (into {} + (keep (fn [[node node-observations]] + (let [fingerprints + (set (map #(snapshot-fingerprint test %) + node-observations))] + (when (< 1 (count fingerprints)) + [node fingerprints])))) + relevant-observations) + expected (expected-state test relevant-snapshots) + expected-view (select-keys expected [:registry :pg]) + views (into {} (map (fn [[node snapshot]] + [node (normalize-view test snapshot)])) + relevant-snapshots) + mismatches (into {} (remove (comp #(= expected-view %) val) views)) + peer-mismatches + (into {} + (keep (fn [[node snapshot]] + (let [expected-peers (->> required-nodes + (remove #(= node %)) + (map #(str "group@" %)) + set) + actual-peers (set (:peers snapshot))] + (when (not= expected-peers actual-peers) + [node {:expected expected-peers, :actual actual-peers}])))) + relevant-snapshots) + transport-events + (reduce #(merge-with + %1 %2) + {} + (map #(or (:transport-events %) {}) (vals relevant-snapshots))) + required-transport-events + (get test :required-transport-events + default-required-transport-events) + missing-transport-events + (set (remove #(pos? (get transport-events % 0)) required-transport-events)) + delta-run-records-peak + (reduce max 0 + (map #(get-in % [:transport-events :delta-run-records-peak] 0) + (vals relevant-snapshots))) + min-delta-run-records (get test :min-delta-run-records 0) + delta-run-coverage? (>= delta-run-records-peak min-delta-run-records) + expected-profile (keyword (:transport test)) + transport-profile-mismatches + (into {} + (keep (fn [[node snapshot]] + (when (not= expected-profile (:transport-profile snapshot)) + [node (:transport-profile snapshot)]))) + relevant-snapshots) + internal-errors + (into {} + (keep (fn [[node snapshot]] + (let [internal (:internal snapshot)] + (when (or (not= true (:healthy internal)) + (seq (:errors internal)) + (not= 0 (:snapshot-staging-count internal))) + [node internal])))) + relevant-snapshots) + unexpected-deaths (->> relevant-snapshots vals (mapcat :unexpected-deaths) set) + live-tokens (set (keys (:owners expected))) + actual-tokens (->> views + vals + (mapcat (fn [{:keys [registry pg]}] + (concat + (->> registry vals (mapcat vals) (remove nil?)) + (->> pg vals (mapcat vals) (mapcat identity))))) + set) + expected-tokens + (set/union + (->> (:registry expected) vals (mapcat vals) (remove nil?) set) + (->> (:pg expected) vals (mapcat vals) (mapcat identity) set)) + orphaned (set/difference actual-tokens live-tokens) + missing-live (set/difference expected-tokens actual-tokens) + latencies (operation-latencies history) + max-latency-us (if (seq latencies) (apply max latencies) 0) + latency-limit-us (* 1000 (get test :max-operation-latency-ms 2000)) + latency-violation? (> max-latency-us latency-limit-us) + valid? (and (empty? missing-nodes) + (empty? insufficient-observations) + (empty? unstable-observations) + (empty? peer-mismatches) + (empty? missing-transport-events) + delta-run-coverage? + (empty? transport-profile-mismatches) + (empty? internal-errors) + (empty? (:conflicts expected)) + (empty? mismatches) + (empty? unexpected-deaths) + (empty? orphaned) + (empty? missing-live) + (not latency-violation?))] + {:valid? valid? + :snapshots (set (keys relevant-snapshots)) + :missing-nodes missing-nodes + :insufficient-terminal-observations insufficient-observations + :unstable-terminal-observations unstable-observations + :peer-mismatches peer-mismatches + :transport-events transport-events + :missing-transport-events missing-transport-events + :delta-run-records-peak delta-run-records-peak + :min-delta-run-records min-delta-run-records + :transport-profile-mismatches transport-profile-mismatches + :internal-invariant-errors internal-errors + :max-group-operation-latency-ms (/ max-latency-us 1000.0) + :group-operation-latency-limit-ms (/ latency-limit-us 1000.0) + :live-owner-count (count live-tokens) + :live-registry-conflicts (:conflicts expected) + :mismatched-views mismatches + :unexpected-owner-deaths unexpected-deaths + :orphaned-owner-tokens orphaned + :missing-live-owner-tokens missing-live + :expected expected-view})) + +(defrecord LifecycleChecker [] + checker/Checker + (check [_this test history _opts] + (analyze test history))) + +(defn checker [], (LifecycleChecker.)) diff --git a/test/jepsen/src/group/jepsen/nemesis.clj b/test/jepsen/src/group/jepsen/nemesis.clj new file mode 100644 index 0000000..e11c40f --- /dev/null +++ b/test/jepsen/src/group/jepsen/nemesis.clj @@ -0,0 +1,166 @@ +(ns group.jepsen.nemesis + (:require [group.jepsen.client :as group-client] + [group.jepsen.docker :as docker] + [jepsen.db :as db] + [jepsen.nemesis :as nemesis])) + +(defn ordered-pairs [nodes] + (for [source nodes, target nodes :when (not= source target)] [source target])) + +(defn partition-shape [nodes requested] + (let [nodes (vec nodes) + shape (or requested (rand-nth [:isolate :all :asymmetric]))] + (case shape + :all {:shape :all, :edges (vec (ordered-pairs nodes))} + :asymmetric + (let [source (rand-nth nodes) + target (rand-nth (vec (remove #(= source %) nodes)))] + {:shape :asymmetric, :edges [[source target]]}) + :isolate + (let [isolated (rand-nth nodes)] + {:shape :isolate + :isolated isolated + :edges (vec (filter (fn [[source target]] + (or (= source isolated) (= target isolated))) + (ordered-pairs nodes)))})))) + +(defn logical-block! [edges] + (doseq [[source target] edges] + (when (docker/running? source) + (group-client/request! source ["transport" "block" (str "group@" (name target))])))) + +(defn logical-heal! [nodes] + (doseq [node nodes] + (when (docker/running? node) + (try + (group-client/request! node ["transport" "heal"]) + (catch Exception _ nil))))) + +(defrecord PartitionNemesis [isolated] + nemesis/Nemesis + (setup! [this test] + (docker/heal-full! (:nodes test)) + this) + + (invoke! [_this test op] + (case (:f op) + :start + (if @isolated + (assoc op :type :info, :value {:already-isolated @isolated}) + (let [node (rand-nth (vec (:nodes test)))] + (docker/isolate! (:nodes test) node) + (reset! isolated node) + (assoc op :type :info, :value {:isolated node}))) + + :stop + (do + (docker/heal-full! (:nodes test)) + (let [node @isolated] + (reset! isolated nil) + (assoc op :type :info, :value {:healed node}))))) + + (teardown! [_this test] + (docker/heal-full! (:nodes test)))) + +(defrecord ReplicaNemesis [active] + nemesis/Nemesis + (setup! [this test] + (docker/heal-replica! (:nodes test)) + (logical-heal! (:nodes test)) + this) + + (invoke! [_this test op] + (case (:f op) + :start + (if @active + (assoc op :type :info, :value {:already-active @active}) + (let [requested (get-in op [:value :shape]) + fault (partition-shape (:nodes test) requested)] + (if (= "tcp" (:transport test)) + (docker/partition-replica! (:nodes test) (:edges fault)) + (logical-block! (:edges fault))) + (reset! active fault) + (assoc op :type :info, :value fault))) + + :stop + (do + (docker/heal-replica! (:nodes test)) + (logical-heal! (:nodes test)) + (let [fault @active] + (reset! active nil) + (assoc op :type :info, :value {:healed fault}))) + + :reset + (let [nodes (vec (filter docker/running? (:nodes test))) + source (when (seq nodes) (rand-nth nodes)) + targets (when source (vec (remove #(= source %) nodes))) + target (when (seq targets) (rand-nth targets))] + (when (and source target) + (group-client/request! + source + ["transport" "reset" (str "group@" (name target))])) + (assoc op :type :info, :value {:source source, :target target})))) + + (teardown! [_this test] + (docker/heal-replica! (:nodes test)) + (logical-heal! (:nodes test)))) + +(defrecord ProcessNemesis [db killed] + nemesis/Nemesis + (setup! [this _test] this) + + (invoke! [_this test op] + (case (:f op) + :start + (if @killed + (assoc op :type :info, :value {:already-killed @killed}) + (let [node (or (get-in op [:value :node]) + (rand-nth (vec (:nodes test))))] + (db/kill! db test node) + (reset! killed node) + (assoc op :type :info, :value {:killed node}))) + + :stop + (if-let [node @killed] + (do + (db/start! db test node) + (reset! killed nil) + (assoc op :type :info, :value {:restarted node})) + (assoc op :type :info, :value {:restarted nil})))) + + (teardown! [_this test] + (when-let [node @killed] + (db/start! db test node) + (reset! killed nil)))) + +(defrecord RetirementNemesis [db retired] + nemesis/Nemesis + (setup! [this _test] this) + + (invoke! [_this test op] + (let [node (or (get-in op [:value :node]) (first (:nodes test)))] + (when (and (nil? @retired) (docker/running? node)) + (db/kill! db test node) + (reset! retired node)) + (assoc op :type :info, :value {:retired @retired}))) + + (teardown! [_this test] + (when-let [node @retired] + (db/start! db test node) + (reset! retired nil)))) + +(defn nemesis [db] + (nemesis/compose + {{:partition-start :start, :partition-stop :stop} + (PartitionNemesis. (atom nil)) + + {:replica-partition-start :start, + :replica-partition-stop :stop, + :replica-reset :reset} + (ReplicaNemesis. (atom nil)) + + {:kill-node :start, :restart-node :stop} + (ProcessNemesis. db (atom nil)) + + {:retire-node :retire} + (RetirementNemesis. db (atom nil))})) diff --git a/test/jepsen/test/group/jepsen/model_test.clj b/test/jepsen/test/group/jepsen/model_test.clj new file mode 100644 index 0000000..81e5263 --- /dev/null +++ b/test/jepsen/test/group/jepsen/model_test.clj @@ -0,0 +1,254 @@ +(ns group.jepsen.model-test + (:require [clojure.test :refer :all] + [group.jepsen.model :as model])) + +(def test-map + {:nodes ["n1" "n2" "n3"] + :terminal-nodes ["n1" "n2" "n3"] + :key-count 2 + :clusters ["red"] + :transport "distribution" + :terminal-snapshots-per-node 1 + :required-transport-events #{}}) + +(defn peers-for [node nodes] + (->> nodes + (remove #(= node %)) + (map #(str "group@" %)) + sort + vec)) + +(defn public-view [root red] + {"root" root, "red" red}) + +(defn healthy-internal [] + {:healthy true + :errors [] + :snapshot-staging-count 0 + :oplog-entries 2}) + +(defn snapshot-op + ([index node owners registry pg] + (snapshot-op index node (:terminal-nodes test-map) owners registry pg)) + ([index node nodes owners registry pg] + {:index index + :process index + :type :ok + :f :snapshot + :value {:node node + :peers (peers-for node nodes) + :owners owners + :unexpected-deaths [] + :transport-events {} + :transport-profile :distribution + :internal (healthy-internal) + :registry registry + :pg pg}})) + +(defn owner [token registrations memberships] + {:token token, :registrations registrations, :memberships memberships}) + +(defn registration [cluster key revision] + {:cluster cluster, :key key, :revision revision}) + +(defn membership [cluster key revision] + {:cluster cluster, :key key, :revision revision}) + +(defn empty-registry [] + (public-view {0 nil, 1 nil} {0 nil, 1 nil})) + +(defn empty-pg [] + (public-view {0 [], 1 []} {0 [], 1 []})) + +(defn with-unexpected-death [op token] + (assoc-in op [:value :unexpected-deaths] [{:token token, :reason ":boom"}])) + +(deftest accepts-an-exact-converged-multi-cluster-view + (let [owners [(owner "a" [(registration nil 0 1) (registration "red" 1 2)] []) + (owner "b" [] [(membership nil 1 2) (membership "red" 0 3)])] + registry (public-view {0 "a", 1 nil} {0 nil, 1 "a"}) + pg (public-view {0 [], 1 ["b"]} {0 ["b"], 1 []}) + history [(snapshot-op 1 "n1" owners registry pg) + (snapshot-op 2 "n2" [] registry pg) + (snapshot-op 3 "n3" [] registry pg)]] + (is (:valid? (model/analyze test-map history))))) + +(deftest does-not-require-an-owner-without-group-intent + (let [idle-owner (owner "idle" [] []) + history [(snapshot-op 1 "n1" [idle-owner] (empty-registry) (empty-pg)) + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))]] + (is (:valid? (model/analyze test-map history))))) + +(deftest rejects-an-incomplete-terminal-observation + (let [history [(snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg))] + result (model/analyze test-map history)] + (is (false? (:valid? result))) + (is (= #{"n3"} (:missing-nodes result))))) + +(deftest accepts-a-permanently-retired-node-and-requires-its-absence + (let [survivors ["n2" "n3"] + permanent-test (assoc test-map :terminal-nodes survivors) + history [(snapshot-op 1 "n2" survivors [] (empty-registry) (empty-pg)) + (snapshot-op 2 "n3" survivors [] (empty-registry) (empty-pg))]] + (is (:valid? (model/analyze permanent-test history))))) + +(deftest rejects-zombies-missing-live-owners-and-divergence + (let [live (owner "live" [(registration nil 0 1)] []) + stale-registry (assoc-in (empty-registry) ["root" 0] "dead") + result (model/analyze + test-map + [(snapshot-op 1 "n1" [live] stale-registry (empty-pg)) + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))])] + (is (false? (:valid? result))) + (is (= #{"dead"} (:orphaned-owner-tokens result))) + (is (= #{"live"} (:missing-live-owner-tokens result))) + (is (seq (:mismatched-views result))))) + +(deftest rejects-a-live-unresolved-registry-conflict + (let [owners [(owner "a" [(registration nil 0 1)] []) + (owner "b" [(registration nil 0 2)] [])] + registry (assoc-in (empty-registry) ["root" 0] "b") + history [(snapshot-op 1 "n1" owners registry (empty-pg)) + (snapshot-op 2 "n2" [] registry (empty-pg)) + (snapshot-op 3 "n3" [] registry (empty-pg))] + result (model/analyze test-map history)] + (is (false? (:valid? result))) + (is (= {["root" 0] #{"a" "b"}} (:live-registry-conflicts result))))) + +(deftest rejects-an-unexpected-owner-death-even-after-cleanup + (let [result (model/analyze + test-map + [(with-unexpected-death + (snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) + "lost-owner") + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))])] + (is (false? (:valid? result))) + (is (= #{{:token "lost-owner", :reason ":boom"}} + (:unexpected-owner-deaths result))))) + +(deftest rejects-terminal-state-which-keeps-changing + (let [stale-registry (assoc-in (empty-registry) ["root" 0] "stale") + history [(snapshot-op 1 "n1" [] stale-registry (empty-pg)) + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg)) + (snapshot-op 4 "n1" [] (empty-registry) (empty-pg)) + (snapshot-op 5 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 6 "n3" [] (empty-registry) (empty-pg))] + result (model/analyze + (assoc test-map :terminal-snapshots-per-node 2) + history)] + (is (false? (:valid? result))) + (is (contains? (:unstable-terminal-observations result) "n1")))) + +(deftest rejects-a-node-without-all-control-plane-peers + (let [history [(assoc-in (snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) + [:value :peers] + ["group@n2"]) + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))] + result (model/analyze test-map history)] + (is (false? (:valid? result))) + (is (= #{"group@n2" "group@n3"} + (get-in result [:peer-mismatches "n1" :expected]))))) + +(deftest rejects-a-run-which-did-not-exercise-required-repair-paths + (let [history [(snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))] + result (model/analyze + (assoc test-map + :required-transport-events + model/default-required-transport-events) + history)] + (is (false? (:valid? result))) + (is (= model/default-required-transport-events + (:missing-transport-events result))))) + +(deftest accepts-the-transport-event-names-emitted-by-the-live-nodes + (let [events {:delta-batch 1 + :snapshot-chunk 2 + :multi-chunk-snapshot 1 + :registry-conflict-death 1} + history [(assoc-in (snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) + [:value :transport-events] + events) + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))] + result (model/analyze + (dissoc test-map :required-transport-events) + history)] + (is (:valid? result)) + (is (empty? (:missing-transport-events result))))) + +(deftest rejects-a-profile-which-never-repairs-a-multi-record-delta-run + (let [single-record-events {:delta-batch 3 + :snapshot-chunk 2 + :multi-chunk-snapshot 1 + :registry-conflict-death 1 + :delta-run-records-peak 1} + with-events #(assoc-in % [:value :transport-events] single-record-events) + history [(with-events (snapshot-op 1 "n1" [] (empty-registry) (empty-pg))) + (with-events (snapshot-op 2 "n2" [] (empty-registry) (empty-pg))) + (with-events (snapshot-op 3 "n3" [] (empty-registry) (empty-pg)))] + result (model/analyze (assoc test-map :min-delta-run-records 2) history)] + (is (false? (:valid? result))) + (is (= 1 (:delta-run-records-peak result))) + (is (= 2 (:min-delta-run-records result))))) + +(deftest accepts-a-profile-which-repairs-a-multi-record-delta-run + (let [events {:delta-batch 1 + :snapshot-chunk 1 + :multi-chunk-snapshot 1 + :registry-conflict-death 1 + :delta-run-records-peak 8} + history [(assoc-in (snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) + [:value :transport-events] + events) + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))] + result (model/analyze (assoc test-map :min-delta-run-records 2) history)] + (is (:valid? result)) + (is (= 8 (:delta-run-records-peak result))))) + +(deftest rejects-internal-corruption-or-leftover-snapshot-staging + (let [bad (-> (snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) + (assoc-in [:value :internal :healthy] false) + (assoc-in [:value :internal :snapshot-staging-count] 1) + (assoc-in [:value :internal :errors] ["broken index"])) + result (model/analyze + test-map + [bad + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))])] + (is (false? (:valid? result))) + (is (= 1 (get-in result [:internal-invariant-errors "n1" + :snapshot-staging-count]))))) + +(deftest rejects-the-wrong-transport-profile + (let [wrong (assoc-in (snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) + [:value :transport-profile] + :tcp) + result (model/analyze + test-map + [wrong + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))])] + (is (false? (:valid? result))) + (is (= {"n1" :tcp} (:transport-profile-mismatches result))))) + +(deftest rejects-a-blocking-group-operation + (let [base [(snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))] + slow {:index 4 + :process 0 + :type :ok + :f :register + :value {:response {:latency-us 2500000}}} + result (model/analyze test-map (conj base slow))] + (is (false? (:valid? result))) + (is (= 2500.0 (:max-group-operation-latency-ms result))))) diff --git a/test/mutation/README.md b/test/mutation/README.md new file mode 100644 index 0000000..3eb0d6d --- /dev/null +++ b/test/mutation/README.md @@ -0,0 +1,47 @@ +# Replica mutation campaign + +This campaign calibrates the anti-entropy tests against deliberate failures of +catastrophic protocol obligations. It covers generation and epoch fencing, +contiguous sequence application, exact registry and PG snapshots, below-floor +repair, process-down sequencing, conflict-loser retirement, authority fanout, +per-lane authority installation, periodic head advertisement, interrupted +journal/index repair, and named-cluster close completion. Snapshot calibration +also covers missing/incomplete terminal commit, conflicting retransmission rows +and manifests, source mutation during a single-pass scan, newer-snapshot +supersession, stale-authority fencing, and staging expiry. +Restart calibration covers per-lane eviction breadcrumbs and partially observed +authority, while wire calibration rejects wrong-shard rows and unsequenced +cluster lifecycle messages. The 68-mutant campaign also independently removes +the generation and epoch fences, races authority changes against local-owner +retirement, skips conflict reprojection after exact authority returns, bypasses +shard-zero authority serialization, separates exact authority from its shared +cluster projection, interrupts local cluster activation/deactivation before +shard notification, deletes terminal close/peer breadcrumbs before their route +cleanup, erases a suspended lane's nodedown breadcrumb, retains a dead peer's +authority-repair obligation, lets stale restart or duplicate-close cleanup +erase reactivated epochs, accepts the wrong epoch's close acknowledgement, +allows newer heartbeats or lane hellos to leave old lane views unfenced, retains +cursorless claims, admits incremental authority across a revision race or gap, +lets an unknown post-retirement hint recreate authority or an unleased lane +route, delays a pre-authority lane's rediscovery until the periodic probe, and +disables bounded snapshot-send/receive progress. + +The runner first verifies every unmodified regression target. It then copies +the current checkout once per mutant, changes only that copy, recompiles it, +and runs the designated real multi-node regression. A compiling mutant is +`killed` only when the regression fails. Any surviving or non-compiling mutant +makes the campaign fail. +Before either listing or running mutants, it also requires every production +replacement to match exactly once and every test selector to point at the test +declaration itself. Source edits therefore fail loudly instead of turning a +stale mutation into a placebo run of an adjacent test. + +```bash +mix run --no-start test/mutation/run.exs + +# List or run selected mutations +mix run --no-start test/mutation/run.exs --list +mix run --no-start test/mutation/run.exs disable_below_floor_snapshot +``` + +Artifacts and complete logs are written below `tmp/mutation/`. diff --git a/test/mutation/run.exs b/test/mutation/run.exs new file mode 100644 index 0000000..a7f9d79 --- /dev/null +++ b/test/mutation/run.exs @@ -0,0 +1,1145 @@ +defmodule Group.MutationCampaign do + @moduledoc """ + Runs protocol mutations in isolated repository copies. + + A mutant is useful only when it compiles and its designated regression test + fails. The source checkout is never edited. + """ + + @repo Path.expand("../..", __DIR__) + @timeout_seconds "120" + + # Each entry replaces one correct production fragment with an intentionally + # faulty fragment, but only inside an isolated campaign checkout. + @mutations [ + %{ + name: "accept_old_generation", + file: "lib/group/replica.ex", + correct_source: + " WireProtocol.stream_generation(stream_id) ==\n" <> + " Data.remote_generation(state.name, source_node) and", + faulty_source: " true and", + test: ["test/distributed_test.exs:5450"] + }, + %{ + name: "accept_old_epoch", + file: "lib/group/replica.ex", + correct_source: + " WireProtocol.stream_epoch(stream_id) ==\n" <> + " Data.remote_cluster_epoch(state.name, source_node, cluster) and", + faulty_source: " true and", + test: ["test/distributed_test.exs:4853"] + }, + %{ + name: "advance_cursor_across_gap", + file: "lib/group/replica.ex", + correct_source: """ + [{first_seq, _mutations} | _] when first_seq > cursor + 1 -> + request_replica_need(state, source_node, stream_id, cursor + 1) + """, + faulty_source: """ + [{first_seq, _mutations} | _] when first_seq > cursor + 1 -> + :ok = + Data.put_replica_cursor( + state.name, + state.shard_index, + stream_id, + first_seq - 1 + ) + + apply_replica_delta_run( + state, + source_node, + stream_id, + records, + advertised_head + ) + """, + test: ["test/distributed_test.exs:5527"] + }, + %{ + name: "registry_snapshot_is_additive", + file: "lib/group/replica/data.ex", + correct_source: """ + acc = + fold_registry_claims_for_stream(name, shard, stream_id, acc, fn + """, + faulty_source: """ + acc = + Enum.reduce([], acc, fn + """, + test: [ + "test/replica_snapshot_distributed_test.exs:16", + "test/distributed_test.exs:4057" + ] + }, + %{ + name: "pg_snapshot_is_additive", + file: "lib/group/replica.ex", + correct_source: " if Snapshot.member_pg?(staging_table, key, pid) do", + faulty_source: " if Process.alive?(self()) do", + test: [ + "test/replica_snapshot_distributed_test.exs:16", + "test/distributed_test.exs:4057" + ] + }, + %{ + name: "commit_incomplete_snapshot", + file: "lib/group/replica.ex", + correct_source: + " if MapSet.size(transfer.received) == chunk_count and\n" <> + " transfer.registry_seen == registry_count and transfer.pg_seen == pg_count do", + faulty_source: + " if MapSet.size(transfer.received) >= 1 and chunk_count >= 1 and\n" <> + " registry_count >= 0 and pg_count >= 0 do", + test: ["test/replica_snapshot_distributed_test.exs:223"] + }, + %{ + name: "commit_snapshot_without_terminal_manifest", + file: "lib/group/replica.ex", + correct_source: + " nil ->\n" <> + " state\n" <> + " end\n" <> + " end\n\n" <> + " defp snapshot_chunk_within_manifest?", + faulty_source: + " nil ->\n" <> + " chunk_count = MapSet.size(transfer.received)\n" <> + " transfer = %{transfer | manifest: {chunk_count, transfer.registry_seen, transfer.pg_seen}}\n" <> + " commit_snapshot_transfer(state, key, source_node, stream_id, transfer)\n" <> + " end\n" <> + " end\n\n" <> + " defp snapshot_chunk_within_manifest?", + test: ["test/replica_snapshot_distributed_test.exs:16"] + }, + %{ + name: "accept_conflicting_snapshot_chunk_retransmission", + file: "lib/group/replica.ex", + correct_source: + " MapSet.member?(transfer.received, chunk_index) and\n" <> + " Snapshot.chunk_matches?(transfer.table, chunk_index, reg_data, pg_data) ->", + faulty_source: + " MapSet.member?(transfer.received, chunk_index) and\n" <> + " Process.alive?(self()) ->", + test: ["test/replica_snapshot_distributed_test.exs:385"] + }, + %{ + name: "retain_conflicting_snapshot_manifest", + file: "lib/group/replica.ex", + correct_source: """ + {:ok, state, _conflicting_transfer} -> + discard_snapshot_transfer(state, key) + """, + faulty_source: """ + {:ok, state, _conflicting_transfer} -> + state + """, + test: ["test/replica_snapshot_distributed_test.exs:191"] + }, + %{ + name: "commit_snapshot_after_source_changes_during_scan", + file: "lib/group/replica.ex", + correct_source: + " if current_snapshot_send?(state, target_node, stream_id, head) do\n" <> + " send_replica_snapshot_commit(state, target_node, stream_id, head, manifest)\n" <> + " else\n" <> + " :complete\n" <> + " end\n" <> + " catch", + faulty_source: + " if Process.alive?(self()) do\n" <> + " send_replica_snapshot_commit(state, target_node, stream_id, head, manifest)\n" <> + " else\n" <> + " :complete\n" <> + " end\n" <> + " catch", + test: ["test/replica_snapshot_distributed_test.exs:101"] + }, + %{ + name: "drop_final_snapshot_event_batch", + file: "lib/group/replica.ex", + correct_source: " _event_buffer = Snapshot.finish_event_buffer(event_buffer)", + faulty_source: " _event_buffer = event_buffer", + test: ["test/replica_snapshot_distributed_test.exs:223"] + }, + %{ + name: "allow_duplicate_snapshot_rows", + file: "lib/group/replica/snapshot.ex", + correct_source: """ + if :ets.insert_new(table, objects) and + :ets.info(table, :size) - size_before == length(objects) do + """, + faulty_source: """ + if :ets.insert(table, objects) and size_before >= 0 do + """, + test: ["test/replica_snapshot_distributed_test.exs:385"] + }, + %{ + name: "do_not_supersede_partial_snapshot", + file: "lib/group/replica.ex", + correct_source: + " %{snapshot_seq: existing_seq} when existing_seq < snapshot_seq ->\n" <> + " state = discard_snapshot_transfer(state, key)\n" <> + " {state, transfer} = new_snapshot_transfer(state, snapshot_seq)\n" <> + " {:ok, put_snapshot_transfer(state, key, transfer), transfer}", + faulty_source: + " %{snapshot_seq: existing_seq} when existing_seq < snapshot_seq ->\n" <> + " _ = existing_seq\n" <> + " {:ignore, state}", + test: ["test/replica_snapshot_distributed_test.exs:328"] + }, + %{ + name: "accept_stale_snapshot_authority", + file: "lib/group/replica.ex", + correct_source: """ + defp valid_snapshot_stream?(state, source_node, stream_id, snapshot_seq) do + valid_remote_stream?(state, source_node, stream_id) and + snapshot_seq > Data.replica_cursor(state.name, state.shard_index, stream_id) + end + """, + faulty_source: """ + defp valid_snapshot_stream?(state, _source_node, stream_id, snapshot_seq) do + snapshot_seq > Data.replica_cursor(state.name, state.shard_index, stream_id) + end + """, + test: ["test/replica_snapshot_distributed_test.exs:556"] + }, + %{ + name: "disable_snapshot_staging_expiry", + file: "lib/group/replica.ex", + correct_source: """ + if now - transfer.last_progress > acc.replicated_peer_lease_timeout do + discard_snapshot_transfer(acc, key) + else + acc + end + """, + faulty_source: """ + _ = {now, transfer} + + if Process.get(:force_snapshot_staging_expiry, false) do + discard_snapshot_transfer(acc, key) + else + acc + end + """, + test: ["test/replica_snapshot_distributed_test.exs:482"] + }, + %{ + name: "disable_below_floor_snapshot", + file: "lib/group/replica.ex", + correct_source: """ + true -> + {:state, send_replica_snapshot(state, target_node, stream_id, head)} + """, + faulty_source: """ + true -> + {:state, state} + """, + test: ["test/replica_model_property_test.exs:174"] + }, + %{ + name: "do_not_sequence_process_down", + file: "lib/group/replica.ex", + correct_source: + " sequenced_downs =\n" <> + " append_process_down_records(state, reason_by_pid, pending_reg, pending_pg)\n", + faulty_source: + " sequenced_downs =\n if false,\n do: append_process_down_records(state, reason_by_pid, pending_reg, pending_pg),\n else: []\n", + test: ["test/distributed_test.exs:3967"] + }, + %{ + name: "do_not_exit_conflict_loser", + file: "lib/group/replica.ex", + correct_source: + " winner_meta = if winner, do: elem(winner, 1), else: nil\n" <> + " exit_local_conflict_loser(pid, key, winner_meta)\n" <> + " acc\n", + faulty_source: + " _winner_meta = if winner, do: elem(winner, 1), else: nil\n" <> + " _ = Process.alive?(pid)\n" <> + " acc\n", + test: ["test/replica_model_property_test.exs:71"] + }, + %{ + name: "heartbeat_does_not_fence_newer_authority", + file: "lib/group/replica/data.ex", + correct_source: " hint_generation == generation and\n", + faulty_source: " false and hint_generation == generation and\n", + test: ["test/anti_entropy_fault_regression_test.exs:2059"] + }, + %{ + name: "heartbeat_does_not_fence_newer_generation", + file: "lib/group/replica/data.ex", + correct_source: + " not is_nil(hint_generation) and\n" <> + " WireProtocol.generation_newer?(generation, hint_generation) ->\n", + faulty_source: + " not is_nil(hint_generation) and\n" <> + " WireProtocol.generation_newer?(generation, hint_generation) and\n" <> + " Process.get(:fence_newer_generation, false) ->\n", + test: ["test/anti_entropy_fault_regression_test.exs:2222"] + }, + %{ + name: "drop_new_generation_authority_hint", + file: "lib/group/replica/data.ex", + correct_source: + " # below are being updated.\n" <> + " put_remote_authority_hint(state.name, remote_node, generation, revision)", + faulty_source: + " # below are being updated.\n" <> + " _ = {state.name, remote_node, generation, revision}", + test: ["test/anti_entropy_fault_regression_test.exs:2222"] + }, + %{ + name: "accept_authority_older_than_generation_hint", + file: "lib/group/replica/data.ex", + correct_source: " hinted_stale? or known_stale? or revision_stale?", + faulty_source: + " _ = hinted_stale?\n" <> + " known_stale? or revision_stale?", + test: ["test/anti_entropy_fault_regression_test.exs:2222"] + }, + %{ + name: "install_lane_view_behind_generation_hint", + file: "lib/group/replica/data.ex", + correct_source: + " remote_replica_authority_hint(state.name, remote_node) == {generation, observed} do", + faulty_source: + " elem(remote_replica_authority_hint(state.name, remote_node), 1) == observed do", + test: ["test/group_test.exs:2897"] + }, + %{ + name: "install_incremental_after_newer_hint", + file: "lib/group/replica/data.ex", + correct_source: + " remote_cluster_epoch_observed_revision(name, remote_node) == expected_revision and\n" <> + " remote_replica_authority_hint(name, remote_node) == {generation, expected_revision}\n", + faulty_source: + " Process.get(:ignore_incremental_authority_race, true) and\n" <> + " is_tuple(remote_replica_authority_hint(name, remote_node))\n", + test: ["test/group_test.exs:2831"] + }, + %{ + name: "accept_hint_without_exact_authority", + file: "lib/group/replica/data.ex", + correct_source: + " not is_nil(hint_generation) and\n" <> + " WireProtocol.generation_newer?(generation, hint_generation) ->\n", + faulty_source: + " (is_nil(hint_generation) or\n" <> + " WireProtocol.generation_newer?(generation, hint_generation)) ->\n", + test: ["test/anti_entropy_fault_regression_test.exs:3318"] + }, + %{ + name: "admit_retired_lane_route_without_authority", + file: "lib/group/replica.ex", + correct_source: + " # A lane hello is only a hint until node-wide exact authority exists.\n" <> + " # In particular, a delayed hello after retirement must not recreate\n" <> + " # an unleased route that can live forever and suppress rediscovery.\n" <> + " {:noreply, request_replica_authority(state, remote_node)}", + faulty_source: + " state = put_remote_shard(state, remote_node, remote_pid)\n" <> + " {:noreply, request_replica_authority(state, remote_node)}", + test: ["test/anti_entropy_fault_regression_test.exs:3318"] + }, + %{ + name: "do_not_restore_hint_lease_after_lane_restart", + file: "lib/group/replica/data.ex", + correct_source: + " {{{:remote_view_info, shard, :\"$1\"}, :_, :_, :_}, [], [:\"$1\"]},\n" <> + " # A hint is the durable fence left before the observing lane records its\n" <> + " # in-memory lease deadline. Every restarting lane must recognize it so a\n" <> + " # crash in that window cannot strand the peer forever.\n" <> + " {{{:remote_authority_hint, :\"$1\"}, :_, :_}, [], [:\"$1\"]}\n", + faulty_source: " {{{:remote_view_info, shard, :\"$1\"}, :_, :_, :_}, [], [:\"$1\"]}\n", + test: ["test/group_test.exs:2770"] + }, + %{ + name: "retain_retired_authority_repair", + file: "lib/group/replica.ex", + correct_source: + " is_nil(Data.remote_generation(state.name, remote_node)) and\n" <> + " is_nil(Data.remote_replica_authority_hint(state.name, remote_node)) ->\n" <> + " acc\n", + faulty_source: + " is_nil(Data.remote_generation(state.name, remote_node)) and\n" <> + " is_nil(Data.remote_replica_authority_hint(state.name, remote_node)) ->\n" <> + " Map.put(acc, remote_node, last_activity)\n", + test: ["test/anti_entropy_fault_regression_test.exs:3318"] + }, + %{ + name: "skip_authority_fanout", + file: "lib/group/replica.ex", + correct_source: """ + fan_out_to_siblings( + state, + {:replica_authority_installed_local, remote_node, generation, epoch_revision, + old_generation, stale_epochs} + ) + """, + faulty_source: """ + :ok + """, + test: ["test/distributed_test.exs:5672"] + }, + %{ + name: "wait_for_periodic_lane_probe_after_authority_fanout", + file: "lib/group/replica.ex", + correct_source: """ + else + # A lane hello can legitimately outrun shard zero's exact authority. + # The hello is not retained as a route, because a delayed hello after + # retirement must not recreate an unleased peer. Once exact authority + # reaches this lane, repeat shard-local discovery immediately instead + # of waiting for the next anti-entropy probe. + send_remote_shard_message( + state, + remote_node, + {:peer_connect, self(), state.shard_index, state.num_shards, + Data.my_clusters(state.name)} + ) + + state + end + """, + faulty_source: """ + else + state + end + """, + test: ["test/anti_entropy_fault_regression_test.exs:3823"] + }, + %{ + name: "assume_authority_fanout_reaches_late_lane", + file: "lib/group/replica.ex", + correct_source: """ + defp install_current_replica_lane(state, remote_node, generation) do + old_generation = + Data.remote_view_generation(state.name, state.shard_index, remote_node) + + state = maybe_purge_remote_generation(state, remote_node, old_generation, generation) + state = purge_remote_streams_outside_authority(state, remote_node) + state = install_replica_view(state, remote_node, generation) + + if replica_view_current?(state, remote_node) do + state + |> touch_replica_peer(remote_node) + |> Map.update!(:cluster_control_dirty, &Map.delete(&1, remote_node)) + |> send_replica_heads(remote_node) + else + state + end + end + """, + faulty_source: """ + defp install_current_replica_lane(state, remote_node, generation) do + _ = {remote_node, generation} + state + end + """, + test: ["test/anti_entropy_fault_regression_test.exs:3823"] + }, + %{ + name: "skip_generation_purge", + file: "lib/group/replica.ex", + correct_source: """ + defp maybe_purge_remote_generation(state, remote_node, _old_generation, _generation) do + state = discard_snapshot_transfers_for_source(state, remote_node) + state = discard_pending_registry_reprojections(state, remote_node) + Data.delete_replica_cursors_for_origin(state.name, state.shard_index, remote_node) + {_reg, _pg} = Data.purge_node(state.name, state.shard_index, remote_node) + + affected = + Data.purge_registry_claims_for_origin( + state.name, + state.shard_index, + remote_node + ) + + {state, events} = + Enum.reduce(affected, {state, []}, fn {cluster, key}, {acc, inner_events} -> + reconcile_registry_projection(acc, cluster, key, :nodedown, inner_events) + end) + + notify_monitors(state.name, events) + state + end + """, + faulty_source: """ + defp maybe_purge_remote_generation(state, _remote_node, _old_generation, _generation), + do: state + """, + test: ["test/distributed_test.exs:5672"] + }, + %{ + name: "disable_periodic_heads", + file: "lib/group/replica.ex", + correct_source: """ + defp broadcast_replica_heads(state) do + peers = Map.keys(state.peer_last_seen) + """, + faulty_source: """ + defp broadcast_replica_heads(state) do + _ = state.peer_last_seen + peers = [] + """, + test: ["test/distributed_test.exs:3967"] + }, + %{ + name: "skip_journal_crash_repair", + file: "lib/group/replica.ex", + correct_source: ":ok = Data.repair_local_replica_journal(name, shard_index)", + faulty_source: ":ok", + test: ["test/group_test.exs:2434"] + }, + %{ + name: "skip_index_crash_repair", + file: "lib/group/replica.ex", + correct_source: ":ok = Data.repair_shard_indexes(name, shard_index)", + faulty_source: ":ok", + test: ["test/group_test.exs:2480"] + }, + %{ + name: "skip_inactive_cluster_repair", + file: "lib/group/replica/data.ex", + correct_source: " repair_primary_replica_rows(name, shard)", + faulty_source: + " if Process.get(:run_primary_replica_repair, false),\n" <> + " do: repair_primary_replica_rows(name, shard),\n" <> + " else: :ok", + test: ["test/group_test.exs:2574"] + }, + %{ + name: "skip_closed_cluster_completion", + file: "lib/group/replica.ex", + correct_source: """ + _completed_clusters = + Data.mark_closed_cluster_shard( + name, + Data.closed_local_cluster_epochs(name), + shard_index + ) + """, + faulty_source: """ + _completed_clusters = [] + """, + test: ["test/group_test.exs:2574"] + }, + %{ + name: "accept_unfenced_cluster_disconnect", + file: "lib/group/replica.ex", + correct_source: + " _ ->\n" <> + " false\n" <> + " end)\n\n" <> + " case epochs do\n", + faulty_source: + " _ ->\n" <> + " Process.get(:accept_unfenced_cluster_disconnect, true)\n" <> + " end)\n\n" <> + " case epochs do\n", + test: ["test/group_test.exs:1485"] + }, + %{ + name: "accept_completed_cluster_disconnect", + file: "lib/group/replica.ex", + correct_source: + " Data.closed_local_cluster_pending?(\n" <> + " state.name,\n" <> + " cluster,\n" <> + " epoch,\n" <> + " state.shard_index\n" <> + " )", + faulty_source: + " _ = cluster\n" <> + " Process.get(:accept_completed_cluster_disconnect, true)", + test: ["test/group_test.exs:1485"] + }, + %{ + name: "acknowledge_wrong_cluster_close_epoch", + file: "lib/group/replica/data.ex", + correct_source: " [{^cluster, ^request_epoch, pending_shards}] ->\n", + faulty_source: " [{^cluster, _stored_epoch, pending_shards}] ->\n", + test: ["test/group_test.exs:2615"] + }, + %{ + name: "accept_shared_authority_before_lane_install", + file: "lib/group/replica.ex", + correct_source: + " WireProtocol.stream_shard(stream_id) == state.shard_index and\n" <> + " replica_view_current?(state, source_node) and", + faulty_source: + " WireProtocol.stream_shard(stream_id) == state.shard_index and\n" <> + " true and", + test: ["test/anti_entropy_fault_regression_test.exs:1173"] + }, + %{ + name: "apply_incremental_authority_across_revision_gap", + file: "lib/group/replica.ex", + correct_source: " if contiguous_cluster_controls?(accepted, next_revision) do", + faulty_source: + " if contiguous_cluster_controls?(accepted, next_revision) or accepted != [] do", + test: ["test/anti_entropy_fault_regression_test.exs:1365"] + }, + %{ + name: "allow_non_owner_lane_to_mutate_shared_authority", + file: "lib/group/replica.ex", + correct_source: " if state.shard_index == 0 do\n remote_node = node(remote_pid)", + faulty_source: " if true do\n remote_node = node(remote_pid)", + test: ["test/anti_entropy_fault_regression_test.exs:1365"] + }, + %{ + name: "crash_lane_when_local_authority_owner_is_missing", + file: "lib/group/replica.ex", + correct_source: " _ = send_local_control_message(state, control)", + faulty_source: " send(shard_name(state.name, 0), control)", + test: ["test/anti_entropy_fault_regression_test.exs:1312"] + }, + %{ + name: "retire_local_owner_after_remote_authority_changed", + file: "lib/group/replica.ex", + correct_source: " registry_winner_authoritative?(state, cluster, winner) ->", + faulty_source: + " Process.get(:skip_remote_registry_authority, true) or\n" <> + " registry_winner_authoritative?(state, cluster, winner) ->", + test: ["test/anti_entropy_fault_regression_test.exs:1524"] + }, + %{ + name: "skip_registry_reprojection_after_authority_restore", + file: "lib/group/replica.ex", + correct_source: """ + defp reproject_pending_registry_keys(state, remote_node) do + case Map.pop(state.pending_registry_reprojections, remote_node) do + {nil, _pending} -> + state + + {keys, pending} -> + state = %{state | pending_registry_reprojections: pending} + {state, events} = reconcile_registry_keys(state, keys, :reconcile, []) + notify_monitors(state.name, events) + state + end + end + """, + faulty_source: """ + defp reproject_pending_registry_keys(state, remote_node) do + _ = remote_node + state + end + """, + test: ["test/anti_entropy_fault_regression_test.exs:1707"] + }, + %{ + name: "retain_registry_reprojection_after_peer_expiry", + file: "lib/group/replica.ex", + correct_source: """ + defp expire_replica_peer(state, remote_node) do + state = discard_snapshot_transfers_for_source(state, remote_node) + state = discard_snapshot_send_offsets_for_target(state, remote_node) + state = discard_pending_registry_reprojections(state, remote_node) + """, + faulty_source: """ + defp expire_replica_peer(state, remote_node) do + state = discard_snapshot_transfers_for_source(state, remote_node) + state = discard_snapshot_send_offsets_for_target(state, remote_node) + """, + test: ["test/anti_entropy_fault_regression_test.exs:1883"] + }, + %{ + name: "retain_registry_reprojection_after_nodedown", + file: "lib/group/replica.ex", + correct_source: """ + def handle_info({:nodedown, dead_node}, state) do + state = flush_pending_replicated_message_barrier(state) + state = discard_snapshot_transfers_for_source(state, dead_node) + state = discard_snapshot_send_offsets_for_target(state, dead_node) + state = discard_pending_registry_reprojections(state, dead_node) + """, + faulty_source: """ + def handle_info({:nodedown, dead_node}, state) do + state = flush_pending_replicated_message_barrier(state) + state = discard_snapshot_transfers_for_source(state, dead_node) + state = discard_snapshot_send_offsets_for_target(state, dead_node) + """, + test: ["test/anti_entropy_fault_regression_test.exs:3601"] + }, + %{ + name: "separate_exact_authority_from_cluster_projection", + file: "lib/group/replica/data.ex", + correct_source: + " replace_remote_cluster_projection(state.name, remote_node, current_epochs)\n", + faulty_source: + " _ = {&replace_remote_cluster_projection/3, state.name, remote_node, current_epochs}\n", + test: ["test/anti_entropy_fault_regression_test.exs:3637"] + }, + %{ + name: "separate_local_activation_from_cluster_projection", + file: "lib/group/replica/data.ex", + correct_source: + " if durable?, do: project_activated_local_clusters(state.name, clusters)\n", + faulty_source: + " _ = {durable?, &project_activated_local_clusters/2, state.name, clusters}\n", + test: ["test/anti_entropy_fault_regression_test.exs:3692"] + }, + %{ + name: "drop_durable_cluster_deactivation_cleanup", + file: "lib/group/replica/data.ex", + correct_source: + " cast_cluster_lifecycle(\n" <> + " state.name,\n" <> + " 0..(state.num_shards - 1),\n" <> + " {:cluster_disconnect, clusters, epochs}\n" <> + " )\n", + faulty_source: + " _ = {&cast_cluster_lifecycle/3, state.name, state.num_shards, clusters, epochs}\n", + test: ["test/anti_entropy_fault_regression_test.exs:3741"] + }, + %{ + name: "delete_close_marker_before_terminal_route_cleanup", + file: "lib/group/replica/data.ex", + correct_source: + " :ok = delete_cluster_routes(state.name, [cluster])\n" <> + " :ets.delete(closed_local_cluster_epochs_table(state.name), cluster)\n", + faulty_source: + " :ets.delete(closed_local_cluster_epochs_table(state.name), cluster)\n", + test: ["test/group_test.exs:2615"] + }, + %{ + name: "retire_peer_authority_before_terminal_route_cleanup", + file: "lib/group/replica/data.ex", + correct_source: + " :ets.delete(replication_meta_table(name), {:remote_generation, remote_node})\n" <> + " :ok = delete_peer_routes(name, remote_node)\n", + faulty_source: + " :ets.delete(replication_meta_table(name), {:remote_generation, remote_node})\n", + test: ["test/group_test.exs:2669"] + }, + %{ + name: "stale_peer_cleanup_removes_rediscovered_routes", + file: "lib/group/replica/data.ex", + correct_source: + " if is_nil(remote_generation(state.name, dead_node)) and\n" <> + " is_nil(remote_replica_authority_hint(state.name, dead_node)) do\n", + faulty_source: + " if Process.get(:purge_rediscovered_peer_routes, true) or\n" <> + " (is_nil(remote_generation(state.name, dead_node)) and\n" <> + " is_nil(remote_replica_authority_hint(state.name, dead_node))) do\n", + test: ["test/group_test.exs:2722"] + }, + %{ + name: "stale_restart_cleanup_removes_reactivated_routes", + file: "lib/group/replica/data.ex", + correct_source: + " Enum.filter(clusters, &is_nil(local_cluster_epoch(state.name, &1)))\n", + faulty_source: " clusters\n", + test: ["test/group_test.exs:2649"] + }, + %{ + name: "retain_authority_repair_after_nodedown", + file: "lib/group/replica.ex", + correct_source: + " cluster_control_dirty: Map.delete(state.cluster_control_dirty, dead_node),\n" <> + " authority_dirty_notified: MapSet.delete(state.authority_dirty_notified, dead_node)\n", + faulty_source: + " cluster_control_dirty: state.cluster_control_dirty,\n" <> + " authority_dirty_notified: MapSet.delete(state.authority_dirty_notified, dead_node)\n", + test: ["test/group_test.exs:2708"] + }, + %{ + name: "retain_receive_cursor_for_inactive_local_cluster", + file: "lib/group/replica/data.ex", + correct_source: + " WireProtocol.stream_origin(stream_id) != node() and\n" <> + " active_local_cluster?(name, WireProtocol.stream_cluster(stream_id)) and", + faulty_source: + " WireProtocol.stream_origin(stream_id) != node() and\n" <> + " true and", + test: ["test/anti_entropy_fault_regression_test.exs:2736"] + }, + %{ + name: "retire_shared_authority_with_live_lanes", + file: "lib/group/replica/data.ex", + correct_source: " result =\n if remaining_lanes == 0 do", + faulty_source: " _ = remaining_lanes\n\n result =\n if true do", + test: ["test/anti_entropy_fault_regression_test.exs:857"] + }, + %{ + name: "shard_zero_deletes_sibling_restart_views", + file: "lib/group/replica/data.ex", + correct_source: """ + if shard == 0 do + delete_remote_authority(state.name, remote_node) + end + """, + faulty_source: """ + if shard == 0 do + if state.num_shards > 1 do + for view_shard <- 1..(state.num_shards - 1) do + :ets.delete( + replication_meta_table(state.name), + {:remote_view_info, view_shard, remote_node} + ) + end + end + + delete_remote_authority(state.name, remote_node) + end + """, + test: ["test/anti_entropy_fault_regression_test.exs:240"] + }, + %{ + name: "forget_partial_authority_after_restart", + file: "lib/group/replica.ex", + correct_source: """ + cluster_control_dirty = + if shard_index == 0 do + Enum.reduce(retained_origins, %{}, fn origin, dirty -> + exact = Data.remote_cluster_epoch_exact_revision(name, origin) + observed = Data.remote_cluster_epoch_observed_revision(name, origin) + known_generation = Data.remote_generation(name, origin) + hint = Data.remote_replica_authority_hint(name, origin) + + if (not is_nil(observed) and observed != exact) or + (not is_nil(hint) and elem(hint, 0) != known_generation) do + Map.put(dirty, origin, restarted_at) + else + dirty + end + end) + else + %{} + end + """, + faulty_source: "cluster_control_dirty = %{}\n", + test: ["test/anti_entropy_fault_regression_test.exs:350"] + }, + %{ + name: "accept_wrong_shard_delta_rows", + file: "lib/group/replica.ex", + correct_source: + " Enum.split_while(contiguous, fn {_seq, mutations} ->\n" <> + " valid_replica_mutations?(state, stream_id, mutations)\n" <> + " end)", + faulty_source: + " Enum.split_while(contiguous, fn {_seq, mutations} ->\n" <> + " valid_replica_mutations?(%{state | num_shards: 1}, stream_id, mutations)\n" <> + " end)", + test: ["test/anti_entropy_fault_regression_test.exs:512"] + }, + %{ + name: "restore_unsequenced_cluster_disconnect", + file: "lib/group/replica.ex", + correct_source: """ + def handle_info(_msg, state) do + state = flush_pending_replicated_message_barrier(state) + {:noreply, state} + end + """, + faulty_source: """ + def handle_info({:cluster_disconnect, clusters, remote_pid}, state) do + Enum.each(clusters, fn cluster -> + Data.purge_registry_claims_for_cluster( + state.name, + state.shard_index, + cluster, + node(remote_pid) + ) + + purge_cluster_entries( + state.name, + state.shard_index, + cluster, + node(remote_pid) + ) + end) + + {:noreply, state} + end + + def handle_info(_msg, state) do + state = flush_pending_replicated_message_barrier(state) + {:noreply, state} + end + """, + test: ["test/group_test.exs:2280"] + }, + %{ + name: "skip_cursorless_restart_authority_repair", + file: "lib/group/replica/data.ex", + correct_source: " repair_primary_replica_rows(name, shard)", + faulty_source: + " if Process.get(:run_primary_replica_repair, false),\n" <> + " do: repair_primary_replica_rows(name, shard),\n" <> + " else: :ok", + test: ["test/anti_entropy_fault_regression_test.exs:3026"] + }, + %{ + name: "project_stale_claims_before_restart_repair", + file: "lib/group/replica.ex", + correct_source: """ + state = replay_local_journal(state) + :ok = Data.repair_shard_indexes(name, shard_index) + {state, _events} = rebuild_registry_projections(state) + """, + faulty_source: """ + state = replay_local_journal(state) + {state, _events} = rebuild_registry_projections(state) + :ok = Data.repair_shard_indexes(name, shard_index) + """, + test: ["test/anti_entropy_fault_regression_test.exs:3458"] + }, + %{ + name: "skip_interrupted_snapshot_install_repair", + file: "lib/group/replica/data.ex", + correct_source: " repair_interrupted_snapshot_installs(name, shard)", + faulty_source: + " if Process.get(:run_snapshot_install_repair, false),\n" <> + " do: repair_interrupted_snapshot_installs(name, shard),\n" <> + " else: :ok", + test: ["test/anti_entropy_fault_regression_test.exs:3138"] + }, + %{ + name: "retain_cursorless_remote_registry_claims", + file: "lib/group/replica/data.ex", + correct_source: + " :ets.member(replica_cursor_table(name, shard), stream_id)\n else\n false\n end\n end\n\n defp valid_remote_pg_authority?", + faulty_source: + " is_tuple(stream_id)\n else\n false\n end\n end\n\n defp valid_remote_pg_authority?", + test: ["test/anti_entropy_fault_regression_test.exs:3026"] + }, + %{ + name: "restart_snapshot_from_first_chunk_after_busy", + file: "lib/group/replica.ex", + correct_source: " resume = Map.get(offsets, snapshot_key, {:chunk, 1})", + faulty_source: """ + resume = + case Map.get(offsets, snapshot_key) do + {:commit, _manifest} = commit -> commit + _chunk_resume -> {:chunk, 1} + end + """, + test: ["test/replica_snapshot_distributed_test.exs:877"] + }, + %{ + name: "drain_oversized_ingress_batch_without_yield", + file: "lib/group/replica.ex", + correct_source: " {turn, remaining} = Enum.split(messages, @incoming_batch_quota)", + faulty_source: " _ = @incoming_batch_quota\n turn = messages\n remaining = []", + test: ["test/group_test.exs:37"] + } + ] + + def run(["--list"]) do + verify_mutation_definitions!(@mutations) + Enum.each(@mutations, &IO.puts(&1.name)) + end + + def run(args) do + selected = select_mutations(args) + verify_mutation_definitions!(selected) + campaign_dir = campaign_dir() + File.mkdir_p!(campaign_dir) + + IO.puts("mutation artifacts: #{campaign_dir}") + verify_baselines!(selected, campaign_dir) + + results = Enum.map(selected, &run_mutant(&1, campaign_dir)) + print_summary(results) + + if Enum.any?(results, fn {_name, status, _path} -> status != :killed end) do + System.halt(1) + end + end + + defp select_mutations([]), do: @mutations + + defp select_mutations(names) do + by_name = Map.new(@mutations, &{&1.name, &1}) + unknown = names -- Map.keys(by_name) + + if unknown != [] do + raise "unknown mutations: #{Enum.join(unknown, ", ")}" + end + + Enum.map(names, &Map.fetch!(by_name, &1)) + end + + defp verify_mutation_definitions!(mutations) do + invalid = + Enum.flat_map(mutations, fn mutation -> + verify_test_selectors!(mutation.test) + + source = mutation.file |> then(&Path.join(@repo, &1)) |> File.read!() + matches = :binary.matches(source, mutation.correct_source) + + if length(matches) == 1, do: [], else: [{mutation.name, length(matches)}] + end) + + if invalid != [] do + details = Enum.map_join(invalid, ", ", fn {name, count} -> "#{name}=#{count}" end) + raise "mutation definitions must match exactly one production fragment: #{details}" + end + end + + defp verify_baselines!(mutations, campaign_dir) do + mutations + |> Enum.map(& &1.test) + |> Enum.uniq() + |> Enum.each(fn test -> + label = test |> hd() |> String.replace(~r/[^A-Za-z0-9_.-]/, "_") + log = Path.join(campaign_dir, "baseline-#{label}.log") + IO.write("baseline #{Enum.join(test, " ")} ... ") + {output, status} = run_test(@repo, test) + File.write!(log, output) + + if status == 0 do + IO.puts("pass") + else + IO.puts("FAIL") + raise "baseline failed; see #{log}" + end + end) + end + + # ExUnit accepts a line anywhere inside a test body, which can silently run + # the preceding test after source edits shift declarations. Mutation tests + # must point at the declaration itself so a stale selector is invalid rather + # than being mistaken for evidence that a mutant survived. + defp verify_test_selectors!(selectors) do + Enum.each(selectors, fn selector -> + with [_, relative, line] <- Regex.run(~r/^(.*):(\d+)$/, selector), + {line, ""} <- Integer.parse(line), + source when is_binary(source) <- File.read!(Path.join(@repo, relative)), + declaration when is_binary(declaration) <- + Enum.at(String.split(source, "\n"), line - 1), + true <- + String.starts_with?(String.trim_leading(declaration), ["test \"", "property \""]) do + :ok + else + _ -> + raise "mutation selector must name an exact test/property declaration: #{selector}" + end + end) + end + + defp run_mutant(mutation, campaign_dir) do + work = Path.join(campaign_dir, mutation.name) + File.mkdir_p!(work) + copy_checkout!(work) + + source_path = Path.join(work, mutation.file) + source = File.read!(source_path) + matches = :binary.matches(source, mutation.correct_source) + + if length(matches) != 1 do + log = Path.join(work, "mutation-error.log") + File.write!(log, "expected one match, found #{length(matches)}\n") + IO.puts("#{mutation.name}: INVALID (replacement matched #{length(matches)} times)") + {mutation.name, :invalid, work} + else + File.write!( + source_path, + String.replace(source, mutation.correct_source, mutation.faulty_source) + ) + + compile_log = Path.join(work, "compile.log") + {compile_output, compile_status} = run_mix(work, ["compile", "--warnings-as-errors"]) + File.write!(compile_log, compile_output) + + if compile_status != 0 do + IO.puts("#{mutation.name}: INVALID (does not compile)") + {mutation.name, :invalid, work} + else + test_log = Path.join(work, "test.log") + {test_output, test_status} = run_test(work, mutation.test) + File.write!(test_log, test_output) + + case test_status do + 0 -> + IO.puts("#{mutation.name}: SURVIVED") + {mutation.name, :survived, work} + + 124 -> + IO.puts("#{mutation.name}: killed (timeout)") + {mutation.name, :killed, work} + + _ -> + IO.puts("#{mutation.name}: killed") + {mutation.name, :killed, work} + end + end + end + end + + defp copy_checkout!(target) do + rsync = System.find_executable("rsync") || raise "rsync is required" + + {_output, 0} = + System.cmd( + rsync, + [ + "-a", + "--exclude=.git", + "--exclude=deps", + "--exclude=tmp", + # Jepsen histories and caches are runtime artifacts. Copying them into + # every mutant can multiply a long soak's disk usage by the number of + # mutations without contributing anything to compilation or tests. + "--exclude=test/jepsen/store", + "--exclude=test/jepsen/.cache", + "#{@repo}/", + "#{target}/" + ], + stderr_to_stdout: true + ) + + File.ln_s!(Path.join(@repo, "deps"), Path.join(target, "deps")) + end + + defp run_test(directory, test) do + run_with_timeout(directory, ["mix", "test" | test], + env: [ + {"GROUP_MODEL_RUNS", "1"}, + {"GROUP_MODEL_COMMANDS", "8"}, + {"GROUP_JEPSEN_SKIP_CHECKER", "1"} + ] + ) + end + + defp run_mix(directory, args) do + System.cmd("mix", args, + cd: directory, + env: [{"MIX_ENV", "test"}], + stderr_to_stdout: true + ) + end + + defp run_with_timeout(directory, command, opts) do + timeout = System.find_executable("timeout") || raise "timeout is required" + env = Keyword.fetch!(opts, :env) + + System.cmd(timeout, [@timeout_seconds | command], + cd: directory, + env: [{"MIX_ENV", "test"} | env], + stderr_to_stdout: true + ) + end + + defp campaign_dir do + stamp = Calendar.strftime(DateTime.utc_now(), "%Y%m%dT%H%M%S") + Path.join([@repo, "tmp", "mutation", "#{stamp}-#{System.unique_integer([:positive])}"]) + end + + defp print_summary(results) do + IO.puts("\nmutation summary") + + Enum.each(results, fn {name, status, path} -> + IO.puts(" #{String.pad_trailing(name, 38)} #{status} #{path}") + end) + end +end + +Group.MutationCampaign.run(System.argv()) diff --git a/test/replica_adversarial_test.exs b/test/replica_adversarial_test.exs new file mode 100644 index 0000000..3b5003f --- /dev/null +++ b/test/replica_adversarial_test.exs @@ -0,0 +1,374 @@ +defmodule Group.ReplicaAdversarialTest do + use ExUnit.Case + + @moduletag :capture_log + @moduletag timeout: 120_000 + + alias Group.TestCluster + + @seeds [10_007, 20_011, 40_009] + @clusters ["red", "blue"] + + for seed <- @seeds do + @seed seed + @tag chaos_seed: seed + + test "seeded three-node transport chaos converges without zombies (seed #{@seed})" do + seed = @seed + :rand.seed(:exsss, {seed, seed * 3 + 1, seed * 7 + 2}) + + peers = TestCluster.start_peers(3) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + nodes = Enum.map(peers, &elem(&1, 1)) + name = :"replica_chaos_#{seed}_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 3, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 2, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 2_000, + replicated_oplog_max_entries: 12 + ] + + for {_peer, node} <- peers do + {:ok, _pid} = TestCluster.start_group(node, opts) + :ok = TestCluster.rpc!(node, Group, :connect, [name, @clusters]) + end + + TestCluster.assert_eventually( + fn -> + Enum.all?(nodes, fn node -> + length(TestCluster.rpc!(node, Group, :nodes, [name])) == length(nodes) - 1 and + Enum.all?(@clusters, fn cluster -> + length(TestCluster.rpc!(node, Group, :nodes, [name, cluster])) == length(nodes) + end) + end) + end, + timeout: 10_000 + ) + + chaos_modes = [ + {:chaos, [drop_every: 5, duplicate_every: 7, max_delay: 40]}, + {:chaos, [drop_every: 7, duplicate_every: 5, max_delay: 55]}, + {:chaos, [drop_every: 4, duplicate_every: 9, max_delay: 70]} + ] + + Enum.zip(nodes, chaos_modes) + |> Enum.each(fn {node, mode} -> + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, mode]) + end) + + initial = %{ + active: Map.new(nodes, &{&1, MapSet.new(@clusters)}), + counter: 0, + pg_keys: MapSet.new(), + pids: [], + reg_keys: MapSet.new(), + trace: [] + } + + state = + Enum.reduce(1..72, initial, fn step, state -> + apply_random_operation(state, step, seed, name, nodes) + end) + + for node <- nodes do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :pass]) + :ok = TestCluster.rpc!(node, Group, :connect, [name, @clusters]) + end + + assert_converges(name, nodes, state) + + # Let delayed frames from the chaos phase arrive, then prove they are + # duplicates/stale rather than a source of resurrection. + Process.sleep(150) + Enum.each(nodes, &TestCluster.flush_shards(&1, name)) + assert_converges(name, nodes, state) + + for node <- nodes do + assert :ok = + TestCluster.rpc!(node, Group.TestCluster, :assert_replica_consistent, [name]) + end + + TestCluster.assert_eventually( + fn -> retained_owners_alive?(name, nodes) end, + timeout: 15_000 + ) + end + end + + defp apply_random_operation(state, step, seed, name, nodes) do + [node_a, node_b | _rest] = nodes + + case :rand.uniform(12) do + choice when choice in 1..3 -> + add_registration(state, step, seed, name, random(nodes)) + + choice when choice in 4..6 -> + add_membership(state, step, seed, name, random(nodes)) + + 7 -> + kill_random_owner(state) + + 8 -> + add_registry_conflict(state, step, seed, name, node_a, node_b) + + 9 -> + toggle_cluster(state, step, name, random(nodes), random(@clusters)) + + 10 -> + change_transport_mode(state, step, name, random(nodes)) + + _ -> + Enum.reduce(1..3, state, fn offset, acc -> + add_registration(acc, step * 10 + offset, seed, name, random(nodes)) + end) + end + end + + defp add_registration(state, step, seed, name, origin) do + cluster = random([nil | MapSet.to_list(state.active[origin])]) + {key, state} = next_key(state, seed, "reg", step) + meta = %{seed: seed, step: step, origin: origin} + + pid = + if cluster do + TestCluster.spawn_register_in_cluster(origin, name, key, meta, cluster) + else + TestCluster.spawn_register(origin, name, key, meta) + end + + state + |> Map.update!(:pids, &[{origin, pid} | &1]) + |> Map.update!(:reg_keys, &MapSet.put(&1, {cluster, key})) + |> trace({:register, origin, cluster, key, pid}) + end + + defp add_membership(state, step, seed, name, origin) do + cluster = random([nil | MapSet.to_list(state.active[origin])]) + {key, state} = next_key(state, seed, "pg", step) + meta = %{seed: seed, step: step, origin: origin} + + pid = + if cluster do + TestCluster.spawn_join_in_cluster(origin, name, key, meta, cluster) + else + TestCluster.spawn_join(origin, name, key, meta) + end + + state + |> Map.update!(:pids, &[{origin, pid} | &1]) + |> Map.update!(:pg_keys, &MapSet.put(&1, {cluster, key})) + |> trace({:join, origin, cluster, key, pid}) + end + + defp kill_random_owner(%{pids: []} = state), do: trace(state, :kill_noop) + + defp kill_random_owner(state) do + {origin, pid} = random(state.pids) + TestCluster.rpc!(origin, Process, :exit, [pid, :kill]) + + state + |> Map.update!(:pids, &List.delete(&1, {origin, pid})) + |> trace({:kill, origin, pid}) + end + + defp add_registry_conflict(state, step, seed, name, node_a, node_b) do + {key, state} = next_key(state, seed, "conflict", step) + + # Establish the competing claims before either origin can observe the + # other. Drain any already-scheduled delayed frame first; otherwise this + # operation nondeterministically degenerates into a local :taken result. + for node <- [node_a, node_b] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :drop]) + end + + Process.sleep(75) + pid_a = TestCluster.spawn_register(node_a, name, key, %{side: :a, seed: seed}) + pid_b = TestCluster.spawn_register(node_b, name, key, %{side: :b, seed: seed}) + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:chaos, [drop_every: 5, duplicate_every: 7, max_delay: 40]} + ]) + + :ok = + TestCluster.rpc!(node_b, Group.TestReplicaTransport, :set_mode, [ + name, + {:chaos, [drop_every: 7, duplicate_every: 5, max_delay: 55]} + ]) + + state + |> Map.update!(:pids, &[{node_a, pid_a}, {node_b, pid_b} | &1]) + |> Map.update!(:reg_keys, &MapSet.put(&1, {nil, key})) + |> trace({:conflict, key, pid_a, pid_b}) + end + + defp toggle_cluster(state, step, name, origin, cluster) do + if MapSet.member?(state.active[origin], cluster) do + :ok = TestCluster.rpc!(origin, Group, :disconnect, [name, cluster]) + + state + |> put_in([:active, origin], MapSet.delete(state.active[origin], cluster)) + |> trace({:disconnect, step, origin, cluster}) + else + :ok = TestCluster.rpc!(origin, Group, :connect, [name, cluster]) + + state + |> put_in([:active, origin], MapSet.put(state.active[origin], cluster)) + |> trace({:connect, step, origin, cluster}) + end + end + + defp change_transport_mode(state, step, name, origin) do + mode = + random([ + :drop, + :busy, + {:chaos, [drop_every: 4, duplicate_every: 3, max_delay: 65]}, + {:chaos, [drop_every: 9, duplicate_every: 2, max_delay: 30]} + ]) + + :ok = TestCluster.rpc!(origin, Group.TestReplicaTransport, :set_mode, [name, mode]) + trace(state, {:transport, step, origin, mode}) + end + + defp next_key(state, seed, kind, step) do + counter = state.counter + 1 + {"chaos/#{seed}/#{kind}/#{step}/#{counter}", %{state | counter: counter}} + end + + defp assert_converges(name, nodes, state) do + TestCluster.assert_eventually( + fn -> + Enum.all?(nodes, fn node -> + length(TestCluster.rpc!(node, Group, :nodes, [name])) == length(nodes) - 1 + end) and + Enum.all?(@clusters, fn cluster -> + Enum.all?(nodes, fn node -> + length(TestCluster.rpc!(node, Group, :nodes, [name, cluster])) == length(nodes) + end) + end) and + registry_equal?(name, nodes, state.reg_keys) and + memberships_equal?(name, nodes, state.pg_keys) + end, + timeout: 20_000, + interval: 75 + ) + rescue + error -> + flunk( + "chaos convergence failed: #{Exception.message(error)}\n" <> + "differences=#{inspect(convergence_differences(name, nodes, state), limit: :infinity)}\n" <> + "recent operations=#{inspect(Enum.take(state.trace, 20), limit: :infinity)}" + ) + end + + defp convergence_differences(name, nodes, state) do + registry = + state.reg_keys + |> Enum.flat_map(fn {cluster, key} -> + args = [name, key, cluster_opts(cluster)] + values = Map.new(nodes, &{&1, TestCluster.rpc!(&1, Group, :lookup, args)}) + + if values |> Map.values() |> Enum.uniq() |> length() == 1 do + [] + else + replica = + Map.new(nodes, fn node -> + {node, + TestCluster.rpc!(node, Group.TestCluster, :replica_registry_key_state, [ + name, + cluster, + key + ])} + end) + + [{:registry, cluster, key, values, replica}] + end + end) + |> Enum.take(10) + + pg = + state.pg_keys + |> Enum.flat_map(fn {cluster, key} -> + args = [name, key, cluster_opts(cluster)] + values = Map.new(nodes, &{&1, TestCluster.rpc!(&1, Group, :members, args) |> Enum.sort()}) + + if values |> Map.values() |> Enum.uniq() |> length() == 1, + do: [], + else: [{:pg, cluster, key, values}] + end) + |> Enum.take(10) + + topology = + for cluster <- [nil | @clusters] do + {cluster, Map.new(nodes, &{&1, group_nodes(&1, name, cluster)})} + end + + cluster_trace = + state.trace + |> Enum.filter(fn + {:connect, _step, _origin, _cluster} -> true + {:disconnect, _step, _origin, _cluster} -> true + _ -> false + end) + + protocol = + for node <- nodes do + {node, TestCluster.rpc!(node, Group.TestCluster, :replica_protocol_state, [name])} + end + + [ + nodes: topology, + registry: registry, + pg: pg, + cluster_trace: cluster_trace, + protocol: protocol + ] + end + + defp group_nodes(node, name, nil), do: TestCluster.rpc!(node, Group, :nodes, [name]) + + defp group_nodes(node, name, cluster), + do: TestCluster.rpc!(node, Group, :nodes, [name, cluster]) + + defp registry_equal?(name, nodes, keys) do + Enum.all?(keys, fn {cluster, key} -> + args = [name, key, cluster_opts(cluster)] + + nodes |> Enum.map(&TestCluster.rpc!(&1, Group, :lookup, args)) |> Enum.uniq() |> length() == + 1 + end) + end + + defp memberships_equal?(name, nodes, keys) do + Enum.all?(keys, fn {cluster, key} -> + args = [name, key, cluster_opts(cluster)] + + nodes + |> Enum.map(&(TestCluster.rpc!(&1, Group, :members, args) |> Enum.sort())) + |> Enum.uniq() + |> length() == 1 + end) + end + + defp retained_owners_alive?(name, nodes) do + nodes + |> Enum.flat_map(fn node -> + TestCluster.rpc!(node, Group.TestCluster, :replica_owner_pids, [name]) + end) + |> Enum.uniq() + |> Enum.all?(fn pid -> TestCluster.rpc!(node(pid), Process, :alive?, [pid]) end) + end + + defp cluster_opts(nil), do: [] + defp cluster_opts(cluster), do: [cluster: cluster] + + defp trace(state, operation), do: Map.update!(state, :trace, &[operation | &1]) + defp random(values), do: Enum.at(values, :rand.uniform(length(values)) - 1) +end diff --git a/test/replica_model_property_test.exs b/test/replica_model_property_test.exs new file mode 100644 index 0000000..e4175d2 --- /dev/null +++ b/test/replica_model_property_test.exs @@ -0,0 +1,392 @@ +defmodule Group.ReplicaModelPropertyTest do + use ExUnit.Case, async: false + use ExUnitProperties + + alias Group.{ + ControlledReplicaTransport, + ModelConflictResolver, + ReplicaModelScheduler, + TestCluster + } + + @moduletag :capture_log + @moduletag timeout: 180_000 + + @model_runs System.get_env("GROUP_MODEL_RUNS", "12") |> String.to_integer() + @max_commands System.get_env("GROUP_MODEL_COMMANDS", "30") |> String.to_integer() + + setup_all do + peers = TestCluster.start_peers(3, schedulers: 4) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}, {_, node_c}] = peers + {:ok, nodes: %{a: node_a, b: node_b, c: node_c}} + end + + property "accepted owner lifecycles converge without permanent orphans or zombies", %{ + nodes: nodes + } do + check all( + commands <- command_history(), + max_runs: @model_runs, + max_shrinking_steps: 100 + ) do + name = :"replica_model_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 2, + resolve_registry_conflict: {ModelConflictResolver, :resolve, []}, + replica_transport: {ControlledReplicaTransport, controller: self()}, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000, + replicated_snapshot_chunk_target_bytes: 256, + replicated_oplog_max_entries: 4 + ] + + Enum.each(nodes, fn {_id, node} -> + {:ok, _pid} = TestCluster.start_group(node, opts) + end) + + scheduler = ReplicaModelScheduler.new(name, nodes, opts) + + try do + await_discovery(nodes, name) + + scheduler = + commands + |> Enum.reduce(ReplicaModelScheduler.sync(scheduler), fn command, state -> + ReplicaModelScheduler.execute(state, command) + end) + |> ReplicaModelScheduler.stabilize_and_assert!() + + assert scheduler.model != nil + after + ReplicaModelScheduler.cleanup(scheduler) + end + end + end + + property "concurrent claims converge to one live winner and permanently retire the loser", %{ + nodes: nodes + } do + check all( + key_slot <- integer(0..2), + winner <- member_of([:a, :b]), + schedule <- list_of(network_command(), min_length: 0, max_length: 20), + max_runs: max(div(@model_runs, 2), 1), + max_shrinking_steps: 100 + ) do + name = :"replica_conflict_model_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 2, + resolve_registry_conflict: {ModelConflictResolver, :resolve, []}, + replica_transport: {ControlledReplicaTransport, controller: self()}, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000, + replicated_snapshot_chunk_target_bytes: 256, + replicated_oplog_max_entries: 2 + ] + + Enum.each(nodes, fn {_id, node} -> + {:ok, _pid} = TestCluster.start_group(node, opts) + end) + + scheduler = ReplicaModelScheduler.new(name, nodes, opts) + + try do + await_discovery(nodes, name) + + {rank_a, rank_b} = if winner == :a, do: {2, 1}, else: {1, 2} + + scheduler = + scheduler + |> ReplicaModelScheduler.sync() + |> ReplicaModelScheduler.execute({:register, 12, :c, key_slot + 10, 1}) + |> ReplicaModelScheduler.execute({:join, 12, :c, key_slot + 10, 1}) + |> ReplicaModelScheduler.execute({:claim, 10, :a, key_slot, rank_a}) + |> ReplicaModelScheduler.execute({:claim, 11, :b, key_slot, rank_b}) + + scheduler = + schedule + |> Enum.reduce(scheduler, fn command, state -> + ReplicaModelScheduler.execute(state, command) + end) + |> ReplicaModelScheduler.stabilize_and_assert!() + + assert scheduler.model != nil + after + ReplicaModelScheduler.cleanup(scheduler) + end + end + end + + property "old frames cannot survive a real Group restart and generation change", %{nodes: nodes} do + check all( + before_restart <- list_of(network_command(), max_length: 8), + after_restart <- list_of(network_command(), max_length: 12), + max_runs: transition_runs(), + max_shrinking_steps: 100 + ) do + name = :"replica_restart_model_#{System.unique_integer([:positive])}" + opts = model_opts(name, self(), shards: 2, oplog: 2) + start_groups(nodes, opts) + scheduler = ReplicaModelScheduler.new(name, nodes, opts) + + try do + await_discovery(nodes, name) + + scheduler = + scheduler + |> ReplicaModelScheduler.sync() + |> ReplicaModelScheduler.execute({:transport, :a, :pass}) + |> ReplicaModelScheduler.execute({:transport, :b, :pass}) + |> ReplicaModelScheduler.execute({:transport, :c, :pass}) + |> ReplicaModelScheduler.execute({:register, 30, :a, 0, 1}) + |> ReplicaModelScheduler.execute({:join, 30, :a, 0, 1}) + |> ReplicaModelScheduler.execute({:register, 32, :c, 1, 1}) + |> ReplicaModelScheduler.execute({:join, 32, :c, 1, 1}) + |> ReplicaModelScheduler.stabilize_and_assert!() + |> ReplicaModelScheduler.execute({:transport, :a, :capture}) + |> ReplicaModelScheduler.execute({:transport, :b, :capture}) + |> ReplicaModelScheduler.execute({:transport, :c, :capture}) + |> ReplicaModelScheduler.execute({:register, 30, :a, 0, 2}) + |> ReplicaModelScheduler.execute({:join, 30, :a, 0, 2}) + |> run_schedule(before_restart) + |> ReplicaModelScheduler.execute({:restart, :a}) + |> ReplicaModelScheduler.execute(:deliver_all) + |> ReplicaModelScheduler.execute({:register, 31, :a, 1, 2}) + |> ReplicaModelScheduler.execute({:join, 31, :a, 1, 2}) + |> run_schedule(after_restart) + |> ReplicaModelScheduler.stabilize_and_assert!() + + assert scheduler.model != nil + after + ReplicaModelScheduler.cleanup(scheduler) + end + end + end + + property "a receiver below the real oplog floor converges through an exact snapshot", %{ + nodes: nodes + } do + check all( + schedule <- list_of(network_command(), max_length: 10), + max_runs: transition_runs(), + max_shrinking_steps: 100 + ) do + name = :"replica_pruning_model_#{System.unique_integer([:positive])}" + opts = model_opts(name, self(), shards: 1, oplog: 2) + start_groups(nodes, opts) + scheduler = ReplicaModelScheduler.new(name, nodes, opts) + + try do + await_discovery(nodes, name) + + scheduler = + scheduler + |> ReplicaModelScheduler.sync() + |> ReplicaModelScheduler.execute({:transport, :a, :pass}) + |> ReplicaModelScheduler.execute({:transport, :b, :pass}) + |> ReplicaModelScheduler.execute({:transport, :c, :pass}) + |> ReplicaModelScheduler.execute({:register, 40, :a, 0, 1}) + |> ReplicaModelScheduler.execute({:join, 40, :a, 0, 1}) + |> ReplicaModelScheduler.execute({:register, 47, :c, 1, 1}) + |> ReplicaModelScheduler.execute({:join, 47, :c, 1, 1}) + |> ReplicaModelScheduler.stabilize_and_assert!() + |> ReplicaModelScheduler.execute({:transport, :a, :drop}) + |> ReplicaModelScheduler.execute({:kill, 40}) + + scheduler = + Enum.reduce(41..46, scheduler, fn owner_id, state -> + state = + ReplicaModelScheduler.execute( + state, + {:register, owner_id, :a, 0, owner_id} + ) + + if rem(owner_id, 2) == 0 do + ReplicaModelScheduler.execute(state, {:unregister, owner_id, 0}) + else + state + end + end) + + scheduler = + scheduler + |> run_schedule(schedule) + |> ReplicaModelScheduler.stabilize_and_assert!() + + assert scheduler.model != nil + after + ReplicaModelScheduler.cleanup(scheduler) + end + end + end + + property "named-cluster close and reopen fences stale epoch frames on real shards", %{ + nodes: nodes + } do + check all( + cluster_slot <- integer(0..2), + key_slot <- integer(0..2), + before_reopen <- list_of(network_command(), max_length: 8), + after_reopen <- list_of(network_command(), max_length: 12), + max_runs: transition_runs(), + max_shrinking_steps: 100 + ) do + name = :"replica_authority_model_#{System.unique_integer([:positive])}" + cluster = "model_cluster_#{cluster_slot}" + opts = model_opts(name, self(), shards: 2, oplog: 2) + start_groups(nodes, opts) + scheduler = ReplicaModelScheduler.new(name, nodes, opts) + + try do + await_discovery(nodes, name) + + scheduler = + scheduler + |> ReplicaModelScheduler.sync() + |> ReplicaModelScheduler.execute({:connect, :a, cluster}) + |> ReplicaModelScheduler.execute({:connect, :b, cluster}) + |> ReplicaModelScheduler.execute({:connect, :c, cluster}) + |> ReplicaModelScheduler.execute({:register_cluster, 50, :a, cluster, key_slot, 1}) + |> ReplicaModelScheduler.execute({:join_cluster, 50, :a, cluster, key_slot, 1}) + |> ReplicaModelScheduler.execute({ + :register_cluster, + 52, + :c, + cluster, + key_slot + 10, + 1 + }) + |> ReplicaModelScheduler.execute({ + :join_cluster, + 52, + :c, + cluster, + key_slot + 10, + 1 + }) + |> run_schedule(before_reopen) + |> ReplicaModelScheduler.execute({:disconnect, :a, cluster}) + |> ReplicaModelScheduler.execute({:connect, :a, cluster}) + |> ReplicaModelScheduler.execute(:deliver_all) + |> ReplicaModelScheduler.execute({:register_cluster, 51, :a, cluster, key_slot, 2}) + |> ReplicaModelScheduler.execute({:join_cluster, 51, :a, cluster, key_slot, 2}) + |> run_schedule(after_reopen) + |> ReplicaModelScheduler.stabilize_and_assert!() + + assert scheduler.model != nil + after + ReplicaModelScheduler.cleanup(scheduler) + end + end + end + + defp command_history do + list_of(command(), min_length: 1, max_length: @max_commands) + end + + defp command do + one_of([ + owner_command(:register), + owner_command(:join), + owner_key_command(:unregister), + owner_key_command(:leave), + map(owner_id(), &{:kill, &1}), + transport_mode_command(), + map(non_negative_integer(), &{:deliver, &1}), + map(non_negative_integer(), &{:duplicate, &1}), + map(non_negative_integer(), &{:drop, &1}), + constant(:deliver_all), + constant(:anti_entropy), + constant(:flush) + ]) + end + + defp network_command do + one_of([ + transport_mode_command(), + map(non_negative_integer(), &{:deliver, &1}), + map(non_negative_integer(), &{:duplicate, &1}), + map(non_negative_integer(), &{:drop, &1}), + constant(:anti_entropy), + constant(:flush) + ]) + end + + defp owner_command(operation) do + gen all( + owner_id <- owner_id(), + node_id <- member_of([:a, :b, :c]), + slot <- integer(0..1), + revision <- integer(0..3) + ) do + {operation, owner_id, node_id, slot, revision} + end + end + + defp owner_key_command(operation) do + gen all( + owner_id <- owner_id(), + slot <- integer(0..1) + ) do + {operation, owner_id, slot} + end + end + + defp transport_mode_command do + gen all( + node_id <- member_of([:a, :b, :c]), + mode <- member_of([:capture, :busy, :drop]) + ) do + {:transport, node_id, mode} + end + end + + defp owner_id, do: integer(0..5) + + defp transition_runs, do: max(div(@model_runs, 3), 1) + + defp run_schedule(state, commands) do + Enum.reduce(commands, state, fn command, acc -> + ReplicaModelScheduler.execute(acc, command) + end) + end + + defp model_opts(name, controller, overrides) do + [ + name: name, + shards: Keyword.fetch!(overrides, :shards), + resolve_registry_conflict: {ModelConflictResolver, :resolve, []}, + replica_transport: {ControlledReplicaTransport, controller: controller}, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000, + replicated_snapshot_chunk_target_bytes: 256, + replicated_oplog_max_entries: Keyword.fetch!(overrides, :oplog) + ] + end + + defp start_groups(nodes, opts) do + Enum.each(nodes, fn {_id, node} -> + {:ok, _pid} = TestCluster.start_group(node, opts) + end) + end + + defp await_discovery(nodes, name) do + TestCluster.assert_eventually( + fn -> + Enum.all?(nodes, fn {_id, node} -> + length(TestCluster.rpc!(node, Group, :nodes, [name])) == map_size(nodes) - 1 + end) + end, + timeout: 10_000 + ) + end +end diff --git a/test/replica_snapshot_distributed_test.exs b/test/replica_snapshot_distributed_test.exs new file mode 100644 index 0000000..e94a736 --- /dev/null +++ b/test/replica_snapshot_distributed_test.exs @@ -0,0 +1,1142 @@ +defmodule Group.ReplicaSnapshotDistributedTest do + use ExUnit.Case, async: false + + @moduletag :capture_log + @moduletag timeout: 120_000 + + alias Group.TestCluster + + setup_all do + peers = TestCluster.start_peers(3, schedulers: 4) + on_exit(fn -> TestCluster.stop_peers(peers) end) + [{_, node_a}, {_, node_b}, {_, node_c}] = peers + {:ok, node_a: node_a, node_b: node_b, node_c: node_c} + end + + test "complete provisional chunks expose nothing until terminal commit", context do + %{name: name, node_a: node_a, node_b: node_b} = start_pair(context) + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + entries = + for index <- 1..8 do + key = "snapshot/terminal-commit/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("c", 160) + })} + end + + TestCluster.flush_shards(node_a, name) + stream_id = local_stream(node_a, name, nil) + {chunks, commit} = capture_snapshot_with_commit(node_a, node_b, name, stream_id, 1) + assert length(chunks) > 1 + + deliver_frames(node_b, node_a, name, Enum.reverse(chunks)) + TestCluster.flush_shards(node_b, name) + + assert replica_cursor(node_b, name, stream_id) == 0 + + assert Enum.all?(entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) + + deliver_frames(node_b, node_a, name, [commit]) + TestCluster.flush_shards(node_b, name) + snapshot_seq = elem(commit, 3) + + assert replica_cursor(node_b, name, stream_id) == snapshot_seq + + assert Enum.all?(entries, fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + end + + test "exact PG install repairs an impossible conflicting stored origin", context do + %{name: name, node_a: node_a, node_b: node_b, node_c: node_c} = start_pair(context) + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + key = "snapshot/pg-origin-collision" + member = TestCluster.spawn_join(node_a, name, key, %{source: true}) + + # Move the source stream floor past the membership so requesting sequence + # one necessarily exercises an exact snapshot rather than a delta. + for index <- 1..4 do + TestCluster.spawn_register(node_a, name, "snapshot/pg-origin-filler/#{index}", %{}) + end + + TestCluster.flush_shards(node_a, name) + stream_id = local_stream(node_a, name, nil) + {chunks, commit} = capture_snapshot_with_commit(node_a, node_b, name, stream_id, 1) + + :ok = + TestCluster.rpc!(node_b, Group.Replica.Data, :pg_insert_many, [ + name, + 0, + [{nil, key, member, %{corrupt: true}, 0, node_c}] + ]) + + shard = TestCluster.rpc!(node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + monitor = Process.monitor(shard) + + deliver_frames(node_b, node_a, name, chunks ++ [commit]) + TestCluster.flush_shards(node_b, name) + + refute_receive {:DOWN, ^monitor, :process, ^shard, _reason}, 250 + assert TestCluster.rpc!(node_b, Process, :alive?, [shard]) + assert [{^member, %{source: true}}] = TestCluster.rpc!(node_b, Group, :members, [name, key]) + + assert {%{source: true}, _time, ^node_a} = + TestCluster.rpc!(node_b, Group.Replica.Data, :pg_lookup, [ + name, + 0, + nil, + key, + member + ]) + + assert :ok = TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + test "a source mutation during a single-pass scan prevents terminal commit", context do + %{name: name, node_a: node_a, node_b: node_b} = start_pair(context) + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + entries = + for index <- 1..8 do + key = "snapshot/concurrent-write/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("s", 160) + })} + end + + TestCluster.flush_shards(node_a, name) + stream_id = local_stream(node_a, name, nil) + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :clear_captured, [name]) + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:capture_drop_pause_once, [:snapshot_chunk, :snapshot_commit], self()} + ]) + + :ok = + TestCluster.rpc!(node_a, Group.Transport, :incoming, [ + name, + node_b, + 0, + {:need, Group.Replica.WireProtocol.version(), stream_id, 1} + ]) + + assert_receive {:replica_transport_paused, worker, ^name, :snapshot_chunk}, 5_000 + + extra_key = "snapshot/concurrent-write/after-scan-started" + extra_pid = TestCluster.spawn_register(node_a, name, extra_key, %{after_start: true}) + TestCluster.flush_shards(node_a, name) + send(worker, {:resume_replica_transport, name}) + + source = TestCluster.rpc!(node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_a, :sys, :get_state, [source]).snapshot_send == nil + end) + + captured = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) + + chunks = + Enum.flat_map(captured, fn + {^node_b, 0, {:snapshot_chunk, _, ^stream_id, _, _, _, _} = chunk} -> [chunk] + _other -> [] + end) + + assert chunks != [] + + refute Enum.any?(captured, fn + {^node_b, 0, {:snapshot_commit, _, ^stream_id, _, _, _, _}} -> true + _other -> false + end) + + deliver_frames(node_b, node_a, name, chunks) + assert replica_cursor(node_b, name, stream_id) == 0 + + assert Enum.all?(entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) + + assert TestCluster.rpc!(node_b, Group, :lookup, [name, extra_key]) == nil + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + :ok = + TestCluster.rpc!(node_a, Group.Transport, :incoming, [ + name, + node_b, + 0, + {:need, Group.Replica.WireProtocol.version(), stream_id, 1} + ]) + + TestCluster.assert_eventually(fn -> + Enum.all?(entries, fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) and + match?( + {^extra_pid, _}, + TestCluster.rpc!(node_b, Group, :lookup, [name, extra_key]) + ) + end) + end + + test "conflicting terminal manifests discard the candidate instead of manufacturing commit", + context do + %{name: name, node_a: node_a, node_b: node_b} = start_pair(context) + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + entries = + for index <- 1..8 do + key = "snapshot/conflicting-commit/#{index}" + {key, TestCluster.spawn_register(node_a, name, key, %{index: index})} + end + + TestCluster.flush_shards(node_a, name) + stream_id = local_stream(node_a, name, nil) + {chunks, commit} = capture_snapshot_with_commit(node_a, node_b, name, stream_id, 1) + conflicting_commit = put_elem(commit, 5, elem(commit, 5) + 1) + + deliver_frames(node_b, node_a, name, chunks ++ [conflicting_commit]) + assert replica_cursor(node_b, name, stream_id) == 0 + assert snapshot_transfer_count(node_b, name) == 1 + + deliver_frames(node_b, node_a, name, [commit]) + assert replica_cursor(node_b, name, stream_id) == 0 + assert snapshot_transfer_count(node_b, name) == 0 + + deliver_frames(node_b, node_a, name, chunks ++ [commit]) + assert replica_cursor(node_b, name, stream_id) == elem(commit, 3) + + assert Enum.all?(entries, fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + end + + test "loss, reordering, and duplication expose nothing until exact commit", context do + %{name: name, node_a: node_a, node_b: node_b} = start_pair(context) + + stale_reg_key = "snapshot/stale-reg" + stale_pg_key = "snapshot/stale-pg" + stale_reg_pid = TestCluster.spawn_register(node_a, name, stale_reg_key, %{stale: true}) + stale_pg_pid = TestCluster.spawn_join(node_a, name, stale_pg_key, %{stale: true}) + + TestCluster.assert_eventually(fn -> + match?({^stale_reg_pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, stale_reg_key])) and + match?( + [{^stale_pg_pid, _}], + TestCluster.rpc!(node_b, Group, :members, [name, stale_pg_key]) + ) + end) + + forwarder = TestCluster.spawn_monitor_forwarder(node_b, name, :all, self()) + assert_receive {:monitor_ready, ^forwarder}, 5_000 + + stream_id = local_stream(node_a, name, nil) + old_cursor = replica_cursor(node_b, name, stream_id) + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + TestCluster.rpc!(node_a, Process, :exit, [stale_reg_pid, :kill]) + TestCluster.rpc!(node_a, Process, :exit, [stale_pg_pid, :kill]) + + fresh_reg = + for index <- 1..8 do + key = "snapshot/fresh-reg/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("r", 160) + })} + end + + fresh_pg_key = "snapshot/fresh-pg" + + fresh_pg = + for _index <- 1..8 do + TestCluster.spawn_join(node_a, name, fresh_pg_key, %{ + payload: String.duplicate("p", 160) + }) + end + + TestCluster.flush_shards(node_a, name) + frames = capture_snapshot(node_a, node_b, name, stream_id, old_cursor + 1) + assert length(frames) > 2 + [missing | delivered] = frames + + deliver_frames(node_b, node_a, name, Enum.reverse(delivered)) + deliver_frames(node_b, node_a, name, [List.last(delivered)]) + TestCluster.flush_shards(node_b, name) + + assert replica_cursor(node_b, name, stream_id) == old_cursor + + assert match?( + {^stale_reg_pid, _}, + TestCluster.rpc!(node_b, Group, :lookup, [name, stale_reg_key]) + ) + + assert match?( + [{^stale_pg_pid, _}], + TestCluster.rpc!(node_b, Group, :members, [name, stale_pg_key]) + ) + + assert Enum.all?(fresh_reg, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) + + assert TestCluster.rpc!(node_b, Group, :members, [name, fresh_pg_key]) == [] + + deliver_frames(node_b, node_a, name, [missing]) + TestCluster.flush_shards(node_b, name) + snapshot_seq = elem(missing, 3) + + assert replica_cursor(node_b, name, stream_id) == snapshot_seq + assert TestCluster.rpc!(node_b, Group, :lookup, [name, stale_reg_key]) == nil + assert TestCluster.rpc!(node_b, Group, :members, [name, stale_pg_key]) == [] + + assert Enum.all?(fresh_reg, fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + + assert MapSet.new( + Enum.map( + TestCluster.rpc!(node_b, Group, :members, [name, fresh_pg_key]), + &elem(&1, 0) + ) + ) == + MapSet.new(fresh_pg) + + {fresh_reg_key, fresh_reg_pid} = hd(fresh_reg) + fresh_pg_pid = hd(fresh_pg) + + assert_receive {:got_event, + %Group.Event{type: :registered, key: ^fresh_reg_key, pid: ^fresh_reg_pid}}, + 5_000 + + assert_receive {:got_event, + %Group.Event{type: :joined, key: ^fresh_pg_key, pid: ^fresh_pg_pid}}, + 5_000 + + assert snapshot_transfer_count(node_b, name) == 0 + end + + test "a newer exact snapshot supersedes an incomplete older one", context do + %{name: name, node_a: node_a, node_b: node_b} = start_pair(context) + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + old_entries = + for index <- 1..8 do + key = "snapshot/superseded/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("o", 160) + })} + end + + TestCluster.flush_shards(node_a, name) + stream_id = local_stream(node_a, name, nil) + older = capture_snapshot(node_a, node_b, name, stream_id, 1) + assert length(older) > 1 + + Enum.each(old_entries, fn {_key, pid} -> + TestCluster.rpc!(node_a, Process, :exit, [pid, :kill]) + end) + + fresh_entries = + for index <- 1..8 do + key = "snapshot/replacement/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("n", 160) + })} + end + + TestCluster.flush_shards(node_a, name) + newer = capture_snapshot(node_a, node_b, name, stream_id, 1) + assert elem(hd(newer), 3) > elem(hd(older), 3) + + deliver_frames(node_b, node_a, name, [hd(older)]) + assert snapshot_transfer_count(node_b, name) == 1 + + deliver_frames(node_b, node_a, name, Enum.reverse(newer)) + deliver_frames(node_b, node_a, name, tl(older)) + TestCluster.flush_shards(node_b, name) + + assert replica_cursor(node_b, name, stream_id) == elem(hd(newer), 3) + + assert Enum.all?(old_entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) + + assert Enum.all?(fresh_entries, fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + + assert snapshot_transfer_count(node_b, name) == 0 + end + + test "conflicting retransmission chunks cannot manufacture an exact snapshot", context do + %{name: name, node_a: node_a, node_b: node_b} = start_pair(context) + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + entries = + for index <- 1..8 do + key = "snapshot/conflicting/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("m", 160) + })} + end + + TestCluster.flush_shards(node_a, name) + stream_id = local_stream(node_a, name, nil) + [first, second | rest] = capture_snapshot(node_a, node_b, name, stream_id, 1) + [first_row | first_tail] = elem(first, 5) + [second_row | second_tail] = elem(second, 5) + conflicting_first = put_elem(first, 5, [second_row | first_tail]) + conflicting_second = put_elem(second, 5, [first_row | second_tail]) + + deliver_frames(node_b, node_a, name, [first, conflicting_first]) + + assert replica_cursor(node_b, name, stream_id) == 0 + assert snapshot_transfer_count(node_b, name) == 0 + + assert Enum.all?(entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) + + # A different chunk index cannot reuse an identity already staged by the + # first chunk to satisfy the terminal row count while omitting another row. + deliver_frames(node_b, node_a, name, [first, conflicting_second | rest]) + assert replica_cursor(node_b, name, stream_id) == 0 + + assert Enum.all?(entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) + + deliver_frames(node_b, node_a, name, [first, second | rest]) + + assert replica_cursor(node_b, name, stream_id) == elem(first, 3) + + assert Enum.all?(entries, fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + end + + test "rejected first chunks clear and reuse their private staging tables", context do + %{name: name, node_a: node_a, node_b: node_b} = start_pair(context) + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + for index <- 1..4 do + TestCluster.spawn_register(node_a, name, "snapshot/rejected/#{index}", %{ + payload: String.duplicate("x", 160) + }) + end + + TestCluster.flush_shards(node_a, name) + stream_id = local_stream(node_a, name, nil) + frames = capture_snapshot(node_a, node_b, name, stream_id, 1) + + rows = + frames + |> Enum.filter(&(elem(&1, 0) == :snapshot_chunk)) + |> Enum.flat_map(&elem(&1, 5)) + |> Enum.take(2) + + assert [first_row, second_row] = rows + {:snapshot_chunk, version, ^stream_id, snapshot_seq, _, _, _} = hd(frames) + assert snapshot_staging_tables(node_b, name) == [] + + commit = {:snapshot_commit, version, stream_id, snapshot_seq, 1, 1, 0} + deliver_frames(node_b, node_a, name, [commit]) + assert snapshot_transfer_count(node_b, name) == 1 + first_tables = snapshot_transfer_tables(node_b, name) + + overflow = + {:snapshot_chunk, version, stream_id, snapshot_seq, 1, [first_row, second_row], []} + + deliver_frames(node_b, node_a, name, [overflow]) + assert snapshot_transfer_count(node_b, name) == 0 + assert snapshot_staging_tables(node_b, name) == [] + + duplicate = + {:snapshot_chunk, version, stream_id, snapshot_seq, 1, [first_row, first_row], []} + + deliver_frames(node_b, node_a, name, [duplicate]) + assert snapshot_transfer_count(node_b, name) == 0 + assert snapshot_staging_tables(node_b, name) == [] + + deliver_frames(node_b, node_a, name, [hd(frames)]) + assert snapshot_transfer_count(node_b, name) == 1 + assert snapshot_transfer_tables(node_b, name) == first_tables + end + + test "an incomplete current-authority snapshot expires without touching visible state", + context do + %{name: name, node_a: node_a, node_b: node_b} = + start_pair(context, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 250 + ) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + entries = + for index <- 1..8 do + key = "snapshot/expiry/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("x", 160) + })} + end + + TestCluster.flush_shards(node_a, name) + stream_id = local_stream(node_a, name, nil) + frames = capture_snapshot(node_a, node_b, name, stream_id, 1) + assert length(frames) > 1 + + deliver_frames(node_b, node_a, name, [hd(frames)]) + assert snapshot_transfer_count(node_b, name) == 1 + assert replica_cursor(node_b, name, stream_id) == 0 + + assert Enum.all?(entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) + + TestCluster.assert_eventually( + fn -> snapshot_transfer_count(node_b, name) == 0 end, + timeout: 2_000, + interval: 25 + ) + + assert replica_cursor(node_b, name, stream_id) == 0 + + assert Enum.all?(entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) + end + + test "nodedown immediately destroys partial staging owned by the retired source", context do + %{name: name, node_a: node_a, node_b: node_b} = start_pair(context) + on_exit(fn -> TestCluster.reconnect_nodes(node_a, node_b) end) + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + for index <- 1..4 do + TestCluster.spawn_register(node_a, name, "snapshot/nodedown/#{index}", %{ + payload: String.duplicate("n", 160) + }) + end + + TestCluster.flush_shards(node_a, name) + stream_id = local_stream(node_a, name, nil) + frames = capture_snapshot(node_a, node_b, name, stream_id, 1) + assert length(frames) > 1 + deliver_frames(node_b, node_a, name, [hd(frames)]) + assert snapshot_transfer_count(node_b, name) == 1 + + TestCluster.disconnect_nodes(node_a, node_b) + + TestCluster.assert_eventually(fn -> + node_a not in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end) + + assert snapshot_transfer_count(node_b, name) == 0 + assert snapshot_staging_tables(node_b, name) == [] + end + + test "an authority epoch change fences partial chunks and their staging expires", context do + cluster = "snapshot-epoch" + + %{name: name, node_a: node_a, node_b: node_b} = + start_pair(context, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 250 + ) + + :ok = TestCluster.rpc!(node_a, Group, :connect, [name, cluster]) + :ok = TestCluster.rpc!(node_b, Group, :connect, [name, cluster]) + + TestCluster.assert_eventually(fn -> + length(TestCluster.rpc!(node_a, Group, :nodes, [name, cluster])) == 2 and + length(TestCluster.rpc!(node_b, Group, :nodes, [name, cluster])) == 2 + end) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + old_entries = + for index <- 1..8 do + key = "snapshot/epoch/#{index}" + + {key, + TestCluster.spawn_register_in_cluster( + node_a, + name, + key, + %{payload: String.duplicate("e", 160)}, + cluster + )} + end + + TestCluster.flush_shards(node_a, name) + old_stream = local_stream(node_a, name, cluster) + old_epoch = Group.Replica.WireProtocol.stream_epoch(old_stream) + frames = capture_snapshot(node_a, node_b, name, old_stream, 1) + assert length(frames) > 1 + {partial, [last]} = Enum.split(frames, -1) + deliver_frames(node_b, node_a, name, partial) + assert snapshot_transfer_count(node_b, name) == 1 + + :ok = TestCluster.rpc!(node_a, Group, :disconnect, [name, cluster]) + :ok = TestCluster.rpc!(node_a, Group, :connect, [name, cluster]) + + TestCluster.assert_eventually(fn -> + epoch = + TestCluster.rpc!(node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + node_a, + cluster + ]) + + not is_nil(epoch) and epoch != old_epoch + end) + + # Replay the entire retired snapshot. A receiver that checks authority only + # when the transfer was first created would now stage and commit it. + deliver_frames(node_b, node_a, name, partial ++ [last]) + TestCluster.flush_shards(node_b, name) + + assert Enum.all?(old_entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key, [cluster: cluster]]) == nil + end) + + assert replica_cursor(node_b, name, old_stream) == 0 + + TestCluster.assert_eventually( + fn -> snapshot_transfer_count(node_b, name) == 0 end, + timeout: 2_000, + interval: 25 + ) + end + + test "an origin restart fences a partial old generation and commits the new exact snapshot", + context do + %{name: name, node_a: node_a, node_b: node_b, opts: opts} = + start_pair(context, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 2_000 + ) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + old_entries = + for index <- 1..8 do + key = "snapshot/restart/old/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("o", 160) + })} + end + + TestCluster.flush_shards(node_a, name) + old_stream = local_stream(node_a, name, nil) + old_frames = capture_snapshot(node_a, node_b, name, old_stream, 1) + assert length(old_frames) > 1 + + {old_partial, _old_tail} = Enum.split(old_frames, -1) + deliver_frames(node_b, node_a, name, old_partial) + assert snapshot_transfer_count(node_b, name) == 1 + assert replica_cursor(node_b, name, old_stream) == 0 + + supervisor = TestCluster.rpc!(node_a, Process, :whereis, [:"#{name}_group_sup"]) + :ok = TestCluster.rpc!(node_a, Supervisor, :stop, [supervisor, :normal, 5_000]) + {:ok, _pid} = TestCluster.start_group(node_a, opts) + + new_stream = local_stream(node_a, name, nil) + refute new_stream == old_stream + new_generation = Group.Replica.WireProtocol.stream_generation(new_stream) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_b, Group.Replica.Data, :remote_generation, [name, node_a]) == + new_generation + end) + + assert snapshot_transfer_count(node_b, name) == 0 + assert snapshot_staging_tables(node_b, name) == [] + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + new_entries = + for index <- 1..8 do + key = "snapshot/restart/new/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("n", 160) + })} + end + + TestCluster.flush_shards(node_a, name) + new_frames = capture_snapshot(node_a, node_b, name, new_stream, 1) + assert length(new_frames) > 1 + + deliver_frames(node_b, node_a, name, Enum.reverse(new_frames)) + new_snapshot_seq = new_frames |> hd() |> elem(3) + + assert replica_cursor(node_b, name, new_stream) == new_snapshot_seq + + assert Enum.all?(new_entries, fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + + assert Enum.all?(old_entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) + + # Replaying every old-generation chunk after the new exact commit must not + # resurrect the old slice or advance its cursor. + deliver_frames(node_b, node_a, name, old_frames) + assert replica_cursor(node_b, name, old_stream) == 0 + assert replica_cursor(node_b, name, new_stream) == new_snapshot_seq + + assert Enum.all?(new_entries, fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + + assert Enum.all?(old_entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) + + TestCluster.assert_eventually( + fn -> snapshot_transfer_count(node_b, name) == 0 end, + timeout: 5_000, + interval: 25 + ) + end + + test "a receiver shard crash destroys partial staging and anti-entropy rebuilds exactly", + context do + %{name: name, node_a: node_a, node_b: node_b} = + start_pair(context, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000 + ) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + entries = + for index <- 1..8 do + key = "snapshot/crash/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("c", 160) + })} + end + + TestCluster.flush_shards(node_a, name) + stream_id = local_stream(node_a, name, nil) + frames = capture_snapshot(node_a, node_b, name, stream_id, 1) + assert length(frames) > 1 + deliver_frames(node_b, node_a, name, [hd(frames)]) + + {old_shard, staging_info_after_crash} = + TestCluster.rpc!(node_b, TestCluster, :kill_shard_with_snapshot_staging, [name, 0]) + + assert staging_info_after_crash == :undefined + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) do + pid when is_pid(pid) -> pid != old_shard + nil -> false + end + end) + + assert snapshot_transfer_count(node_b, name) == 0 + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + TestCluster.assert_eventually( + fn -> + Enum.all?(entries, fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + end, + timeout: 10_000, + interval: 25 + ) + + assert :ok = TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + test "authority fanout tolerates a sibling that is not registered", context do + name = :"authority_startup_fanout_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 2, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + for node <- [context.node_a, context.node_b, context.node_c] do + {:ok, _pid} = TestCluster.start_group(node, opts) + end + + TestCluster.assert_eventually(fn -> + context.node_b in TestCluster.rpc!(context.node_a, Group, :nodes, [name]) + end) + + replica_supervisor = + TestCluster.rpc!(context.node_b, Process, :whereis, [:"#{name}_replica_sup"]) + + :ok = TestCluster.rpc!(context.node_b, :sys, :suspend, [replica_supervisor]) + + on_exit(fn -> + TestCluster.rpc!(context.node_b, Group.TestCluster, :resume_if_alive, [replica_supervisor]) + end) + + sibling = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + sibling_monitor = Process.monitor(sibling) + TestCluster.rpc!(context.node_b, Process, :exit, [sibling, :kill]) + assert_receive {:DOWN, ^sibling_monitor, :process, ^sibling, :killed}, 5_000 + + assert TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) == + nil + + source_control = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + target_control = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + :ok = TestCluster.rpc!(context.node_a, Group, :connect, [name, "late-lane-authority"]) + + {generation, revision, epochs} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_authority, [name]) + + control_monitor = Process.monitor(target_control) + + send( + target_control, + {:replica_hello, source_control, Group.Replica.WireProtocol.version(), generation, revision, + epochs, Group.Transport.DistErl.id(), Group.Transport.DistErl.descriptor(name, [])} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + refute_receive {:DOWN, ^control_monitor, :process, ^target_control, _reason}, 250 + assert TestCluster.rpc!(context.node_b, Process, :alive?, [target_control]) + + :ok = TestCluster.rpc!(context.node_b, :sys, :resume, [replica_supervisor]) + + TestCluster.assert_eventually(fn -> + sibling = + TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, 1) + ]) + + view_generation = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 1, + context.node_a + ]) + + view_exact_revision = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_view_cluster_epoch_revision, + [name, 1, context.node_a] + ) + + exact_revision = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_exact_revision, + [name, context.node_a] + ) + + is_pid(sibling) and view_generation == generation and + view_exact_revision == exact_revision + end) + end + + test "a bounded transport makes forward progress across a snapshot larger than its window", + context do + %{name: name, node_a: node_a, node_b: node_b} = + start_pair(context, + replicated_sender_buffer_size: 1, + replicated_oplog_max_entries: 2, + replicated_snapshot_chunk_target_bytes: 700, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + entries = + for index <- 1..20 do + key = "snapshot/window/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("w", 160) + })} + end + + TestCluster.flush_shards(node_a, name) + stream_id = local_stream(node_a, name, nil) + + {chunk_count, registry_count, pg_count} = + drive_snapshot_chunks_until_commit_pending(node_a, node_b, name, stream_id, 100) + + TestCluster.flush_shards(node_b, name) + {received, manifest} = snapshot_transfer_progress(node_b, name) + assert received == chunk_count + assert manifest == nil + assert registry_count == length(entries) + assert pg_count == 0 + assert replica_cursor(node_b, name, stream_id) == 0 + + assert Enum.all?(entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) + + # The sender retained only the tiny terminal manifest after backpressure. + # Its next repair attempt sends that commit directly without rescanning or + # retransmitting the already staged chunks. + request_snapshot_window(node_a, node_b, name, stream_id, 1) + + TestCluster.assert_eventually( + fn -> + Enum.all?(entries, fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + end, + timeout: 5_000, + interval: 25 + ) + + assert snapshot_transfer_count(node_b, name) == 0 + assert :ok = TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + defp start_pair(context, extra_opts \\ []) do + name = :"snapshot_chunks_#{System.unique_integer([:positive])}" + + opts = + Keyword.merge( + [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_oplog_max_entries: 2, + replicated_snapshot_chunk_target_bytes: 700, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ], + extra_opts + ) + + for node <- [context.node_a, context.node_b, context.node_c] do + {:ok, _pid} = TestCluster.start_group(node, opts) + end + + TestCluster.assert_eventually(fn -> + context.node_b in TestCluster.rpc!(context.node_a, Group, :nodes, [name]) and + context.node_c in TestCluster.rpc!(context.node_a, Group, :nodes, [name]) and + context.node_a in TestCluster.rpc!(context.node_b, Group, :nodes, [name]) and + context.node_a in TestCluster.rpc!(context.node_c, Group, :nodes, [name]) + end) + + %{ + name: name, + node_a: context.node_a, + node_b: context.node_b, + node_c: context.node_c, + opts: opts + } + end + + defp capture_snapshot(node_a, node_b, name, stream_id, next_seq) do + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :clear_captured, [name]) + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:capture_drop, [:snapshot_chunk, :snapshot_commit]} + ]) + + :ok = + TestCluster.rpc!(node_a, Group.Transport, :incoming, [ + name, + node_b, + 0, + {:needs, Group.Replica.WireProtocol.version(), [{stream_id, next_seq}]} + ]) + + TestCluster.assert_eventually( + fn -> + captured = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) + + snapshot_send = + TestCluster.rpc!(node_a, :sys, :get_state, [Group.Replica.shard_name(name, 0)]).snapshot_send + + has_chunk? = + Enum.any?(captured, fn + {^node_b, 0, {:snapshot_chunk, _, ^stream_id, _, _, _, _}} -> true + _ -> false + end) + + has_commit? = + Enum.any?(captured, fn + {^node_b, 0, {:snapshot_commit, _, ^stream_id, _, _, _, _}} -> true + _ -> false + end) + + has_chunk? and has_commit? and is_nil(snapshot_send) + end, + timeout: 5_000, + interval: 10 + ) + + messages = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) + |> Enum.flat_map(fn + {^node_b, 0, {:snapshot_chunk, _, ^stream_id, _, _, _, _} = frame} -> [frame] + {^node_b, 0, {:snapshot_commit, _, ^stream_id, _, _, _, _} = frame} -> [frame] + _other -> [] + end) + + chunks = + messages + |> Enum.filter(&(elem(&1, 0) == :snapshot_chunk)) + |> Enum.sort_by(&elem(&1, 4)) + + [commit] = Enum.filter(messages, &(elem(&1, 0) == :snapshot_commit)) + chunks ++ [commit] + end + + defp capture_snapshot_with_commit(node_a, node_b, name, stream_id, next_seq) do + frames = capture_snapshot(node_a, node_b, name, stream_id, next_seq) + {Enum.drop(frames, -1), List.last(frames)} + end + + defp request_snapshot_window(node_a, node_b, name, stream_id, limit) do + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:accept_types_up_to, [:snapshot_chunk, :snapshot_commit], limit} + ]) + + :ok = + TestCluster.rpc!(node_a, Group.Transport, :incoming, [ + name, + node_b, + 0, + {:needs, Group.Replica.WireProtocol.version(), [{stream_id, 1}]} + ]) + + source = TestCluster.rpc!(node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + _state = TestCluster.rpc!(node_a, :sys, :get_state, [source]) + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!(node_a, :sys, :get_state, [source]).snapshot_send == nil + end, + timeout: 5_000, + interval: 10 + ) + end + + defp snapshot_transfer_progress(node, name) do + state = + TestCluster.rpc!(node, :sys, :get_state, [Group.Replica.shard_name(name, 0)]) + + [{_key, transfer}] = Map.to_list(state.snapshot_transfers) + {MapSet.size(transfer.received), transfer.manifest} + end + + defp drive_snapshot_chunks_until_commit_pending( + node_a, + node_b, + name, + stream_id, + attempts_left + ) + when attempts_left > 0 do + request_snapshot_window(node_a, node_b, name, stream_id, 1) + TestCluster.flush_shards(node_b, name) + + state = + TestCluster.rpc!(node_a, :sys, :get_state, [Group.Replica.shard_name(name, 0)]) + + case Map.values(state.snapshot_send_offsets) do + [{:commit, manifest}] -> + manifest + + [{:chunk, _next_index}] -> + drive_snapshot_chunks_until_commit_pending( + node_a, + node_b, + name, + stream_id, + attempts_left - 1 + ) + end + end + + defp deliver_frames(node_b, node_a, name, frames) do + Enum.each(frames, fn frame -> + :ok = + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ + name, + node_a, + 0, + frame + ]) + end) + + TestCluster.flush_shards(node_b, name) + end + + defp local_stream(node, name, cluster) do + TestCluster.rpc!(node, Group.Replica.Data, :local_stream_id, [name, 0, cluster]) + end + + defp replica_cursor(node, name, stream_id) do + TestCluster.rpc!(node, Group.Replica.Data, :replica_cursor, [name, 0, stream_id]) + end + + defp snapshot_transfer_count(node, name) do + TestCluster.rpc!(node, :erlang, :map_size, [ + TestCluster.rpc!(node, :sys, :get_state, [Group.Replica.shard_name(name, 0)]).snapshot_transfers + ]) + end + + defp snapshot_transfer_tables(node, name) do + state = + TestCluster.rpc!(node, :sys, :get_state, [Group.Replica.shard_name(name, 0)]) + + [{_key, transfer}] = Map.to_list(state.snapshot_transfers) + {transfer.table, transfer.events} + end + + defp snapshot_staging_tables(node, name) do + TestCluster.rpc!(node, TestCluster, :snapshot_staging_tables, [name, 0]) + end +end diff --git a/test/replica_snapshot_test.exs b/test/replica_snapshot_test.exs new file mode 100644 index 0000000..957b232 --- /dev/null +++ b/test/replica_snapshot_test.exs @@ -0,0 +1,159 @@ +defmodule Group.ReplicaSnapshotTest do + use ExUnit.Case, async: true + + alias Group.Replica.Snapshot + + test "streams a complete exact slice in byte-bounded chunks without retaining prior chunks" do + pid = self() + metadata = %{payload: String.duplicate("x", 96)} + + registry_rows = + for index <- 1..80 do + {"registry/#{index}", pid, metadata, index} + end + + pg_rows = + for index <- 1..80 do + {"pg/#{index}", pid, metadata, index} + end + + target = 2_048 + stream_id = {:group, node(), make_ref(), 0, nil, make_ref()} + envelope = Snapshot.stream_envelope_bytes(stream_id, 123) + owner = self() + + emit = fn registry, pg, index -> + send(owner, {:chunk, index, registry, pg}) + :ok + end + + stream = Snapshot.new_stream(target, envelope, 1, emit) + stream = Snapshot.stream_registry_many(registry_rows, stream) + stream = Snapshot.stream_pg_many(pg_rows, stream) + stream = Snapshot.finish_stream(stream) + + assert stream.registry_count == 80 + assert stream.pg_count == 80 + assert stream.chunk_count > 1 + assert stream.registry == [] + assert stream.pg == [] + + chunks = + for index <- 1..stream.chunk_count do + assert_receive {:chunk, ^index, registry, pg} + + {registry, pg} + end + + assert chunks |> Enum.flat_map(&elem(&1, 0)) == registry_rows + assert chunks |> Enum.flat_map(&elem(&1, 1)) == pg_rows + + Enum.with_index(chunks, 1) + |> Enum.each(fn {{registry, pg}, index} -> + frame = + {:snapshot_chunk, Group.Replica.WireProtocol.version(), stream_id, 123, index, registry, + pg} + + assert :erlang.external_size(frame) <= target + end) + end + + test "streams an empty exact slice and permits one intrinsically oversized row" do + owner = self() + + emit = fn registry, pg, index -> + send(owner, {:chunk, index, registry, pg}) + :ok + end + + empty = Snapshot.new_stream(1_024, 512, 1, emit) |> Snapshot.finish_stream() + assert empty.chunk_count == 1 + assert_receive {:chunk, 1, [], []} + + row = {"large", self(), String.duplicate("x", 4_096), 1} + large = Snapshot.new_stream(1_024, 512, 1, emit) + large = Snapshot.stream_registry_many([row], large) |> Snapshot.finish_stream() + assert large.chunk_count == 1 + assert_receive {:chunk, 1, [^row], []} + end + + test "staging is set-valued across chunks and remains private to its owner" do + table = Snapshot.new_staging_table() + registry = {"registry", self(), %{v: 1}, 1} + pg = {"pg", self(), %{v: 2}, 2} + + assert :ok = Snapshot.stage_rows(table, 1, [registry], [pg]) + assert :ets.info(table, :size) == 3 + assert {:error, :duplicate_row} = Snapshot.stage_rows(table, 2, [registry], []) + + assert Snapshot.fold_registry(table, 1, [], fn row, acc -> [row | acc] end) == [registry] + assert Snapshot.fold_pg(table, 1, [], fn row, acc -> [row | acc] end) == [pg] + assert Snapshot.member_pg?(table, "pg", self()) + + assert :ok = Snapshot.delete_staging_table(table) + assert :ets.info(table) == :undefined + end + + test "a resumed stream recounts the exact snapshot but emits only the requested suffix" do + pid = self() + metadata = %{payload: String.duplicate("x", 96)} + + registry_rows = + for index <- 1..80 do + {"registry/#{index}", pid, metadata, index} + end + + target = 2_048 + stream_id = {:group, node(), make_ref(), 0, nil, make_ref()} + envelope = Snapshot.stream_envelope_bytes(stream_id, 123) + + emit_all = fn _registry, _pg, index -> + send(self(), {:first_pass, index}) + :ok + end + + first = Snapshot.new_stream(target, envelope, 1, emit_all) + first = Snapshot.stream_registry_many(registry_rows, first) |> Snapshot.finish_stream() + assert first.chunk_count > 3 + + for index <- 1..first.chunk_count do + assert_receive {:first_pass, ^index} + end + + emit_suffix = fn registry, pg, index -> + send(self(), {:suffix, index, registry, pg}) + :ok + end + + resumed = Snapshot.new_stream(target, envelope, 3, emit_suffix) + resumed = Snapshot.stream_registry_many(registry_rows, resumed) |> Snapshot.finish_stream() + + assert resumed.chunk_count == first.chunk_count + assert resumed.registry_count == 80 + refute_receive {:suffix, 1, _, _} + refute_receive {:suffix, 2, _, _} + + for index <- 3..resumed.chunk_count do + assert_receive {:suffix, ^index, _, _} + end + end + + test "event buffering preserves every event across bounded ETS chunks" do + table = Snapshot.new_event_table() + events = Enum.map(1..1_025, &{:event, &1}) + + buffer = Snapshot.new_event_buffer(table) + buffer = Snapshot.buffer_events(events, buffer) + buffer = Snapshot.finish_event_buffer(buffer) + + assert buffer.events == [] + assert buffer.count == 0 + assert buffer.chunk_index == 3 + assert :ets.info(table, :size) == 3 + + assert Snapshot.fold_events(table, [], fn event, acc -> [event | acc] end) + |> Enum.reverse() == events + + assert :ok = Snapshot.delete_staging_table(table) + end +end diff --git a/test/replica_transport_outbox_test.exs b/test/replica_transport_outbox_test.exs new file mode 100644 index 0000000..bec5990 --- /dev/null +++ b/test/replica_transport_outbox_test.exs @@ -0,0 +1,217 @@ +defmodule Group.ReplicaTransportOutboxTest do + use ExUnit.Case, async: true + + alias Group.Transport.Outbox + + defmodule Backend do + @behaviour Outbox + + @impl true + def init_outbox(group, shard, opts) do + {:ok, + %{ + controller: Keyword.fetch!(opts, :controller), + group: group, + shard: shard, + result: Keyword.get(opts, :backend_result, :ok), + sleep: Keyword.get(opts, :backend_sleep, 0) + }} + end + + @impl true + def send_batch(target_node, messages, deadline, state) do + send( + state.controller, + {:outbox_batch, state.group, state.shard, target_node, messages, deadline} + ) + + if state.sleep > 0, do: Process.sleep(state.sleep) + {state.result, state} + end + end + + test "batches messages per target while preserving per-target order" do + group = unique_group(:batch) + target_a = :"outbox-a@test" + target_b = :"outbox-b@test" + + start_outboxes(group, + outbox_batch_size: 3, + outbox_flush_interval: 1_000 + ) + + assert :ok = Outbox.push(group, target_a, 0, {:message, 1}, outbox_deadline: 1_000) + assert :ok = Outbox.push(group, target_b, 0, {:message, 2}, outbox_deadline: 1_000) + assert :ok = Outbox.push(group, target_a, 0, {:message, 3}, outbox_deadline: 1_000) + + batches = + for _ <- 1..2, into: %{} do + assert_receive {:outbox_batch, ^group, 0, target, messages, deadline}, 1_000 + assert deadline > Outbox.monotonic_ms() + {target, messages} + end + + assert batches == %{ + target_a => [{:message, 1}, {:message, 3}], + target_b => [{:message, 2}] + } + end + + test "a blocked backend never blocks the Group-facing local send" do + group = unique_group(:blocked) + target = :"outbox-blocked@test" + + start_outboxes(group, + outbox_batch_size: 1, + backend_sleep: 200 + ) + + assert :ok = Outbox.push(group, target, 0, :first, outbox_deadline: 1_000) + assert_receive {:outbox_batch, ^group, 0, ^target, [:first], _deadline}, 1_000 + + caller = self() + + spawn(fn -> + result = Outbox.push(group, target, 0, :expires_behind_backend, outbox_deadline: 10) + send(caller, {:push_returned, result}) + end) + + assert_receive {:push_returned, :ok}, 100 + refute_receive {:outbox_batch, ^group, 0, ^target, [:expires_behind_backend], _deadline}, 300 + end + + test "bounded admission drops excess messages before the local mailbox grows" do + group = unique_group(:bounded_admission) + target = :"outbox-bounded@test" + + start_outboxes(group, + outbox_max_messages: 8, + outbox_batch_size: 8, + outbox_flush_interval: 1_000 + ) + + outbox = Process.whereis(Outbox.name(group, 0)) + :ok = :sys.suspend(outbox) + + results = + for message <- 1..32 do + Outbox.push(group, target, 0, message, outbox_deadline: 1_000) + end + + assert Enum.count(results, &(&1 == :ok)) == 8 + assert Enum.count(results, &(&1 == :busy)) == 24 + assert {:message_queue_len, 8} = Process.info(outbox, :message_queue_len) + + :ok = :sys.resume(outbox) + end + + test "expired messages and backend backpressure are dropped without local retries" do + expired_group = unique_group(:expired) + target = :"outbox-expired@test" + + start_outboxes(expired_group, + outbox_flush_interval: 50 + ) + + assert :ok = + Outbox.push(expired_group, target, 0, :expired, outbox_deadline: 5) + + refute_receive {:outbox_batch, ^expired_group, 0, ^target, [:expired], _deadline}, 100 + + busy_group = unique_group(:busy) + + start_outboxes(busy_group, + outbox_batch_size: 1, + backend_result: :busy + ) + + assert :ok = Outbox.push(busy_group, target, 0, :busy, outbox_deadline: 1_000) + assert_receive {:outbox_batch, ^busy_group, 0, ^target, [:busy], _deadline}, 1_000 + refute_receive {:outbox_batch, ^busy_group, 0, ^target, [:busy], _deadline}, 100 + end + + test "complete incoming batches use one local mailbox operation" do + group = unique_group(:deliver) + source_node = :"outbox-source@test" + parent = self() + shard_name = Group.Replica.shard_name(group, 0) + + receiver = + spawn(fn -> + Process.register(self(), shard_name) + send(parent, :receiver_ready) + + receive do + message -> send(parent, {:receiver_message, message}) + end + end) + + assert_receive :receiver_ready + + assert :ok = + Group.Transport.incoming_batch( + group, + source_node, + 0, + [{:heads, 1, []}, {:needs, 1, []}] + ) + + assert_receive {:receiver_message, + {:group_replica_batch, ^source_node, [{:heads, 1, []}, {:needs, 1, []}]}} + + refute Process.alive?(receiver) + end + + test "incoming messages are dropped when the destination shard is unavailable" do + group = unique_group(:missing_ingress) + source_node = :"missing-ingress@test" + + assert :disconnected = Group.Transport.incoming(group, source_node, 0, {:heads, 1, []}) + + assert :disconnected = + Group.Transport.incoming_batch(group, source_node, 0, [ + {:heads, 1, []}, + {:needs, 1, []} + ]) + end + + test "invalid outbox deadlines are rejected when the outbox supervisor boots" do + group = unique_group(:invalid_deadline) + + assert_raise ArgumentError, ~r/:outbox_deadline/, fn -> + Group.Transport.Outbox.Supervisor.init( + name: group, + num_shards: 1, + backend: Backend, + controller: self(), + outbox_deadline: 0 + ) + end + + assert_raise ArgumentError, ~r/:outbox_max_messages/, fn -> + Group.Transport.Outbox.Supervisor.init( + name: group, + num_shards: 1, + backend: Backend, + controller: self(), + outbox_max_messages: 0 + ) + end + end + + defp start_outboxes(group, opts) do + base = [ + name: group, + num_shards: 1, + backend: Backend, + controller: self(), + outbox_deadline: 100 + ] + + start_supervised!(Outbox.child_spec(Keyword.merge(base, opts))) + end + + defp unique_group(suffix) do + :"outbox_#{suffix}_#{System.unique_integer([:positive])}" + end +end diff --git a/test/support/controlled_replica_transport.ex b/test/support/controlled_replica_transport.ex new file mode 100644 index 0000000..aecef9b --- /dev/null +++ b/test/support/controlled_replica_transport.ex @@ -0,0 +1,49 @@ +defmodule Group.ControlledReplicaTransport do + @moduledoc false + @behaviour Group.Transport + + @impl true + def id, do: :group_controlled_replica_transport + + @impl true + def descriptor(_group, _opts), do: :group_controlled_replica_transport + + def set_mode(group, mode) when mode in [:capture, :pass, :busy, :drop] do + :persistent_term.put({__MODULE__, group, :mode}, mode) + :ok + end + + def clear(group) do + :persistent_term.erase({__MODULE__, group, :mode}) + :ok + end + + @impl true + def outgoing(group, target_node, shard, message, opts) do + case :persistent_term.get({__MODULE__, group, :mode}, :capture) do + :capture -> + controller = Keyword.fetch!(opts, :controller) + send(controller, {__MODULE__, :message, group, node(), target_node, shard, message}) + :ok + + :pass -> + forward(group, target_node, shard, message) + + :busy -> + :busy + + :drop -> + :ok + end + end + + defp forward(group, target_node, shard, replica_message) do + destination = {Group.Replica.shard_name(group, shard), target_node} + message = {:group_replica_frame, node(), replica_message} + + case :erlang.send_nosuspend(destination, message, [:noconnect]) do + true -> :ok + false -> :busy + end + end +end diff --git a/test/support/cyclic_conflict_resolver.ex b/test/support/cyclic_conflict_resolver.ex new file mode 100644 index 0000000..ba25950 --- /dev/null +++ b/test/support/cyclic_conflict_resolver.ex @@ -0,0 +1,19 @@ +defmodule Group.CyclicConflictResolver do + @moduledoc false + + # New contract: rank one claim. Erlang term ordering plus Group's PID + # tiebreaker makes max/2 associative and independent of arrival order. + def resolve(_name, _key, {_pid, %{rank: rank}, _time}), do: rank + + # Old contract: a deterministic but cyclic pairwise choice. This exists only + # so the regression demonstrates why arbitrary pairwise resolvers are unsafe. + def resolve(_name, _key, {pid1, %{rank: rank1}, _time1}, {pid2, %{rank: rank2}, _time2}) do + if beats?(rank1, rank2), do: pid1, else: pid2 + end + + defp beats?(:a, :b), do: true + defp beats?(:b, :c), do: true + defp beats?(:c, :a), do: true + defp beats?(rank, rank), do: true + defp beats?(_left, _right), do: false +end diff --git a/test/support/model_conflict_resolver.ex b/test/support/model_conflict_resolver.ex new file mode 100644 index 0000000..58ed671 --- /dev/null +++ b/test/support/model_conflict_resolver.ex @@ -0,0 +1,5 @@ +defmodule Group.ModelConflictResolver do + @moduledoc false + + def resolve(_name, _key, {_pid, meta, _time}), do: Map.fetch!(meta, :rank) +end diff --git a/test/support/pausing_conflict_resolver.ex b/test/support/pausing_conflict_resolver.ex new file mode 100644 index 0000000..16e3cfd --- /dev/null +++ b/test/support/pausing_conflict_resolver.ex @@ -0,0 +1,19 @@ +defmodule Group.PausingConflictResolver do + @moduledoc false + + # Test-only deterministic scheduling point. The resolver still returns the + # normal total rank; a marked claim merely waits until the test releases the + # replica shard that is evaluating it. + def resolve(_name, key, {pid, %{rank: rank} = meta, _time}, controller) do + if Map.get(meta, :pause, false) do + ref = make_ref() + send(controller, {:conflict_resolver_waiting, self(), ref, key, pid}) + + receive do + {:continue_conflict_resolution, ^ref} -> :ok + end + end + + rank + end +end diff --git a/test/support/replica_lifecycle_model.ex b/test/support/replica_lifecycle_model.ex new file mode 100644 index 0000000..abb0dc1 --- /dev/null +++ b/test/support/replica_lifecycle_model.ex @@ -0,0 +1,223 @@ +defmodule Group.ReplicaLifecycleModel do + @moduledoc """ + Independent application-level oracle for replica convergence tests. + + This model deliberately knows nothing about Group's oplog, receive cursors, + batching, authority tables, or wire frames. It records only operations that + the public API accepted and the owner lifecycle consequences that follow. + """ + + defstruct owners: %{}, + registrations: %{}, + memberships: MapSet.new(), + seen_registration_keys: MapSet.new(), + seen_membership_keys: MapSet.new() + + @type owner_id :: non_neg_integer() + @type cluster :: term() + @type key :: binary() + @type scoped_key :: {cluster(), key()} + @type owner :: %{node: atom(), alive?: boolean()} + @type t :: %__MODULE__{ + owners: %{optional(owner_id()) => owner()}, + registrations: %{optional(scoped_key()) => %{optional(owner_id()) => map()}}, + memberships: MapSet.t({cluster(), key(), owner_id(), map()}), + seen_registration_keys: MapSet.t(scoped_key()), + seen_membership_keys: MapSet.t(scoped_key()) + } + + def new, do: %__MODULE__{} + + def owner(%__MODULE__{} = model, owner_id), do: Map.get(model.owners, owner_id) + + def put_owner(%__MODULE__{} = model, owner_id, node) do + case owner(model, owner_id) do + nil -> + put_in(model.owners[owner_id], %{node: node, alive?: true}) + + %{node: ^node} -> + model + + %{node: other_node} -> + raise ArgumentError, + "logical owner #{inspect(owner_id)} moved from #{inspect(other_node)} to #{inspect(node)}" + end + end + + def record_register(%__MODULE__{} = model, owner_id, key, meta, :ok) do + model + |> ensure_alive!(owner_id) + |> Map.update!(:seen_registration_keys, &MapSet.put(&1, key)) + |> Map.update!(:registrations, fn registrations -> + Map.update(registrations, key, %{owner_id => meta}, &Map.put(&1, owner_id, meta)) + end) + end + + def record_register(%__MODULE__{} = model, _owner_id, key, _meta, {:error, :taken}) do + Map.update!(model, :seen_registration_keys, &MapSet.put(&1, key)) + end + + def record_unregister(%__MODULE__{} = model, owner_id, key, :ok) do + registrations = + model.registrations + |> Map.update(key, %{}, &Map.delete(&1, owner_id)) + |> drop_empty_claim_sets() + + %{model | registrations: registrations} + end + + def record_unregister(%__MODULE__{} = model, _owner_id, _key, {:error, _reason}), do: model + + def record_join(%__MODULE__{} = model, owner_id, {cluster, key} = scope, meta, :ok) do + model + |> ensure_alive!(owner_id) + |> Map.update!(:seen_membership_keys, &MapSet.put(&1, scope)) + |> Map.update!(:memberships, fn memberships -> + memberships + |> Enum.reject(fn {member_cluster, member_key, member_owner, _old_meta} -> + member_cluster == cluster and member_key == key and member_owner == owner_id + end) + |> MapSet.new() + |> MapSet.put({cluster, key, owner_id, meta}) + end) + end + + def record_leave(%__MODULE__{} = model, owner_id, {cluster, key}, :ok) do + memberships = + model.memberships + |> Enum.reject(fn {member_cluster, member_key, member_owner, _meta} -> + member_cluster == cluster and member_key == key and member_owner == owner_id + end) + |> MapSet.new() + + %{model | memberships: memberships} + end + + def record_leave(%__MODULE__{} = model, _owner_id, _key, {:error, _reason}), do: model + + def kill(%__MODULE__{} = model, owner_id) do + case owner(model, owner_id) do + nil -> + model + + owner -> + registrations = + model.registrations + |> Map.new(fn {key, claims} -> {key, Map.delete(claims, owner_id)} end) + |> drop_empty_claim_sets() + + memberships = + model.memberships + |> Enum.reject(fn {_cluster, _key, member_owner, _meta} -> + member_owner == owner_id + end) + |> MapSet.new() + + %{ + model + | owners: Map.put(model.owners, owner_id, %{owner | alive?: false}), + registrations: registrations, + memberships: memberships + } + end + end + + def restart_node(%__MODULE__{} = model, node) do + remove_owned_scope(model, fn owner -> owner.node == node end, fn _scope -> true end) + end + + def disconnect_cluster(%__MODULE__{} = model, node, cluster) do + remove_owned_scope( + model, + fn owner -> owner.node == node end, + fn {entry_cluster, _key} -> entry_cluster == cluster end + ) + end + + def expected_registrations(%__MODULE__{} = model) do + Map.new(model.registrations, fn {key, claims} -> + {owner_id, meta} = + Enum.max_by(claims, fn {owner_id, meta} -> + {Map.get(meta, :rank, owner_id), owner_id} + end) + + {key, {owner_id, meta}} + end) + end + + def expected_memberships(%__MODULE__{} = model) do + model.memberships + |> Enum.group_by( + fn {cluster, key, _owner_id, _meta} -> {cluster, key} end, + fn {_cluster, _key, owner_id, meta} -> {owner_id, meta} end + ) + |> Map.new(fn {key, members} -> {key, Enum.sort(members)} end) + end + + def resolve_registry_conflicts(%__MODULE__{} = model) do + losing_owners = + model.registrations + |> Enum.flat_map(fn {_key, claims} -> + if map_size(claims) > 1 do + {winner, _meta} = + Enum.max_by(claims, fn {owner_id, meta} -> + {Map.get(meta, :rank, owner_id), owner_id} + end) + + Map.keys(claims) -- [winner] + else + [] + end + end) + |> MapSet.new() + + if MapSet.size(losing_owners) == 0 do + model + else + losing_owners + |> Enum.reduce(model, &kill(&2, &1)) + |> resolve_registry_conflicts() + end + end + + defp ensure_alive!(model, owner_id) do + case owner(model, owner_id) do + %{alive?: true} -> model + other -> raise ArgumentError, "owner #{inspect(owner_id)} is not alive: #{inspect(other)}" + end + end + + defp drop_empty_claim_sets(registrations) do + Map.reject(registrations, fn {_key, claims} -> map_size(claims) == 0 end) + end + + defp remove_owned_scope(model, owner_filter, scope_filter) do + owner_ids = + model.owners + |> Enum.filter(fn {_owner_id, owner} -> owner_filter.(owner) end) + |> MapSet.new(&elem(&1, 0)) + + registrations = + model.registrations + |> Map.new(fn {scope, claims} -> + claims = + if scope_filter.(scope) do + Map.reject(claims, fn {owner_id, _meta} -> MapSet.member?(owner_ids, owner_id) end) + else + claims + end + + {scope, claims} + end) + |> drop_empty_claim_sets() + + memberships = + model.memberships + |> Enum.reject(fn {cluster, key, owner_id, _meta} -> + scope_filter.({cluster, key}) and MapSet.member?(owner_ids, owner_id) + end) + |> MapSet.new() + + %{model | registrations: registrations, memberships: memberships} + end +end diff --git a/test/support/replica_model_scheduler.ex b/test/support/replica_model_scheduler.ex new file mode 100644 index 0000000..fc766ff --- /dev/null +++ b/test/support/replica_model_scheduler.ex @@ -0,0 +1,521 @@ +defmodule Group.ReplicaModelScheduler do + @moduledoc false + + alias Group.{ControlledReplicaTransport, ReplicaLifecycleModel, TestCluster} + + defmodule Envelope do + @moduledoc false + defstruct [:id, :source, :target, :shard, :message] + end + + defstruct [:name, :nodes, :model, :group_opts, owners: %{}, queue: [], next_message_id: 1] + + def new(name, nodes, group_opts \\ []) do + %__MODULE__{ + name: name, + nodes: Map.new(nodes), + model: ReplicaLifecycleModel.new(), + group_opts: group_opts + } + end + + def execute(%__MODULE__{} = state, {:register, owner_id, node_id, slot, revision}) do + with_owner(state, owner_id, node_id, fn state, _pid -> + key = registration_key(owner_id, slot) + meta = %{owner: owner_id, rank: owner_id, revision: revision} + result = owner_call(state, owner_id, {:register, state.name, key, meta, []}) + + model = + ReplicaLifecycleModel.record_register(state.model, owner_id, {nil, key}, meta, result) + + sync(%{state | model: model}) + end) + end + + def execute(%__MODULE__{} = state, {:claim, owner_id, node_id, key_slot, rank}) do + with_owner(state, owner_id, node_id, fn state, _pid -> + key = "model/conflict/#{key_slot}" + meta = %{owner: owner_id, rank: rank} + result = owner_call(state, owner_id, {:register, state.name, key, meta, []}) + + model = + ReplicaLifecycleModel.record_register(state.model, owner_id, {nil, key}, meta, result) + + sync(%{state | model: model}) + end) + end + + def execute(%__MODULE__{} = state, {:unregister, owner_id, slot}) do + with_existing_owner(state, owner_id, fn state, _pid -> + key = registration_key(owner_id, slot) + result = owner_call(state, owner_id, {:unregister, state.name, key, []}) + model = ReplicaLifecycleModel.record_unregister(state.model, owner_id, {nil, key}, result) + sync(%{state | model: model}) + end) + end + + def execute(%__MODULE__{} = state, {:join, owner_id, node_id, slot, revision}) do + with_owner(state, owner_id, node_id, fn state, _pid -> + key = membership_key(owner_id, slot) + meta = %{owner: owner_id, revision: revision} + result = owner_call(state, owner_id, {:join, state.name, key, meta, []}) + model = ReplicaLifecycleModel.record_join(state.model, owner_id, {nil, key}, meta, result) + sync(%{state | model: model}) + end) + end + + def execute(%__MODULE__{} = state, {:leave, owner_id, slot}) do + with_existing_owner(state, owner_id, fn state, _pid -> + key = membership_key(owner_id, slot) + result = owner_call(state, owner_id, {:leave, state.name, key, []}) + model = ReplicaLifecycleModel.record_leave(state.model, owner_id, {nil, key}, result) + sync(%{state | model: model}) + end) + end + + def execute( + %__MODULE__{} = state, + {:register_cluster, owner_id, node_id, cluster, slot, revision} + ) do + with_owner(state, owner_id, node_id, fn state, _pid -> + key = cluster_registration_key(owner_id, slot) + meta = %{owner: owner_id, rank: owner_id, revision: revision, cluster: cluster} + opts = [cluster: cluster] + result = owner_call(state, owner_id, {:register, state.name, key, meta, opts}) + scope = {cluster, key} + model = ReplicaLifecycleModel.record_register(state.model, owner_id, scope, meta, result) + sync(%{state | model: model}) + end) + end + + def execute(%__MODULE__{} = state, {:join_cluster, owner_id, node_id, cluster, slot, revision}) do + with_owner(state, owner_id, node_id, fn state, _pid -> + key = cluster_membership_key(owner_id, slot) + meta = %{owner: owner_id, revision: revision, cluster: cluster} + opts = [cluster: cluster] + result = owner_call(state, owner_id, {:join, state.name, key, meta, opts}) + scope = {cluster, key} + model = ReplicaLifecycleModel.record_join(state.model, owner_id, scope, meta, result) + sync(%{state | model: model}) + end) + end + + def execute(%__MODULE__{} = state, {:connect, node_id, cluster}) do + node = Map.fetch!(state.nodes, node_id) + :ok = TestCluster.rpc!(node, Group, :connect, [state.name, cluster]) + sync(state) + end + + def execute(%__MODULE__{} = state, {:disconnect, node_id, cluster}) do + node = Map.fetch!(state.nodes, node_id) + :ok = TestCluster.rpc!(node, Group, :disconnect, [state.name, cluster]) + model = ReplicaLifecycleModel.disconnect_cluster(state.model, node, cluster) + sync(%{state | model: model}) + end + + def execute(%__MODULE__{} = state, {:kill, owner_id}) do + case Map.get(state.owners, owner_id) do + nil -> + state + + %{pid: pid, node: node} -> + if remote_alive?(node, pid) do + true = TestCluster.rpc!(node, Process, :exit, [pid, :kill]) + end + + state + |> Map.update!(:model, &ReplicaLifecycleModel.kill(&1, owner_id)) + |> sync() + end + end + + def execute(%__MODULE__{} = state, {:transport, node_id, mode}) do + node = Map.fetch!(state.nodes, node_id) + :ok = TestCluster.rpc!(node, ControlledReplicaTransport, :set_mode, [state.name, mode]) + state + end + + def execute(%__MODULE__{} = state, {:deliver, selector}) do + state + |> sync() + |> take_envelope(selector, fn state, envelope -> + deliver_envelope(state, envelope, 1) + end) + end + + def execute(%__MODULE__{} = state, {:duplicate, selector}) do + state + |> sync() + |> take_envelope(selector, fn state, envelope -> + deliver_envelope(state, envelope, 2) + end) + end + + def execute(%__MODULE__{} = state, {:drop, selector}) do + state + |> sync() + |> take_envelope(selector, fn state, _envelope -> state end) + end + + def execute(%__MODULE__{} = state, :deliver_all) do + state = sync(state) + envelopes = state.queue + + Enum.reduce(envelopes, %{state | queue: []}, fn envelope, acc -> + deliver_envelope(acc, envelope, 1) + end) + end + + def execute(%__MODULE__{} = state, {:restart, node_id}) do + state = sync(state) + node = Map.fetch!(state.nodes, node_id) + :ok = stop_group_local(node, state.name) + model = ReplicaLifecycleModel.restart_node(state.model, node) + {:ok, _pid} = TestCluster.start_group(node, state.group_opts) + state = %{state | model: model} + + TestCluster.assert_eventually( + fn -> + Enum.all?(state.nodes, fn {_id, peer} -> + expected = map_size(state.nodes) - 1 + length(TestCluster.rpc!(peer, Group, :nodes, [state.name])) == expected + end) + end, + timeout: 10_000, + interval: 25 + ) + + sync(state) + end + + def execute(%__MODULE__{} = state, :anti_entropy) do + Enum.each(state.nodes, fn {_id, node} -> + TestCluster.rpc!(node, __MODULE__, :trigger_anti_entropy_local, [state.name]) + end) + + sync(state) + end + + def execute(%__MODULE__{} = state, :flush), do: sync(state) + + def stabilize_and_assert!(%__MODULE__{} = state) do + state = sync(state) + + Enum.each(state.nodes, fn {_id, node} -> + :ok = TestCluster.rpc!(node, ControlledReplicaTransport, :set_mode, [state.name, :pass]) + end) + + expected = ReplicaLifecycleModel.resolve_registry_conflicts(state.model) + state = %{state | model: expected, queue: []} + + TestCluster.assert_eventually( + fn -> + pump_anti_entropy(state) + converged?(state) + end, + timeout: 15_000, + interval: 25 + ) + + Enum.each(state.nodes, fn {_id, node} -> + :ok = TestCluster.rpc!(node, TestCluster, :assert_replica_consistent, [state.name]) + end) + + assert_expected_owner_lifecycle!(state) + assert_no_dead_retained_owners!(state) + state + end + + def cleanup(%__MODULE__{} = state) do + Enum.each(state.nodes, fn {_id, node} -> + TestCluster.rpc!(node, __MODULE__, :cleanup_owners_local, [state.name]) + TestCluster.rpc!(node, ControlledReplicaTransport, :clear, [state.name]) + stop_group_local(node, state.name) + end) + + drain_transport_messages(state.name) + :ok + end + + def sync(%__MODULE__{} = state) do + Enum.each(state.nodes, fn {_id, node} -> + TestCluster.flush_shards(node, state.name) + end) + + drain(state, 2) + end + + def drain(%__MODULE__{} = state, wait_ms \\ 0) do + receive do + {ControlledReplicaTransport, :message, group, source, target, shard, message} + when group == state.name -> + envelope = %Envelope{ + id: state.next_message_id, + source: source, + target: target, + shard: shard, + message: message + } + + drain( + %{state | queue: state.queue ++ [envelope], next_message_id: state.next_message_id + 1}, + wait_ms + ) + after + wait_ms -> state + end + end + + def trigger_anti_entropy_local(name) do + num_shards = Group.get_config(name).num_shards + + Enum.each(0..(num_shards - 1), fn shard -> + shard_name = Group.Replica.shard_name(name, shard) + state = :sys.get_state(shard_name) + send(shard_name, {:group_replica_anti_entropy, state.anti_entropy_ref}) + end) + + :ok + end + + def spawn_owner(name) do + pid = spawn(fn -> owner_loop() end) + key = {__MODULE__, :owners, name} + owners = :persistent_term.get(key, []) + :persistent_term.put(key, [pid | owners]) + pid + end + + def cleanup_owners_local(name) do + key = {__MODULE__, :owners, name} + + key + |> :persistent_term.get([]) + |> Enum.each(fn pid -> + if Process.alive?(pid), do: Process.exit(pid, :kill) + end) + + :persistent_term.erase(key) + :ok + end + + def call_owner(pid, operation) do + if Process.alive?(pid) do + ref = make_ref() + send(pid, {__MODULE__, :call, self(), ref, operation}) + + receive do + {__MODULE__, :reply, ^ref, result} -> result + after + 5_000 -> raise "model owner #{inspect(pid)} did not answer #{inspect(operation)}" + end + else + {:error, :owner_dead} + end + end + + defp owner_loop do + receive do + {__MODULE__, :call, caller, ref, operation} -> + result = apply_owner_operation(operation) + send(caller, {__MODULE__, :reply, ref, result}) + owner_loop() + end + end + + defp apply_owner_operation({:register, name, key, meta, opts}), + do: Group.register(name, key, meta, opts) + + defp apply_owner_operation({:unregister, name, key, opts}), + do: Group.unregister(name, key, opts) + + defp apply_owner_operation({:join, name, key, meta, opts}), + do: Group.join(name, key, meta, opts) + + defp apply_owner_operation({:leave, name, key, opts}), + do: Group.leave(name, key, opts) + + defp with_owner(state, owner_id, node_id, fun) do + case Map.get(state.owners, owner_id) do + nil -> + node = Map.fetch!(state.nodes, node_id) + pid = TestCluster.rpc!(node, __MODULE__, :spawn_owner, [state.name]) + model = ReplicaLifecycleModel.put_owner(state.model, owner_id, node) + + state = %{ + state + | owners: Map.put(state.owners, owner_id, %{node: node, pid: pid}), + model: model + } + + fun.(state, pid) + + %{node: node, pid: pid} -> + if remote_alive?(node, pid), do: fun.(state, pid), else: state + end + end + + defp with_existing_owner(state, owner_id, fun) do + case Map.get(state.owners, owner_id) do + nil -> + state + + %{node: node, pid: pid} -> + if remote_alive?(node, pid), do: fun.(state, pid), else: state + end + end + + defp owner_call(state, owner_id, operation) do + %{node: node, pid: pid} = Map.fetch!(state.owners, owner_id) + + case TestCluster.rpc!(node, __MODULE__, :call_owner, [pid, operation]) do + {:error, :owner_dead} -> + raise "model owner #{owner_id} died outside an expected lifecycle transition" + + result -> + result + end + end + + defp take_envelope(%{queue: []} = state, _selector, _fun), do: state + + defp take_envelope(state, selector, fun) do + index = rem(selector, length(state.queue)) + {envelope, queue} = List.pop_at(state.queue, index) + fun.(%{state | queue: queue}, envelope) + end + + defp deliver_envelope(state, envelope, times) do + Enum.each(1..times, fn _ -> + :ok = + TestCluster.rpc!( + envelope.target, + Group.Transport, + :incoming, + [state.name, envelope.source, envelope.shard, envelope.message] + ) + + TestCluster.flush_shards(envelope.target, state.name) + end) + + drain(state, 2) + end + + defp pump_anti_entropy(state) do + Enum.each(state.nodes, fn {_id, node} -> + TestCluster.rpc!(node, __MODULE__, :trigger_anti_entropy_local, [state.name]) + end) + + Enum.each(1..2, fn _ -> + Enum.each(state.nodes, fn {_id, node} -> + TestCluster.flush_shards(node, state.name) + end) + end) + end + + defp converged?(state) do + expected_registrations = ReplicaLifecycleModel.expected_registrations(state.model) + expected_memberships = ReplicaLifecycleModel.expected_memberships(state.model) + pid_to_owner = Map.new(state.owners, fn {owner_id, %{pid: pid}} -> {pid, owner_id} end) + + Enum.all?(state.nodes, fn {_node_id, node} -> + registrations_match?( + node, + state.name, + state.model.seen_registration_keys, + expected_registrations, + pid_to_owner + ) and + memberships_match?( + node, + state.name, + state.model.seen_membership_keys, + expected_memberships, + pid_to_owner + ) + end) + end + + defp registrations_match?(node, name, keys, expected, pid_to_owner) do + Enum.all?(keys, fn {cluster, key} = scope -> + actual = + case TestCluster.rpc!(node, Group, :lookup, [name, key, cluster_opts(cluster)]) do + nil -> nil + {pid, meta} -> {Map.get(pid_to_owner, pid, {:unknown_pid, pid}), meta} + end + + actual == Map.get(expected, scope) + end) + end + + defp memberships_match?(node, name, keys, expected, pid_to_owner) do + Enum.all?(keys, fn {cluster, key} = scope -> + actual = + node + |> TestCluster.rpc!(Group, :members, [name, key, cluster_opts(cluster)]) + |> Enum.map(fn {pid, meta} -> {Map.get(pid_to_owner, pid, {:unknown_pid, pid}), meta} end) + |> Enum.sort() + + actual == Map.get(expected, scope, []) + end) + end + + defp assert_no_dead_retained_owners!(state) do + retained = + state.nodes + |> Enum.flat_map(fn {_id, node} -> + TestCluster.rpc!(node, TestCluster, :replica_owner_pids, [state.name]) + end) + |> Enum.uniq() + + dead = + Enum.reject(retained, fn pid -> TestCluster.rpc!(node(pid), Process, :alive?, [pid]) end) + + if dead != [], do: raise("dead owners retained after convergence: #{inspect(dead)}") + end + + defp assert_expected_owner_lifecycle!(state) do + mismatches = + Enum.flat_map(state.model.owners, fn {owner_id, %{alive?: expected_alive?}} -> + %{node: node, pid: pid} = Map.fetch!(state.owners, owner_id) + actual_alive? = remote_alive?(node, pid) + + if expected_alive? == actual_alive? do + [] + else + [{owner_id, expected_alive?, actual_alive?, pid}] + end + end) + + if mismatches != [] do + raise "owner lifecycle diverged from model: #{inspect(mismatches)}" + end + end + + defp remote_alive?(node, pid), do: TestCluster.rpc!(node, Process, :alive?, [pid]) + + defp stop_group_local(node, name) do + case TestCluster.rpc!(node, Process, :whereis, [:"#{name}_group_sup"]) do + nil -> :ok + pid -> TestCluster.rpc!(node, Supervisor, :stop, [pid, :normal, 5_000]) + end + catch + :exit, _ -> :ok + end + + defp drain_transport_messages(name) do + receive do + {ControlledReplicaTransport, :message, ^name, _source, _target, _shard, _message} -> + drain_transport_messages(name) + after + 0 -> :ok + end + end + + defp registration_key(owner_id, slot), do: "model/reg/#{owner_id}/#{slot}" + defp membership_key(owner_id, slot), do: "model/pg/#{owner_id}/#{slot}" + defp cluster_registration_key(owner_id, slot), do: "model/cluster/reg/#{owner_id}/#{slot}" + defp cluster_membership_key(owner_id, slot), do: "model/cluster/pg/#{owner_id}/#{slot}" + defp cluster_opts(nil), do: [] + defp cluster_opts(cluster), do: [cluster: cluster] +end diff --git a/test/support/test_cluster.ex b/test/support/test_cluster.ex index f9b8c1e..1bfc9d2 100644 --- a/test/support/test_cluster.ex +++ b/test/support/test_cluster.ex @@ -5,16 +5,36 @@ defmodule Group.TestCluster do def start_peers(count, opts \\ []) do cookie = Keyword.get(opts, :cookie, Node.get_cookie()) code_paths = :code.get_path() + schedulers = Keyword.get(opts, :schedulers) + + scheduler_args = + if schedulers, do: [~c"+S", ~c"#{schedulers}:#{schedulers}"], else: [] args = - [~c"-setcookie", ~c"#{cookie}", ~c"-kernel", ~c"prevent_overlapping_partitions", ~c"false"] ++ + scheduler_args ++ + [ + ~c"-setcookie", + ~c"#{cookie}", + ~c"-kernel", + ~c"prevent_overlapping_partitions", + ~c"false" + ] ++ Enum.flat_map(code_paths, fn p -> [~c"-pa", p] end) for _i <- 1..count do name = :"peer#{System.unique_integer([:positive])}" + # A fixed inet_dist_listen_min/max inherited through ERL_AFLAGS makes + # every child contend for the parent VM's distribution port. Peer args + # above carry every setting the test nodes require explicitly. {:ok, pid, node} = - :peer.start(%{name: name, host: ~c"127.0.0.1", longnames: true, args: args}) + :peer.start(%{ + name: name, + host: ~c"127.0.0.1", + longnames: true, + args: args, + env: [{~c"ERL_AFLAGS", ~c""}] + }) {:ok, _} = :rpc.call(node, :application, :ensure_all_started, [:elixir]) {:ok, _} = :rpc.call(node, :application, :ensure_all_started, [:group]) @@ -34,6 +54,45 @@ defmodule Group.TestCluster do end) end + @doc false + def resume_if_alive(pid) when is_pid(pid) do + if Process.alive?(pid), do: :sys.resume(pid) + :ok + end + + @doc false + def expire_replica_lane(name, shard, remote_node) do + replica = Process.whereis(Group.Replica.shard_name(name, shard)) + now = System.monotonic_time(:millisecond) + + :sys.replace_state(replica, fn state -> + expired_at = now - state.replicated_peer_lease_timeout - 1 + %{state | peer_last_seen: Map.put(state.peer_last_seen, remote_node, expired_at)} + end) + + state = :sys.get_state(replica) + send(replica, {:group_replica_anti_entropy, state.anti_entropy_ref}) + _state = :sys.get_state(replica) + :ok + end + + @doc false + def put_pending_registry_reprojection(replica, remote_node, cluster, key) do + :sys.replace_state(replica, fn state -> + pending = + Map.update( + state.pending_registry_reprojections, + remote_node, + MapSet.new([{cluster, key}]), + &MapSet.put(&1, {cluster, key}) + ) + + %{state | pending_registry_reprojections: pending} + end) + + :ok + end + @doc "Call a function on a remote node, raise on badrpc" def rpc!(node, mod, fun, args) do case :rpc.call(node, mod, fun, args) do @@ -42,6 +101,19 @@ defmodule Group.TestCluster do end end + @doc false + def spawn_trace_forwarder(node, target_pid) do + :erpc.call(node, fn -> spawn(fn -> forward_trace_messages(target_pid) end) end) + end + + defp forward_trace_messages(target_pid) do + receive do + message -> + send(target_pid, {:forwarded_trace, message}) + forward_trace_messages(target_pid) + end + end + @doc "Start Group on a remote node" def start_group(node, opts) do opts = Keyword.put_new(opts, :log, false) @@ -87,7 +159,7 @@ defmodule Group.TestCluster do spawn(fn -> :ok = Group.register(name, key, meta) send(parent, {:registered, self()}) - Process.sleep(:infinity) + registration_owner_loop(name, key) end) receive do @@ -106,6 +178,26 @@ defmodule Group.TestCluster do end) end + @doc false + def unregister_owner(pid) when is_pid(pid) do + send(pid, {:unregister, self()}) + + receive do + {:unregistered, ^pid, result} -> result + after + 5_000 -> raise "unregister_owner timed out" + end + end + + defp registration_owner_loop(name, key) do + receive do + {:unregister, reply_to} when is_pid(reply_to) -> + result = Group.unregister(name, key) + send(reply_to, {:unregistered, self(), result}) + registration_owner_loop(name, key) + end + end + @doc "Spawn a process on a remote node that joins and sleeps forever. Waits for the join to complete before returning." def spawn_join(node, name, key, meta, opts \\ []) do @@ -129,14 +221,14 @@ defmodule Group.TestCluster do @doc "Spawn a process on a remote node that registers, joins, and sleeps forever. Waits for both operations to complete before returning." - def spawn_register_and_join(node, name, reg_key, reg_meta, join_key, join_meta) do + def spawn_register_and_join(node, name, reg_key, reg_meta, join_key, join_meta, opts \\ []) do :erpc.call(node, fn -> parent = self() pid = spawn(fn -> - :ok = Group.register(name, reg_key, reg_meta) - :ok = Group.join(name, join_key, join_meta) + :ok = Group.register(name, reg_key, reg_meta, opts) + :ok = Group.join(name, join_key, join_meta, opts) send(parent, {:ready, self()}) Process.sleep(:infinity) end) @@ -299,6 +391,92 @@ defmodule Group.TestCluster do end) end + @doc "Spawn a process on a remote node that joins in a named cluster and sleeps." + def spawn_join_in_cluster(node, name, key, meta, cluster) do + :erpc.call(node, fn -> + parent = self() + + pid = + spawn(fn -> + :ok = Group.join(name, key, meta, cluster: cluster) + send(parent, {:joined, self()}) + Process.sleep(:infinity) + end) + + receive do + {:joined, ^pid} -> pid + after + 5000 -> raise "spawn_join_in_cluster timed out" + end + end) + end + + @doc "Connects every cluster through an independent concurrent caller." + def connect_many_concurrently(node, name, clusters) do + :erpc.call(node, __MODULE__, :do_connect_many_concurrently, [name, clusters], 60_000) + end + + @doc false + def do_connect_many_concurrently(name, clusters) do + clusters + |> Task.async_stream( + fn cluster -> Group.connect(name, cluster) end, + max_concurrency: 64, + ordered: false, + timeout: 30_000 + ) + |> Enum.each(fn {:ok, :ok} -> :ok end) + + :ok + end + + @doc "Spawns one long-lived registration owner in each named cluster." + def spawn_register_many_clusters(node, name, clusters, key_prefix) do + :erpc.call( + node, + __MODULE__, + :do_spawn_register_many_clusters, + [name, clusters, key_prefix], + 60_000 + ) + end + + @doc false + def do_spawn_register_many_clusters(name, clusters, key_prefix) do + parent = self() + + entries = + Enum.map(clusters, fn cluster -> + key = "#{key_prefix}/#{cluster}" + + pid = + spawn(fn -> + :ok = Group.register(name, key, %{cluster: cluster}, cluster: cluster) + send(parent, {:registered_many, self()}) + Process.sleep(:infinity) + end) + + {cluster, key, pid} + end) + + Enum.each(entries, fn {_cluster, _key, pid} -> + receive do + {:registered_many, ^pid} -> :ok + after + 30_000 -> raise "spawn_register_many_clusters timed out" + end + end) + + entries + end + + @doc false + def registry_entries_present?(name, entries) do + Enum.all?(entries, fn {cluster, key, pid} -> + match?({^pid, %{cluster: ^cluster}}, Group.lookup(name, key, cluster: cluster)) + end) + end + @doc "Monitor nodedown events from a remote node, forwarding to caller" def monitor_nodes_on(node, target_pid) do :erpc.call(node, fn -> @@ -397,6 +575,42 @@ defmodule Group.TestCluster do end) end + @doc false + def kill_shard_with_snapshot_staging(name, shard_index) do + shard = Process.whereis(Group.Replica.shard_name(name, shard_index)) + state = :sys.get_state(shard) + {_key, transfer} = Enum.at(state.snapshot_transfers, 0) + monitor = Process.monitor(shard) + Process.exit(shard, :kill) + + receive do + {:DOWN, ^monitor, :process, ^shard, :killed} -> + {shard, :ets.info(transfer.table)} + after + 5_000 -> raise "snapshot staging owner did not terminate" + end + end + + @doc false + def snapshot_staging_tables(name, shard_index) do + owner = Process.whereis(Group.Replica.shard_name(name, shard_index)) + + :ets.all() + |> Enum.filter(fn table -> + :ets.info(table, :owner) == owner and :ets.info(table, :size) > 0 + end) + end + + @doc false + def delete_remote_view_info(name, shard_index, remote_node) do + :ets.delete( + Group.Replica.Data.replication_meta_table(name), + {:remote_view_info, shard_index, remote_node} + ) + + :ok + end + @doc "Returns the current message_queue_len for a shard on a remote node." def shard_message_queue_len(node, name, shard) do :erpc.call(node, __MODULE__, :do_shard_message_queue_len, [name, shard]) @@ -553,27 +767,494 @@ defmodule Group.TestCluster do :ok end + @doc """ + Asserts the replica-only authority and journal invariants in addition to the + public dual-index invariants checked by `assert_ets_consistent/1`. + + This is intended for quiescent convergence points in adversarial tests. + """ + def assert_replica_consistent(name) do + :ok = assert_ets_consistent(name) + num_shards = Group.get_config(name).num_shards + + for shard <- 0..(num_shards - 1) do + assert_rows_on_matching_shard(name, shard, num_shards) + assert_registry_claim_indexes(name, shard) + assert_registry_projection_has_authority(name, shard) + assert_registry_claim_authority(name, shard) + assert_pg_row_authority(name, shard) + assert_oplog_indexes(name, shard) + assert_replica_cursor_authority(name, shard) + end + + :ok + end + + defp assert_rows_on_matching_shard(name, shard, num_shards) do + checks = [ + {Group.Replica.Data.reg_by_key_table(name, shard), + fn + {{cluster, key}, _pid, _meta, _time, _origin} -> {cluster, key} + end}, + {Group.Replica.Data.reg_claim_by_key_table(name, shard), + fn + {{cluster, key, _origin, _generation, _epoch}, _pid, _meta, _time, _seq} -> + {cluster, key} + end}, + {Group.Replica.Data.pg_by_key_table(name, shard), + fn + {{cluster, key, _pid}, _meta, _time, _origin} -> {cluster, key} + end} + ] + + Enum.each(checks, fn {table, key_fun} -> + case Enum.find(:ets.tab2list(table), fn row -> + {cluster, key} = key_fun.(row) + Group.Replica.shard_index_for(cluster, key, num_shards) != shard + end) do + nil -> + :ok + + row -> + raise "row stored on wrong shard in #{name} shard #{shard}: #{inspect(row)}" + end + end) + end + + @doc false + def assert_replica_origin_purged(name, origin) do + num_shards = Group.get_config(name).num_shards + + for shard <- 0..(num_shards - 1) do + retained_claims = + Group.Replica.Data.reg_claim_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.filter(fn {{_cluster, _key, row_origin, _generation, _epoch}, _pid, _meta, _time, + _seq} -> + row_origin == origin + end) + + retained_registry = + Group.Replica.Data.reg_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.filter(fn {_key, _pid, _meta, _time, entry_node} -> entry_node == origin end) + + retained_pg = + Group.Replica.Data.pg_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.filter(fn {_key, _meta, _time, entry_node} -> entry_node == origin end) + + retained_cursors = + Group.Replica.Data.replica_cursor_table(name, shard) + |> :ets.tab2list() + |> Enum.filter(fn {stream_id, _seq} -> + Group.Replica.WireProtocol.stream_origin(stream_id) == origin + end) + + retained_view = Group.Replica.Data.remote_view_generation(name, shard, origin) + + unless retained_claims == [] and retained_registry == [] and retained_pg == [] and + retained_cursors == [] and is_nil(retained_view) do + raise "replica origin was not fully purged from #{name} shard #{shard}: " <> + inspect(%{ + claims: retained_claims, + registry: retained_registry, + pg: retained_pg, + cursors: retained_cursors, + view_generation: retained_view + }) + end + end + + unless is_nil(Group.Replica.Data.remote_generation(name, origin)) and + is_nil(Group.Replica.Data.remote_replica_authority_hint(name, origin)) and + Group.Replica.Data.clusters_for_node(name, origin) == [] do + raise "replica origin retained shared authority after purge: #{inspect(origin)}" + end + + :ok + end + + @doc """ + Returns every PID currently retained as replica authority or visible PG state. + + Adversarial tests use this at a quiescent convergence point to prove that no + dead owner remains hidden behind otherwise-consistent dual indexes. + """ + def replica_owner_pids(name) do + num_shards = Group.get_config(name).num_shards + + 0..(num_shards - 1) + |> Enum.flat_map(fn shard -> + registry_pids = + Group.Replica.Data.reg_claim_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.map(fn {{_cluster, _key, _origin, _generation, _epoch}, pid, _meta, _time, _seq} -> + pid + end) + + pg_pids = + Group.Replica.Data.pg_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.map(fn {{_cluster, _key, pid}, _meta, _time, _node} -> pid end) + + registry_pids ++ pg_pids + end) + |> Enum.uniq() + end + + @doc false + def replica_protocol_state(name) do + num_shards = Group.get_config(name).num_shards + + for shard <- 0..(num_shards - 1) do + %{ + shard: shard, + heads: Group.Replica.Data.replica_stream_heads(name, shard), + cursors: + Group.Replica.Data.replica_cursor_table(name, shard) + |> :ets.tab2list() + |> Enum.sort() + } + end + end + + @doc false + def replica_registry_key_state(name, cluster, key) do + num_shards = Group.get_config(name).num_shards + shard = Group.Replica.shard_index_for(cluster, key, num_shards) + + %{ + shard: shard, + projection: Group.Replica.Data.registry_lookup(name, shard, cluster, key), + claims: Group.Replica.Data.registry_claims(name, shard, cluster, key) + } + end + + @doc false + def replica_registry_replication_state(name, cluster, key, origin) do + num_shards = Group.get_config(name).num_shards + shard = Group.Replica.shard_index_for(cluster, key, num_shards) + replica = Process.whereis(Group.Replica.shard_name(name, shard)) + replica_state = :sys.get_state(replica) + local_stream_id = Group.Replica.Data.local_stream_id(name, shard, cluster) + + streams = + Group.Replica.Data.replica_cursor_streams_for_origin_cluster( + name, + shard, + origin, + cluster + ) + + peer_authority = + Node.list() + |> Map.new(fn peer -> + {peer, + %{ + generation: Group.Replica.Data.remote_generation(name, peer), + epoch: Group.Replica.Data.remote_cluster_epoch(name, peer, cluster), + revision: Group.Replica.Data.remote_cluster_epoch_revision(name, peer), + exact: Group.Replica.Data.remote_cluster_epoch_exact_revision(name, peer), + observed: Group.Replica.Data.remote_cluster_epoch_observed_revision(name, peer), + hint: Group.Replica.Data.remote_replica_authority_hint(name, peer), + lane_view: { + Group.Replica.Data.remote_view_generation(name, shard, peer), + Group.Replica.Data.remote_view_cluster_epoch_revision(name, shard, peer), + Group.Replica.Data.remote_view_observed_revision(name, shard, peer) + } + }} + end) + + %{ + key: replica_registry_key_state(name, cluster, key), + origin: origin, + cluster_nodes: Group.nodes(name, cluster), + local_generation: Group.Replica.Data.generation(name), + local_epoch: Group.Replica.Data.local_cluster_epoch(name, cluster), + local_revision: Group.Replica.Data.local_cluster_epoch_revision(name), + local_stream: + if(local_stream_id, + do: + {local_stream_id, + Group.Replica.Data.replica_stream_head(name, shard, local_stream_id)}, + else: nil + ), + peer_authority: peer_authority, + remote_generation: Group.Replica.Data.remote_generation(name, origin), + remote_epoch: Group.Replica.Data.remote_cluster_epoch(name, origin, cluster), + remote_revision: Group.Replica.Data.remote_cluster_epoch_revision(name, origin), + remote_exact_revision: Group.Replica.Data.remote_cluster_epoch_exact_revision(name, origin), + remote_observed_revision: + Group.Replica.Data.remote_cluster_epoch_observed_revision(name, origin), + remote_authority_hint: Group.Replica.Data.remote_replica_authority_hint(name, origin), + lane_view: { + Group.Replica.Data.remote_view_generation(name, shard, origin), + Group.Replica.Data.remote_view_cluster_epoch_revision(name, shard, origin), + Group.Replica.Data.remote_view_observed_revision(name, shard, origin) + }, + cursors: + Enum.map(streams, fn stream_id -> + {stream_id, Group.Replica.Data.replica_cursor(name, shard, stream_id)} + end), + remote_shards: Map.keys(replica_state.remote_shards), + peer_last_seen_nodes: Map.keys(replica_state.peer_last_seen), + peer_last_seen: Map.get(replica_state.peer_last_seen, origin), + pending_reprojection?: + replica_state.pending_registry_reprojections + |> Map.get(origin, MapSet.new()) + |> MapSet.member?({cluster, key}) + } + end + + defp assert_registry_claim_indexes(name, shard) do + by_key = Group.Replica.Data.reg_claim_by_key_table(name, shard) + by_pid = Group.Replica.Data.reg_claim_by_pid_table(name, shard) + + key_set = + :ets.tab2list(by_key) + |> MapSet.new(fn + {{cluster, key, origin, generation, epoch}, pid, meta, time, seq} -> + {cluster, key, pid, meta, time, origin, generation, epoch, seq} + end) + + pid_set = + :ets.tab2list(by_pid) + |> MapSet.new(fn + {{pid, cluster, key, origin, generation, epoch}, meta, time, seq} -> + {cluster, key, pid, meta, time, origin, generation, epoch, seq} + end) + + if key_set != pid_set do + raise "registry claim index inconsistency in #{name} shard #{shard}: " <> + "by_key_only=#{inspect(MapSet.difference(key_set, pid_set) |> MapSet.to_list())} " <> + "by_pid_only=#{inspect(MapSet.difference(pid_set, key_set) |> MapSet.to_list())}" + end + + case Enum.find(key_set, fn {_cluster, _key, pid, _meta, _time, origin, _gen, _epoch, _seq} -> + node(pid) != origin + end) do + nil -> :ok + invalid -> raise "registry claim has invalid origin authority: #{inspect(invalid)}" + end + end + + defp assert_registry_projection_has_authority(name, shard) do + claim_rows = + Group.Replica.Data.reg_claim_by_key_table(name, shard) + |> :ets.tab2list() + + resolver = Map.get(Group.get_config(name), :resolve_registry_conflict) + + expected = + claim_rows + |> Enum.group_by(fn + {{cluster, key, _origin, _generation, _epoch}, _pid, _meta, _time, _seq} -> + {cluster, key} + end) + |> MapSet.new(fn {{cluster, key}, claims} -> + {{^cluster, ^key, origin, _generation, _epoch}, pid, meta, time, _seq} = + Enum.max_by(claims, fn + {{^cluster, ^key, _origin, _generation, _epoch}, claim_pid, claim_meta, claim_time, + _seq} -> + registry_claim_order_key( + name, + key, + {claim_pid, claim_meta, claim_time}, + resolver + ) + end) + + {cluster, key, pid, meta, time, origin} + end) + + visible = + Group.Replica.Data.reg_by_key_table(name, shard) + |> :ets.tab2list() + |> MapSet.new(fn {{cluster, key}, pid, meta, time, origin} -> + {cluster, key, pid, meta, time, origin} + end) + + if visible != expected do + raise "registry projection does not match the deterministic claim winner in #{name} " <> + "shard #{shard}: expected=#{inspect(MapSet.to_list(expected))} " <> + "visible=#{inspect(MapSet.to_list(visible))}" + end + end + + defp registry_claim_order_key(_name, _key, {pid, _meta, time}, nil), do: {time, pid} + + defp registry_claim_order_key(name, key, {pid, _meta, _time} = claim, { + mod, + func, + extra_args + }) do + {apply(mod, func, [name, key, claim | extra_args]), pid} + end + + defp assert_registry_claim_authority(name, shard) do + Group.Replica.Data.reg_claim_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.each(fn + {{cluster, key, origin, generation, epoch}, pid, _meta, _time, seq} = claim -> + valid? = + if origin == node() do + generation == Group.Replica.Data.generation(name) and + epoch == Group.Replica.Data.local_cluster_epoch(name, cluster) + else + generation == Group.Replica.Data.remote_generation(name, origin) and + remote_lane_current?(name, shard, origin) and + epoch == Group.Replica.Data.remote_cluster_epoch(name, origin, cluster) + end + + unless valid? and node(pid) == origin and seq > 0 do + raise "registry claim is not fenced by current authority in #{name} shard #{shard}: " <> + inspect({claim, key}) + end + end) + end + + defp assert_pg_row_authority(name, shard) do + Group.Replica.Data.pg_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.each(fn {{cluster, key, pid}, _meta, _time, origin} = row -> + valid? = + node(pid) == origin and + if origin == node() do + not is_nil(Group.Replica.Data.local_cluster_epoch(name, cluster)) + else + generation = Group.Replica.Data.remote_generation(name, origin) + + not is_nil(generation) and + remote_lane_current?(name, shard, origin) and + not is_nil(Group.Replica.Data.remote_cluster_epoch(name, origin, cluster)) + end + + unless valid? do + raise "PG row is not fenced by current authority in #{name} shard #{shard}: " <> + inspect({row, key}) + end + end) + end + + defp assert_oplog_indexes(name, shard) do + oplog = + Group.Replica.Data.replica_oplog_table(name, shard) + |> :ets.tab2list() + |> MapSet.new(fn {{stream_id, seq}, append_id, _mutations} -> + {append_id, stream_id, seq} + end) + + order = + Group.Replica.Data.replica_oplog_order_table(name, shard) + |> :ets.tab2list() + |> MapSet.new() + + if oplog != order do + raise "oplog/order index inconsistency in #{name} shard #{shard}: " <> + "oplog_only=#{inspect(MapSet.difference(oplog, order) |> MapSet.to_list())} " <> + "order_only=#{inspect(MapSet.difference(order, oplog) |> MapSet.to_list())}" + end + + max_entries = Group.get_config(name).replicated_oplog_max_entries + + if MapSet.size(order) > max_entries do + raise "oplog bound exceeded in #{name} shard #{shard}: " <> + "size=#{MapSet.size(order)} max=#{max_entries}" + end + + Group.Replica.Data.replica_stream_meta_table(name, shard) + |> :ets.tab2list() + |> Enum.each(fn {stream_id, head, floor, applied} -> + unless floor >= 1 and floor <= head + 1 and applied >= 0 and applied <= head do + raise "invalid stream bounds in #{name} shard #{shard}: " <> + inspect({stream_id, head, floor, applied}) + end + + retained = + oplog + |> Enum.filter(fn {_append_id, row_stream, _seq} -> row_stream == stream_id end) + |> Enum.map(&elem(&1, 2)) + |> Enum.sort() + + expected = if floor <= head, do: Enum.to_list(floor..head), else: [] + + if retained != expected do + raise "non-contiguous retained oplog in #{name} shard #{shard}: " <> + "stream=#{inspect(stream_id)} retained=#{inspect(retained)} " <> + "expected=#{inspect(expected)}" + end + end) + end + + defp assert_replica_cursor_authority(name, shard) do + Group.Replica.Data.replica_cursor_table(name, shard) + |> :ets.tab2list() + |> Enum.each(fn {stream_id, seq} -> + origin = Group.Replica.WireProtocol.stream_origin(stream_id) + generation = Group.Replica.WireProtocol.stream_generation(stream_id) + cluster = Group.Replica.WireProtocol.stream_cluster(stream_id) + epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) + + valid? = + Group.Replica.WireProtocol.stream_name(stream_id) == name and + Group.Replica.WireProtocol.stream_shard(stream_id) == shard and + origin != node() and + generation == Group.Replica.Data.remote_generation(name, origin) and + remote_lane_current?(name, shard, origin) and + epoch == Group.Replica.Data.remote_cluster_epoch(name, origin, cluster) and + seq >= 0 + + unless valid? do + raise "replica cursor is not fenced by current authority in #{name} shard #{shard}: " <> + inspect({stream_id, seq}) + end + end) + end + + defp remote_lane_current?(name, shard, origin) do + generation = Group.Replica.Data.remote_generation(name, origin) + observed = Group.Replica.Data.remote_cluster_epoch_observed_revision(name, origin) + + Group.Replica.Data.remote_replica_authority_hint(name, origin) == + {generation, observed} and + Group.Replica.Data.remote_cluster_epoch_revision(name, origin) == observed and + Group.Replica.Data.remote_view_generation(name, shard, origin) == generation and + Group.Replica.Data.remote_view_cluster_epoch_revision(name, shard, origin) == + Group.Replica.Data.remote_cluster_epoch_exact_revision(name, origin) and + Group.Replica.Data.remote_view_observed_revision(name, shard, origin) == + observed + end + @doc "Wait for a condition to become true, with retries" def assert_eventually(fun, opts \\ []) do timeout = Keyword.get(opts, :timeout, 2000) interval = Keyword.get(opts, :interval, 50) + diagnostic = Keyword.get(opts, :diagnostic) deadline = System.monotonic_time(:millisecond) + timeout - do_assert_eventually(fun, interval, deadline) + do_assert_eventually(fun, interval, deadline, diagnostic) end - defp do_assert_eventually(fun, interval, deadline) do + defp do_assert_eventually(fun, interval, deadline, diagnostic) do case fun.() do true -> true false -> if System.monotonic_time(:millisecond) >= deadline do - raise "assert_eventually timed out" + details = + if is_function(diagnostic, 0) do + "\ndiagnostic: " <> inspect(diagnostic.(), pretty: true, limit: :infinity) + else + "" + end + + raise "assert_eventually timed out#{details}" end Process.sleep(interval) - do_assert_eventually(fun, interval, deadline) + do_assert_eventually(fun, interval, deadline, diagnostic) end end end diff --git a/test/support/test_conflict_resolver.ex b/test/support/test_conflict_resolver.ex index ce41c34..c75e4e3 100644 --- a/test/support/test_conflict_resolver.ex +++ b/test/support/test_conflict_resolver.ex @@ -4,10 +4,5 @@ defmodule Group.TestConflictResolver do # A compiled module for conflict resolution that records calls to an ETS table. # Must be in test/support/ so it's compiled to beam and available on peer nodes. - def resolve(_name, _key, {pid1, _meta1, time1}, {pid2, _meta2, time2}) do - # Keep the one with the higher time (more recent wins). - # On equal timestamps, use pid ordering for a deterministic tiebreaker - # that doesn't depend on which node is resolving (avoids mutual kill). - if time2 > time1 or (time2 == time1 and pid2 > pid1), do: pid2, else: pid1 - end + def resolve(_name, _key, {_pid, _meta, time}), do: time end diff --git a/test/support/test_replica_transport.ex b/test/support/test_replica_transport.ex new file mode 100644 index 0000000..4821fc6 --- /dev/null +++ b/test/support/test_replica_transport.ex @@ -0,0 +1,198 @@ +defmodule Group.TestReplicaTransport do + @moduledoc false + @behaviour Group.Transport + + @impl true + def id, do: :group_test_transport + + @impl true + def descriptor(_group, _opts), do: :group_test_transport + + @simple_modes [:pass, :drop, :busy, :duplicate] + + def set_mode(group, mode) + when mode in @simple_modes or + (is_tuple(mode) and tuple_size(mode) == 2 and + elem(mode, 0) in [:drop_types, :duplicate_types, :capture_drop, :capture_pass]) or + (is_tuple(mode) and tuple_size(mode) == 3 and elem(mode, 0) == :delay_types) or + (is_tuple(mode) and tuple_size(mode) == 3 and + elem(mode, 0) == :accept_types_up_to) or + (is_tuple(mode) and tuple_size(mode) == 3 and + elem(mode, 0) == :capture_drop_pause_once) or + (is_tuple(mode) and tuple_size(mode) == 2 and elem(mode, 0) == :chaos) do + :persistent_term.put({__MODULE__, group}, mode) + + if is_tuple(mode) and tuple_size(mode) == 3 and elem(mode, 0) == :accept_types_up_to do + :persistent_term.put({__MODULE__, group, :accepted}, 0) + end + + if is_tuple(mode) and tuple_size(mode) == 3 and elem(mode, 0) == :capture_drop_pause_once do + :persistent_term.erase({__MODULE__, group, :paused}) + end + + :ok + end + + def captured(group) do + :persistent_term.get({__MODULE__, group, :captured}, []) |> Enum.reverse() + end + + def clear_captured(group) do + :persistent_term.erase({__MODULE__, group, :captured}) + :ok + end + + def clear(group) do + :persistent_term.erase({__MODULE__, group}) + :persistent_term.erase({__MODULE__, group, :captured}) + :persistent_term.erase({__MODULE__, group, :accepted}) + :persistent_term.erase({__MODULE__, group, :paused}) + :ok + end + + @impl true + def outgoing(group, target_node, shard, message, _opts) do + case :persistent_term.get({__MODULE__, group}, :pass) do + :drop -> + :ok + + :busy -> + :busy + + :duplicate -> + forward(group, target_node, shard, message) + forward(group, target_node, shard, message) + + {:drop_types, types} -> + if message_type(message) in types, + do: :ok, + else: forward(group, target_node, shard, message) + + {:duplicate_types, types} -> + if message_type(message) in types do + forward(group, target_node, shard, message) + forward(group, target_node, shard, message) + else + forward(group, target_node, shard, message) + end + + {:delay_types, delays, default_delay} -> + delay = Map.get(delays, message_type(message), default_delay) + delayed_forward(group, target_node, shard, message, delay) + + {:accept_types_up_to, types, limit} + when is_list(types) and is_integer(limit) and limit > 0 -> + if message_type(message) in types do + key = {__MODULE__, group, :accepted} + accepted = :persistent_term.get(key, 0) + + if accepted < limit do + :persistent_term.put(key, accepted + 1) + forward(group, target_node, shard, message) + else + :busy + end + else + forward(group, target_node, shard, message) + end + + {:capture_drop, types} -> + if message_type(message) in types, do: capture(group, target_node, shard, message) + :ok + + {:capture_pass, types} -> + if message_type(message) in types, do: capture(group, target_node, shard, message) + forward(group, target_node, shard, message) + + {:capture_drop_pause_once, types, observer} -> + if message_type(message) in types do + capture(group, target_node, shard, message) + pause_once(group, message_type(message), observer) + end + + :ok + + {:chaos, opts} -> + chaos_forward(group, target_node, shard, message, opts) + + :pass -> + forward(group, target_node, shard, message) + end + end + + defp pause_once(group, message_type, observer) do + key = {__MODULE__, group, :paused} + + if :persistent_term.get(key, false) do + :ok + else + :persistent_term.put(key, true) + send(observer, {:replica_transport_paused, self(), group, message_type}) + + receive do + {:resume_replica_transport, ^group} -> :ok + after + 5_000 -> :ok + end + end + end + + defp chaos_forward(group, target_node, shard, message, opts) do + hash = :erlang.phash2({target_node, shard, message}, 1_000_003) + drop_every = Keyword.get(opts, :drop_every, 0) + duplicate_every = Keyword.get(opts, :duplicate_every, 0) + max_delay = Keyword.get(opts, :max_delay, 0) + + cond do + drop_every > 0 and rem(hash, drop_every) == 0 -> + :ok + + duplicate_every > 0 and rem(hash, duplicate_every) == 0 -> + delay = if max_delay > 0, do: rem(hash, max_delay + 1), else: 0 + delayed_forward(group, target_node, shard, message, delay) + delayed_forward(group, target_node, shard, message, max(max_delay - delay, 0)) + + true -> + delay = if max_delay > 0, do: rem(hash, max_delay + 1), else: 0 + delayed_forward(group, target_node, shard, message, delay) + end + end + + defp delayed_forward(group, target_node, shard, message, delay) when delay <= 0, + do: forward(group, target_node, shard, message) + + defp delayed_forward(group, target_node, shard, message, delay) do + source_node = node() + + spawn(fn -> + receive do + after + delay -> forward(group, target_node, shard, message, source_node) + end + end) + + :ok + end + + defp capture(group, target_node, shard, message) do + key = {__MODULE__, group, :captured} + captured = :persistent_term.get(key, []) + :persistent_term.put(key, [{target_node, shard, message} | captured]) + end + + defp forward(group, target_node, shard, message), + do: forward(group, target_node, shard, message, node()) + + defp forward(group, target_node, shard, replica_message, source_node) do + destination = {Group.Replica.shard_name(group, shard), target_node} + message = {:group_replica_frame, source_node, replica_message} + + case :erlang.send_nosuspend(destination, message, [:noconnect]) do + true -> :ok + false -> :busy + end + end + + defp message_type(message) when is_tuple(message), do: elem(message, 0) + defp message_type(_message), do: :unknown +end diff --git a/test/support/test_tcp_transport.ex b/test/support/test_tcp_transport.ex new file mode 100644 index 0000000..82e8ca7 --- /dev/null +++ b/test/support/test_tcp_transport.ex @@ -0,0 +1,498 @@ +defmodule Group.TestTCPTransport do + @moduledoc false + + use GenServer + + @behaviour Group.Transport + @behaviour Group.Transport.Outbox + + alias Group.Transport.Outbox + + @impl true + def id, do: :group_test_sideband_tcp_v2 + + @impl true + def child_spec(opts) do + name = Keyword.fetch!(opts, :name) + + %{ + id: {__MODULE__, name}, + start: {Group.TestTCPTransport.Supervisor, :start_link, [opts]}, + type: :supervisor, + restart: :permanent, + shutdown: :infinity + } + end + + def start_link(opts) do + name = Keyword.fetch!(opts, :name) + GenServer.start_link(__MODULE__, opts, name: server_name(name)) + end + + @impl true + def descriptor(group, _opts) do + :persistent_term.get({__MODULE__, group, :descriptor}) + end + + @impl true + def outgoing(group, target_node, shard, message, opts), + do: Outbox.push(group, target_node, shard, message, opts) + + @impl Group.Transport.Outbox + def init_outbox(group, shard, _opts), do: {:ok, %{group: group, shard: shard}} + + @impl Group.Transport.Outbox + def send_batch(target_node, messages, deadline, %{group: group, shard: shard} = state) do + result = + try do + case :ets.lookup(route_table(group), target_node) do + [{^target_node, writer, queued, max_queue}] -> + if :atomics.add_get(queued, 1, 1) <= max_queue do + send(writer, {:replica_batch, deadline, shard, messages}) + :ok + else + :atomics.sub(queued, 1, 1) + :busy + end + + [] -> + :disconnected + end + rescue + ArgumentError -> :disconnected + end + + {result, state} + end + + @impl true + def peer_up(group, remote_node, shard, descriptor, _opts) do + send_manager(group, {:peer_up, remote_node, shard, descriptor}) + end + + @impl true + def peer_down(group, remote_node, shard, _opts) do + send_manager(group, {:peer_down, remote_node, shard}) + end + + @doc false + def disconnect_peer(group, remote_node) do + GenServer.call(server_name(group), {:disable_peer, remote_node}) + end + + @doc false + def reconnect_peer(group, remote_node) do + GenServer.call(server_name(group), {:enable_peer, remote_node}) + end + + @doc false + def connected?(group, remote_node) do + :ets.member(route_table(group), remote_node) + end + + @doc false + def status(group) do + GenServer.call(server_name(group), :status) + end + + @impl true + def init(opts) do + Process.flag(:trap_exit, true) + group = Keyword.fetch!(opts, :name) + ip = Keyword.get(opts, :ip, {127, 0, 0, 1}) + advertised_ip = Keyword.get(opts, :advertised_ip, ip) + port = Keyword.get(opts, :port, 0) + + {:ok, listener} = + :gen_tcp.listen(port, [ + :binary, + packet: 4, + active: false, + reuseaddr: true, + ip: ip + ]) + + {:ok, {_listen_ip, listen_port}} = :inet.sockname(listener) + + capability = :erlang.term_to_binary({node(), make_ref(), System.unique_integer()}) + descriptor = {:group_test_sideband_tcp_v2, advertised_ip, listen_port, capability} + :persistent_term.put({__MODULE__, group, :descriptor}, descriptor) + + :ets.new(route_table(group), [ + :named_table, + :public, + :set, + read_concurrency: true, + write_concurrency: true + ]) + + manager = self() + acceptor = spawn_link(fn -> accept_loop(listener, manager, group, capability) end) + + {:ok, + %{ + group: group, + listener: listener, + acceptor: acceptor, + peers: %{}, + writers: %{}, + inbound: %{}, + disabled: MapSet.new(), + max_queue: Keyword.get(opts, :max_queue, 1_024), + connect_timeout: Keyword.get(opts, :connect_timeout, 1_000), + send_timeout: Keyword.get(opts, :send_timeout, 1_000), + reconnect_interval: Keyword.get(opts, :reconnect_interval, 50) + }} + end + + @impl true + def handle_info({:peer_up, remote_node, shard, descriptor}, state) do + peer = + state.peers + |> Map.get(remote_node, %{descriptor: descriptor, lanes: MapSet.new()}) + |> Map.put(:descriptor, descriptor) + |> Map.update!(:lanes, &MapSet.put(&1, shard)) + + state = %{state | peers: Map.put(state.peers, remote_node, peer)} + + state = + if MapSet.member?(state.disabled, remote_node) do + state + else + ensure_writer(state, remote_node) + end + + {:noreply, state} + end + + def handle_info({:peer_down, remote_node, shard}, state) do + case Map.get(state.peers, remote_node) do + nil -> + {:noreply, state} + + peer -> + lanes = MapSet.delete(peer.lanes, shard) + + if MapSet.size(lanes) == 0 do + {:noreply, drop_peer(state, remote_node, true)} + else + peer = %{peer | lanes: lanes} + {:noreply, %{state | peers: Map.put(state.peers, remote_node, peer)}} + end + end + end + + def handle_info({:writer_ready, remote_node, writer, queued}, state) do + if Map.get(state.writers, remote_node) == writer and + not MapSet.member?(state.disabled, remote_node) do + :ets.insert( + route_table(state.group), + {remote_node, writer, queued, state.max_queue} + ) + end + + {:noreply, state} + end + + def handle_info({:writer_failed, remote_node, writer}, state) do + {:noreply, writer_failed(state, remote_node, writer)} + end + + def handle_info({:reader_ready, source_node, reader}, state) do + {:noreply, %{state | inbound: Map.put(state.inbound, source_node, reader)}} + end + + def handle_info({:reconnect, remote_node}, state) do + {:noreply, ensure_writer(state, remote_node)} + end + + def handle_info({:EXIT, pid, _reason}, %{acceptor: pid} = state) do + {:stop, :acceptor_stopped, state} + end + + def handle_info({:EXIT, writer, _reason}, state) do + case Enum.find(state.writers, fn {_node, pid} -> pid == writer end) do + {remote_node, ^writer} -> + {:noreply, writer_failed(state, remote_node, writer)} + + nil -> + {:noreply, state} + end + end + + @impl true + def handle_call({:disable_peer, remote_node}, _from, state) do + state = %{state | disabled: MapSet.put(state.disabled, remote_node)} + {:reply, :ok, drop_writer(state, remote_node)} + end + + def handle_call({:enable_peer, remote_node}, _from, state) do + state = %{state | disabled: MapSet.delete(state.disabled, remote_node)} + {:reply, :ok, ensure_writer(state, remote_node)} + end + + def handle_call(:status, _from, state) do + {:reply, + %{ + peers: Map.keys(state.peers), + writers: Map.keys(state.writers), + connected: :ets.tab2list(route_table(state.group)) |> Enum.map(&elem(&1, 0)), + inbound: state.inbound, + disabled: MapSet.to_list(state.disabled) + }, state} + end + + @impl true + def terminate(_reason, state) do + :persistent_term.erase({__MODULE__, state.group, :descriptor}) + :gen_tcp.close(state.listener) + :ok + end + + defp send_manager(group, message) do + case Process.whereis(server_name(group)) do + nil -> + :ok + + pid -> + send(pid, message) + :ok + end + end + + defp ensure_writer(state, remote_node) do + cond do + MapSet.member?(state.disabled, remote_node) -> + state + + Map.has_key?(state.writers, remote_node) -> + state + + peer = Map.get(state.peers, remote_node) -> + manager = self() + + writer = + spawn_link(fn -> + writer_connect( + manager, + state.group, + remote_node, + peer.descriptor, + state.connect_timeout, + state.send_timeout + ) + end) + + %{state | writers: Map.put(state.writers, remote_node, writer)} + + true -> + state + end + end + + defp writer_failed(state, remote_node, writer) do + if Map.get(state.writers, remote_node) == writer do + :ets.delete(route_table(state.group), remote_node) + state = %{state | writers: Map.delete(state.writers, remote_node)} + + if Map.has_key?(state.peers, remote_node) and + not MapSet.member?(state.disabled, remote_node) do + Process.send_after(self(), {:reconnect, remote_node}, state.reconnect_interval) + end + + state + else + state + end + end + + defp drop_peer(state, remote_node, remove_descriptor?) do + state = drop_writer(state, remote_node) + + if remove_descriptor? do + %{state | peers: Map.delete(state.peers, remote_node)} + else + state + end + end + + defp drop_writer(state, remote_node) do + :ets.delete(route_table(state.group), remote_node) + + case Map.pop(state.writers, remote_node) do + {nil, writers} -> + %{state | writers: writers} + + {writer, writers} -> + Process.exit(writer, :shutdown) + %{state | writers: writers} + end + end + + defp writer_connect( + manager, + group, + remote_node, + {:group_test_sideband_tcp_v2, host, port, capability}, + connect_timeout, + send_timeout + ) do + opts = [ + :binary, + packet: 4, + active: false, + send_timeout: send_timeout, + send_timeout_close: true + ] + + case :gen_tcp.connect(host, port, opts, connect_timeout) do + {:ok, socket} -> + case :gen_tcp.send(socket, :erlang.term_to_binary({:hello, group, node(), capability})) do + :ok -> + queued = :atomics.new(1, signed: false) + send(manager, {:writer_ready, remote_node, self(), queued}) + writer_loop(socket, manager, remote_node, queued) + + {:error, _reason} -> + :gen_tcp.close(socket) + send(manager, {:writer_failed, remote_node, self()}) + end + + {:error, _reason} -> + send(manager, {:writer_failed, remote_node, self()}) + end + end + + defp writer_connect(manager, _group, remote_node, _descriptor, _connect_timeout, _send_timeout) do + send(manager, {:writer_failed, remote_node, self()}) + end + + defp writer_loop(socket, manager, remote_node, queued) do + receive do + {:replica_batch, deadline, shard, messages} -> + result = + if deadline <= Outbox.monotonic_ms() do + :expired + else + :gen_tcp.send(socket, :erlang.term_to_binary({:batch, shard, messages})) + end + + :atomics.sub(queued, 1, 1) + + case result do + result when result in [:ok, :expired] -> + writer_loop(socket, manager, remote_node, queued) + + {:error, _reason} -> + :gen_tcp.close(socket) + send(manager, {:writer_failed, remote_node, self()}) + end + end + end + + defp accept_loop(listener, manager, group, capability) do + case :gen_tcp.accept(listener) do + {:ok, socket} -> + reader = + spawn(fn -> + receive do + {:accepted_socket, accepted} -> + reader_handshake(accepted, manager, group, capability) + end + end) + + :ok = :gen_tcp.controlling_process(socket, reader) + send(reader, {:accepted_socket, socket}) + accept_loop(listener, manager, group, capability) + + {:error, :closed} -> + :ok + + {:error, _reason} -> + send(manager, {:EXIT, self(), :accept_failed}) + end + end + + defp reader_handshake(socket, manager, group, capability) do + with {:ok, payload} <- :gen_tcp.recv(socket, 0), + {:ok, {:hello, ^group, source_node, ^capability}} <- decode(payload), + true <- is_atom(source_node) do + send(manager, {:reader_ready, source_node, self()}) + reader_loop(socket, group, source_node) + else + _ -> :gen_tcp.close(socket) + end + end + + defp reader_loop(socket, group, source_node) do + case :gen_tcp.recv(socket, 0) do + {:ok, payload} -> + case decode_authenticated_frame(payload) do + {:ok, {:batch, shard, messages}} + when is_integer(shard) and shard >= 0 and is_list(messages) -> + case Group.Transport.incoming_batch(group, source_node, shard, messages) do + result when result in [:ok, :disconnected] -> + reader_loop(socket, group, source_node) + end + + _ -> + :gen_tcp.close(socket) + end + + {:error, _reason} -> + :ok + end + end + + defp decode(payload) do + {:ok, :erlang.binary_to_term(payload, [:safe])} + rescue + ArgumentError -> :error + end + + # The capability handshake above establishes the same trusted-cluster + # boundary as Erlang distribution. Replica metadata is an arbitrary BEAM + # term and may legitimately contain atoms not yet loaded on this node. + defp decode_authenticated_frame(payload) do + {:ok, :erlang.binary_to_term(payload)} + rescue + ArgumentError -> :error + end + + defp server_name(group), do: :"#{group}_replica_tcp_transport" + defp route_table(group), do: :"#{group}_replica_tcp_routes" +end + +defmodule Group.TestTCPTransport.Supervisor do + @moduledoc false + use Supervisor + + alias Group.TestTCPTransport, as: TCP + alias Group.Transport.Outbox + + def start_link(opts), do: Supervisor.start_link(__MODULE__, opts) + + @impl true + def init(opts) do + group = Keyword.fetch!(opts, :name) + + manager = %{ + id: {TCP, group, :manager}, + start: {TCP, :start_link, [opts]}, + type: :worker, + restart: :transient, + shutdown: 5_000, + significant: true + } + + outboxes = + opts + |> Keyword.put(:backend, TCP) + |> Outbox.child_spec() + + Supervisor.init([manager, outboxes], + strategy: :rest_for_one, + auto_shutdown: :any_significant + ) + end +end